diff --git a/.gitmodules b/.gitmodules index 006735f5..e58ad745 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,9 @@ [submodule "lib/tarantool-python"] path = lib/tarantool-python url = https://github.com/tarantool/tarantool-python.git +[submodule "lib/checks"] + path = lib/checks + url = https://github.com/tarantool/checks.git +[submodule "lib/luatest"] + path = lib/luatest + url = https://github.com/tarantool/luatest.git diff --git a/.luacheckrc b/.luacheckrc index 8df2789f..bdb965a5 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -1,4 +1,4 @@ -globals = {"box", "_TARANTOOL", "tonumber64"} +globals = {"box", "_TARANTOOL", "tonumber64", "os"} ignore = { -- Accessing an undefined field of a global variable . "143/debug", @@ -27,4 +27,6 @@ include_files = { exclude_files = { "lib/tarantool-python", "test/test-tarantool/*.test.lua", + "lib/luatest/**", + "lib/checks/**", } diff --git a/lib/__init__.py b/lib/__init__.py index dcfe149e..449f653a 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -6,6 +6,7 @@ from lib.tarantool_server import TarantoolServer from lib.unittest_server import UnittestServer from lib.app_server import AppServer +from lib.luatest_server import LuatestServer from lib.utils import warn_unix_sockets_at_start @@ -31,6 +32,7 @@ def module_init(): # If script executed with (python test-run.py) dirname is '' # so we need to make it . path = os.path.dirname(sys.argv[0]) + os.environ['TEST_RUN_DIR'] = os.path.dirname(os.path.realpath(sys.argv[0])) if not path: path = '.' os.chdir(path) @@ -56,14 +58,25 @@ def module_init(): os.environ["SOURCEDIR"] = SOURCEDIR os.environ["BUILDDIR"] = BUILDDIR soext = sys.platform == 'darwin' and 'dylib' or 'so' - os.environ["LUA_PATH"] = SOURCEDIR+"/?.lua;"+SOURCEDIR+"/?/init.lua;;" + + os.environ['LUA_PATH'] = ( + SOURCEDIR + '/?.lua;' + SOURCEDIR + '/?/init.lua;' + + os.environ['TEST_RUN_DIR'] + '/lib/checks/?.lua;' + + os.environ['TEST_RUN_DIR'] + '/lib/luatest/?/init.lua;' + + os.environ['TEST_RUN_DIR'] + '/lib/luatest/?.lua;;' + ) + os.environ["LUA_CPATH"] = BUILDDIR+"/?."+soext+";;" os.environ["REPLICATION_SYNC_TIMEOUT"] = str(args.replication_sync_timeout) os.environ['MEMTX_ALLOCATOR'] = args.memtx_allocator + os.environ['LUATEST_BIN'] = os.path.join( + os.environ['TEST_RUN_DIR'], 'lib/luatest/bin/luatest') + TarantoolServer.find_exe(args.builddir) UnittestServer.find_exe(args.builddir) AppServer.find_exe(args.builddir) + LuatestServer.find_exe(args.builddir) Options().check_schema_upgrade_option(TarantoolServer.debug) diff --git a/lib/checks b/lib/checks new file mode 160000 index 00000000..eb49c159 --- /dev/null +++ b/lib/checks @@ -0,0 +1 @@ +Subproject commit eb49c159ac1f81d37420454c6574cc003aa8569d diff --git a/lib/error.py b/lib/error.py new file mode 100644 index 00000000..b2246008 --- /dev/null +++ b/lib/error.py @@ -0,0 +1,3 @@ +class TestRunInitError(Exception): + def __init__(self, *args, **kwargs): + super(TestRunInitError, self).__init__(*args, **kwargs) diff --git a/lib/luatest b/lib/luatest new file mode 160000 index 00000000..10590e70 --- /dev/null +++ b/lib/luatest @@ -0,0 +1 @@ +Subproject commit 10590e70204d6e49a96e77cce2e8d80783f19000 diff --git a/lib/luatest_server.py b/lib/luatest_server.py new file mode 100644 index 00000000..47b7c01b --- /dev/null +++ b/lib/luatest_server.py @@ -0,0 +1,133 @@ +import glob +import os +import re +import sys + +from subprocess import Popen, PIPE +from subprocess import STDOUT + +from lib.error import TestRunInitError +from lib.sampler import sampler +from lib.server import Server +from lib.tarantool_server import Test +from lib.tarantool_server import TarantoolServer + + +class LuatestTest(Test): + """ Handle *_test.lua. + + Provide method for executing luatest _test.lua test. + """ + + def __init__(self, *args, **kwargs): + super(LuatestTest, self).__init__(*args, **kwargs) + self.valgrind = kwargs.get('valgrind', False) + + def execute(self, server): + """Execute test by luatest command + + Execute 'luatest -c -v _test.lua -o tap --shuffle none' + Provide a verbose output in the tap format. + Use shuffle option in none mode for avoiding mixing tests. + Use capture mode. + """ + server.current_test = self + script = os.path.join(os.path.basename(server.testdir), self.name) + command = [os.environ['LUATEST_BIN'], '-c', '-v', script, '-o', 'tap', + '--shuffle', 'none'] + + # Tarantool's build directory is added to PATH in + # TarantoolServer.find_exe(). + # + # We start luatest from the project source directory, it + # is the usual way to use luatest. + # + # VARDIR (${BUILDDIR}/test/var/001_foo) will be used for + # write ahead logs, snapshots, logs, unix domain sockets + # and so on. + os.environ['VARDIR'] = server.vardir + project_dir = os.environ['SOURCEDIR'] + proc = Popen(command, cwd=project_dir, stdout=PIPE, stderr=STDOUT) + sampler.register_process(proc.pid, self.id, server.name) + sys.stdout.write_bytes(proc.communicate()[0]) + + +class LuatestServer(Server): + """A dummy server implementation for luatest server tests""" + + def __new__(cls, ini=None, *args, **kwargs): + cls = Server.get_mixed_class(cls, ini) + return object.__new__(cls) + + def __init__(self, _ini=None, test_suite=None): + if _ini is None: + _ini = {} + ini = {'vardir': None} + ini.update(_ini) + super(LuatestServer, self).__init__(ini, test_suite) + self.testdir = os.path.abspath(os.curdir) + self.vardir = ini['vardir'] + self.builddir = ini['builddir'] + self.name = 'luatest_server' + + @property + def logfile(self): + return self.current_test.tmp_result + + @property + def binary(self): + return LuatestServer.prepare_args(self)[0] + + def deploy(self, vardir=None, silent=True, wait=True): + self.vardir = vardir + if not os.access(self.vardir, os.F_OK): + os.makedirs(self.vardir) + + @classmethod + def find_exe(cls, builddir): + cls.builddir = builddir + cls.binary = TarantoolServer.binary + cls.debug = bool(re.findall(r'-Debug', str(cls.version()), + re.I)) + + @classmethod + def verify_luatest_exe(cls): + """Verify that luatest executable is available.""" + try: + # Just check that the command returns zero exit code. + with open(os.devnull, 'w') as devnull: + returncode = Popen([os.environ['LUATEST_BIN'], '--version'], + stdout=devnull, + stderr=devnull).wait() + if returncode != 0: + raise TestRunInitError('Unable to run `luatest --version`', + {'returncode': returncode}) + except OSError as e: + # Python 2 raises OSError if the executable is not + # found or if it has no executable bit. Python 3 + # raises FileNotFoundError and PermissionError in + # those cases, which are childs of OSError anyway. + raise TestRunInitError('Unable to find luatest executable', e) + + @staticmethod + def find_tests(test_suite, suite_path): + """Looking for *_test.lua, which are can be executed by luatest.""" + + def patterned(test, patterns): + answer = [] + for i in patterns: + if test.name.find(i) != -1: + answer.append(test) + return answer + + test_suite.ini['suite'] = suite_path + tests = glob.glob(os.path.join(suite_path, '*_test.lua')) + + tests = Server.exclude_tests(tests, test_suite.args.exclude) + test_suite.tests = [LuatestTest(k, test_suite.args, test_suite.ini) + for k in sorted(tests)] + test_suite.tests = sum([patterned(x, test_suite.args.tests) + for x in test_suite.tests], []) + + def print_log(self, lines): + pass diff --git a/lib/test.py b/lib/test.py index d29896ae..69a53cd7 100644 --- a/lib/test.py +++ b/lib/test.py @@ -94,7 +94,13 @@ def flush(self): def get_filename_by_test(postfix, test_name): - rg = re.compile(r'\.test.*') + """For <..>/_test.* or <..>/.test.* return + postfix + + Examples: + postfix='.result', test_name='foo/bar.test.lua' => return 'bar.result' + postfix='.reject', test_name='bar_test.lua' => return 'bar.reject' + """ + rg = re.compile(r'[._]test.*') return os.path.basename(rg.sub(postfix, test_name)) diff --git a/lib/test_suite.py b/lib/test_suite.py index c6e81baf..97845651 100644 --- a/lib/test_suite.py +++ b/lib/test_suite.py @@ -13,6 +13,7 @@ from lib import Options from lib.app_server import AppServer +from lib.luatest_server import LuatestServer from lib.colorer import color_stdout from lib.inspector import TarantoolInspector from lib.server import Server @@ -146,12 +147,17 @@ def __init__(self, suite_path, args): # rid of all other side effects. self.tests_are_collected = False + if self.ini['core'] == 'luatest': + LuatestServer.verify_luatest_exe() + def collect_tests(self): if self.tests_are_collected: return self.tests if self.ini['core'] == 'tarantool': TarantoolServer.find_tests(self, self.suite_path) + elif self.ini['core'] == 'luatest': + LuatestServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'app': AppServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'unittest': @@ -162,7 +168,8 @@ def collect_tests(self): self.tests_are_collected = True return self.tests else: - raise ValueError('Cannot collect tests of unknown type') + raise ValueError( + 'Cannot collect tests of unknown type: %s' % self.ini['core']) # In given cases, this large output looks redundant. if not Options().args.reproduce and not Options().args.show_tags: diff --git a/test-run.py b/test-run.py index cc27751f..4abae405 100755 --- a/test-run.py +++ b/test-run.py @@ -56,6 +56,7 @@ from lib import Options from lib.colorer import color_stdout from lib.utils import find_tags +from lib.error import TestRunInitError from lib.utils import print_tail_n from lib.utils import PY3 from lib.worker import get_task_groups @@ -70,6 +71,7 @@ EXIT_INTERRUPTED = 2 EXIT_FAILED_TEST = 3 EXIT_NOTDONE_TEST = 4 +EXIT_INIT_ERROR = 5 EXIT_UNKNOWN_ERROR = 50 @@ -293,10 +295,14 @@ def open_as_utf8(*args, **kwargs): show_tags() exit(status) - force_parallel = bool(Options().args.reproduce) - if not force_parallel and Options().args.jobs == -1: - status = main_consistent() - else: - status = main_parallel() + try: + force_parallel = bool(Options().args.reproduce) + if not force_parallel and Options().args.jobs == -1: + status = main_consistent() + else: + status = main_parallel() + except TestRunInitError as e: + color_stdout(str(e), '\n', schema='error') + status = EXIT_INIT_ERROR exit(status) diff --git a/test/instances/default.lua b/test/instances/default.lua new file mode 100755 index 00000000..9c5f4878 --- /dev/null +++ b/test/instances/default.lua @@ -0,0 +1,19 @@ +#!/usr/bin/env tarantool + +local helpers = require('test.luatest_helpers') + +box.cfg(helpers.box_cfg()) +box.schema.user.grant('guest', 'super', nil, nil, {if_not_exists = true}) + +-- luatest_helpers.Server:start() unblocks only when this variable +-- becomes true. +-- +-- Set it when the instance is fully operable: +-- +-- * The server listens for requests. +-- * The database is bootstrapped. +-- * Permissions are granted. +-- +-- Use luatest_helpers.Server:start({wait_for_readiness = false}) +-- to don't wait for setting of this variable. +_G.ready = true diff --git a/test/luatest_helpers.lua b/test/luatest_helpers.lua new file mode 100644 index 00000000..9b1d0c00 --- /dev/null +++ b/test/luatest_helpers.lua @@ -0,0 +1,44 @@ +local fun = require('fun') +local json = require('json') +local fio = require('fio') + +local luatest_helpers = { + SOCKET_DIR = fio.abspath(os.getenv('VARDIR') or 'test/var') +} + +luatest_helpers.Server = require('test.luatest_helpers.server') + +local function default_cfg() + return { + work_dir = os.getenv('TARANTOOL_WORKDIR'), + listen = os.getenv('TARANTOOL_LISTEN'), + } +end + +local function env_cfg() + local src = os.getenv('TARANTOOL_BOX_CFG') + if src == nil then + return {} + end + local res = json.decode(src) + assert(type(res) == 'table') + return res +end + +-- Collect box.cfg table from values passed through +-- luatest_helpers.Server({<...>}) and from the given argument. +-- +-- Use it from inside an instance script. +function luatest_helpers.box_cfg(cfg) + return fun.chain(default_cfg(), env_cfg(), cfg or {}):tomap() +end + +function luatest_helpers.instance_uri(alias, instance_id) + if instance_id == nil then + instance_id = '' + end + instance_id = tostring(instance_id) + return ('%s/%s%s.iproto'):format(luatest_helpers.SOCKET_DIR, alias, instance_id); +end + +return luatest_helpers diff --git a/test/luatest_helpers/server.lua b/test/luatest_helpers/server.lua new file mode 100644 index 00000000..128063bd --- /dev/null +++ b/test/luatest_helpers/server.lua @@ -0,0 +1,141 @@ +local clock = require('clock') +local digest = require('digest') +local ffi = require('ffi') +local fiber = require('fiber') +local fio = require('fio') +local fun = require('fun') +local json = require('json') + +local checks = require('checks') +local luatest = require('luatest') + +ffi.cdef([[ + int kill(pid_t pid, int sig); +]]) + +local Server = luatest.Server:inherit({}) + +local WAIT_TIMEOUT = 60 +local WAIT_DELAY = 0.1 + +-- Differences from luatest.Server: +-- +-- * 'alias' is mandatory. +-- * 'command' is optional, assumed test/instances/default.lua by +-- default. +-- * 'workdir' is optional, determined by 'alias'. +-- * The new 'box_cfg' parameter. +-- * engine - provides engine for parameterized tests +Server.constructor_checks = fun.chain(Server.constructor_checks, { + alias = 'string', + command = '?string', + workdir = '?string', + box_cfg = '?table', + engine = '?string', +}):tomap() + +function Server:initialize() + local vardir = fio.abspath(os.getenv('VARDIR') or 'test/var') + + if self.id == nil then + local random = digest.urandom(9) + self.id = digest.base64_encode(random, {urlsafe = true}) + end + if self.command == nil then + self.command = 'test/instances/default.lua' + end + if self.workdir == nil then + self.workdir = ('%s/%s-%s'):format(vardir, self.alias, self.id) + fio.rmtree(self.workdir) + fio.mktree(self.workdir) + end + if self.net_box_port == nil and self.net_box_uri == nil then + self.net_box_uri = ('%s/%s.iproto'):format(vardir, self.alias) + fio.mktree(vardir) + end + + -- AFAIU, the inner getmetatable() returns our helpers.Server + -- class, the outer one returns luatest.Server class. + getmetatable(getmetatable(self)).initialize(self) +end + +--- Generates environment to run process with. +-- The result is merged into os.environ(). +-- @return map +function Server:build_env() + local res = getmetatable(getmetatable(self)).build_env(self) + if self.box_cfg ~= nil then + res.TARANTOOL_BOX_CFG = json.encode(self.box_cfg) + end + res.TARANTOOL_ENGINE = self.engine + return res +end + +function Server:wait_for_readiness() + local alias = self.alias + local id = self.id + local pid = self.process.pid + + local deadline = clock.time() + WAIT_TIMEOUT + while true do + local ok, is_ready = pcall(function() + self:connect_net_box() + return self.net_box:eval('return _G.ready') == true + end) + if ok and is_ready then + break + end + if clock.time() > deadline then + error(('Starting of server %s-%s (PID %d) was timed out'):format( + alias, id, pid)) + end + fiber.sleep(WAIT_DELAY) + end +end + +-- Unlike the original luatest.Server function it waits for +-- starting the server. +function Server:start(opts) + checks('table', { + wait_for_readiness = '?boolean', + }) + getmetatable(getmetatable(self)).start(self) + + -- The option is true by default. + local wait_for_readiness = true + if opts ~= nil and opts.wait_for_readiness ~= nil then + wait_for_readiness = opts.wait_for_readiness + end + + if wait_for_readiness then + self:wait_for_readiness() + end +end + +-- TODO: Add the 'wait_for_readiness' parameter for the restart() +-- method. + +-- Unlike the original luatest.Server function it waits until +-- the server will stop. +function Server:stop() + local alias = self.alias + local id = self.id + if self.process then + local pid = self.process.pid + getmetatable(getmetatable(self)).stop(self) + + local deadline = clock.time() + WAIT_TIMEOUT + while true do + if ffi.C.kill(pid, 0) ~= 0 then + break + end + if clock.time() > deadline then + error(('Stopping of server %s-%s (PID %d) was timed out'):format( + alias, id, pid)) + end + fiber.sleep(WAIT_DELAY) + end + end +end + +return Server diff --git a/test/test-luatest/smoke_check_test.lua b/test/test-luatest/smoke_check_test.lua new file mode 100644 index 00000000..702407d2 --- /dev/null +++ b/test/test-luatest/smoke_check_test.lua @@ -0,0 +1,35 @@ +local t = require('luatest') +local g = t.group() + +local luatest_helpers = require('test.luatest_helpers') + +g.before_all = function() + g.server = luatest_helpers.Server:new({ + alias = 'my_server', + env = {MY_ENV_VAR = 'test_value'}, + box_cfg = {memtx_memory = 100 * 1024 ^ 2}, + }) + g.server:start() +end + +g.after_all = function() + g.server:stop() +end + +g.test_smoke = function() + -- The server is started and operable. + local res = g.server:eval('return 42') + t.assert_equals(res, 42) + + -- The database is bootstrapped and accessible. + local res = g.server:eval('return box.info.status') + t.assert_equals(res, 'running') + + -- The environment variable is passed. + local res = g.server:eval('return os.getenv("MY_ENV_VAR")') + t.assert_equals(res, 'test_value') + + -- The box.cfg() values are passed as well. + local res = g.server:eval('return box.cfg.memtx_memory') + t.assert_equals(res, 100 * 1024 ^ 2) +end diff --git a/test/test-luatest/suite.ini b/test/test-luatest/suite.ini new file mode 100644 index 00000000..5b75a3c0 --- /dev/null +++ b/test/test-luatest/suite.ini @@ -0,0 +1,3 @@ +[default] +core = luatest +description = luatest