Add luatest server

Provide luatest server, which can find tests by luatest pattern *_test.lua
and execute them luatest -c -v <name>_test.lua -o tap --shuffle none.
Use luatest bin from a submodule.

Was changed get_filename_by_test for right working not only with *.test.*,
but also with *_test.*.

Smoke check lua-test was added with starting server and checks of
box_cfg args.

Show a test suite type when it is unknown.

Part of: #304

Author: VitaliyaIoffe

.luacheckrc

@@ -1,4 +1,4 @@
-globals = {"box", "_TARANTOOL", "tonumber64"}
+globals = {"box", "_TARANTOOL", "tonumber64", "os"}
 ignore = {
     -- Accessing an undefined field of a global variable <debug>.
     "143/debug",

lib/__init__.py

@@ -6,6 +6,7 @@
 from lib.tarantool_server import TarantoolServer
 from lib.unittest_server import UnittestServer
 from lib.app_server import AppServer
+from lib.luatest_server import LuatestServer
 from lib.utils import warn_unix_sockets_at_start
 
 
@@ -31,6 +32,7 @@ def module_init():
     # If script executed with (python test-run.py) dirname is ''
     # so we need to make it .
     path = os.path.dirname(sys.argv[0])
+    os.environ['TEST_RUN_DIR'] = os.path.dirname(os.path.realpath(sys.argv[0]))
     if not path:
         path = '.'
     os.chdir(path)
@@ -56,14 +58,25 @@ def module_init():
     os.environ["SOURCEDIR"] = SOURCEDIR
     os.environ["BUILDDIR"] = BUILDDIR
     soext = sys.platform == 'darwin' and 'dylib' or 'so'
-    os.environ["LUA_PATH"] = SOURCEDIR+"/?.lua;"+SOURCEDIR+"/?/init.lua;;"
+
+    os.environ['LUA_PATH'] = (
+            SOURCEDIR + '/?.lua;' + SOURCEDIR + '/?/init.lua;'
+            + os.environ['TEST_RUN_DIR'] + '/lib/checks/?.lua;'
+            + os.environ['TEST_RUN_DIR'] + '/lib/luatest/?/init.lua;'
+            + os.environ['TEST_RUN_DIR'] + '/lib/luatest/?.lua;;'
+    )
+
     os.environ["LUA_CPATH"] = BUILDDIR+"/?."+soext+";;"
     os.environ["REPLICATION_SYNC_TIMEOUT"] = str(args.replication_sync_timeout)
     os.environ['MEMTX_ALLOCATOR'] = args.memtx_allocator
 
+    os.environ['LUATEST_BIN'] = os.path.join(
+        os.environ['TEST_RUN_DIR'], 'lib/luatest/bin/luatest')
+
     TarantoolServer.find_exe(args.builddir)
     UnittestServer.find_exe(args.builddir)
     AppServer.find_exe(args.builddir)
+    LuatestServer.find_exe(args.builddir)
 
     Options().check_schema_upgrade_option(TarantoolServer.debug)
 

lib/error.py

@@ -0,0 +1,3 @@
+class TestRunInitError(Exception):
+    def __init__(self, *args, **kwargs):
+        super(TestRunInitError, self).__init__(*args, **kwargs)

lib/luatest_server.py

@@ -0,0 +1,133 @@
+import glob
+import os
+import re
+import sys
+
+from subprocess import Popen, PIPE
+from subprocess import STDOUT
+
+from lib.error import TestRunInitError
+from lib.sampler import sampler
+from lib.server import Server
+from lib.tarantool_server import Test
+from lib.tarantool_server import TarantoolServer
+
+
+class LuatestTest(Test):
+    """ Handle *_test.lua.
+
+    Provide method for executing luatest <name>_test.lua test.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(LuatestTest, self).__init__(*args, **kwargs)
+        self.valgrind = kwargs.get('valgrind', False)
+
+    def execute(self, server):
+        """Execute test by luatest command
+
+        Execute 'luatest -c -v <name>_test.lua -o tap --shuffle none'
+        Provide a verbose output in the tap format.
+        Use shuffle option in none mode for avoiding mixing tests.
+        Use capture mode.
+        """
+        server.current_test = self
+        script = os.path.join(os.path.basename(server.testdir), self.name)
+        command = [os.environ['LUATEST_BIN'], '-c', '-v', script, '-o', 'tap',
+                   '--shuffle', 'none']
+
+        # Tarantool's build directory is added to PATH in
+        # TarantoolServer.find_exe().
+        #
+        # We start luatest from the project source directory, it
+        # is the usual way to use luatest.
+        #
+        # VARDIR (${BUILDDIR}/test/var/001_foo) will be used for
+        # write ahead logs, snapshots, logs, unix domain sockets
+        # and so on.
+        os.environ['VARDIR'] = server.vardir
+        project_dir = os.environ['SOURCEDIR']
+        proc = Popen(command, cwd=project_dir, stdout=PIPE, stderr=STDOUT)
+        sampler.register_process(proc.pid, self.id, server.name)
+        sys.stdout.write_bytes(proc.communicate()[0])
+
+
+class LuatestServer(Server):
+    """A dummy server implementation for luatest server tests"""
+
+    def __new__(cls, ini=None, *args, **kwargs):
+        cls = Server.get_mixed_class(cls, ini)
+        return object.__new__(cls)
+
+    def __init__(self, _ini=None, test_suite=None):
+        if _ini is None:
+            _ini = {}
+        ini = {'vardir': None}
+        ini.update(_ini)
+        super(LuatestServer, self).__init__(ini, test_suite)
+        self.testdir = os.path.abspath(os.curdir)
+        self.vardir = ini['vardir']
+        self.builddir = ini['builddir']
+        self.name = 'luatest_server'
+
+    @property
+    def logfile(self):
+        return self.current_test.tmp_result
+
+    @property
+    def binary(self):
+        return LuatestServer.prepare_args(self)[0]
+
+    def deploy(self, vardir=None, silent=True, wait=True):
+        self.vardir = vardir
+        if not os.access(self.vardir, os.F_OK):
+            os.makedirs(self.vardir)
+
+    @classmethod
+    def find_exe(cls, builddir):
+        cls.builddir = builddir
+        cls.binary = TarantoolServer.binary
+        cls.debug = bool(re.findall(r'-Debug', str(cls.version()),
+                                    re.I))
+
+    @classmethod
+    def verify_luatest_exe(cls):
+        """Verify that luatest executable is available."""
+        try:
+            # Just check that the command returns zero exit code.
+            with open(os.devnull, 'w') as devnull:
+                returncode = Popen([os.environ['LUATEST_BIN'], '--version'],
+                                   stdout=devnull,
+                                   stderr=devnull).wait()
+            if returncode != 0:
+                raise TestRunInitError('Unable to run `luatest --version`',
+                                       {'returncode': returncode})
+        except OSError as e:
+            # Python 2 raises OSError if the executable is not
+            # found or if it has no executable bit. Python 3
+            # raises FileNotFoundError and PermissionError in
+            # those cases, which are childs of OSError anyway.
+            raise TestRunInitError('Unable to find luatest executable', e)
+
+    @staticmethod
+    def find_tests(test_suite, suite_path):
+        """Looking for *_test.lua, which are can be executed by luatest."""
+
+        def patterned(test, patterns):
+            answer = []
+            for i in patterns:
+                if test.name.find(i) != -1:
+                    answer.append(test)
+            return answer
+
+        test_suite.ini['suite'] = suite_path
+        tests = glob.glob(os.path.join(suite_path, '*_test.lua'))
+
+        tests = Server.exclude_tests(tests, test_suite.args.exclude)
+        test_suite.tests = [LuatestTest(k, test_suite.args, test_suite.ini)
+                            for k in sorted(tests)]
+        test_suite.tests = sum([patterned(x, test_suite.args.tests)
+                                for x in test_suite.tests], [])
+
+    def print_log(self, lines):
+        pass

lib/test.py

@@ -94,7 +94,13 @@ def flush(self):
 
 
 def get_filename_by_test(postfix, test_name):
-    rg = re.compile(r'\.test.*')
+    """For <..>/<name>_test.* or <..>/<name>.test.* return <name> + postfix
+
+    Examples:
+        postfix='.result', test_name='foo/bar.test.lua' => return 'bar.result'
+        postfix='.reject', test_name='bar_test.lua' => return 'bar.reject'
+    """
+    rg = re.compile(r'[._]test.*')
     return os.path.basename(rg.sub(postfix, test_name))
 
 

lib/test_suite.py

@@ -13,6 +13,7 @@
 
 from lib import Options
 from lib.app_server import AppServer
+from lib.luatest_server import LuatestServer
 from lib.colorer import color_stdout
 from lib.inspector import TarantoolInspector
 from lib.server import Server
@@ -146,12 +147,17 @@ def __init__(self, suite_path, args):
         # rid of all other side effects.
         self.tests_are_collected = False
 
+        if self.ini['core'] == 'luatest':
+            LuatestServer.verify_luatest_exe()
+
     def collect_tests(self):
         if self.tests_are_collected:
             return self.tests
 
         if self.ini['core'] == 'tarantool':
             TarantoolServer.find_tests(self, self.suite_path)
+        elif self.ini['core'] == 'luatest':
+            LuatestServer.find_tests(self, self.suite_path)
         elif self.ini['core'] == 'app':
             AppServer.find_tests(self, self.suite_path)
         elif self.ini['core'] == 'unittest':
@@ -162,7 +168,8 @@ def collect_tests(self):
             self.tests_are_collected = True
             return self.tests
         else:
-            raise ValueError('Cannot collect tests of unknown type')
+            raise ValueError(
+                'Cannot collect tests of unknown type: %s' % self.ini['core'])
 
         # In given cases, this large output looks redundant.
         if not Options().args.reproduce and not Options().args.show_tags:

test-run.py

@@ -56,6 +56,7 @@
 from lib import Options
 from lib.colorer import color_stdout
 from lib.utils import find_tags
+from lib.error import TestRunInitError
 from lib.utils import print_tail_n
 from lib.utils import PY3
 from lib.worker import get_task_groups
@@ -70,6 +71,7 @@
 EXIT_INTERRUPTED = 2
 EXIT_FAILED_TEST = 3
 EXIT_NOTDONE_TEST = 4
+EXIT_INIT_ERROR = 5
 EXIT_UNKNOWN_ERROR = 50
 
 
@@ -293,10 +295,14 @@ def open_as_utf8(*args, **kwargs):
         show_tags()
         exit(status)
 
-    force_parallel = bool(Options().args.reproduce)
-    if not force_parallel and Options().args.jobs == -1:
-        status = main_consistent()
-    else:
-        status = main_parallel()
+    try:
+        force_parallel = bool(Options().args.reproduce)
+        if not force_parallel and Options().args.jobs == -1:
+            status = main_consistent()
+        else:
+            status = main_parallel()
+    except TestRunInitError as e:
+        color_stdout(str(e), '\n', schema='error')
+        status = EXIT_INIT_ERROR
 
     exit(status)

test/instances/default.lua

@@ -0,0 +1,19 @@
+#!/usr/bin/env tarantool
+
+local helpers = require('test.luatest_helpers')
+
+box.cfg(helpers.box_cfg())
+box.schema.user.grant('guest', 'super', nil, nil, {if_not_exists = true})
+
+-- luatest_helpers.Server:start() unblocks only when this variable
+-- becomes true.
+--
+-- Set it when the instance is fully operable:
+--
+-- * The server listens for requests.
+-- * The database is bootstrapped.
+-- * Permissions are granted.
+--
+-- Use luatest_helpers.Server:start({wait_for_readiness = false})
+-- to don't wait for setting of this variable.
+_G.ready = true

test/luatest_helpers.lua

@@ -0,0 +1,33 @@
+local fun = require('fun')
+local json = require('json')
+
+local luatest_helpers = {}
+
+luatest_helpers.Server = require('test.luatest_helpers.server')
+
+local function default_cfg()
+    return {
+        work_dir = os.getenv('TARANTOOL_WORKDIR'),
+        listen = os.getenv('TARANTOOL_LISTEN'),
+    }
+end
+
+local function env_cfg()
+    local src = os.getenv('TARANTOOL_BOX_CFG')
+    if src == nil then
+        return {}
+    end
+    local res = json.decode(src)
+    assert(type(res) == 'table')
+    return res
+end
+
+-- Collect box.cfg table from values passed through
+-- luatest_helpers.Server({<...>}) and from the given argument.
+--
+-- Use it from inside an instance script.
+function luatest_helpers.box_cfg(cfg)
+    return fun.chain(default_cfg(), env_cfg(), cfg or {}):tomap()
+end
+
+return luatest_helpers