From 5de98f01ca34c1759e1eed64ac1e1c8470be1c4e Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Fri, 27 Aug 2021 12:16:19 +0200
Subject: [PATCH 01/17] - updated to match new monetdbe_prepare signature -
 fixed connection boilerplate code - fixed/updated unit tests

---
 monetdbe/_cffi/convert/bind.py  |   4 +-
 monetdbe/_cffi/embed.h.j2       |   5 +-
 monetdbe/_cffi/internal.py      |  27 ++--
 monetdbe/cursors.py             |   6 +
 tests/test_cffi.py              | 124 ++++++++---------
 tests/test_csv.py               |  23 ++--
 tests/test_dataframe.py         |  10 +-
 tests/test_dbapi.py             | 231 +++++++++++++++++---------------
 tests/test_lite/conftest.py     |  39 +++---
 tests/test_lite/test_dbapi03.py |  26 ++--
 tests/test_regression.py        |  28 +++-
 tests/test_transactions.py      |   2 +
 12 files changed, 286 insertions(+), 239 deletions(-)

diff --git a/monetdbe/_cffi/convert/bind.py b/monetdbe/_cffi/convert/bind.py
index 27b4054..28eb9a7 100644
--- a/monetdbe/_cffi/convert/bind.py
+++ b/monetdbe/_cffi/convert/bind.py
@@ -10,8 +10,8 @@ def monetdbe_int(data: int) -> ffi.CData:
         return ffi.new("int *", data)
 
 
-def bind_str(data: str) -> bytes:
-    return str(data).encode()
+def bind_str(data: str) -> ffi.CData:
+    return ffi.new("char[]", str(data).encode())
 
 
 def bind_float(data: float) -> ffi.CData:
diff --git a/monetdbe/_cffi/embed.h.j2 b/monetdbe/_cffi/embed.h.j2
index b2cd683..1a214fc 100644
--- a/monetdbe/_cffi/embed.h.j2
+++ b/monetdbe/_cffi/embed.h.j2
@@ -25,8 +25,7 @@ typedef struct {
 
 typedef enum {
  monetdbe_bool, monetdbe_int8_t, monetdbe_int16_t, monetdbe_int32_t, monetdbe_int64_t,
- {% if not win32 %}
- {# windows does not support 128-bit #}
+ {% if HAVE_HGE %}
  monetdbe_int128_t,
  {% endif %}
  monetdbe_size_t, monetdbe_float, monetdbe_double,
@@ -115,7 +114,7 @@ extern char* monetdbe_query(monetdbe_database dbhdl, char* query, monetdbe_resul
 extern char* monetdbe_result_fetch(monetdbe_result *mres, monetdbe_column** res, size_t column_index);
 extern char* monetdbe_cleanup_result(monetdbe_database dbhdl, monetdbe_result* result);
 
-extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt);
+extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt, monetdbe_result** result);
 extern char* monetdbe_bind(monetdbe_statement *stmt, void *data, size_t parameter_nr);
 extern char* monetdbe_execute(monetdbe_statement *stmt, monetdbe_result **result, monetdbe_cnt* affected_rows);
 extern char* monetdbe_cleanup_statement(monetdbe_database dbhdl, monetdbe_statement *stmt);
diff --git a/monetdbe/_cffi/internal.py b/monetdbe/_cffi/internal.py
index 92d96be..c24f124 100644
--- a/monetdbe/_cffi/internal.py
+++ b/monetdbe/_cffi/internal.py
@@ -113,18 +113,14 @@ def __init__(
         self.nr_threads = nr_threads
         self.have_hge = have_hge
         self._switch()
+        self._monetdbe_database = self.open()
 
     @classmethod
     def set_active_context(cls, active_context: Optional['Internal']):
         cls._active_context = active_context
 
-    @classmethod
-    def set_active_connection(cls, active_connection: Optional['Connection']):
-        cls._active_connection = active_connection
-
-    @classmethod
-    def set_monetdbe_database(cls, connection: Optional[monetdbe_database]):
-        cls._monetdbe_database = connection
+    def set_monetdbe_database(self, connection: Optional[monetdbe_database]):
+        self._monetdbe_database = connection
 
     def __del__(self):
         if self._active_context == self:
@@ -135,17 +131,10 @@ def _switch(self):
         if self._active_context == self:
             return
 
-        # this is a bit scary but just to make sure the previous connection
-        # can't touch us anymore
-        if self._active_connection:
-            self._active_connection._internal = None
-
-        self.close()
-        self.set_monetdbe_database(self.open())
         self.set_active_context(self)
-        self.set_active_connection(self._connection)
 
     def cleanup_result(self, result: monetdbe_result):
+        self._switch()
         _logger.info("cleanup_result called")
         if result and self._monetdbe_database:
             check_error(lib.monetdbe_cleanup_result(self._monetdbe_database, result))
@@ -180,11 +169,13 @@ def open(self) -> monetdbe_database:
                 lib.monetdbe_close(connection)
             else:
                 error = errors.get(result_code, "unknown error")
-            raise exceptions.OperationalError(f"Failed to open database: {error} (code {result_code})")
+            msg = "Failed to open database: {error} (code {result_code})".format(error=error, result_code=result_code)
+            raise exceptions.OperationalError(msg)
 
         return connection
 
     def close(self) -> None:
+        self._switch()
         if self._monetdbe_database:
             if lib.monetdbe_close(self._monetdbe_database):
                 raise exceptions.OperationalError("Failed to close database")
@@ -193,8 +184,6 @@ def close(self) -> None:
         if self._active_context:
             self.set_active_context(None)
 
-        self.set_active_connection(None)
-
     def query(self, query: str, make_result: bool = False) -> Tuple[Optional[Any], int]:
         """
         Execute a query.
@@ -268,7 +257,7 @@ def append(self, table: str, data: Mapping[str, np.ndarray], schema: str = 'sys'
     def prepare(self, query: str) -> monetdbe_statement:
         self._switch()
         stmt = ffi.new("monetdbe_statement **")
-        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
+        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
         return stmt[0]
 
     def cleanup_statement(self, statement: monetdbe_statement) -> None:
diff --git a/monetdbe/cursors.py b/monetdbe/cursors.py
index 09d5049..bd6a737 100644
--- a/monetdbe/cursors.py
+++ b/monetdbe/cursors.py
@@ -37,6 +37,12 @@ def __init__(self, con: 'Connection'):
 
         self._fetch_generator: Optional[Iterator['Row']] = None
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
     def __del__(self):
         self.close()
 
diff --git a/tests/test_cffi.py b/tests/test_cffi.py
index 56e49ae..ec95ea7 100644
--- a/tests/test_cffi.py
+++ b/tests/test_cffi.py
@@ -15,81 +15,81 @@ def test_cffi(self):
             self.assertEqual(lib.monetdbe_type_unknown, 14)
 
     def test_append_too_many_columns(self):
-        con = connect()
-        con.execute("CREATE TABLE test (i int)")
-        data = {'i': np.array([1, 2, 3]), 'j': np.array([1, 2, 3])}
-        with self.assertRaises(ProgrammingError):
-            con._internal.append(table='test', data=data)
+        with connect() as con:
+            con.execute("CREATE TABLE test (i int)")
+            data = {'i': np.array([1, 2, 3]), 'j': np.array([1, 2, 3])}
+            with self.assertRaises(ProgrammingError):
+                con._internal.append(table='test', data=data)
 
     def test_append_too_little_columns(self):
-        con = connect()
-        con.execute("CREATE TABLE test (i int, j int)")
-        data = {'i': np.array([1, 2, 3])}
-        with self.assertRaises(ProgrammingError):
-            con._internal.append(table='test', data=data)
+        with connect() as con:
+            con.execute("CREATE TABLE test (i int, j int)")
+            data = {'i': np.array([1, 2, 3])}
+            with self.assertRaises(ProgrammingError):
+                con._internal.append(table='test', data=data)
 
     def test_append_wrong_type(self):
-        con = connect()
-        con.execute("CREATE TABLE test (i int)")
-        data = {'i': np.array([0.1, 0.2, 0.3], dtype=np.float32)}
-        with self.assertRaises(ProgrammingError):
-            con._internal.append(table='test', data=data)
+        with connect() as con:
+            con.execute("CREATE TABLE test (i int)")
+            data = {'i': np.array([0.1, 0.2, 0.3], dtype=np.float32)}
+            with self.assertRaises(ProgrammingError):
+                con._internal.append(table='test', data=data)
 
     def test_append_wrong_size(self):
-        con = connect()
-        con.execute("CREATE TABLE test (i int)")  # SQL int is 32 bit
-        data = {'i': np.array([1, 2, 3], dtype=np.int64)}
-        with self.assertRaises(ProgrammingError):
-            con._internal.append(table='test', data=data)
+        with connect() as con:
+            con.execute("CREATE TABLE test (i int)")  # SQL int is 32 bit
+            data = {'i': np.array([1, 2, 3], dtype=np.int64)}
+            with self.assertRaises(ProgrammingError):
+                con._internal.append(table='test', data=data)
 
     def test_append_supported_types(self):
-        con = connect()
-        con.execute("CREATE TABLE test (t tinyint, s smallint, i int, b bigint, r real, f float)")
-        con.execute(
-            """
-            INSERT INTO test VALUES (2^8,  2^16,  2^32,  2^64,  0.12345,  0.123456789),
-                                    (NULL, NULL,  NULL,  NULL,  NULL,     NULL),
-                                    (0,    0,     0,     0,     0.0,      0.0),
-                                    (-2^8, -2^16, -2^32, -2^64, -0.12345, -0.123456789)
-            """
-        )
-        data = con.execute("select * from test").fetchnumpy()
-        con._internal.append(schema='sys', table='test', data=data)
-        con.cursor().insert(table='test', values=data)
+        with connect() as con:
+            con.execute("CREATE TABLE test (t tinyint, s smallint, i int, b bigint, r real, f float)")
+            con.execute(
+                """
+                INSERT INTO test VALUES (2^8,  2^16,  2^32,  2^64,  0.12345,  0.123456789),
+                                        (NULL, NULL,  NULL,  NULL,  NULL,     NULL),
+                                        (0,    0,     0,     0,     0.0,      0.0),
+                                        (-2^8, -2^16, -2^32, -2^64, -0.12345, -0.123456789)
+                """
+            )
+            data = con.execute("select * from test").fetchnumpy()
+            con._internal.append(schema='sys', table='test', data=data)
+            con.cursor().insert(table='test', values=data)
 
     def test_append_unsupported_types(self):
-        con = connect()
-        con.execute("CREATE TABLE test (s string, b blob, d date, t time, ts timestamp)")
-        con.execute(
-            """
-            INSERT INTO test VALUES ('hi',    '01020308', '2020-01-02', '10:20:30', '2020-01-02 10:20:30' ),
-                                    ('World', NULL,       NULL,         NULL,       NULL )
-            """
-        )
+        with connect() as con:
+            con.execute("CREATE TABLE test (s string, b blob, d date, t time, ts timestamp)")
+            con.execute(
+                """
+                INSERT INTO test VALUES ('hi',    '01020308', '2020-01-02', '10:20:30', '2020-01-02 10:20:30' ),
+                                        ('World', NULL,       NULL,         NULL,       NULL )
+                """
+            )
 
-        data = con.execute("select * from test").fetchnumpy()
-        with self.assertRaises(con.ProgrammingError):
-            con._internal.append(schema='sys', table='test', data=data)
-        con.cursor().insert(table='test', values=data)
+            data = con.execute("select * from test").fetchnumpy()
+            with self.assertRaises(con.ProgrammingError):
+                con._internal.append(schema='sys', table='test', data=data)
+            con.cursor().insert(table='test', values=data)
 
     def test_append_blend(self):
-        con = connect()
-        con.execute("CREATE TABLE test (i int, f float, s string, ts timestamp)")
-        con.execute(
-            """
-            INSERT INTO test VALUES (1, 1.2, 'bla', '2020-01-02 10:20:30'),
-                                    (NULL, NULL, NULL, NULL)
-            """
-        )
+        with connect() as con:
+            con.execute("CREATE TABLE test (i int, f float, s string, ts timestamp)")
+            con.execute(
+                """
+                INSERT INTO test VALUES (1, 1.2, 'bla', '2020-01-02 10:20:30'),
+                                        (NULL, NULL, NULL, NULL)
+                """
+            )
 
-        data = con.execute("select * from test").fetchnumpy()
-        with self.assertRaises(con.ProgrammingError):
-            con._internal.append(schema='sys', table='test', data=data)
-        con.cursor().insert(table='test', values=data)
+            data = con.execute("select * from test").fetchnumpy()
+            with self.assertRaises(con.ProgrammingError):
+                con._internal.append(schema='sys', table='test', data=data)
+            con.cursor().insert(table='test', values=data)
 
     def test_get_columns(self):
-        con = connect()
-        con.execute("CREATE TABLE test (i int)")
-        con.execute("INSERT INTO test VALUES (1)")
-        result = list(con._internal.get_columns(table='test'))
-        self.assertEqual(result, [('i', 3)])
+        with connect() as con:
+            con.execute("CREATE TABLE test (i int)")
+            con.execute("INSERT INTO test VALUES (1)")
+            result = list(con._internal.get_columns(table='test'))
+            self.assertEqual(result, [('i', 3)])
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 371f7c2..d35b574 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -7,18 +7,17 @@
 class TestCsv(TestCase):
     def test_read_csv(self):
         table = 'test'
-        con = connect()
-
-        con.read_csv(
-            table=table,
-            filepath_or_buffer=Path(__file__).parent / "example.csv",
-            names=['i', 's', 'i2', 'f'],
-            dtype={"i1": int, 's': str, 'i2': int, 'f': float},
-        )
-        x = con.execute(f'select * from {table}').fetchall()
+        with connect(autocommit=True) as con:
+            con.read_csv(
+                table=table,
+                filepath_or_buffer=Path(__file__).parent / "example.csv",
+                names=['i', 's', 'i2', 'f'],
+                dtype={"i1": int, 's': str, 'i2': int, 'f': float},
+            )
+            x = con.execute(f'select * from {table}').fetchall()
 
     def test_write_csv(self):
-        con = connect()
-        t = TemporaryDirectory()
+        with connect(autocommit=True) as con:
+            t = TemporaryDirectory()
 
-        con.write_csv(table='tables', path_or_buf=Path(t.name) / 'output.csv')
+            con.write_csv(table='tables', path_or_buf=Path(t.name) / 'output.csv')
diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py
index d4a1f7c..0ee29f7 100644
--- a/tests/test_dataframe.py
+++ b/tests/test_dataframe.py
@@ -7,11 +7,11 @@
 
 
 def _connect(values: List[Any], type: str) -> DataFrame:
-    con = connect(autocommit=True)
-    cur = con.execute(f"create table example(d {type})")
-    cur.executemany("insert into example(d) values (?)", ((v,) for v in values))
-    cur.execute("select * from example")
-    return cur.fetchdf()
+    with connect(autocommit=True) as con:
+        cur = con.execute(f"create table example(d {type})")
+        cur.executemany("insert into example(d) values (?)", ((v,) for v in values))
+        cur.execute("select * from example")
+        return cur.fetchdf()
 
 
 class TestDataFrame(TestCase):
diff --git a/tests/test_dbapi.py b/tests/test_dbapi.py
index 62035a7..8d3fa87 100644
--- a/tests/test_dbapi.py
+++ b/tests/test_dbapi.py
@@ -122,11 +122,6 @@ def test_RollbackAfterNoChanges(self):
     def test_Cursor(self):
         cu = self.cx.cursor()
 
-    def test_FailedOpen(self):
-        YOU_CANNOT_OPEN_THIS = "/foo/bar/bla/23534/mydb.db"
-        with self.assertRaises(monetdbe.OperationalError):
-            con = monetdbe.connect(YOU_CANNOT_OPEN_THIS)
-
     def test_Close(self):
         self.cx.close()
 
@@ -143,28 +138,22 @@ def test_Exceptions(self):
         self.assertEqual(self.cx.ProgrammingError, monetdbe.ProgrammingError)
         self.assertEqual(self.cx.NotSupportedError, monetdbe.NotSupportedError)
 
-    def test_InTransaction(self):
-        # Can't use db from setUp because we want to test initial state.
-        cx = monetdbe.connect(":memory:")
-        cu = cx.cursor()
-        self.assertEqual(cx.in_transaction, False)
-        cu.execute("create table transactiontest(id integer auto_increment primary key, name text)")
-        self.assertEqual(cx.in_transaction, True)
-        cu.execute("insert into transactiontest(name) values (?)", ("foo",))
-        self.assertEqual(cx.in_transaction, True)
-        cu.execute("select name from transactiontest where name=?", ["foo"])
-        row = cu.fetchone()
-        self.assertEqual(cx.in_transaction, True)
-        cx.commit()
-        self.assertEqual(cx.in_transaction, False)
-        cu.execute("select name from transactiontest where name=?", ["foo"])
-        row = cu.fetchone()
-        self.assertEqual(cx.in_transaction, True)
-
     def test_InTransactionRO(self):
         with self.assertRaises(AttributeError):
             self.cx.in_transaction = True
 
+    @unittest.skip("TODO: Not yet implemented, see issue #22")
+    def test_OpenUri(self):
+        self.addCleanup(rmtree, TESTFN)
+        with monetdbe.connect(TESTFN) as cx:
+            cx.execute('create table test(id integer)')
+        with monetdbe.connect('file:' + TESTFN, uri=True) as cx:
+            cx.execute('insert into test(id) values(0)')
+        with monetdbe.connect('file:' + TESTFN + '?mode=ro', uri=True) as cx:
+            with self.assertRaises(monetdbe.OperationalError):
+                cx.execute('insert into test(id) values(1)')
+
+class VariousConnectionTests(unittest.TestCase):
     def test_OpenWithPathLikeObject(self):
         """ Checks that we can successfully connect to a database using an object that
             is PathLike, i.e. has __fspath__(). """
@@ -178,17 +167,47 @@ def __fspath__(self):
         with monetdbe.connect(path) as cx:
             cx.execute('create table test(id integer)')
 
-    @unittest.skip("TODO: Not yet implemented, see issue #22")
-    def test_OpenUri(self):
-        self.addCleanup(rmtree, TESTFN)
-        with monetdbe.connect(TESTFN) as cx:
-            cx.execute('create table test(id integer)')
-        with monetdbe.connect('file:' + TESTFN, uri=True) as cx:
-            cx.execute('insert into test(id) values(0)')
-        with monetdbe.connect('file:' + TESTFN + '?mode=ro', uri=True) as cx:
+    def test_InTransaction(self):
+        # Can't use db from setUp because we want to test initial state.
+        with monetdbe.connect(":memory:") as cx:
+            cu = cx.cursor()
+            self.assertEqual(cx.in_transaction, False)
+            cu.execute("create table transactiontest(id integer auto_increment primary key, name text)")
+            self.assertEqual(cx.in_transaction, True)
+            cu.execute("insert into transactiontest(name) values (?)", ("foo",))
+            self.assertEqual(cx.in_transaction, True)
+            cu.execute("select name from transactiontest where name=?", ["foo"])
+            row = cu.fetchone()
+            self.assertEqual(cx.in_transaction, True)
+            cx.commit()
+            self.assertEqual(cx.in_transaction, False)
+            cu.execute("select name from transactiontest where name=?", ["foo"])
+            row = cu.fetchone()
+            self.assertEqual(cx.in_transaction, True)
+
+    def test_FailedOpen(self):
+        YOU_CANNOT_OPEN_THIS = "/foo/bar/bla/23534/mydb.db"
+        with self.assertRaises(monetdbe.OperationalError):
+            con = monetdbe.connect(YOU_CANNOT_OPEN_THIS)
+
+    def test_empty_query(self):
+        with monetdbe.connect(':memory:') as conn:
             with self.assertRaises(monetdbe.OperationalError):
-                cx.execute('insert into test(id) values(1)')
+                conn.execute("")
 
+            with self.assertRaises(monetdbe.OperationalError):
+                conn.execute(";")
+
+def prepare_cursor(cx):
+    cu = cx.cursor()
+
+    # todo/note (Gijs): changed income type from number to float and made ID auto_increment
+    cu.execute(
+        "create table test(id integer auto_increment primary key, name text, "
+        "income float, unique_test text unique)"
+    )
+    cu.execute("insert into test(name) values (?)", ("foo",))
+    return cu
 
 class CursorTests(unittest.TestCase):
     def setUp(self):
@@ -231,12 +250,11 @@ def test_ExecuteTooMuchSql3(self):
             """)
 
     def test_empty_query(self):
-        conn = monetdbe.connect(':memory:')
         with self.assertRaises(monetdbe.OperationalError):
-            conn.execute("")
+            self.cx.execute("")
 
         with self.assertRaises(monetdbe.OperationalError):
-            conn.execute(";")
+            self.cx.execute(";")
 
     def test_ExecuteWrongSqlArg(self):
         with self.assertRaises(monetdbe.OperationalError):
@@ -419,6 +437,10 @@ def test_Fetchone(self):
         row = self.cu.fetchone()
         self.assertEqual(row, None)
 
+    @unittest.skip(
+        """
+        monetdbe returns the parameter description table for PREPARED statements.
+        Not really sure if it is a bad thing that the cursor can fetch that information""")
     def test_FetchoneNoStatement(self):
         cur = self.cx.cursor()
         row = cur.fetchone()
@@ -530,7 +552,6 @@ def test_LastRowIDInsertOR(self):
         ]
         self.assertEqual(results, expected)
 
-
 @unittest.skip("todo (gijs): for now we dont support or check for multi-threading")
 class ThreadTests(unittest.TestCase):
     def setUp(self):
@@ -716,84 +737,84 @@ def test_Binary(self):
 
 class ExtensionTests(unittest.TestCase):
     def test_ScriptStringSql(self):
-        con = monetdbe.connect(":memory:")
-        cur = con.cursor()
-        cur.executescript("""
-            -- bla bla
-            /* a stupid comment */
-            create table a(i int);
-            insert into a(i) values (5);
-            """)
-        cur.execute("select i from a")
-        res = cur.fetchone()[0]
-        self.assertEqual(res, 5)
+        with monetdbe.connect(":memory:") as con:
+            cur = con.cursor()
+            cur.executescript("""
+                -- bla bla
+                /* a stupid comment */
+                create table a(i int);
+                insert into a(i) values (5);
+                """)
+            cur.execute("select i from a")
+            res = cur.fetchone()[0]
+            self.assertEqual(res, 5)
 
     def test_ScriptSyntaxError(self):
-        con = monetdbe.connect(":memory:")
-        cur = con.cursor()
-        with self.assertRaises(monetdbe.OperationalError):
-            cur.executescript("create table test(x); asdf; create table test2(x)")
+        with monetdbe.connect(":memory:") as con:
+            cur = con.cursor()
+            with self.assertRaises(monetdbe.OperationalError):
+                cur.executescript("create table test(x); asdf; create table test2(x)")
 
     def test_ScriptErrorNormal(self):
-        con = monetdbe.connect(":memory:")
-        cur = con.cursor()
-        with self.assertRaises(monetdbe.OperationalError):
-            cur.executescript("create table test(sadfsadfdsa); select foo from hurz;")
+        with monetdbe.connect(":memory:") as con:
+            cur = con.cursor()
+            with self.assertRaises(monetdbe.OperationalError):
+                cur.executescript("create table test(sadfsadfdsa); select foo from hurz;")
 
     def test_CursorExecutescriptAsBytes(self):
-        con = monetdbe.connect(":memory:")
-        cur = con.cursor()
-        with self.assertRaises(ValueError) as cm:
-            cur.executescript(b"create table test(foo); insert into test(foo) values (5);")
-        self.assertEqual(str(cm.exception), 'script argument must be unicode.')
+        with monetdbe.connect(":memory:") as con:
+            cur = con.cursor()
+            with self.assertRaises(ValueError) as cm:
+                cur.executescript(b"create table test(foo); insert into test(foo) values (5);")
+            self.assertEqual(str(cm.exception), 'script argument must be unicode.')
 
     def test_ConnectionExecute(self):
-        con = monetdbe.connect(":memory:")
-        result = con.execute("select 5").fetchone()[0]
-        self.assertEqual(result, 5, "Basic test of Connection.execute")
+        with monetdbe.connect(":memory:") as con:
+            result = con.execute("select 5").fetchone()[0]
+            self.assertEqual(result, 5, "Basic test of Connection.execute")
 
     def test_ConnectionExecutemany(self):
-        con = monetdbe.connect(":memory:")
-        # NOTE: (gijs) added type int, required for MonetDB
-        con.execute("create table test(foo int)")
-        con.executemany("insert into test(foo) values (?)", [(3,), (4,)])
-        result = con.execute("select foo from test order by foo").fetchall()
-        self.assertEqual(result[0][0], 3, "Basic test of Connection.executemany")
-        self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany")
+        with monetdbe.connect(":memory:") as con:
+            # NOTE: (gijs) added type int, required for MonetDB
+            con.execute("create table test(foo int)")
+            con.executemany("insert into test(foo) values (?)", [(3,), (4,)])
+            result = con.execute("select foo from test order by foo").fetchall()
+            self.assertEqual(result[0][0], 3, "Basic test of Connection.executemany")
+            self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany")
 
     def test_ConnectionExecutescript(self):
-        con = monetdbe.connect(":memory:")
-        # NOTE: (gijs) added type int, required for MonetDB
-        con.executescript("create table test(foo int); insert into test(foo) values (5);")
-        result = con.execute("select foo from test").fetchone()[0]
-        self.assertEqual(result, 5, "Basic test of Connection.executescript")
+        with monetdbe.connect(":memory:") as con:
+            # NOTE: (gijs) added type int, required for MonetDB
+            con.executescript("create table test(foo int); insert into test(foo) values (5);")
+            result = con.execute("select foo from test").fetchone()[0]
+            self.assertEqual(result, 5, "Basic test of Connection.executescript")
 
 
 class ClosedConTests(unittest.TestCase):
     def test_ClosedConCursor(self):
-        con = monetdbe.connect(":memory:")
-        con.close()
-        with self.assertRaises(monetdbe.ProgrammingError):
-            cur = con.cursor()
+        with monetdbe.connect(":memory:") as con:
+            con.close()
+            with self.assertRaises(monetdbe.ProgrammingError):
+                cur = con.cursor()
 
     def test_ClosedConCommit(self):
-        con = monetdbe.connect(":memory:")
-        con.close()
-        with self.assertRaises(monetdbe.ProgrammingError):
-            con.commit()
+        with monetdbe.connect(":memory:") as con:
+            con.close()
+            with self.assertRaises(monetdbe.ProgrammingError):
+                con.commit()
 
     def test_ClosedConRollback(self):
-        con = monetdbe.connect(":memory:")
-        con.close()
-        with self.assertRaises(monetdbe.ProgrammingError):
-            con.rollback()
+        with monetdbe.connect(":memory:") as con:
+            con.close()
+            with self.assertRaises(monetdbe.ProgrammingError):
+                con.rollback()
 
     def test_ClosedCurExecute(self):
-        con = monetdbe.connect(":memory:")
-        cur = con.cursor()
-        con.close()
-        with self.assertRaises(monetdbe.ProgrammingError):
-            cur.execute("select 4")
+        with monetdbe.connect(":memory:") as con:
+            cur = con.cursor()
+            con.close()
+            with self.assertRaises(monetdbe.ProgrammingError):
+                cur.execute("select 4")
 
     def test_ClosedCreateFunction(self):
         con = monetdbe.connect(":memory:")
@@ -851,21 +872,21 @@ def test_ClosedCall(self):
 
 class ClosedCurTests(unittest.TestCase):
     def test_Closed(self):
-        con = monetdbe.connect(":memory:")
-        cur = con.cursor()
-        cur.close()
+        with monetdbe.connect(":memory:") as con:
+            cur = con.cursor()
+            cur.close()
 
-        for method_name in ("execute", "executemany", "executescript", "fetchall", "fetchmany", "fetchone"):
-            if method_name in ("execute", "executescript"):
-                params = ("select 4 union select 5",)
-            elif method_name == "executemany":
-                params = ("insert into foo(bar) values (?)", [(3,), (4,)])
-            else:
-                params = []
+            for method_name in ("execute", "executemany", "executescript", "fetchall", "fetchmany", "fetchone"):
+                if method_name in ("execute", "executescript"):
+                    params = ("select 4 union select 5",)
+                elif method_name == "executemany":
+                    params = ("insert into foo(bar) values (?)", [(3,), (4,)])
+                else:
+                    params = []
 
-            with self.assertRaises(monetdbe.ProgrammingError):
-                method = getattr(cur, method_name)
-                method(*params)
+                with self.assertRaises(monetdbe.ProgrammingError):
+                    method = getattr(cur, method_name)
+                    method(*params)
 
 
 @unittest.skip("We don't support INSERT OR syntax")
diff --git a/tests/test_lite/conftest.py b/tests/test_lite/conftest.py
index 7aced62..6d3f2a0 100644
--- a/tests/test_lite/conftest.py
+++ b/tests/test_lite/conftest.py
@@ -3,18 +3,17 @@
 from shutil import rmtree
 import monetdbe
 
-
 @pytest.fixture(scope="function")
 def monetdbe_empty_cursor(request, tmp_path):
     test_dbfarm = tmp_path.resolve().as_posix()
+    connection = monetdbe.make_connection(test_dbfarm)
 
     def finalizer():
         if tmp_path.is_dir():
-            monetdbe.connect(tmp_path).close()
+            connection.close()
             rmtree(test_dbfarm, ignore_errors=True)
 
     request.addfinalizer(finalizer)
-    connection = monetdbe.make_connection(test_dbfarm)
     cursor = connection.cursor()
     return cursor
 
@@ -23,14 +22,15 @@ def finalizer():
 def monetdbe_cursor(request, tmp_path):
     test_dbfarm = tmp_path.resolve().as_posix()
 
+    connection = monetdbe.make_connection(test_dbfarm)
+
     def finalizer():
         if tmp_path.is_dir():
-            monetdbe.connect(tmp_path).close()
+            connection.close()
             rmtree(test_dbfarm, ignore_errors=True)
 
     request.addfinalizer(finalizer)
 
-    connection = monetdbe.make_connection(test_dbfarm)
     cursor = connection.cursor()
     cursor.create('integers', {'i': numpy.arange(10)})
     cursor.execute('INSERT INTO integers VALUES(NULL)')
@@ -38,28 +38,37 @@ def finalizer():
 
 
 @pytest.fixture(scope="function")
-def monetdbe_cursor_autocommit(request, tmp_path):
+def monetdbe_cursor_autocommit(tmp_path):
     test_dbfarm = tmp_path.resolve().as_posix()
-
-    def finalizer():
-        if tmp_path.is_dir():
-            monetdbe.connect(tmp_path).close()
-            rmtree(test_dbfarm, ignore_errors=True)
-
-    request.addfinalizer(finalizer)
     connection = monetdbe.connect(test_dbfarm)
+
     connection.set_autocommit(True)
     cursor = connection.cursor()
-    return cursor, connection, test_dbfarm
+
+    class Context:
+        def __init__(self, cursor, connection, dbfarm):
+            self.cursor = cursor
+            self.connection = connection
+            self.dbfarm = dbfarm
+
+    context = Context(cursor, connection, test_dbfarm)
+
+    yield context
+
+    if tmp_path.is_dir():
+            context.connection.close()
+            rmtree(context.dbfarm, ignore_errors=True)
+
 
 
 @pytest.fixture(scope="function")
 def initialize_monetdbe(request, tmp_path):
     test_dbfarm = tmp_path.resolve().as_posix()
+    connection = monetdbe.connect(test_dbfarm)
 
     def finalizer():
         if tmp_path.is_dir():
-            monetdbe.connect(tmp_path).close()
+            connection.close()
             rmtree(test_dbfarm, ignore_errors=True)
 
     request.addfinalizer(finalizer)
diff --git a/tests/test_lite/test_dbapi03.py b/tests/test_lite/test_dbapi03.py
index 05581a5..aae85eb 100644
--- a/tests/test_lite/test_dbapi03.py
+++ b/tests/test_lite/test_dbapi03.py
@@ -5,7 +5,8 @@
 
 class TestShutdown:
     def test_commited_on_restart(self, monetdbe_cursor_autocommit):
-        (cursor, connection, dbfarm) = monetdbe_cursor_autocommit
+        context = monetdbe_cursor_autocommit
+        cursor = context.cursor
         cursor.transaction()
         cursor.execute('CREATE TABLE integers (i INTEGER)')
         cursor.executemany('INSERT INTO integers VALUES (%s)', [[x] for x in range(3)])
@@ -13,30 +14,32 @@ def test_commited_on_restart(self, monetdbe_cursor_autocommit):
         result = cursor.fetchall()
         assert result == [(0,), (1,), (2,)], "Incorrect result returned"
         cursor.commit()
-        connection.close()
+        context.connection.close()
 
-        connection = monetdbe.make_connection(dbfarm)
-        cursor = connection.cursor()
+        context.connection = monetdbe.make_connection(context.dbfarm)
+        cursor = context.connection.cursor()
         cursor.execute('SELECT * FROM integers')
         assert result == [(0,), (1,), (2,)], "Incorrect result returned"
 
     def test_transaction_aborted_on_shutdown(self, monetdbe_cursor_autocommit):
-        (cursor, connection, dbfarm) = monetdbe_cursor_autocommit
+        context = monetdbe_cursor_autocommit
+        cursor = context.cursor
         cursor.transaction()
         cursor.execute('CREATE TABLE integers (i INTEGER)')
         cursor.executemany('INSERT INTO integers VALUES (%s)', [[x] for x in range(3)])
         cursor.execute('SELECT * FROM integers')
         result = cursor.fetchall()
         assert result == [(0,), (1,), (2,)], "Incorrect result returned"
-        connection.close()
+        context.connection.close()
 
-        connection = monetdbe.make_connection(dbfarm)
-        cursor = connection.cursor()
+        context.connection = monetdbe.make_connection(context.dbfarm)
+        cursor = context.connection.cursor()
         with pytest.raises(monetdbe.DatabaseError):
             cursor.execute('SELECT * FROM integers')
 
     def test_many_shutdowns(self, monetdbe_cursor_autocommit):
-        (cursor, connection, dbfarm) = monetdbe_cursor_autocommit
+        context = monetdbe_cursor_autocommit
+        cursor = context.cursor
         for i in range(10):
             cursor.transaction()
             cursor.execute('CREATE TABLE integers (i INTEGER)')
@@ -44,9 +47,10 @@ def test_many_shutdowns(self, monetdbe_cursor_autocommit):
             cursor.execute('SELECT MIN(i * 3 + 5) FROM integers')
             result = cursor.fetchall()
             assert result == [(5,)], "Incorrect result returned"
-            connection.close()
+            context.connection.close()
 
-            connection = monetdbe.make_connection(dbfarm)
+            context.connection = monetdbe.make_connection(context.dbfarm)
+            connection = context.connection
             connection.set_autocommit(True)
             cursor = connection.cursor()
 
diff --git a/tests/test_regression.py b/tests/test_regression.py
index fe062a7..0ea5486 100644
--- a/tests/test_regression.py
+++ b/tests/test_regression.py
@@ -89,9 +89,9 @@ def test_StatementFinalizationOnCloseDb(self):
         # cache when closing the database. statements that were still
         # referenced in cursors weren't closed and could provoke "
         # "OperationalError: Unable to close due to unfinalised statements".
-        con = monetdbe.connect(":memory:")
         cursors = []
         # default statement cache size is 100
+        con = self.con
         for i in range(105):
             cur = con.cursor()
             cursors.append(cur)
@@ -100,7 +100,7 @@ def test_StatementFinalizationOnCloseDb(self):
 
     @unittest.skip("syntax not supported")
     def test_OnConflictRollback(self):
-        con = monetdbe.connect(":memory:")
+        con = self.con
         con.execute("create table foo(x int, unique(x) on conflict rollback)")
         con.execute("insert into foo(x) values (1)")
         try:
@@ -137,6 +137,9 @@ def test_TypeMapUsage(self):
         a statement. This test exhibits the problem.
         """
         SELECT = "select * from foo"
+
+        con = self.con
+        con.close()
         con = monetdbe.connect(":memory:", detect_types=monetdbe.PARSE_DECLTYPES)
         con.execute("create table foo(bar timestamp)")
         con.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),))
@@ -145,6 +148,7 @@ def test_TypeMapUsage(self):
         con.execute("create table foo(bar integer)")
         con.execute("insert into foo(bar) values (5)")
         con.execute(SELECT)
+        self.con = con
 
     @unittest.skip("todo/note (gijs): disable this for now since the monetdbe engine sees this as valid")
     def test_ErrorMsgDecodeError(self):
@@ -295,7 +299,8 @@ def collation_cb(a, b):
 
     def test_RecursiveCursorUse(self):
         # note: modified test slighty since we actually just handle this fine.
-        con = monetdbe.connect(":memory:")
+        
+        con = self.con
 
         cur = con.cursor()
         cur.execute("create table a (bar int)")
@@ -317,6 +322,8 @@ def test_ConvertTimestampMicrosecondPadding(self):
         since the microsecond string "456" actually represents "456000".
         """
 
+        con = self.con
+        con.close()
         con = monetdbe.connect(":memory:", detect_types=monetdbe.PARSE_DECLTYPES)
         cur = con.cursor()
         cur.execute("CREATE TABLE t (x TIMESTAMP)")
@@ -334,9 +341,12 @@ def test_ConvertTimestampMicrosecondPadding(self):
             datetime.datetime(2012, 4, 4, 15, 6, 0, 456000),
             datetime.datetime(2012, 4, 4, 15, 6, 0, 123456),
         ])
+        self.con = con
 
     def test_InvalidIsolationLevelType(self):
         # isolation level is a string, not an integer
+
+        self.con.close()
         self.assertRaises(TypeError,
                           monetdbe.connect, ":memory:", isolation_level=123)
 
@@ -399,6 +409,7 @@ def callback(*args):
         # The interpreter shouldn't crash when ref is collected.
         del ref
         gc.collect()
+        con.close()
 
     @unittest.skip("not supported (yet)")
     def test_DelIsolation_levelSegfault(self):
@@ -431,11 +442,11 @@ def test_multiple_memory_db_issue60(self):
         m = monetdbe.connect()
         c = m.cursor()
         c.execute(q)
-        del m._internal
-        del m
+        m.close()
         m = monetdbe.connect()
         c = m.cursor()
         c.execute(q)
+        m.close()
 
     def test_proper_error_on_empty_query_issue63(self):
         conn = monetdbe.connect(':memory:')
@@ -444,6 +455,7 @@ def test_proper_error_on_empty_query_issue63(self):
 
         with self.assertRaises(monetdbe.OperationalError):
             conn.execute(";")
+        conn.close()
 
     def test_real_issue83(self):
         conn = monetdbe.connect(':memory:')
@@ -456,6 +468,7 @@ def test_real_issue83(self):
         cursor.execute('SELECT * FROM "test"')
         df_out = cursor.fetchdf()
         pd.testing.assert_frame_equal(df, df_out)
+        conn.close()
 
     def test_relative_path(self):
         path = Path('this_folder_can_be_removed')
@@ -488,6 +501,7 @@ def test_copy_into_issue84(self):
 
         path = str((Path(__file__).parent / "example.csv").resolve().absolute())
         cur.execute(f"COPY  INTO test FROM '{path}' delimiters ',','\n'  best effort")
+        con.close()
 
     @unittest.skip("Disabled since takes quite long")
     def test_crash_loop(self):
@@ -496,6 +510,7 @@ def test_crash_loop(self):
             cu = cx.cursor()
             cu.execute("create table test(id integer auto_increment primary key, name text)")
             cu.execute("insert into test(name) values (?)", ("foo",))
+            cx.close()
 
     def test_issue127(self):
         conn = monetdbe.connect(':memory:')
@@ -504,6 +519,7 @@ def test_issue127(self):
         res = cur.execute("insert into tmp values(123, 'hello''world'':\n ERROR');")
         rows = res.fetchall()
         print(rows)
+        conn.close()
 
     @unittest.skip("Disabled until issue #118 is solved")
     def test_issue118_explain(self):
@@ -512,6 +528,7 @@ def test_issue118_explain(self):
         res = cur.execute('explain select 1')
         tbl = res.fetchall()
         print(tbl)
+        conn.close()
 
     @unittest.skipIf(OCT2020, "This issue was not fixed on Oct2020")
     def test_issue_136_bigint_result(self):
@@ -534,3 +551,4 @@ def test_issue_136_bigint_result(self):
         res5 = cur.execute('select * from test where col1 = -781377998').fetchall()
         assert (len(res5) == 1)
         assert (len(res5[0]) == 4)
+        con.close()
diff --git a/tests/test_transactions.py b/tests/test_transactions.py
index 7a4c04b..c8a247b 100644
--- a/tests/test_transactions.py
+++ b/tests/test_transactions.py
@@ -140,6 +140,7 @@ def test_Locking(self):
         # NO self.con2.rollback() HERE!!!
         self.con1.commit()
 
+    @unittest.skip("disabled: MonetDBe does not yet support concurrent connections to different dbfarm's")
     def test_RollbackCursorConsistency(self):
         """
         Checks if cursors on the connection are set into a "reset" state
@@ -154,6 +155,7 @@ def test_RollbackCursorConsistency(self):
         con.rollback()
         with self.assertRaises(monetdbe.InterfaceError):
             cur.fetchall()
+        con.close()
 
 
 class SpecialCommandTests(unittest.TestCase):

From e3f4e479e7588380ca064efd2eda90dd0207e125 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Mon, 6 Sep 2021 11:04:37 +0200
Subject: [PATCH 02/17] Accomodate API change in monetdbe_prepare for released
 versions.

---
 monetdbe/_cffi/builder.py  | 4 +++-
 monetdbe/_cffi/embed.h.j2  | 4 ++++
 monetdbe/_cffi/internal.py | 6 +++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/monetdbe/_cffi/builder.py b/monetdbe/_cffi/builder.py
index 49a68fb..6976dfa 100644
--- a/monetdbe/_cffi/builder.py
+++ b/monetdbe/_cffi/builder.py
@@ -14,10 +14,12 @@
 print("**MONETDB**: If this is incorrect, set the MONETDB_BRANCH environment variable during monetdbe-python build\n")
 
 branch_file = str(Path(__file__).parent / 'branch.py')
+newer_then_jul2021 = monetdb_branch.lower() not in ("oct2020", "jul2021")
 
 with open(branch_file, 'w') as f:
     f.write("# this file is created by the cffi interface builder and contains the monetdb branch env variable.\n\n")
     f.write(f"monetdb_branch = '{monetdb_branch}'\n")
+    f.write(f"newer_then_jul2021 = {newer_then_jul2021}\n")
 
 
 default = monetdb_branch.lower() in ("default", "jul2021")
@@ -38,7 +40,7 @@
 with open(embed_path, 'r') as f:
     content = f.read()
     template = Template(content)
-    cdef = template.render(win32=win32, default=default)
+    cdef = template.render(win32=win32, default=default, newer_then_jul2021=newer_then_jul2021)
     ffibuilder.cdef(cdef)
 
 
diff --git a/monetdbe/_cffi/embed.h.j2 b/monetdbe/_cffi/embed.h.j2
index 1a214fc..8675a16 100644
--- a/monetdbe/_cffi/embed.h.j2
+++ b/monetdbe/_cffi/embed.h.j2
@@ -114,7 +114,11 @@ extern char* monetdbe_query(monetdbe_database dbhdl, char* query, monetdbe_resul
 extern char* monetdbe_result_fetch(monetdbe_result *mres, monetdbe_column** res, size_t column_index);
 extern char* monetdbe_cleanup_result(monetdbe_database dbhdl, monetdbe_result* result);
 
+{% if newer_then_jul2021 %}
 extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt, monetdbe_result** result);
+{% else %}
+extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt);
+{% endif %}
 extern char* monetdbe_bind(monetdbe_statement *stmt, void *data, size_t parameter_nr);
 extern char* monetdbe_execute(monetdbe_statement *stmt, monetdbe_result **result, monetdbe_cnt* affected_rows);
 extern char* monetdbe_cleanup_statement(monetdbe_database dbhdl, monetdbe_statement *stmt);
diff --git a/monetdbe/_cffi/internal.py b/monetdbe/_cffi/internal.py
index c24f124..2bb62ab 100644
--- a/monetdbe/_cffi/internal.py
+++ b/monetdbe/_cffi/internal.py
@@ -257,7 +257,11 @@ def append(self, table: str, data: Mapping[str, np.ndarray], schema: str = 'sys'
     def prepare(self, query: str) -> monetdbe_statement:
         self._switch()
         stmt = ffi.new("monetdbe_statement **")
-        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
+        from monetdbe._cffi.branch import newer_then_jul2021
+        if newer_then_jul2021:
+            check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
+        else:
+            check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
         return stmt[0]
 
     def cleanup_statement(self, statement: monetdbe_statement) -> None:

From 8ea25406e4805b46977aaf5cfc3eee8022db077d Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Mon, 6 Sep 2021 11:51:28 +0200
Subject: [PATCH 03/17] Code style fixes.

---
 tests/test_dbapi.py         | 4 ++++
 tests/test_lite/conftest.py | 6 +++---
 tests/test_regression.py    | 1 -
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/tests/test_dbapi.py b/tests/test_dbapi.py
index 8d3fa87..781fe03 100644
--- a/tests/test_dbapi.py
+++ b/tests/test_dbapi.py
@@ -153,6 +153,7 @@ def test_OpenUri(self):
             with self.assertRaises(monetdbe.OperationalError):
                 cx.execute('insert into test(id) values(1)')
 
+
 class VariousConnectionTests(unittest.TestCase):
     def test_OpenWithPathLikeObject(self):
         """ Checks that we can successfully connect to a database using an object that
@@ -198,6 +199,7 @@ def test_empty_query(self):
             with self.assertRaises(monetdbe.OperationalError):
                 conn.execute(";")
 
+
 def prepare_cursor(cx):
     cu = cx.cursor()
 
@@ -209,6 +211,7 @@ def prepare_cursor(cx):
     cu.execute("insert into test(name) values (?)", ("foo",))
     return cu
 
+
 class CursorTests(unittest.TestCase):
     def setUp(self):
         self.cx = monetdbe.connect(":memory:")
@@ -552,6 +555,7 @@ def test_LastRowIDInsertOR(self):
         ]
         self.assertEqual(results, expected)
 
+
 @unittest.skip("todo (gijs): for now we dont support or check for multi-threading")
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/tests/test_lite/conftest.py b/tests/test_lite/conftest.py
index 6d3f2a0..1ffef01 100644
--- a/tests/test_lite/conftest.py
+++ b/tests/test_lite/conftest.py
@@ -3,6 +3,7 @@
 from shutil import rmtree
 import monetdbe
 
+
 @pytest.fixture(scope="function")
 def monetdbe_empty_cursor(request, tmp_path):
     test_dbfarm = tmp_path.resolve().as_posix()
@@ -56,9 +57,8 @@ def __init__(self, cursor, connection, dbfarm):
     yield context
 
     if tmp_path.is_dir():
-            context.connection.close()
-            rmtree(context.dbfarm, ignore_errors=True)
-
+        context.connection.close()
+        rmtree(context.dbfarm, ignore_errors=True)
 
 
 @pytest.fixture(scope="function")
diff --git a/tests/test_regression.py b/tests/test_regression.py
index 0ea5486..b6aae4a 100644
--- a/tests/test_regression.py
+++ b/tests/test_regression.py
@@ -299,7 +299,6 @@ def collation_cb(a, b):
 
     def test_RecursiveCursorUse(self):
         # note: modified test slighty since we actually just handle this fine.
-        
         con = self.con
 
         cur = con.cursor()

From c3da377dafe30ed2d8e18f1c1535decbd992f99c Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Mon, 6 Sep 2021 13:45:06 +0200
Subject: [PATCH 04/17] Override the prepare method at a more static time of
 execution.

---
 monetdbe/_cffi/internal.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/monetdbe/_cffi/internal.py b/monetdbe/_cffi/internal.py
index 2bb62ab..3450685 100644
--- a/monetdbe/_cffi/internal.py
+++ b/monetdbe/_cffi/internal.py
@@ -257,11 +257,7 @@ def append(self, table: str, data: Mapping[str, np.ndarray], schema: str = 'sys'
     def prepare(self, query: str) -> monetdbe_statement:
         self._switch()
         stmt = ffi.new("monetdbe_statement **")
-        from monetdbe._cffi.branch import newer_then_jul2021
-        if newer_then_jul2021:
-            check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
-        else:
-            check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
+        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
         return stmt[0]
 
     def cleanup_statement(self, statement: monetdbe_statement) -> None:
@@ -289,3 +285,13 @@ def get_columns(self, table: str, schema: str = 'sys') -> Iterator[Tuple[str, in
             name = ffi.string(names_p[0][i]).decode()
             type_ = types_p[0][i]
             yield name, type_
+
+    from monetdbe._cffi.branch import newer_then_jul2021
+    if not newer_then_jul2021:
+        def prepare(_self: Internal, query: str) -> monetdbe_statement:
+            _self._switch()
+            stmt = ffi.new("monetdbe_statement **")
+            check_error(lib.monetdbe_prepare(_self._monetdbe_database, str(query).encode(), stmt))
+            return stmt[0]
+
+        Internal.prepare = prepare

From dbf55a5024a057f035dc75d219c51ad9c020df22 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Tue, 7 Sep 2021 09:54:54 +0200
Subject: [PATCH 05/17] Allow test workflows to be run manually.

---
 .github/workflows/linux.yml   | 2 ++
 .github/workflows/osx.yml     | 2 ++
 .github/workflows/windows.yml | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 8d4940b..d671d43 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -6,6 +6,8 @@ on:
     tags: [ "*" ]
   pull_request:
     branches: [ master ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
   linux-test:
diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index 4008853..bb5387e 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -6,6 +6,8 @@ on:
     tags: [ "*" ]
   pull_request:
     branches: [ master ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
   osx-monetdb-Jul2021:
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index a05fd30..60c604d 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -6,6 +6,8 @@ on:
     tags: [ "*" ]
   pull_request:
     branches: [ master ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 # the prefix and version numbers for the branches need to be manually copied (for now) from:
 # http://monetdb.cwi.nl/testweb/web/status.php

From 392a0902bd054224c543306214451ac198ba9af5 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Tue, 7 Sep 2021 10:57:54 +0200
Subject: [PATCH 06/17] Clean up the version specific method override

---
 monetdbe/_cffi/internal.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/monetdbe/_cffi/internal.py b/monetdbe/_cffi/internal.py
index 3450685..98cd63f 100644
--- a/monetdbe/_cffi/internal.py
+++ b/monetdbe/_cffi/internal.py
@@ -286,12 +286,13 @@ def get_columns(self, table: str, schema: str = 'sys') -> Iterator[Tuple[str, in
             type_ = types_p[0][i]
             yield name, type_
 
-    from monetdbe._cffi.branch import newer_then_jul2021
-    if not newer_then_jul2021:
-        def prepare(_self: Internal, query: str) -> monetdbe_statement:
-            _self._switch()
-            stmt = ffi.new("monetdbe_statement **")
-            check_error(lib.monetdbe_prepare(_self._monetdbe_database, str(query).encode(), stmt))
-            return stmt[0]
-
-        Internal.prepare = prepare
+
+from monetdbe._cffi.branch import newer_then_jul2021
+if not newer_then_jul2021:
+    def prepare(self, query: str) -> monetdbe_statement:
+        self._switch()
+        stmt = ffi.new("monetdbe_statement **")
+        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
+        return stmt[0]
+
+    Internal.prepare = prepare

From 9b2bb674d1b256b2de83fe9def2abc3216194e21 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Tue, 7 Sep 2021 11:08:24 +0200
Subject: [PATCH 07/17] Fix type check.

---
 monetdbe/_cffi/internal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monetdbe/_cffi/internal.py b/monetdbe/_cffi/internal.py
index 98cd63f..11ca476 100644
--- a/monetdbe/_cffi/internal.py
+++ b/monetdbe/_cffi/internal.py
@@ -295,4 +295,4 @@ def prepare(self, query: str) -> monetdbe_statement:
         check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
         return stmt[0]
 
-    Internal.prepare = prepare
+    setattr(Internal, 'prepare', prepare)

From d2a5d592902e2567fbb37b3b24e70a1113f8f7f2 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Tue, 7 Sep 2021 11:29:49 +0200
Subject: [PATCH 08/17] Use newer MonetDB Windows build

---
 .github/workflows/windows.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 60c604d..3bab0d9 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -22,8 +22,8 @@ jobs:
 #            MONETDB_WIN_PREFIX: 82411
 #            MONETDB_WIN_VERSION: c51ff55bd174
           - MONETDB_BRANCH: Jul2021
-            MONETDB_WIN_PREFIX: 82733
-            MONETDB_WIN_VERSION: 3a8a37d1da2b
+            MONETDB_WIN_PREFIX: 82786
+            MONETDB_WIN_VERSION: e39e91cde6b5
 
     env:
       MSI: MonetDB5-SQL-Installer-x86_64-${{ matrix.MONETDB_WIN_VERSION }}.msi

From 60bbfad993f62c2b605b9abf5d7a52ad8fc1fa34 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Tue, 7 Sep 2021 13:13:55 +0200
Subject: [PATCH 09/17] Base macos CI on mercurial source.

---
 .github/workflows/osx.yml | 56 +++++++++++++++------------------------
 1 file changed, 21 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index bb5387e..21f6f73 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -17,46 +17,32 @@ jobs:
     steps:
       -
         name: brew packages
-        run: brew install monetdb
-      - 
+        run: brew install bison
+      -
+        name: download MonetDB
+        run: curl https://dev.monetdb.org/hg/MonetDB/archive/${{ env.MONETDB_BRANCH }}.tar.bz2 -O
+      -
+        name: extract MonetDB
+        run: tar jxvf ${{ env.MONETDB_BRANCH }}.tar.bz2
+      -
+        name: build monetdb
+        run: |
+          mkdir MonetDB-${{ env.MONETDB_BRANCH }}/build
+          cd MonetDB-${{ env.MONETDB_BRANCH }}/build
+          cmake ..  -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/monetdb/11.40.1 \
+            -DWITH_CRYPTO=OFF \
+            -DPY3INTEGRATION=OFF \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DASSERT=OFF \
+            -DRINTEGRATION=OFF \
+            -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
+          make install -j3
+      -
         uses: actions/upload-artifact@v2
         with:
           name: monetdb-osx-${{ env.MONETDB_BRANCH }}
           path: /usr/local/Cellar/monetdb
 
-#  osx-monetdb-default:
-#    runs-on: macos-10.15
-#    env:
-#      MONETDB_BRANCH: default
-#    steps:
-#      -
-#        name: brew packages
-#        run: brew install bison
-#      -
-#        name: download MonetDB
-#        run: curl https://dev.monetdb.org/hg/MonetDB/archive/${{ env.MONETDB_BRANCH }}.tar.bz2 -O
-#      -
-#        name: extract MonetDB
-#        run: tar jxvf ${{ env.MONETDB_BRANCH }}.tar.bz2
-#      -
-#        name: build monetdb
-#        run: |
-#          mkdir MonetDB-${{ env.MONETDB_BRANCH }}/build
-#          cd MonetDB-${{ env.MONETDB_BRANCH }}/build
-#          cmake ..  -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/monetdb/11.40.1 \
-#            -DWITH_CRYPTO=OFF \
-#            -DPY3INTEGRATION=OFF \
-#            -DCMAKE_BUILD_TYPE=Release \
-#            -DASSERT=OFF \
-#            -DRINTEGRATION=OFF \
-#            -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
-#          make install -j3
-#      -
-#        uses: actions/upload-artifact@v2
-#        with:
-#          name: monetdb-osx-${{ env.MONETDB_BRANCH }}
-#          path: /usr/local/Cellar/monetdb
-
   osx-wheel:
 #    continue-on-error: true
     runs-on: macos-10.15

From 1856a6fcaeca87ce8eec9ad792e0b2f36a9d550b Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Wed, 8 Sep 2021 09:44:22 +0200
Subject: [PATCH 10/17] Activate default for osx workflow.

---
 .github/workflows/osx.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index 21f6f73..8aa2dda 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -10,10 +10,13 @@ on:
   workflow_dispatch:
 
 jobs:
-  osx-monetdb-Jul2021:
+  osx-monetdb:
     runs-on: macos-10.15
+    strategy:
+      matrix:
+        branch: [default, Jul2021]
     env:
-      MONETDB_BRANCH: Jul2021
+      MONETDB_BRANCH: ${{ matrix.branch }}
     steps:
       -
         name: brew packages
@@ -49,10 +52,10 @@ jobs:
     strategy:
       matrix:
         python-version: [ 3.6, 3.7, 3.8, 3.9 ]
-        branch: [Jul2021]
+        branch: [default, Jul2021]
     env:
       MONETDB_BRANCH: ${{ matrix.branch }}
-    needs: [osx-monetdb-Jul2021]
+    needs: [osx-monetdb]
     steps:
       -
         name: Print MONETDB_BRANCH to make sure it is set

From b37d4d3a4ea1ed12c1deaace8923e2598f381031 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Wed, 8 Sep 2021 10:15:59 +0200
Subject: [PATCH 11/17] Activate default based CI on Windows and Linux.

---
 .github/workflows/linux.yml   | 4 ++--
 .github/workflows/windows.yml | 9 ++++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index d671d43..7ccc6b0 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        branch: [Jul2021]
+        branch: [default, Jul2021]
     container: monetdb/dev-builds:${{ matrix.branch }}
     steps:
       -
@@ -35,7 +35,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
-        branch: [Jul2021]
+        branch: [default, Jul2021]
     container: monetdb/dev-builds:${{ matrix.branch}}_manylinux
 
     steps:
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 3bab0d9..f091900 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -16,11 +16,10 @@ jobs:
     runs-on: windows-2019
     strategy:
       matrix:
-        MONETDB_BRANCH: [ Jul2021 ]
         include:
-#          - MONETDB_BRANCH: default
-#            MONETDB_WIN_PREFIX: 82411
-#            MONETDB_WIN_VERSION: c51ff55bd174
+          - MONETDB_BRANCH: default
+            MONETDB_WIN_PREFIX: 82777
+            MONETDB_WIN_VERSION: f619cf62c92d
           - MONETDB_BRANCH: Jul2021
             MONETDB_WIN_PREFIX: 82786
             MONETDB_WIN_VERSION: e39e91cde6b5
@@ -55,7 +54,7 @@ jobs:
     strategy:
       matrix:
         python-version: [ 3.6, 3.7, 3.8, 3.9 ]
-        branch: [ Jul2021 ]
+        branch: [ default, Jul2021 ]
     env:
       MONETDB_BRANCH: ${{ matrix.branch }}
     steps:

From 476c4ac68c18c3731e4788cc59883db796464d34 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Wed, 8 Sep 2021 10:32:56 +0200
Subject: [PATCH 12/17] Small improvement.

---
 .github/workflows/linux.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 7ccc6b0..10ec516 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -31,6 +31,7 @@ jobs:
         run: py.test -vv
 
   linux-wheel:
+    needs: linux-test
     runs-on: ubuntu-20.04
     strategy:
       matrix:

From 16561d947406c01c6a9bf51f89e6b9540ed27796 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Wed, 8 Sep 2021 10:34:08 +0200
Subject: [PATCH 13/17] Make step in workflow job a bit more branch-agnostic
 and concise.

---
 .github/workflows/windows.yml | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index f091900..80d0ec1 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -78,21 +78,7 @@ jobs:
         name: Collect DLLs
         run: |
           ls 'C:\Program Files\MonetDB\MonetDB5\bin'
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\bat.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\mapi.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdb5.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdbe.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdbsql.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\stream.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\libcrypto-1_1-x64.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\iconv-2.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\libxml2.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\pcre.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\zlib1.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\lzma.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\charset-1.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\bz2.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\lz4.dll' monetdbe\.
+          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\*.dll' monetdbe\.
           ls monetdbe
       -
         name: Upgrade pip and wheel for all pythons

From d8ed01fc8f4e855da351a8246615b1f73c80cb8c Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Wed, 8 Sep 2021 11:03:38 +0200
Subject: [PATCH 14/17] Recover test_FailedOpen.

---
 tests/test_dbapi.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test_dbapi.py b/tests/test_dbapi.py
index 781fe03..daddba1 100644
--- a/tests/test_dbapi.py
+++ b/tests/test_dbapi.py
@@ -122,6 +122,11 @@ def test_RollbackAfterNoChanges(self):
     def test_Cursor(self):
         cu = self.cx.cursor()
 
+    def test_FailedOpen(self):
+        YOU_CANNOT_OPEN_THIS = "asdiadasdiasjdasoijdasdsdasasl31mydb.db"
+        with self.assertRaises(monetdbe.OperationalError):
+            con = monetdbe.connect(YOU_CANNOT_OPEN_THIS)
+
     def test_Close(self):
         self.cx.close()
 

From 785d5218b1020db5ade8d25a32966e3ecc1c2463 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Thu, 9 Sep 2021 10:16:40 +0200
Subject: [PATCH 15/17] Duplicate test.

---
 tests/test_dbapi.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tests/test_dbapi.py b/tests/test_dbapi.py
index daddba1..b5f0c44 100644
--- a/tests/test_dbapi.py
+++ b/tests/test_dbapi.py
@@ -122,11 +122,6 @@ def test_RollbackAfterNoChanges(self):
     def test_Cursor(self):
         cu = self.cx.cursor()
 
-    def test_FailedOpen(self):
-        YOU_CANNOT_OPEN_THIS = "asdiadasdiasjdasoijdasdsdasasl31mydb.db"
-        with self.assertRaises(monetdbe.OperationalError):
-            con = monetdbe.connect(YOU_CANNOT_OPEN_THIS)
-
     def test_Close(self):
         self.cx.close()
 
@@ -192,7 +187,7 @@ def test_InTransaction(self):
             self.assertEqual(cx.in_transaction, True)
 
     def test_FailedOpen(self):
-        YOU_CANNOT_OPEN_THIS = "/foo/bar/bla/23534/mydb.db"
+        YOU_CANNOT_OPEN_THIS = "asdiadasdiasjdasoijdasdsdasasl31mydb.db"
         with self.assertRaises(monetdbe.OperationalError):
             con = monetdbe.connect(YOU_CANNOT_OPEN_THIS)
 

From acec3f0ecb331439ade77030ba991344ff1f9ec5 Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Thu, 9 Sep 2021 10:25:09 +0200
Subject: [PATCH 16/17] Use a cross-platform dummy path in test.

---
 tests/test_dbapi.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_dbapi.py b/tests/test_dbapi.py
index b5f0c44..1a19c9b 100644
--- a/tests/test_dbapi.py
+++ b/tests/test_dbapi.py
@@ -187,7 +187,8 @@ def test_InTransaction(self):
             self.assertEqual(cx.in_transaction, True)
 
     def test_FailedOpen(self):
-        YOU_CANNOT_OPEN_THIS = "asdiadasdiasjdasoijdasdsdasasl31mydb.db"
+        import os
+        YOU_CANNOT_OPEN_THIS = os.path.join("foo", "bar", "bla", "23534", "mydb.db")
         with self.assertRaises(monetdbe.OperationalError):
             con = monetdbe.connect(YOU_CANNOT_OPEN_THIS)
 

From 931460d975f0e1301677f0fbbad10ad0cf95411a Mon Sep 17 00:00:00 2001
From: Aris Koning <aris.koning@monetdbsolutions.com>
Date: Thu, 9 Sep 2021 10:42:06 +0200
Subject: [PATCH 17/17] Modernize formatted string.

---
 monetdbe/_cffi/internal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monetdbe/_cffi/internal.py b/monetdbe/_cffi/internal.py
index 11ca476..bedd0ef 100644
--- a/monetdbe/_cffi/internal.py
+++ b/monetdbe/_cffi/internal.py
@@ -169,7 +169,7 @@ def open(self) -> monetdbe_database:
                 lib.monetdbe_close(connection)
             else:
                 error = errors.get(result_code, "unknown error")
-            msg = "Failed to open database: {error} (code {result_code})".format(error=error, result_code=result_code)
+            msg = f"Failed to open database: {error} (code {result_code})"
             raise exceptions.OperationalError(msg)
 
         return connection