Merge pull request #153 from MonetDBSolutions/issue151_monetdbe_bind

Issue151 monetdbe bind

Author: gijzelaerr

.github/workflows/linux.yml

@@ -6,13 +6,15 @@ on:
     tags: [ "*" ]
   pull_request:
     branches: [ master ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
   linux-test:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        branch: [Jul2021]
+        branch: [default, Jul2021]
     container: monetdb/dev-builds:${{ matrix.branch }}
     steps:
       -
@@ -29,11 +31,12 @@ jobs:
         run: py.test -vv
 
   linux-wheel:
+    needs: linux-test
     runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
-        branch: [Jul2021]
+        branch: [default, Jul2021]
     container: monetdb/dev-builds:${{ matrix.branch}}_manylinux
 
     steps:

.github/workflows/osx.yml

@@ -6,65 +6,56 @@ on:
     tags: [ "*" ]
   pull_request:
     branches: [ master ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
-  osx-monetdb-Jul2021:
+  osx-monetdb:
     runs-on: macos-10.15
+    strategy:
+      matrix:
+        branch: [default, Jul2021]
     env:
-      MONETDB_BRANCH: Jul2021
+      MONETDB_BRANCH: ${{ matrix.branch }}
     steps:
       -
         name: brew packages
-        run: brew install monetdb
-      - 
+        run: brew install bison
+      -
+        name: download MonetDB
+        run: curl https://dev.monetdb.org/hg/MonetDB/archive/${{ env.MONETDB_BRANCH }}.tar.bz2 -O
+      -
+        name: extract MonetDB
+        run: tar jxvf ${{ env.MONETDB_BRANCH }}.tar.bz2
+      -
+        name: build monetdb
+        run: |
+          mkdir MonetDB-${{ env.MONETDB_BRANCH }}/build
+          cd MonetDB-${{ env.MONETDB_BRANCH }}/build
+          cmake ..  -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/monetdb/11.40.1 \
+            -DWITH_CRYPTO=OFF \
+            -DPY3INTEGRATION=OFF \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DASSERT=OFF \
+            -DRINTEGRATION=OFF \
+            -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
+          make install -j3
+      -
         uses: actions/upload-artifact@v2
         with:
           name: monetdb-osx-${{ env.MONETDB_BRANCH }}
           path: /usr/local/Cellar/monetdb
 
-#  osx-monetdb-default:
-#    runs-on: macos-10.15
-#    env:
-#      MONETDB_BRANCH: default
-#    steps:
-#      -
-#        name: brew packages
-#        run: brew install bison
-#      -
-#        name: download MonetDB
-#        run: curl https://dev.monetdb.org/hg/MonetDB/archive/${{ env.MONETDB_BRANCH }}.tar.bz2 -O
-#      -
-#        name: extract MonetDB
-#        run: tar jxvf ${{ env.MONETDB_BRANCH }}.tar.bz2
-#      -
-#        name: build monetdb
-#        run: |
-#          mkdir MonetDB-${{ env.MONETDB_BRANCH }}/build
-#          cd MonetDB-${{ env.MONETDB_BRANCH }}/build
-#          cmake ..  -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/monetdb/11.40.1 \
-#            -DWITH_CRYPTO=OFF \
-#            -DPY3INTEGRATION=OFF \
-#            -DCMAKE_BUILD_TYPE=Release \
-#            -DASSERT=OFF \
-#            -DRINTEGRATION=OFF \
-#            -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
-#          make install -j3
-#      -
-#        uses: actions/upload-artifact@v2
-#        with:
-#          name: monetdb-osx-${{ env.MONETDB_BRANCH }}
-#          path: /usr/local/Cellar/monetdb
-
   osx-wheel:
 #    continue-on-error: true
     runs-on: macos-10.15
     strategy:
       matrix:
         python-version: [ 3.6, 3.7, 3.8, 3.9 ]
-        branch: [Jul2021]
+        branch: [default, Jul2021]
     env:
       MONETDB_BRANCH: ${{ matrix.branch }}
-    needs: [osx-monetdb-Jul2021]
+    needs: [osx-monetdb]
     steps:
       -
         name: Print MONETDB_BRANCH to make sure it is set

.github/workflows/windows.yml

@@ -6,6 +6,8 @@ on:
     tags: [ "*" ]
   pull_request:
     branches: [ master ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 # the prefix and version numbers for the branches need to be manually copied (for now) from:
 # http://monetdb.cwi.nl/testweb/web/status.php
@@ -14,14 +16,13 @@ jobs:
     runs-on: windows-2019
     strategy:
       matrix:
-        MONETDB_BRANCH: [ Jul2021 ]
         include:
-#          - MONETDB_BRANCH: default
-#            MONETDB_WIN_PREFIX: 82411
-#            MONETDB_WIN_VERSION: c51ff55bd174
+          - MONETDB_BRANCH: default
+            MONETDB_WIN_PREFIX: 82777
+            MONETDB_WIN_VERSION: f619cf62c92d
           - MONETDB_BRANCH: Jul2021
-            MONETDB_WIN_PREFIX: 82733
-            MONETDB_WIN_VERSION: 3a8a37d1da2b
+            MONETDB_WIN_PREFIX: 82786
+            MONETDB_WIN_VERSION: e39e91cde6b5
 
     env:
       MSI: MonetDB5-SQL-Installer-x86_64-${{ matrix.MONETDB_WIN_VERSION }}.msi
@@ -53,7 +54,7 @@ jobs:
     strategy:
       matrix:
         python-version: [ 3.6, 3.7, 3.8, 3.9 ]
-        branch: [ Jul2021 ]
+        branch: [ default, Jul2021 ]
     env:
       MONETDB_BRANCH: ${{ matrix.branch }}
     steps:
@@ -77,21 +78,7 @@ jobs:
         name: Collect DLLs
         run: |
           ls 'C:\Program Files\MonetDB\MonetDB5\bin'
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\bat.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\mapi.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdb5.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdbe.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdbsql.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\stream.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\libcrypto-1_1-x64.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\iconv-2.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\libxml2.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\pcre.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\zlib1.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\lzma.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\charset-1.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\bz2.dll' monetdbe\.
-          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\lz4.dll' monetdbe\.
+          Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\*.dll' monetdbe\.
           ls monetdbe
       -
         name: Upgrade pip and wheel for all pythons

monetdbe/_cffi/builder.py

@@ -14,10 +14,12 @@
 print("**MONETDB**: If this is incorrect, set the MONETDB_BRANCH environment variable during monetdbe-python build\n")
 
 branch_file = str(Path(__file__).parent / 'branch.py')
+newer_then_jul2021 = monetdb_branch.lower() not in ("oct2020", "jul2021")
 
 with open(branch_file, 'w') as f:
     f.write("# this file is created by the cffi interface builder and contains the monetdb branch env variable.\n\n")
     f.write(f"monetdb_branch = '{monetdb_branch}'\n")
+    f.write(f"newer_then_jul2021 = {newer_then_jul2021}\n")
 
 
 default = monetdb_branch.lower() in ("default", "jul2021")
@@ -38,7 +40,7 @@
 with open(embed_path, 'r') as f:
     content = f.read()
     template = Template(content)
-    cdef = template.render(win32=win32, default=default)
+    cdef = template.render(win32=win32, default=default, newer_then_jul2021=newer_then_jul2021)
     ffibuilder.cdef(cdef)
 
 

monetdbe/_cffi/convert/bind.py

@@ -10,8 +10,8 @@ def monetdbe_int(data: int) -> ffi.CData:
         return ffi.new("int *", data)
 
 
-def bind_str(data: str) -> bytes:
-    return str(data).encode()
+def bind_str(data: str) -> ffi.CData:
+    return ffi.new("char[]", str(data).encode())
 
 
 def bind_float(data: float) -> ffi.CData:

monetdbe/_cffi/embed.h.j2

@@ -25,8 +25,7 @@ typedef struct {
 
 typedef enum {
  monetdbe_bool, monetdbe_int8_t, monetdbe_int16_t, monetdbe_int32_t, monetdbe_int64_t,
- {% if not win32 %}
- {# windows does not support 128-bit #}
+ {% if HAVE_HGE %}
  monetdbe_int128_t,
  {% endif %}
  monetdbe_size_t, monetdbe_float, monetdbe_double,
@@ -115,7 +114,11 @@ extern char* monetdbe_query(monetdbe_database dbhdl, char* query, monetdbe_resul
 extern char* monetdbe_result_fetch(monetdbe_result *mres, monetdbe_column** res, size_t column_index);
 extern char* monetdbe_cleanup_result(monetdbe_database dbhdl, monetdbe_result* result);
 
+{% if newer_then_jul2021 %}
+extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt, monetdbe_result** result);
+{% else %}
 extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt);
+{% endif %}
 extern char* monetdbe_bind(monetdbe_statement *stmt, void *data, size_t parameter_nr);
 extern char* monetdbe_execute(monetdbe_statement *stmt, monetdbe_result **result, monetdbe_cnt* affected_rows);
 extern char* monetdbe_cleanup_statement(monetdbe_database dbhdl, monetdbe_statement *stmt);

monetdbe/_cffi/internal.py

@@ -113,18 +113,14 @@ def __init__(
         self.nr_threads = nr_threads
         self.have_hge = have_hge
         self._switch()
+        self._monetdbe_database = self.open()
 
     @classmethod
     def set_active_context(cls, active_context: Optional['Internal']):
         cls._active_context = active_context
 
-    @classmethod
-    def set_active_connection(cls, active_connection: Optional['Connection']):
-        cls._active_connection = active_connection
-
-    @classmethod
-    def set_monetdbe_database(cls, connection: Optional[monetdbe_database]):
-        cls._monetdbe_database = connection
+    def set_monetdbe_database(self, connection: Optional[monetdbe_database]):
+        self._monetdbe_database = connection
 
     def __del__(self):
         if self._active_context == self:
@@ -135,17 +131,10 @@ def _switch(self):
         if self._active_context == self:
             return
 
-        # this is a bit scary but just to make sure the previous connection
-        # can't touch us anymore
-        if self._active_connection:
-            self._active_connection._internal = None
-
-        self.close()
-        self.set_monetdbe_database(self.open())
         self.set_active_context(self)
-        self.set_active_connection(self._connection)
 
     def cleanup_result(self, result: monetdbe_result):
+        self._switch()
         _logger.info("cleanup_result called")
         if result and self._monetdbe_database:
             check_error(lib.monetdbe_cleanup_result(self._monetdbe_database, result))
@@ -180,11 +169,13 @@ def open(self) -> monetdbe_database:
                 lib.monetdbe_close(connection)
             else:
                 error = errors.get(result_code, "unknown error")
-            raise exceptions.OperationalError(f"Failed to open database: {error} (code {result_code})")
+            msg = f"Failed to open database: {error} (code {result_code})"
+            raise exceptions.OperationalError(msg)
 
         return connection
 
     def close(self) -> None:
+        self._switch()
         if self._monetdbe_database:
             if lib.monetdbe_close(self._monetdbe_database):
                 raise exceptions.OperationalError("Failed to close database")
@@ -193,8 +184,6 @@ def close(self) -> None:
         if self._active_context:
             self.set_active_context(None)
 
-        self.set_active_connection(None)
-
     def query(self, query: str, make_result: bool = False) -> Tuple[Optional[Any], int]:
         """
         Execute a query.
@@ -268,7 +257,7 @@ def append(self, table: str, data: Mapping[str, np.ndarray], schema: str = 'sys'
     def prepare(self, query: str) -> monetdbe_statement:
         self._switch()
         stmt = ffi.new("monetdbe_statement **")
-        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
+        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
         return stmt[0]
 
     def cleanup_statement(self, statement: monetdbe_statement) -> None:
@@ -296,3 +285,14 @@ def get_columns(self, table: str, schema: str = 'sys') -> Iterator[Tuple[str, in
             name = ffi.string(names_p[0][i]).decode()
             type_ = types_p[0][i]
             yield name, type_
+
+
+from monetdbe._cffi.branch import newer_then_jul2021
+if not newer_then_jul2021:
+    def prepare(self, query: str) -> monetdbe_statement:
+        self._switch()
+        stmt = ffi.new("monetdbe_statement **")
+        check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
+        return stmt[0]
+
+    setattr(Internal, 'prepare', prepare)

monetdbe/cursors.py

@@ -37,6 +37,12 @@ def __init__(self, con: 'Connection'):
 
         self._fetch_generator: Optional[Iterator['Row']] = None
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
     def __del__(self):
         self.close()