Skip to content

Issue151 monetdbe bind #153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Sep 9, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ on:
tags: [ "*" ]
pull_request:
branches: [ master ]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:

jobs:
linux-test:
runs-on: ubuntu-20.04
strategy:
matrix:
branch: [Jul2021]
branch: [default, Jul2021]
container: monetdb/dev-builds:${{ matrix.branch }}
steps:
-
Expand All @@ -29,11 +31,12 @@ jobs:
run: py.test -vv

linux-wheel:
needs: linux-test
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
branch: [Jul2021]
branch: [default, Jul2021]
container: monetdb/dev-builds:${{ matrix.branch}}_manylinux

steps:
Expand Down
69 changes: 30 additions & 39 deletions .github/workflows/osx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,65 +6,56 @@ on:
tags: [ "*" ]
pull_request:
branches: [ master ]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:

jobs:
osx-monetdb-Jul2021:
osx-monetdb:
runs-on: macos-10.15
strategy:
matrix:
branch: [default, Jul2021]
env:
MONETDB_BRANCH: Jul2021
MONETDB_BRANCH: ${{ matrix.branch }}
steps:
-
name: brew packages
run: brew install monetdb
-
run: brew install bison
-
name: download MonetDB
run: curl https://dev.monetdb.org/hg/MonetDB/archive/${{ env.MONETDB_BRANCH }}.tar.bz2 -O
-
name: extract MonetDB
run: tar jxvf ${{ env.MONETDB_BRANCH }}.tar.bz2
-
name: build monetdb
run: |
mkdir MonetDB-${{ env.MONETDB_BRANCH }}/build
cd MonetDB-${{ env.MONETDB_BRANCH }}/build
cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/monetdb/11.40.1 \
-DWITH_CRYPTO=OFF \
-DPY3INTEGRATION=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DASSERT=OFF \
-DRINTEGRATION=OFF \
-DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
make install -j3
-
uses: actions/upload-artifact@v2
with:
name: monetdb-osx-${{ env.MONETDB_BRANCH }}
path: /usr/local/Cellar/monetdb

# osx-monetdb-default:
# runs-on: macos-10.15
# env:
# MONETDB_BRANCH: default
# steps:
# -
# name: brew packages
# run: brew install bison
# -
# name: download MonetDB
# run: curl https://dev.monetdb.org/hg/MonetDB/archive/${{ env.MONETDB_BRANCH }}.tar.bz2 -O
# -
# name: extract MonetDB
# run: tar jxvf ${{ env.MONETDB_BRANCH }}.tar.bz2
# -
# name: build monetdb
# run: |
# mkdir MonetDB-${{ env.MONETDB_BRANCH }}/build
# cd MonetDB-${{ env.MONETDB_BRANCH }}/build
# cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/monetdb/11.40.1 \
# -DWITH_CRYPTO=OFF \
# -DPY3INTEGRATION=OFF \
# -DCMAKE_BUILD_TYPE=Release \
# -DASSERT=OFF \
# -DRINTEGRATION=OFF \
# -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
# make install -j3
# -
# uses: actions/upload-artifact@v2
# with:
# name: monetdb-osx-${{ env.MONETDB_BRANCH }}
# path: /usr/local/Cellar/monetdb

osx-wheel:
# continue-on-error: true
runs-on: macos-10.15
strategy:
matrix:
python-version: [ 3.6, 3.7, 3.8, 3.9 ]
branch: [Jul2021]
branch: [default, Jul2021]
env:
MONETDB_BRANCH: ${{ matrix.branch }}
needs: [osx-monetdb-Jul2021]
needs: [osx-monetdb]
steps:
-
name: Print MONETDB_BRANCH to make sure it is set
Expand Down
31 changes: 9 additions & 22 deletions .github/workflows/windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ on:
tags: [ "*" ]
pull_request:
branches: [ master ]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:

# the prefix and version numbers for the branches need to be manually copied (for now) from:
# http://monetdb.cwi.nl/testweb/web/status.php
Expand All @@ -14,14 +16,13 @@ jobs:
runs-on: windows-2019
strategy:
matrix:
MONETDB_BRANCH: [ Jul2021 ]
include:
# - MONETDB_BRANCH: default
# MONETDB_WIN_PREFIX: 82411
# MONETDB_WIN_VERSION: c51ff55bd174
- MONETDB_BRANCH: default
MONETDB_WIN_PREFIX: 82777
MONETDB_WIN_VERSION: f619cf62c92d
- MONETDB_BRANCH: Jul2021
MONETDB_WIN_PREFIX: 82733
MONETDB_WIN_VERSION: 3a8a37d1da2b
MONETDB_WIN_PREFIX: 82786
MONETDB_WIN_VERSION: e39e91cde6b5

env:
MSI: MonetDB5-SQL-Installer-x86_64-${{ matrix.MONETDB_WIN_VERSION }}.msi
Expand Down Expand Up @@ -53,7 +54,7 @@ jobs:
strategy:
matrix:
python-version: [ 3.6, 3.7, 3.8, 3.9 ]
branch: [ Jul2021 ]
branch: [ default, Jul2021 ]
env:
MONETDB_BRANCH: ${{ matrix.branch }}
steps:
Expand All @@ -77,21 +78,7 @@ jobs:
name: Collect DLLs
run: |
ls 'C:\Program Files\MonetDB\MonetDB5\bin'
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\bat.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\mapi.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdb5.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdbe.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\monetdbsql.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\stream.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\libcrypto-1_1-x64.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\iconv-2.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\libxml2.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\pcre.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\zlib1.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\lzma.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\charset-1.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\bz2.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\lz4.dll' monetdbe\.
Copy-Item 'C:\Program Files\MonetDB\MonetDB5\bin\*.dll' monetdbe\.
ls monetdbe
-
name: Upgrade pip and wheel for all pythons
Expand Down
4 changes: 3 additions & 1 deletion monetdbe/_cffi/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
print("**MONETDB**: If this is incorrect, set the MONETDB_BRANCH environment variable during monetdbe-python build\n")

branch_file = str(Path(__file__).parent / 'branch.py')
newer_then_jul2021 = monetdb_branch.lower() not in ("oct2020", "jul2021")

with open(branch_file, 'w') as f:
f.write("# this file is created by the cffi interface builder and contains the monetdb branch env variable.\n\n")
f.write(f"monetdb_branch = '{monetdb_branch}'\n")
f.write(f"newer_then_jul2021 = {newer_then_jul2021}\n")


default = monetdb_branch.lower() in ("default", "jul2021")
Expand All @@ -38,7 +40,7 @@
with open(embed_path, 'r') as f:
content = f.read()
template = Template(content)
cdef = template.render(win32=win32, default=default)
cdef = template.render(win32=win32, default=default, newer_then_jul2021=newer_then_jul2021)
ffibuilder.cdef(cdef)


Expand Down
4 changes: 2 additions & 2 deletions monetdbe/_cffi/convert/bind.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ def monetdbe_int(data: int) -> ffi.CData:
return ffi.new("int *", data)


def bind_str(data: str) -> bytes:
return str(data).encode()
def bind_str(data: str) -> ffi.CData:
return ffi.new("char[]", str(data).encode())


def bind_float(data: float) -> ffi.CData:
Expand Down
7 changes: 5 additions & 2 deletions monetdbe/_cffi/embed.h.j2
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ typedef struct {

typedef enum {
monetdbe_bool, monetdbe_int8_t, monetdbe_int16_t, monetdbe_int32_t, monetdbe_int64_t,
{% if not win32 %}
{# windows does not support 128-bit #}
{% if HAVE_HGE %}
monetdbe_int128_t,
{% endif %}
monetdbe_size_t, monetdbe_float, monetdbe_double,
Expand Down Expand Up @@ -115,7 +114,11 @@ extern char* monetdbe_query(monetdbe_database dbhdl, char* query, monetdbe_resul
extern char* monetdbe_result_fetch(monetdbe_result *mres, monetdbe_column** res, size_t column_index);
extern char* monetdbe_cleanup_result(monetdbe_database dbhdl, monetdbe_result* result);

{% if newer_then_jul2021 %}
extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt, monetdbe_result** result);
{% else %}
extern char* monetdbe_prepare(monetdbe_database dbhdl, char *query, monetdbe_statement **stmt);
{% endif %}
extern char* monetdbe_bind(monetdbe_statement *stmt, void *data, size_t parameter_nr);
extern char* monetdbe_execute(monetdbe_statement *stmt, monetdbe_result **result, monetdbe_cnt* affected_rows);
extern char* monetdbe_cleanup_statement(monetdbe_database dbhdl, monetdbe_statement *stmt);
Expand Down
38 changes: 19 additions & 19 deletions monetdbe/_cffi/internal.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,18 +113,14 @@ def __init__(
self.nr_threads = nr_threads
self.have_hge = have_hge
self._switch()
self._monetdbe_database = self.open()

@classmethod
def set_active_context(cls, active_context: Optional['Internal']):
cls._active_context = active_context

@classmethod
def set_active_connection(cls, active_connection: Optional['Connection']):
cls._active_connection = active_connection

@classmethod
def set_monetdbe_database(cls, connection: Optional[monetdbe_database]):
cls._monetdbe_database = connection
def set_monetdbe_database(self, connection: Optional[monetdbe_database]):
self._monetdbe_database = connection

def __del__(self):
if self._active_context == self:
Expand All @@ -135,17 +131,10 @@ def _switch(self):
if self._active_context == self:
return

# this is a bit scary but just to make sure the previous connection
# can't touch us anymore
if self._active_connection:
self._active_connection._internal = None

self.close()
self.set_monetdbe_database(self.open())
self.set_active_context(self)
self.set_active_connection(self._connection)

def cleanup_result(self, result: monetdbe_result):
self._switch()
_logger.info("cleanup_result called")
if result and self._monetdbe_database:
check_error(lib.monetdbe_cleanup_result(self._monetdbe_database, result))
Expand Down Expand Up @@ -180,11 +169,13 @@ def open(self) -> monetdbe_database:
lib.monetdbe_close(connection)
else:
error = errors.get(result_code, "unknown error")
raise exceptions.OperationalError(f"Failed to open database: {error} (code {result_code})")
msg = f"Failed to open database: {error} (code {result_code})"
raise exceptions.OperationalError(msg)

return connection

def close(self) -> None:
self._switch()
if self._monetdbe_database:
if lib.monetdbe_close(self._monetdbe_database):
raise exceptions.OperationalError("Failed to close database")
Expand All @@ -193,8 +184,6 @@ def close(self) -> None:
if self._active_context:
self.set_active_context(None)

self.set_active_connection(None)

def query(self, query: str, make_result: bool = False) -> Tuple[Optional[Any], int]:
"""
Execute a query.
Expand Down Expand Up @@ -268,7 +257,7 @@ def append(self, table: str, data: Mapping[str, np.ndarray], schema: str = 'sys'
def prepare(self, query: str) -> monetdbe_statement:
self._switch()
stmt = ffi.new("monetdbe_statement **")
check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt, ffi.NULL))
return stmt[0]

def cleanup_statement(self, statement: monetdbe_statement) -> None:
Expand Down Expand Up @@ -296,3 +285,14 @@ def get_columns(self, table: str, schema: str = 'sys') -> Iterator[Tuple[str, in
name = ffi.string(names_p[0][i]).decode()
type_ = types_p[0][i]
yield name, type_


from monetdbe._cffi.branch import newer_then_jul2021
if not newer_then_jul2021:
def prepare(self, query: str) -> monetdbe_statement:
self._switch()
stmt = ffi.new("monetdbe_statement **")
check_error(lib.monetdbe_prepare(self._monetdbe_database, str(query).encode(), stmt))
return stmt[0]

setattr(Internal, 'prepare', prepare)
6 changes: 6 additions & 0 deletions monetdbe/cursors.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ def __init__(self, con: 'Connection'):

self._fetch_generator: Optional[Iterator['Row']] = None

def __enter__(self):
return self

def __exit__(self, exc_type, exc_value, traceback):
self.close()

def __del__(self):
self.close()

Expand Down
Loading