Skip to content

Commit 57c18ba

Browse files
prhagibsondan
authored andcommitted
stop using python semver parsing for mysql versions (#11868)
### Summary & Motivation We were using `packaging.parse` to compare mysql server versions, which broke when `0.23.0` was released with dropped support for `LegacyVersion`. This rolls our own custom version compares to find minimum supported versions, which is tolerant of non semver compliant version strings. The regex parses all the numeric values, and does tuple int comparisons. Reported in #11794 Reference links: pypa/packaging#407 ### How I Tested These Changes Added some test cases that override the server version string to exercise the parsing logic.
1 parent 02b9ad0 commit 57c18ba

5 files changed

Lines changed: 57 additions & 7 deletions

File tree

python_modules/libraries/dagster-mysql/dagster_mysql/event_log/event_log.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@
1616
stamp_alembic_rev,
1717
)
1818
from dagster._serdes import ConfigurableClass, ConfigurableClassData
19-
from packaging.version import parse
2019

2120
from ..utils import (
2221
create_mysql_connection,
2322
mysql_alembic_config,
2423
mysql_url_from_config,
24+
parse_mysql_version,
2525
retry_mysql_connection_fn,
2626
retry_mysql_creation_fn,
2727
)
@@ -195,7 +195,9 @@ def end_watch(self, run_id, handler):
195195

196196
@property
197197
def supports_intersect(self):
198-
return parse(self._mysql_version) >= parse(MINIMUM_MYSQL_INTERSECT_VERSION)
198+
return parse_mysql_version(self._mysql_version) >= parse_mysql_version(
199+
MINIMUM_MYSQL_INTERSECT_VERSION
200+
)
199201

200202
@property
201203
def event_watcher(self):

python_modules/libraries/dagster-mysql/dagster_mysql/run_storage/run_storage.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,12 @@
1818
)
1919
from dagster._serdes import ConfigurableClass, ConfigurableClassData, serialize_dagster_namedtuple
2020
from dagster._utils import utc_datetime_from_timestamp
21-
from packaging.version import parse
2221

2322
from ..utils import (
2423
create_mysql_connection,
2524
mysql_alembic_config,
2625
mysql_url_from_config,
26+
parse_mysql_version,
2727
retry_mysql_connection_fn,
2828
retry_mysql_creation_fn,
2929
)
@@ -154,11 +154,15 @@ def supports_bucket_queries(self):
154154
if not self._mysql_version:
155155
return False
156156

157-
return parse(self._mysql_version) >= parse(MINIMUM_MYSQL_BUCKET_VERSION)
157+
return parse_mysql_version(self._mysql_version) >= parse_mysql_version(
158+
MINIMUM_MYSQL_BUCKET_VERSION
159+
)
158160

159161
@property
160162
def supports_intersect(self):
161-
return parse(self._mysql_version) >= parse(MINIMUM_MYSQL_INTERSECT_VERSION)
163+
return parse_mysql_version(self._mysql_version) >= parse_mysql_version(
164+
MINIMUM_MYSQL_INTERSECT_VERSION
165+
)
162166

163167
def add_daemon_heartbeat(self, daemon_heartbeat):
164168
with self.connect() as conn:

python_modules/libraries/dagster-mysql/dagster_mysql/schedule_storage/schedule_storage.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@
1111
stamp_alembic_rev,
1212
)
1313
from dagster._serdes import ConfigurableClass, ConfigurableClassData, serialize_dagster_namedtuple
14-
from packaging.version import parse
1514

1615
from ..utils import (
1716
create_mysql_connection,
1817
mysql_alembic_config,
1918
mysql_url_from_config,
19+
parse_mysql_version,
2020
retry_mysql_connection_fn,
2121
retry_mysql_creation_fn,
2222
)
@@ -117,7 +117,9 @@ def supports_batch_queries(self):
117117
if not self._mysql_version:
118118
return False
119119

120-
return parse(self._mysql_version) >= parse(MINIMUM_MYSQL_BATCH_VERSION)
120+
return parse_mysql_version(self._mysql_version) >= parse_mysql_version(
121+
MINIMUM_MYSQL_BATCH_VERSION
122+
)
121123

122124
def get_server_version(self):
123125
rows = self.execute("select version()")

python_modules/libraries/dagster-mysql/dagster_mysql/utils.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import logging
2+
import re
23
import time
34
from contextlib import contextmanager
45
from urllib.parse import (
@@ -46,6 +47,26 @@ def get_conn_string(username, password, hostname, db_name, port="3306"):
4647
)
4748

4849

50+
def parse_mysql_version(version: str) -> tuple:
51+
"""Parse MySQL version into a tuple of ints.
52+
53+
Args:
54+
version (str): MySQL version string.
55+
56+
Returns:
57+
tuple: Tuple of ints representing the MySQL version.
58+
"""
59+
parsed = []
60+
for part in re.split(r"\D+", version):
61+
if len(part) == 0:
62+
continue
63+
try:
64+
parsed.append(int(part))
65+
except ValueError:
66+
continue
67+
return tuple(parsed)
68+
69+
4970
def retry_mysql_creation_fn(fn, retry_limit=5, retry_wait=0.2):
5071
# Retry logic to recover from the case where two processes are creating
5172
# tables at the same time using sqlalchemy

python_modules/libraries/dagster-mysql/dagster_mysql_tests/test_run_storage.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import yaml
55
from dagster._core.test_utils import environ, instance_for_test
66
from dagster_mysql.run_storage import MySQLRunStorage
7+
from dagster_mysql.utils import parse_mysql_version
78
from dagster_tests.storage_tests.utils.run_storage import TestRunStorage
89

910
TestRunStorage.__test__ = False
@@ -83,3 +84,23 @@ def test_load_from_config(self, conn_string):
8384
from_url_instance._run_storage.mysql_url
8485
== from_env_instance._run_storage.mysql_url
8586
)
87+
88+
89+
def test_mysql_version(conn_string):
90+
class FakeNonBucketing(MySQLRunStorage):
91+
def get_server_version(self):
92+
# override the server version to make sure the parsing works
93+
return "5.7.38-log"
94+
95+
storage = FakeNonBucketing(conn_string)
96+
assert parse_mysql_version("5.7.38-log") == (5, 7, 38)
97+
assert not storage.supports_bucket_queries
98+
99+
class FakeBucketing(MySQLRunStorage):
100+
def get_server_version(self):
101+
# override the server version to make sure the parsing works
102+
return "8.0.31-google"
103+
104+
storage = FakeBucketing(conn_string)
105+
assert parse_mysql_version("8.0.31-google") == (8, 0, 31)
106+
assert storage.supports_bucket_queries

0 commit comments

Comments
 (0)