Merge pull request #224 from czgdp1807/upgrade_deps

Author: aktech

.bazelrc

@@ -15,6 +15,7 @@
 # This macro is needed in order for mlmd to build with ZetaSQL which can only
 # be compiled upon c++17 or higher.
 build --cxxopt="-std=c++17"
+build --host_cxxopt="-std=c++17"
 
 # Needed to avoid zetasql proto error.
 build --protocopt=--experimental_allow_proto3_optional
@@ -26,3 +27,28 @@ build --incompatible_require_linker_input_cc_api=false
 
 build:macos --apple_platform_type=macos
 build:macos_arm64 --cpu=darwin_arm64
+build:macos_arm64 --linkopt=-Wl,-undefined,dynamic_lookup
+build:macos_arm64 --host_linkopt=-Wl,-undefined,dynamic_lookup
+build --conlyopt=-std=c11
+build --host_conlyopt=-std=c11
+build --cxxopt=-std=c++17
+build --host_cxxopt=-std=c++17
+build --copt=-Wno-error
+build --cxxopt=-Wno-error
+build --cxxopt=-fpermissive
+build --conlyopt=-Wno-array-parameter
+build --conlyopt=-Wno-implicit-function-declaration
+build --host_copt=-Wno-error
+build --host_cxxopt=-Wno-error
+build --host_cxxopt=-fpermissive
+build --host_conlyopt=-Wno-array-parameter
+build --host_conlyopt=-Wno-implicit-function-declaration
+# Linux-only linker flags (not supported on macOS)
+build:linux --linkopt=-Wl,--no-as-needed
+build:linux --host_linkopt=-Wl,--no-as-needed
+build:macos_arm64 --copt=-Wno-error=c23-extensions
+build:macos_arm64 --host_copt=-Wno-error=c23-extensions
+build:macos_arm64 --copt=-Wno-c23-extensions
+build:macos_arm64 --host_copt=-Wno-c23-extensions
+build:macos_arm64 --action_env=CONDA_PREFIX
+build:macos_arm64 --action_env=CMAKE_ICONV_FLAG="-DCMAKE_DISABLE_FIND_PACKAGE_Iconv=ON -DICONV_LIBRARIES=$CONDA_PREFIX/lib/libiconv.dylib -DICONV_INCLUDE_DIR=$CONDA_PREFIX/include"

.github/workflows/build.yml

@@ -17,8 +17,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - name: Checkout
@@ -27,7 +27,7 @@ jobs:
     - name: Setup Micromamba
       uses: mamba-org/setup-micromamba@v1
       with:
-        environment-file: ci/environment.yml
+        environment-file: ${{ matrix.os == 'macos-latest' && 'ci/environment-macos.yml' || 'ci/environment.yml' }}
         cache-environment: true
         create-args: >-
           python=${{ matrix.python-version }}
@@ -62,7 +62,7 @@ jobs:
       run: |
         python -m build --wheel --no-isolation
 
-    - name: Repair wheel for manylinux
+    - name: Repair wheel (Linux)
       if: runner.os == 'Linux'
       shell: bash -l {0}
       run: |

.github/workflows/conda-build.yml

@@ -15,8 +15,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - name: Checkout
@@ -25,7 +25,7 @@ jobs:
     - name: Setup Micromamba
       uses: mamba-org/setup-micromamba@v1
       with:
-        environment-file: ci/environment.yml
+        environment-file: ${{ matrix.os == 'macos-latest' && 'ci/environment-macos.yml' || 'ci/environment.yml' }}
         cache-environment: true
         create-args: >-
           python=${{ matrix.python-version }}
@@ -55,7 +55,7 @@ jobs:
       run: |
         python setup.py bdist_wheel
 
-    - name: Install built wheel (Linux/macOS)
+    - name: Install built wheel
       shell: bash -l {0}
       run: |
         pip install dist/*.whl

.github/workflows/conda-test.yml

@@ -24,28 +24,23 @@ http_archive(
     sha256 = "5308fc1d8865406a49427ba24a9ab53087f17f5266a7aabbfc28823f3916e1ca",
 )
 
-# Install version 0.9.0 of rules_foreign_cc, as default version causes an
-# invalid escape sequence error to be raised, which can't be avoided with
-# the --incompatible_restrict_string_escapes=false flag (flag was removed in
-# Bazel 5.0).
-RULES_FOREIGN_CC_VERSION = "0.9.0"
+# Install version 0.12.0 of rules_foreign_cc
+RULES_FOREIGN_CC_VERSION = "0.12.0"
 http_archive(
     name = "rules_foreign_cc",
-    sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
+    sha256 = "a2e6fb56e649c1ee79703e99aa0c9d13c6cc53c8d7a0cbb8797ab2888bbc99a3",
     strip_prefix = "rules_foreign_cc-%s" % RULES_FOREIGN_CC_VERSION,
     url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/%s.tar.gz" % RULES_FOREIGN_CC_VERSION,
-    patch_tool = "patch",
-    patches = ["//ml_metadata/third_party:rules_foreign_cc.patch",],
 )
 
 load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 rules_foreign_cc_dependencies()
 
 http_archive(
     name = "com_google_absl",
-    urls = ["https://github.com/abseil/abseil-cpp/archive/940c06c25d2953f44310b68eb8aab6114dba11fb.zip"],
-    strip_prefix = "abseil-cpp-940c06c25d2953f44310b68eb8aab6114dba11fb",
-    sha256 = "0e800799aa64d0b4d354f3ff317bbd5fbf42f3a522ab0456bb749fc8d3b67415",
+    urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz"],
+    strip_prefix = "abseil-cpp-20230802.1",
+    sha256 = "987ce98f02eefbaf930d6e38ab16aa05737234d7afbab2d5c4ea7adbe50c28ed",
 )
 
 http_archive(
@@ -60,18 +55,18 @@ http_archive(
 http_archive(
     name = "org_sqlite",
     build_file = clean_dep("//ml_metadata/third_party:sqlite.BUILD"),
-    sha256 = "87775784f8b22d0d0f1d7811870d39feaa7896319c7c20b849a4181c5a50609b",
-    strip_prefix = "sqlite-amalgamation-3390200",
+    sha256 = "aa73d8748095808471deaa8e6f34aa700e37f2f787f4425744f53fdd15a89c40",
+    strip_prefix = "sqlite-amalgamation-3470200",
     urls = [
-        "https://www.sqlite.org/2022/sqlite-amalgamation-3390200.zip",
+        "https://www.sqlite.org/2024/sqlite-amalgamation-3470200.zip",
     ],
 )
 
 http_archive(
     name = "com_google_googletest",
-    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
-    strip_prefix = "googletest-release-1.12.1",
-    urls = ["https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz"],
+    sha256 = "7b42dc4b2106035276f8f0a5019c929a77d9c606ab43b8e0e1c4b7cc27c8e5ce",
+    strip_prefix = "googletest-release-1.15.2",
+    urls = ["https://github.com/google/googletest/archive/refs/tags/release-1.15.2.tar.gz"],
 )
 
 http_archive(
@@ -84,13 +79,13 @@ http_archive(
     sha256 = "6281aa4eeecb9e932d7091f99872e7b26fa6aacece49c15ce5b14af2b7ec050f",
 )
 
-# 1.5.0
+# 1.7.1
 http_archive(
     name = "bazel_skylib",
-    sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94",
+    sha256 = "bc283cdfcd526a52c3201279cda4bc298652efa898b10b4db0837dc51652756f",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.7.1/bazel-skylib-1.7.1.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.7.1/bazel-skylib-1.7.1.tar.gz",
     ],
 )
 
@@ -117,13 +112,26 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
 
+# Override upb from protobuf_deps() to apply our patch
+http_archive(
+    name = "upb",
+    sha256 = "017a7e8e4e842d01dba5dc8aa316323eee080cd1b75986a7d1f94d87220e6502",
+    strip_prefix = "upb-e4635f223e7d36dfbea3b722a4ca4807a7e882e2",
+    urls = [
+        "https://storage.googleapis.com/grpc-bazel-mirror/github.com/protocolbuffers/upb/archive/e4635f223e7d36dfbea3b722a4ca4807a7e882e2.tar.gz",
+        "https://github.com/protocolbuffers/upb/archive/e4635f223e7d36dfbea3b722a4ca4807a7e882e2.tar.gz",
+    ],
+    patches = ["//ml_metadata/third_party:upb.patch"],
+    patch_args = ["-p0"],
+)
+
 # Needed by Protobuf.
 http_archive(
     name = "zlib",
     build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-    sha256 = "d8688496ea40fb61787500e863cc63c9afcbc524468cedeb478068924eb54932",
-    strip_prefix = "zlib-1.2.12",
-    urls = ["https://github.com/madler/zlib/archive/v1.2.12.tar.gz"],
+    sha256 = "17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
+    strip_prefix = "zlib-1.3.1",
+    urls = ["https://github.com/madler/zlib/archive/v1.3.1.tar.gz"],
 )
 
 http_archive(
@@ -136,9 +144,10 @@ http_archive(
 http_archive(
     name = "pybind11",
     urls = [
-        "https://github.com/pybind/pybind11/archive/v2.10.1.tar.gz",
+        "https://github.com/pybind/pybind11/archive/v2.13.6.tar.gz",
     ],
-    strip_prefix = "pybind11-2.10.1",
+    sha256 = "e08cb87f4773da97fa7b5f035de8763abc656d87d5773e62f6da0587d1f0ec20",
+    strip_prefix = "pybind11-2.13.6",
     build_file = "@pybind11_bazel//:pybind11.BUILD",
 )
 
@@ -247,33 +256,7 @@ http_archive(
     url = "https://github.com/gflags/gflags/archive/a738fdf9338412f83ab3f26f31ac11ed3f3ec4bd.zip",
 )
 
-ZETASQL_COMMIT = "ac37cf5c0d80b5605176fc0f29e87b12f00be693" # 08/10/2022
-http_archive(
-    name = "com_google_zetasql",
-    urls = ["https://github.com/google/zetasql/archive/%s.zip" % ZETASQL_COMMIT],
-    strip_prefix = "googlesql-%s" % ZETASQL_COMMIT,
-    #patches = ["//ml_metadata/third_party:zetasql.patch"],
-    sha256 = '86f81591ab5ec20457a5394eb2c5c981e6f6c89f4c49c211d096c3acffec1eb1'
-)
-
-load("@com_google_zetasql//bazel:zetasql_deps_step_1.bzl", "zetasql_deps_step_1")
-zetasql_deps_step_1()
-load("@com_google_zetasql//bazel:zetasql_deps_step_2.bzl", "zetasql_deps_step_2")
-zetasql_deps_step_2(
-    analyzer_deps = True,
-    evaluator_deps = True,
-    tools_deps = False,
-    java_deps = False,
-    testing_deps = False)
-
-# This is part of what zetasql_deps_step_3() does.
-load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
-switched_rules_by_language(
-    name = "com_google_googleapis_imports",
-    cc = True,
-)
-
-
+# ZetaSQL removed - not needed for core functionality
 
 # Please add all new ML Metadata dependencies in workspace.bzl.
 load("//ml_metadata:workspace.bzl", "ml_metadata_workspace")

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+# Conda environment for building and testing ml-metadata on macOS
+name: mlmd-dev
+channels:
+  - conda-forge
+dependencies:
+  # Note: Bazel is installed separately via official installer (conda package is unreliable)
+  - setuptools<69.3  # Pin to avoid Metadata-Version 2.4 (PyPI only supports up to 2.3)
+  - wheel
+  - pip
+  - numpy>=1.23,<2.0
+  - pytest
+  - pytest-cov
+  - cmake=3.29  # Required for building libmysqlclient (CMake 4.x is incompatible)
+  - six  # Required by ZetaSQL build tools
+  - libiconv  # Required by libmysqlclient for character encoding conversions
+
+  # C/C++ compilers
+  - clang
+  - clangxx
+
+  - pip:
+    - delocate  # For macOS wheel repair

WORKSPACE

@@ -13,9 +13,8 @@ dependencies:
   - patchelf  # For wheel repair on Linux
   - cmake=3.29
 
-  # C/C++ compilers - GCC 8.x to match manylinux2014 devtoolset-8
-  - gcc_linux-64=8.5.0
-  - gxx_linux-64=8.5.0
+  - gcc_linux-64
+  - gxx_linux-64
   - sysroot_linux-64=2.17  # CentOS 7/manylinux2014 compatible glibc headers
 
   - pip:

ci/environment-macos.yml

@@ -39,6 +39,12 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "darwin_arm64",
+    values = {"cpu": "darwin_arm64"},
+    visibility = ["//visibility:public"],
+)
+
 cc_library(
     name = "libmysqlclient",
     srcs = configure_out_srcs + [
@@ -91,7 +97,19 @@ cc_library(
         "-DLIBICONV_PLUG",
         "-DHAVE_OPENSSL",
         "-DHAVE_TLS",
-    ],
+        # Fix implicit function declarations
+        "-D_GNU_SOURCE",
+    ] + select({
+        # On Linux/GCC 13 / Ubuntu 24.04: define legacy MySQL types using stdint.h types
+        # On macOS: these types are already correct size, don't redefine to avoid conflicts
+        ":darwin": [],
+        ":darwin_arm64": [],
+        "//conditions:default": [
+            "-Duint=uint32_t",
+            "-Dushort=uint16_t",
+            "-Dulong=uint64_t",
+        ],
+    }),
     includes = [
         "build/include/",
         "include/",
@@ -103,6 +121,7 @@ cc_library(
         "-lm",
     ] + select({
         ":darwin": ["-liconv"],
+        ":darwin_arm64": ["-liconv"],
         "//conditions:default": [],
     }),
     visibility = ["//visibility:public"],