Adopt Python 3.12

.github/workflows/ci_pipe.yml

@@ -230,7 +230,7 @@ jobs:
       matrix:
         arch: ["amd64", "arm64"]
         include:
-          - runner: linux-amd64-gpu-v100-latest-1
+          - runner: linux-amd64-gpu-l4-latest-1
             arch: "amd64"
           - runner: linux-arm64-cpu16
             arch: "arm64"

.github/workflows/pr.yaml

@@ -95,9 +95,9 @@ jobs:
       conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }}
       # Upload morpheus conda packages only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
-      base_container: rapidsai/ci-conda:cuda12.8.0-ubuntu22.04-py3.10
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-250213
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-250213
+      base_container: rapidsai/ci-conda:cuda12.8.0-ubuntu22.04-py3.12
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-250225
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-250225
     secrets:
       CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
       NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
               - id: isort
                 args: ["--filter-files", "--settings-file=./pyproject.toml"]
       - repo: https://github.com/PyCQA/flake8
-        rev: 6.1.0
+        rev: 7.0.0
         hooks:
               - id: flake8
                 entry: pflake8

ci/conda/recipes/morpheus-libs/conda_build_config.yaml

@@ -26,7 +26,7 @@ cuda_compiler_version:
   - 12.8
 
 python:
-  - 3.10
+  - 3.12
 
 rapids_version:
   - 25.02

ci/conda/recipes/morpheus-libs/meta.yaml

@@ -51,7 +51,7 @@ outputs:
         - cmake =3.27
         - ninja =1.11
         - pkg-config =0.29
-        - sysroot_linux-64 =2.28
+        - sysroot_linux-64>=2.28
       host:
         # CUDA dependencies
         - cuda-cudart-dev {{ cuda_compiler_version }}.*
@@ -66,7 +66,7 @@ outputs:
         - glog >=0.7.1,<0.8
         - indicators=2.3
         - libcudf {{ rapids_version }}
-        - librdkafka >=1.9.2,<1.10.0a0
+        - librdkafka =2.6.1
         - mrc {{ minor_version }}
         - nlohmann_json 3.11.*
         - pip
@@ -75,7 +75,7 @@ outputs:
         - rapidjson 1.1.0
         - scikit-build 0.17.6
         - versioneer-518
-        - zlib 1.3.1 # required to build triton client
+        - zlib >=1.3.1,<2 # required to build triton client
       run:
         # Runtime only requirements. This + setup.py is the definitive runtime requirement list
         # This should be synced with `runtime` in dependencies.yaml
@@ -84,8 +84,8 @@ outputs:
         - click >=8
         - cuda-version {{ cuda_compiler_version }}.*
         - cudf
-        - cupy # Version determined from cudf
-        - datacompy =0.10.*
+        - cupy <13.4
+        - datacompy =0.13.1
         - dill =0.3.7
         - docker-py =5.0.*
         - elasticsearch ==8.9.0
@@ -100,7 +100,7 @@ outputs:
         - pydantic
         - pluggy =1.3.*
         - python
-        - python-confluent-kafka >=1.9.2,<1.10.0a0
+        - python-confluent-kafka =2.6.1
         - python-graphviz
         - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed
         - requests<2.29.0
@@ -222,7 +222,7 @@ outputs:
         - rapidjson 1.1.0
         - scikit-build 0.17.6
         - versioneer-518
-        - zlib 1.3.1 # required to build triton client
+        - zlib >=1.3.1,<2 # required to build triton client
       run:
         - {{ pin_subpackage('morpheus-core', exact=True) }}
 

ci/conda/recipes/morpheus-libs/morpheus_core_test.sh

@@ -19,11 +19,20 @@ if [[ $(arch) == "aarch64" ]]; then
     exit 0
 fi
 
+package_name="morpheus"
+file_name="requirements_morpheus_core_arch-$(arch).txt"
+
+# Install requirements if they are included in the package
 python3 <<EOF
-import importlib.resources
 import subprocess
-requirements_file = importlib.resources.path("morpheus", "requirements_morpheus_core_arch-$(arch).txt")
-subprocess.call(f"pip install -r {requirements_file}".split())
+import sys
+import importlib.resources as ir
+
+package_name = "${package_name}"
+file_name = "${file_name}"
+
+with ir.as_file(ir.files(anchor=package_name).joinpath(file_name)) as requirements_file:
+    subprocess.call(f"pip install -r {requirements_file}".split())
 EOF
 
-pytest tests/morpheus
+pytest tests/${package_name}

ci/conda/recipes/morpheus-libs/morpheus_dfp_build.sh

@@ -56,5 +56,5 @@ cmake -B ${BUILD_DIR} \
 # Build the components
 cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
 
-# Install the mprpheus dfp python wheel components
+# Install the morpheus dfp python wheel components
 ${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_dfp/dist/*.whl

ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh

@@ -19,11 +19,20 @@ if [[ $(arch) == "aarch64" ]]; then
     exit 0
 fi
 
+package_name="morpheus_dfp"
+file_name="requirements_morpheus_dfp_arch-$(arch).txt"
+
+# Install requirements if they are included in the package
 python3 <<EOF
-import importlib.resources
 import subprocess
-requirements_file = importlib.resources.path("morpheus_dfp", "requirements_morpheus_dfp_arch-$(arch).txt")
-subprocess.call(f"pip install -r {requirements_file}".split())
+import sys
+import importlib.resources as ir
+
+package_name = "${package_name}"
+file_name = "${file_name}"
+
+with ir.as_file(ir.files(anchor=package_name).joinpath(file_name)) as requirements_file:
+    subprocess.call(f"pip install -r {requirements_file}".split())
 EOF
 
-pytest tests/morpheus_dfp
+pytest tests/${package_name}

ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh

@@ -19,12 +19,20 @@ if [[ $(arch) == "aarch64" ]]; then
     exit 0
 fi
 
+package_name="morpheus_llm"
+file_name="requirements_morpheus_llm_arch-$(arch).txt"
+
 # Install requirements if they are included in the package
 python3 <<EOF
-import importlib.resources
 import subprocess
-requirements_file = importlib.resources.path("morpheus_llm", "requirements_morpheus_llm_arch-$(arch).txt")
-subprocess.call(f"pip install -r {requirements_file}".split())
+import sys
+import importlib.resources as ir
+
+package_name = "${package_name}"
+file_name = "${file_name}"
+
+with ir.as_file(ir.files(anchor=package_name).joinpath(file_name)) as requirements_file:
+    subprocess.call(f"pip install -r {requirements_file}".split())
 EOF
 
-pytest tests/morpheus_llm
+pytest tests/${package_name}

ci/conda/recipes/morpheus/conda_build_config.yaml

@@ -26,7 +26,7 @@ cuda_compiler_version:
   - 12.8
 
 python:
-  - 3.10
+  - 3.12
 
 rapids_version:
   - 25.02

ci/conda/recipes/morpheus/meta.yaml

@@ -71,7 +71,7 @@ outputs:
         - glog >=0.7.1,<0.8
         - indicators=2.3
         - libcudf {{ rapids_version }}
-        - librdkafka >=1.9.2,<1.10.0a0
+        - librdkafka =2.6.1
         - mrc {{ minor_version }}
         - nlohmann_json 3.11.*
         - pip
@@ -81,7 +81,7 @@ outputs:
         - rdma-core >=48 # Needed for DOCA.
         - scikit-build 0.17.6
         - versioneer-518
-        - zlib 1.3.1 # required to build triton client
+        - zlib >=1.3.1,<2 # required to build triton client
       run:
         # Runtime only requirements. This + setup.py is the definitive runtime requirement list
         # This should be synced with `runtime` in dependencies.yaml
@@ -90,8 +90,8 @@ outputs:
         - click >=8
         - cuda-version {{ cuda_compiler_version }}.*
         - cudf
-        - cupy # Version determined from cudf
-        - datacompy =0.10.*
+        - cupy <13.4
+        - datacompy =0.13.1
         - dill =0.3.7
         - docker-py =5.0.*
         - elasticsearch ==8.9.0
@@ -106,7 +106,7 @@ outputs:
         - pydantic
         - pluggy =1.3.*
         - python
-        - python-confluent-kafka >=1.9.2,<1.10.0a0
+        - python-confluent-kafka =2.6.1
         - python-graphviz
         - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed
         - requests<2.29.0

ci/conda/recipes/python-dbg/meta.yaml

@@ -37,7 +37,7 @@ requirements:
   build:
     - bzip2       [unix]
     - libffi      [unix] # Required for _ctypes, pip will complain that setuptools doesn't exist if _ctypes is missing.
-    - openssl >=1.1.1n,<2.0   # Required for TLS/SSL capabilities in pip.
+    - openssl >=3.0,<3.1   # Required for TLS/SSL capabilities in pip.
     - readline    [unix]
     - sqlite      [unix]
     - tk          [unix]
@@ -46,7 +46,7 @@ requirements:
   run:
     - bzip2       [unix]
     - libffi      [unix]
-    - openssl >=1.1.1n,<2.0   # Required for TLS/SSL capabilities in pip.
+    - openssl >=3.0,<3.1   # Required for TLS/SSL capabilities in pip.
     - readline    [unix]
     - sqlite      [unix]
     - tk          [unix]

ci/conda/recipes/run_conda_build.sh

@@ -121,7 +121,6 @@ CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}rapi
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia")
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/dev")
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}pytorch")
-CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}defaults")
 
 if [[ ${NUMARGS} == 0 ]]; then
    echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}"

ci/runner/Dockerfile

@@ -20,7 +20,7 @@ ARG CUDA_VER=12.8.0
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_VER=22.04
 ARG PROJ_NAME=morpheus
-ARG PYTHON_VER=3.10
+ARG PYTHON_VER=3.12
 ARG REAL_ARCH=x86_64
 
 

ci/scripts/cpp_checks.sh

@@ -60,7 +60,7 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
 
       # Run using a clang-tidy wrapper to allow warnings-as-errors and to eliminate any output except errors (since
       # clang-tidy-diff.py doesn't return the correct error codes)
-      CLANG_TIDY_OUTPUT=`get_unified_diff ${CPP_FILE_REGEX} | ${CLANG_TIDY_DIFF} \
+      CLANG_TIDY_OUTPUT=`get_unified_diff ${CPP_FILE_REGEX} | python -W ignore ${CLANG_TIDY_DIFF} \
          -extra-arg="-Wno-ignored-optimization-argument" -j 0 -path ${BUILD_DIR} -p1 -quiet 2>&1`
 
       if [[ -n "${CLANG_TIDY_OUTPUT}" && ${CLANG_TIDY_OUTPUT} != "No relevant changes found." ]]; then

ci/scripts/github/conda_libs.sh

@@ -31,10 +31,12 @@ fetch_base_branch
 # Print the info just to be sure base is active
 conda info
 
+rapids-logger "Installing Boa and Git LFS"
+conda install --solver=libmamba -c conda-forge boa git-lfs
+
 # Pull down data needed for running the per-library unit tests
 rapids-logger "Pulling LFS assets"
 
-conda install git-lfs
 git lfs install
 ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation
 

ci/scripts/run_ci_local.sh

@@ -59,7 +59,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-250213}
+CONTAINER_VER=${CONTAINER_VER:-250225}
 CUDA_VER=${CUDA_VER:-12.8}
 CUDA_FULL_VER=${CUDA_FULL_VER:-12.8.0}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}

conda/environments/all_cuda-128_arch-aarch64.yaml

@@ -37,7 +37,7 @@ dependencies:
 - cupy<13.4
 - cxx-compiler
 - cython=3.0
-- datacompy=0.10
+- datacompy=0.13.1
 - dill=0.3.7
 - docker-py=5.0
 - doxygen=1.9.2
@@ -63,7 +63,7 @@ dependencies:
 - libcufft-dev
 - libcurand-dev
 - libcusolver-dev
-- librdkafka>=1.9.2,<1.10.0a0
+- librdkafka=2.6.1
 - libtool
 - libwebp=1.3.2
 - libzlib >=1.3.1,<2
@@ -80,7 +80,7 @@ dependencies:
 - numpydoc=1.5
 - onnx=1.15
 - openai==1.13.*
-- papermill=2.4.0
+- papermill=2.6.0
 - pip
 - pkg-config=0.29
 - pluggy=1.3
@@ -95,10 +95,10 @@ dependencies:
 - pytest-benchmark=4.0
 - pytest-cov
 - pytest=7.4.4
-- python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-confluent-kafka=2.6.1
 - python-docx==1.1.0
 - python-graphviz
-- python=3.10
+- python=3.12
 - rapidjson=1.1.0
 - rapids-dask-dependency=25.02
 - rdma-core>=48
@@ -124,12 +124,11 @@ dependencies:
 - yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
-  - --find-links https://data.dgl.ai/wheels-test/repo.html
-  - --find-links https://data.dgl.ai/wheels/cu121/repo.html
+  - --find-links https://data.dgl.ai/wheels/torch-2.3/repo.html
+  - --find-links https://data.dgl.ai/wheels/torch-2.4/cu124/repo.html
   - databricks-cli < 0.100
   - databricks-connect
-  - dgl==2.0.0
-  - dglgo
+  - dgl
   - faiss-cpu
   - google-search-results==2.4
   - gpudb>=7.2.2.3