PYTHON-5217 Update Atlas Data Lake tests (#2209)

Author: blink1073

.evergreen/config.yml

@@ -213,20 +213,6 @@ functions:
       params:
         file: ${DRIVERS_TOOLS}/mo-expansion.yml
 
-  "bootstrap data lake":
-    - command: subprocess.exec
-      type: setup
-      params:
-        binary: bash
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
-    - command: subprocess.exec
-      type: setup
-      params:
-        binary: bash
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
-
   "run doctests":
      - command: subprocess.exec
        type: test
@@ -404,13 +390,6 @@ tasks:
             TOPOLOGY: "replica_set"
         - func: "run tests"
 
-    - name: atlas-data-lake-tests
-      commands:
-        - func: "bootstrap data lake"
-        - func: "run tests"
-          vars:
-            TEST_NAME: "data_lake"
-
     - name: "test-aws-lambda-deployed"
       commands:
         - command: ec2.assume_role

.evergreen/generated_configs/tasks.yml

@@ -8,6 +8,21 @@ tasks:
           TEST_NAME: atlas_connect
     tags: [atlas_connect]
 
+  # Atlas data lake tests
+  - name: test-atlas-data-lake-with_ext
+    commands:
+      - func: run tests
+        vars:
+          TEST_NAME: data_lake
+          NO_EXT: "1"
+    tags: [atlas_data_lake]
+  - name: test-atlas-data-lake-without_ext
+    commands:
+      - func: run tests
+        vars:
+          TEST_NAME: data_lake
+    tags: [atlas_data_lake]
+
   # Aws tests
   - name: test-auth-aws-4.4-regular
     commands:

.evergreen/generated_configs/variants.yml

@@ -65,43 +65,21 @@ buildvariants:
       PYTHON_BINARY: /opt/python/3.13/bin/python3
 
   # Atlas data lake tests
-  - name: atlas-data-lake-ubuntu-22-python3.9-auth-no-c
+  - name: atlas-data-lake-ubuntu-22-python3.9
     tasks:
-      - name: atlas-data-lake-tests
-    display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth No C
+      - name: .atlas_data_lake
+    display_name: Atlas Data Lake Ubuntu-22 Python3.9
     run_on:
       - ubuntu2204-small
     expansions:
-      AUTH: auth
-      NO_EXT: "1"
-      PYTHON_BINARY: /opt/python/3.9/bin/python3
-  - name: atlas-data-lake-ubuntu-22-python3.9-auth
-    tasks:
-      - name: atlas-data-lake-tests
-    display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth
-    run_on:
-      - ubuntu2204-small
-    expansions:
-      AUTH: auth
       PYTHON_BINARY: /opt/python/3.9/bin/python3
-  - name: atlas-data-lake-ubuntu-22-python3.13-auth-no-c
+  - name: atlas-data-lake-ubuntu-22-python3.13
     tasks:
-      - name: atlas-data-lake-tests
-    display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth No C
+      - name: .atlas_data_lake
+    display_name: Atlas Data Lake Ubuntu-22 Python3.13
     run_on:
       - ubuntu2204-small
     expansions:
-      AUTH: auth
-      NO_EXT: "1"
-      PYTHON_BINARY: /opt/python/3.13/bin/python3
-  - name: atlas-data-lake-ubuntu-22-python3.13-auth
-    tasks:
-      - name: atlas-data-lake-tests
-    display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth
-    run_on:
-      - ubuntu2204-small
-    expansions:
-      AUTH: auth
       PYTHON_BINARY: /opt/python/3.13/bin/python3
 
   # Aws auth tests

.evergreen/scripts/generate_config.py

@@ -213,7 +213,7 @@ def zip_cycle(*iterables, empty_default=None):
         yield tuple(next(i, empty_default) for i in cycles)
 
 
-def handle_c_ext(c_ext, expansions):
+def handle_c_ext(c_ext, expansions) -> None:
     """Handle c extension option."""
     if c_ext == C_EXTS[0]:
         expansions["NO_EXT"] = "1"
@@ -600,14 +600,10 @@ def create_no_c_ext_variants():
 def create_atlas_data_lake_variants():
     variants = []
     host = HOSTS["ubuntu22"]
-    for python, c_ext in product(MIN_MAX_PYTHON, C_EXTS):
-        tasks = ["atlas-data-lake-tests"]
-        expansions = dict(AUTH="auth")
-        handle_c_ext(c_ext, expansions)
-        display_name = get_display_name("Atlas Data Lake", host, python=python, **expansions)
-        variant = create_variant(
-            tasks, display_name, host=host, python=python, expansions=expansions
-        )
+    for python in MIN_MAX_PYTHON:
+        tasks = [".atlas_data_lake"]
+        display_name = get_display_name("Atlas Data Lake", host, python=python)
+        variant = create_variant(tasks, display_name, host=host, python=python)
         variants.append(variant)
     return variants
 
@@ -970,6 +966,18 @@ def create_perf_tasks():
     return tasks
 
 
+def create_atlas_data_lake_tasks():
+    tags = ["atlas_data_lake"]
+    tasks = []
+    for c_ext in C_EXTS:
+        vars = dict(TEST_NAME="data_lake")
+        handle_c_ext(c_ext, vars)
+        test_func = FunctionCall(func="run tests", vars=vars)
+        task_name = f"test-atlas-data-lake-{c_ext}"
+        tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func]))
+    return tasks
+
+
 def create_ocsp_tasks():
     tasks = []
     tests = [

.evergreen/scripts/setup_tests.py

@@ -175,6 +175,12 @@ def handle_test_env() -> None:
         if not config:
             AUTH = "noauth"
 
+    if test_name == "data_lake":
+        # Stop any running mongo-orchestration which might be using the port.
+        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh")
+        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas_data_lake/setup.sh")
+        AUTH = "auth"
+
     if AUTH != "noauth":
         if test_name == "data_lake":
             config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh")

.evergreen/scripts/teardown_tests.py

@@ -57,6 +57,10 @@
 
     teardown_mod_wsgi()
 
+# Tear down data_lake if applicable.
+elif TEST_NAME == "data_lake":
+    run_command(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/teardown.sh")
+
 # Tear down coverage if applicable.
 if os.environ.get("COVERAGE"):
     shutil.rmtree(".pytest_cache", ignore_errors=True)

CONTRIBUTING.md

@@ -286,16 +286,23 @@ Note: these tests can only be run from an Evergreen Linux host that has the Pyth
 The `mode` can be `standalone` or `embedded`.  For the `replica_set` version of the tests, use
 `TOPOLOGY=replica_set just run-server`.
 
+### Atlas Data Lake tests.
+
+You must have `docker` or `podman` installed locally.
+
+- Run `just setup-tests data_lake`.
+- Run `just run-tests`.
+
 ### OCSP tests
 
-  - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`.
-    This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`.
-    MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA.
-  - Run `just run-server ocsp`.
-  - Run `just setup-tests ocsp <sub test>` (options are "valid", "revoked", "valid-delegate", "revoked-delegate").
-  - Run `just run-tests`
+- Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`.
+This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`.
+MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA.
+- Run `just run-server ocsp`.
+- Run `just setup-tests ocsp <sub test>` (options are "valid", "revoked", "valid-delegate", "revoked-delegate").
+- Run `just run-tests`
 
-  If you are running one of the `no-responder` tests, omit the `run-server` step.
+If you are running one of the `no-responder` tests, omit the `run-server` step.
 
 ### Perf Tests