@@ -249,7 +249,7 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
249
249
250
250
251
251
def gen_build_base_venvs () -> None :
252
- """Generate component-specific build jobs for virtual environments and native extensions ."""
252
+ """Generate 4-cache build jobs based on product areas and dependencies ."""
253
253
254
254
ci_commit_sha = os .getenv ("CI_COMMIT_SHA" , "default" )
255
255
# Try to get component-specific hashes, fall back to legacy single hash
@@ -269,7 +269,7 @@ def gen_build_base_venvs() -> None:
269
269
# Generate dependency hash for Python environment
270
270
dependency_hash = _get_dependency_hash ()
271
271
272
- # Generate base venv job (Python dependencies, no native extensions )
272
+ # 1. RIOT DEPENDENCIES (test requirements )
273
273
f .write (
274
274
f"""
275
275
build_base_venv:
@@ -284,54 +284,116 @@ def gen_build_base_venvs() -> None:
284
284
DD_BUILD_EXT_EXCLUDES: '*' # Exclude all native extensions
285
285
script: |
286
286
set -e -o pipefail
287
- echo "Building base Python environment (no native extensions)"
287
+ echo "Building riot test dependencies (no native extensions)"
288
288
echo "Dependency hash: { dependency_hash [:16 ]} "
289
289
pip install riot==0.20.1
290
290
riot -P -v generate --python=$PYTHON_VERSION
291
- echo "Base venv created successfully"
291
+ echo "Riot dependencies created successfully"
292
292
cache:
293
- # Cache based on dependency files, not source code
294
- - key: v1-base_venv -${{PYTHON_VERSION}}-deps -{ dependency_hash [:16 ]}
293
+ # Cache based on dependency files (riot requirements)
294
+ - key: v1-riot_deps -${{PYTHON_VERSION}}-{ dependency_hash [:16 ]}
295
295
paths:
296
296
- .cache
297
297
- .riot/venv_*
298
298
artifacts:
299
- name: base_venv_ $PYTHON_VERSION
299
+ name: riot_deps_ $PYTHON_VERSION
300
300
paths:
301
301
- .riot/venv_*
302
302
- ddtrace/_version.py
303
- expire_in: 4 hours # Longer expiry for base venv
303
+ expire_in: 72 hours # Longer expiry for dependencies
304
304
"""
305
305
)
306
306
307
- # Generate component-specific build jobs
308
- for component in all_components :
309
- component_hash = component_hashes .get (component , ci_commit_sha )[:16 ]
310
-
311
- # Define component-specific patterns and artifacts
312
- if ComponentConfig .is_cmake_component (component ):
313
- build_patterns = _get_cmake_build_patterns (component )
314
- artifact_patterns = _get_cmake_artifact_patterns (component )
315
- build_deps = ["apt update && apt install -y sccache cmake" ]
316
- elif ComponentConfig .is_rust_component (component ):
317
- build_patterns = _get_rust_build_patterns (component )
318
- artifact_patterns = _get_rust_artifact_patterns (component )
319
- build_deps = [
320
- "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y" ,
321
- "source ~/.cargo/env" ,
322
- ]
323
- else : # Python components (Cython, C extensions)
324
- build_patterns = _get_python_build_patterns (component )
325
- artifact_patterns = _get_python_artifact_patterns (component )
326
- build_deps = ["apt update && apt install -y build-essential" ]
307
+ # 2. PROFILING NATIVE BUILD (ddtrace.internal.profiling)
308
+ profiling_components = ["cmake_ddup" , "cmake_crashtracker" , "cmake_stack_v2" , "rust_native" ]
309
+ profiling_hash = _get_combined_hash ([component_hashes .get (c , ci_commit_sha ) for c in profiling_components ])
310
+ profiling_patterns = [
311
+ "ddtrace.internal.datadog.profiling.ddup._ddup" ,
312
+ "ddtrace.internal.datadog.profiling.crashtracker._crashtracker" ,
313
+ "ddtrace.internal.datadog.profiling.stack_v2._stack_v2" ,
314
+ "ddtrace.internal.native._native" ,
315
+ ]
327
316
328
- f .write (
329
- f"""
330
- build_{ component } :
317
+ f .write (
318
+ f"""
319
+ build_profiling_native:
320
+ extends: .testrunner
321
+ stage: setup
322
+ needs:
323
+ - job: build_riot_dependencies
324
+ artifacts: true
325
+ parallel:
326
+ matrix:
327
+ - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
328
+ variables:
329
+ CMAKE_BUILD_PARALLEL_LEVEL: '12'
330
+ DD_USE_SCCACHE: '1'
331
+ PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
332
+ SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
333
+ DD_BUILD_EXT_INCLUDES: '{ "," .join (profiling_patterns )} '
334
+ DD_FAST_BUILD: '1'
335
+ rules:
336
+ - if: '$CI_COMMIT_REF_NAME == "main"'
337
+ variables:
338
+ DD_FAST_BUILD: '0'
339
+ - when: always
340
+ script: |
341
+ set -e -o pipefail
342
+ echo "Building profiling native components (hash: { profiling_hash [:16 ]} )"
343
+ apt update && apt install -y sccache cmake
344
+ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
345
+ source ~/.cargo/env
346
+
347
+ # Check if we can use cached artifacts
348
+ if python3 scripts/build_cache.py check cmake_ddup 2>/dev/null; then
349
+ echo "Using cached artifacts for profiling components"
350
+ python3 scripts/build_cache.py restore cmake_ddup || echo "Cache restore failed, will rebuild"
351
+ python3 scripts/build_cache.py restore cmake_crashtracker || echo "Cache restore failed, will rebuild"
352
+ python3 scripts/build_cache.py restore cmake_stack_v2 || echo "Cache restore failed, will rebuild"
353
+ python3 scripts/build_cache.py restore rust_native || echo "Cache restore failed, will rebuild"
354
+ fi
355
+
356
+ # Build profiling extensions
357
+ pip install riot==0.20.1
358
+ riot -P -v generate --python=$PYTHON_VERSION
359
+
360
+ echo "Profiling native components built successfully"
361
+ cache:
362
+ # Cache based on profiling source code hash
363
+ - key: v1-profiling_native-${{PYTHON_VERSION}}-{ profiling_hash [:16 ]}
364
+ paths:
365
+ - .cache
366
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_ddup
367
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_crashtracker
368
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_stack_v2
369
+ - .build_cache/py${{PYTHON_VERSION}}/rust_native
370
+ - target/
371
+ artifacts:
372
+ name: profiling_native_$PYTHON_VERSION
373
+ paths:
374
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_ddup
375
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_crashtracker
376
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_stack_v2
377
+ - .build_cache/py${{PYTHON_VERSION}}/rust_native
378
+ - ddtrace/internal/datadog/profiling/ddup/**/*.so*
379
+ - ddtrace/internal/datadog/profiling/crashtracker/**/*.so*
380
+ - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
381
+ - ddtrace/internal/datadog/profiling/stack_v2/**/*.so*
382
+ - ddtrace/internal/native/**/*.so*
383
+ - target/
384
+ expire_in: 24 hours """
385
+ )
386
+
387
+ # 3. IAST NATIVE BUILD (ddtrace.appsec._iast)
388
+ iast_hash = component_hashes .get ("cmake_iast" , ci_commit_sha )[:16 ]
389
+
390
+ f .write (
391
+ f"""
392
+ build_iast_native:
331
393
extends: .testrunner
332
394
stage: setup
333
395
needs:
334
- - job: build_base_venv
396
+ - job: build_riot_dependencies
335
397
artifacts: true
336
398
parallel:
337
399
matrix:
@@ -341,7 +403,7 @@ def gen_build_base_venvs() -> None:
341
403
DD_USE_SCCACHE: '1'
342
404
PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
343
405
SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
344
- DD_BUILD_EXT_INCLUDES: '{ build_patterns } '
406
+ DD_BUILD_EXT_INCLUDES: 'ddtrace.appsec._iast._taint_tracking._native '
345
407
DD_FAST_BUILD: '1'
346
408
rules:
347
409
- if: '$CI_COMMIT_REF_NAME == "main"'
@@ -350,50 +412,138 @@ def gen_build_base_venvs() -> None:
350
412
- when: always
351
413
script: |
352
414
set -e -o pipefail
353
- echo "Building component: { component } (hash: { component_hash } )"
354
- { chr ( 10 ). join ( build_deps ) }
415
+ echo "Building IAST native component (hash: { iast_hash } )"
416
+ apt update && apt install -y sccache cmake
355
417
356
418
# Check if we can use cached artifacts
357
- if python3 scripts/build_cache.py check { component } 2>/dev/null; then
358
- echo "Using cached artifacts for { component } "
359
- python3 scripts/build_cache.py restore { component } || echo "Cache restore failed, will rebuild"
419
+ if python3 scripts/build_cache.py check cmake_iast 2>/dev/null; then
420
+ echo "Using cached artifacts for IAST component"
421
+ python3 scripts/build_cache.py restore cmake_iast || echo "Cache restore failed, will rebuild"
360
422
fi
361
423
362
- # Build only this component's extensions
424
+ # Build IAST extension
363
425
pip install riot==0.20.1
364
426
riot -P -v generate --python=$PYTHON_VERSION
365
427
366
- echo "Component { component } built successfully"
428
+ echo "IAST native component built successfully"
367
429
cache:
368
- # Component-specific cache based on source code hash
369
- - key: v1-{ component } -${{PYTHON_VERSION}}-src- { component_hash }
430
+ # Cache based on IAST source code hash
431
+ - key: v1-iast_native -${{PYTHON_VERSION}}-{ iast_hash }
370
432
paths:
371
433
- .cache
372
- - .build_cache/py${{PYTHON_VERSION}}/{ ComponentConfig . get_component_cache_dir_name ( component ) }
434
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_iast
373
435
artifacts:
374
- name: { component } _ $PYTHON_VERSION
436
+ name: iast_native_ $PYTHON_VERSION
375
437
paths:
376
- { chr (10 ).join (f" - { pattern } " for pattern in artifact_patterns )}
377
- expire_in: 2 hours
438
+ - .build_cache/py${{PYTHON_VERSION}}/cmake_iast
439
+ - ddtrace/appsec/_iast/_taint_tracking/**/*.so*
440
+ expire_in: 24 hours
378
441
"""
379
- )
442
+ )
443
+
444
+ # 4. BASE DDTRACE BUILD (everything else: hatch.toml + setup.py + cython + vendor + etc.)
445
+ base_components = ["cython_extensions" , "c_extensions" , "vendor_extensions" ]
446
+ base_hash = _get_combined_hash (
447
+ [component_hashes .get (c , ci_commit_sha ) for c in base_components ] + [dependency_hash ]
448
+ )
449
+ base_patterns = [
450
+ "ddtrace.internal._rand,ddtrace.internal._tagset,ddtrace.internal._encoding,ddtrace.profiling.collector.*" ,
451
+ "ddtrace.profiling.collector._memalloc,ddtrace.internal._threads,ddtrace.appsec._iast._stacktrace" ,
452
+ "ddtrace.vendor.*" ,
453
+ ]
380
454
381
- # Generate final assembly job that combines all components
455
+ f .write (
456
+ f"""
457
+ build_base_ddtrace:
458
+ extends: .testrunner
459
+ stage: setup
460
+ needs:
461
+ - job: build_riot_dependencies
462
+ artifacts: true
463
+ parallel:
464
+ matrix:
465
+ - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
466
+ variables:
467
+ CMAKE_BUILD_PARALLEL_LEVEL: '12'
468
+ DD_USE_SCCACHE: '1'
469
+ PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
470
+ SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
471
+ DD_BUILD_EXT_INCLUDES: '{ "," .join (base_patterns )} '
472
+ DD_FAST_BUILD: '1'
473
+ rules:
474
+ - if: '$CI_COMMIT_REF_NAME == "main"'
475
+ variables:
476
+ DD_FAST_BUILD: '0'
477
+ - when: always
478
+ script: |
479
+ set -e -o pipefail
480
+ echo "Building base ddtrace components (hash: { base_hash [:16 ]} )"
481
+ apt update && apt install -y build-essential
482
+
483
+ # Check if we can use cached artifacts
484
+ if python3 scripts/build_cache.py check cython_extensions 2>/dev/null; then
485
+ echo "Using cached artifacts for base ddtrace components"
486
+ python3 scripts/build_cache.py restore cython_extensions || echo "Cache restore failed, will rebuild"
487
+ python3 scripts/build_cache.py restore c_extensions || echo "Cache restore failed, will rebuild"
488
+ python3 scripts/build_cache.py restore vendor_extensions || echo "Cache restore failed, will rebuild"
489
+ fi
490
+
491
+ # Build base ddtrace extensions
492
+ pip install riot==0.20.1
493
+ riot -P -v generate --python=$PYTHON_VERSION
494
+
495
+ echo "Base ddtrace components built successfully"
496
+ cache:
497
+ # Cache based on base ddtrace source + dependency hash
498
+ - key: v1-base_ddtrace-${{PYTHON_VERSION}}-{ base_hash [:16 ]}
499
+ paths:
500
+ - .cache
501
+ - .build_cache/py${{PYTHON_VERSION}}/cython
502
+ - .build_cache/py${{PYTHON_VERSION}}/c_extensions
503
+ - .build_cache/py${{PYTHON_VERSION}}/vendor
504
+ artifacts:
505
+ name: base_ddtrace_$PYTHON_VERSION
506
+ paths:
507
+ - .build_cache/py${{PYTHON_VERSION}}/cython
508
+ - .build_cache/py${{PYTHON_VERSION}}/c_extensions
509
+ - .build_cache/py${{PYTHON_VERSION}}/vendor
510
+ - ddtrace/internal/_rand*.so*
511
+ - ddtrace/internal/_tagset*.so*
512
+ - ddtrace/internal/_encoding*.so*
513
+ - ddtrace/profiling/collector/stack*.so*
514
+ - ddtrace/profiling/collector/_traceback*.so*
515
+ - ddtrace/profiling/_threading*.so*
516
+ - ddtrace/profiling/collector/_task*.so*
517
+ - ddtrace/profiling/collector/_memalloc*.so*
518
+ - ddtrace/internal/_threads*.so*
519
+ - ddtrace/appsec/_iast/_stacktrace*.so*
520
+ - ddtrace/appsec/_iast/_ast/iastpatch*.so*
521
+ - ddtrace/vendor/**/*.so*
522
+ expire_in: 24 hours
523
+ """
524
+ )
525
+
526
+ # Generate final assembly job that combines all 4 caches
382
527
f .write (
383
528
f"""
384
529
build_complete_venv:
385
530
extends: .testrunner
386
531
stage: setup
387
532
needs:
388
- - job: build_base_venv
533
+ - job: build_riot_dependencies
534
+ artifacts: true
535
+ - job: build_profiling_native
536
+ artifacts: true
537
+ - job: build_iast_native
538
+ artifacts: true
539
+ - job: build_base_ddtrace
389
540
artifacts: true
390
- { chr (10 ).join (f" - job: build_{ component } " + chr (10 ) + " artifacts: true" for component in all_components )}
391
541
parallel:
392
542
matrix:
393
543
- PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
394
544
script: |
395
545
set -e -o pipefail
396
- echo "Assembling complete virtual environment from components "
546
+ echo "Assembling complete virtual environment from 4 build caches "
397
547
398
548
# All artifacts are already in place from dependencies
399
549
# Run smoke test to verify everything works
@@ -409,6 +559,7 @@ def gen_build_base_venvs() -> None:
409
559
- ddtrace/**/*.so*
410
560
- ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
411
561
- ddtrace/internal/datadog/profiling/test/test_*
562
+ - target/
412
563
expire_in: 1 day
413
564
"""
414
565
)
@@ -604,6 +755,14 @@ def _get_dependency_hash() -> str:
604
755
return hasher .hexdigest ()
605
756
606
757
758
+ def _get_combined_hash (hashes : list ) -> str :
759
+ """Combine multiple hashes into a single hash."""
760
+ import hashlib
761
+
762
+ combined = "" .join (str (h ) for h in hashes )
763
+ return hashlib .sha256 (combined .encode ()).hexdigest ()
764
+
765
+
607
766
# -----------------------------------------------------------------------------
608
767
609
768
# The code below is the boilerplate that makes the script work. There is
0 commit comments