Skip to content

Commit 0480d38

Browse files
committed
4 keys
1 parent 77d7466 commit 0480d38

File tree

1 file changed

+210
-51
lines changed

1 file changed

+210
-51
lines changed

scripts/gen_gitlab_config.py

Lines changed: 210 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
249249

250250

251251
def gen_build_base_venvs() -> None:
252-
"""Generate component-specific build jobs for virtual environments and native extensions."""
252+
"""Generate 4-cache build jobs based on product areas and dependencies."""
253253

254254
ci_commit_sha = os.getenv("CI_COMMIT_SHA", "default")
255255
# Try to get component-specific hashes, fall back to legacy single hash
@@ -269,7 +269,7 @@ def gen_build_base_venvs() -> None:
269269
# Generate dependency hash for Python environment
270270
dependency_hash = _get_dependency_hash()
271271

272-
# Generate base venv job (Python dependencies, no native extensions)
272+
# 1. RIOT DEPENDENCIES (test requirements)
273273
f.write(
274274
f"""
275275
build_base_venv:
@@ -284,54 +284,116 @@ def gen_build_base_venvs() -> None:
284284
DD_BUILD_EXT_EXCLUDES: '*' # Exclude all native extensions
285285
script: |
286286
set -e -o pipefail
287-
echo "Building base Python environment (no native extensions)"
287+
echo "Building riot test dependencies (no native extensions)"
288288
echo "Dependency hash: {dependency_hash[:16]}"
289289
pip install riot==0.20.1
290290
riot -P -v generate --python=$PYTHON_VERSION
291-
echo "Base venv created successfully"
291+
echo "Riot dependencies created successfully"
292292
cache:
293-
# Cache based on dependency files, not source code
294-
- key: v1-base_venv-${{PYTHON_VERSION}}-deps-{dependency_hash[:16]}
293+
# Cache based on dependency files (riot requirements)
294+
- key: v1-riot_deps-${{PYTHON_VERSION}}-{dependency_hash[:16]}
295295
paths:
296296
- .cache
297297
- .riot/venv_*
298298
artifacts:
299-
name: base_venv_$PYTHON_VERSION
299+
name: riot_deps_$PYTHON_VERSION
300300
paths:
301301
- .riot/venv_*
302302
- ddtrace/_version.py
303-
expire_in: 4 hours # Longer expiry for base venv
303+
expire_in: 72 hours # Longer expiry for dependencies
304304
"""
305305
)
306306

307-
# Generate component-specific build jobs
308-
for component in all_components:
309-
component_hash = component_hashes.get(component, ci_commit_sha)[:16]
310-
311-
# Define component-specific patterns and artifacts
312-
if ComponentConfig.is_cmake_component(component):
313-
build_patterns = _get_cmake_build_patterns(component)
314-
artifact_patterns = _get_cmake_artifact_patterns(component)
315-
build_deps = ["apt update && apt install -y sccache cmake"]
316-
elif ComponentConfig.is_rust_component(component):
317-
build_patterns = _get_rust_build_patterns(component)
318-
artifact_patterns = _get_rust_artifact_patterns(component)
319-
build_deps = [
320-
"curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y",
321-
"source ~/.cargo/env",
322-
]
323-
else: # Python components (Cython, C extensions)
324-
build_patterns = _get_python_build_patterns(component)
325-
artifact_patterns = _get_python_artifact_patterns(component)
326-
build_deps = ["apt update && apt install -y build-essential"]
307+
# 2. PROFILING NATIVE BUILD (ddtrace.internal.profiling)
308+
profiling_components = ["cmake_ddup", "cmake_crashtracker", "cmake_stack_v2", "rust_native"]
309+
profiling_hash = _get_combined_hash([component_hashes.get(c, ci_commit_sha) for c in profiling_components])
310+
profiling_patterns = [
311+
"ddtrace.internal.datadog.profiling.ddup._ddup",
312+
"ddtrace.internal.datadog.profiling.crashtracker._crashtracker",
313+
"ddtrace.internal.datadog.profiling.stack_v2._stack_v2",
314+
"ddtrace.internal.native._native",
315+
]
327316

328-
f.write(
329-
f"""
330-
build_{component}:
317+
f.write(
318+
f"""
319+
build_profiling_native:
320+
extends: .testrunner
321+
stage: setup
322+
needs:
323+
- job: build_riot_dependencies
324+
artifacts: true
325+
parallel:
326+
matrix:
327+
- PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
328+
variables:
329+
CMAKE_BUILD_PARALLEL_LEVEL: '12'
330+
DD_USE_SCCACHE: '1'
331+
PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
332+
SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
333+
DD_BUILD_EXT_INCLUDES: '{",".join(profiling_patterns)}'
334+
DD_FAST_BUILD: '1'
335+
rules:
336+
- if: '$CI_COMMIT_REF_NAME == "main"'
337+
variables:
338+
DD_FAST_BUILD: '0'
339+
- when: always
340+
script: |
341+
set -e -o pipefail
342+
echo "Building profiling native components (hash: {profiling_hash[:16]})"
343+
apt update && apt install -y sccache cmake
344+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
345+
source ~/.cargo/env
346+
347+
# Check if we can use cached artifacts
348+
if python3 scripts/build_cache.py check cmake_ddup 2>/dev/null; then
349+
echo "Using cached artifacts for profiling components"
350+
python3 scripts/build_cache.py restore cmake_ddup || echo "Cache restore failed, will rebuild"
351+
python3 scripts/build_cache.py restore cmake_crashtracker || echo "Cache restore failed, will rebuild"
352+
python3 scripts/build_cache.py restore cmake_stack_v2 || echo "Cache restore failed, will rebuild"
353+
python3 scripts/build_cache.py restore rust_native || echo "Cache restore failed, will rebuild"
354+
fi
355+
356+
# Build profiling extensions
357+
pip install riot==0.20.1
358+
riot -P -v generate --python=$PYTHON_VERSION
359+
360+
echo "Profiling native components built successfully"
361+
cache:
362+
# Cache based on profiling source code hash
363+
- key: v1-profiling_native-${{PYTHON_VERSION}}-{profiling_hash[:16]}
364+
paths:
365+
- .cache
366+
- .build_cache/py${{PYTHON_VERSION}}/cmake_ddup
367+
- .build_cache/py${{PYTHON_VERSION}}/cmake_crashtracker
368+
- .build_cache/py${{PYTHON_VERSION}}/cmake_stack_v2
369+
- .build_cache/py${{PYTHON_VERSION}}/rust_native
370+
- target/
371+
artifacts:
372+
name: profiling_native_$PYTHON_VERSION
373+
paths:
374+
- .build_cache/py${{PYTHON_VERSION}}/cmake_ddup
375+
- .build_cache/py${{PYTHON_VERSION}}/cmake_crashtracker
376+
- .build_cache/py${{PYTHON_VERSION}}/cmake_stack_v2
377+
- .build_cache/py${{PYTHON_VERSION}}/rust_native
378+
- ddtrace/internal/datadog/profiling/ddup/**/*.so*
379+
- ddtrace/internal/datadog/profiling/crashtracker/**/*.so*
380+
- ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
381+
- ddtrace/internal/datadog/profiling/stack_v2/**/*.so*
382+
- ddtrace/internal/native/**/*.so*
383+
- target/
384+
expire_in: 24 hours """
385+
)
386+
387+
# 3. IAST NATIVE BUILD (ddtrace.appsec._iast)
388+
iast_hash = component_hashes.get("cmake_iast", ci_commit_sha)[:16]
389+
390+
f.write(
391+
f"""
392+
build_iast_native:
331393
extends: .testrunner
332394
stage: setup
333395
needs:
334-
- job: build_base_venv
396+
- job: build_riot_dependencies
335397
artifacts: true
336398
parallel:
337399
matrix:
@@ -341,7 +403,7 @@ def gen_build_base_venvs() -> None:
341403
DD_USE_SCCACHE: '1'
342404
PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
343405
SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
344-
DD_BUILD_EXT_INCLUDES: '{build_patterns}'
406+
DD_BUILD_EXT_INCLUDES: 'ddtrace.appsec._iast._taint_tracking._native'
345407
DD_FAST_BUILD: '1'
346408
rules:
347409
- if: '$CI_COMMIT_REF_NAME == "main"'
@@ -350,50 +412,138 @@ def gen_build_base_venvs() -> None:
350412
- when: always
351413
script: |
352414
set -e -o pipefail
353-
echo "Building component: {component} (hash: {component_hash})"
354-
{chr(10).join(build_deps)}
415+
echo "Building IAST native component (hash: {iast_hash})"
416+
apt update && apt install -y sccache cmake
355417
356418
# Check if we can use cached artifacts
357-
if python3 scripts/build_cache.py check {component} 2>/dev/null; then
358-
echo "Using cached artifacts for {component}"
359-
python3 scripts/build_cache.py restore {component} || echo "Cache restore failed, will rebuild"
419+
if python3 scripts/build_cache.py check cmake_iast 2>/dev/null; then
420+
echo "Using cached artifacts for IAST component"
421+
python3 scripts/build_cache.py restore cmake_iast || echo "Cache restore failed, will rebuild"
360422
fi
361423
362-
# Build only this component's extensions
424+
# Build IAST extension
363425
pip install riot==0.20.1
364426
riot -P -v generate --python=$PYTHON_VERSION
365427
366-
echo "Component {component} built successfully"
428+
echo "IAST native component built successfully"
367429
cache:
368-
# Component-specific cache based on source code hash
369-
- key: v1-{component}-${{PYTHON_VERSION}}-src-{component_hash}
430+
# Cache based on IAST source code hash
431+
- key: v1-iast_native-${{PYTHON_VERSION}}-{iast_hash}
370432
paths:
371433
- .cache
372-
- .build_cache/py${{PYTHON_VERSION}}/{ComponentConfig.get_component_cache_dir_name(component)}
434+
- .build_cache/py${{PYTHON_VERSION}}/cmake_iast
373435
artifacts:
374-
name: {component}_$PYTHON_VERSION
436+
name: iast_native_$PYTHON_VERSION
375437
paths:
376-
{chr(10).join(f" - {pattern}" for pattern in artifact_patterns)}
377-
expire_in: 2 hours
438+
- .build_cache/py${{PYTHON_VERSION}}/cmake_iast
439+
- ddtrace/appsec/_iast/_taint_tracking/**/*.so*
440+
expire_in: 24 hours
378441
"""
379-
)
442+
)
443+
444+
# 4. BASE DDTRACE BUILD (everything else: hatch.toml + setup.py + cython + vendor + etc.)
445+
base_components = ["cython_extensions", "c_extensions", "vendor_extensions"]
446+
base_hash = _get_combined_hash(
447+
[component_hashes.get(c, ci_commit_sha) for c in base_components] + [dependency_hash]
448+
)
449+
base_patterns = [
450+
"ddtrace.internal._rand,ddtrace.internal._tagset,ddtrace.internal._encoding,ddtrace.profiling.collector.*",
451+
"ddtrace.profiling.collector._memalloc,ddtrace.internal._threads,ddtrace.appsec._iast._stacktrace",
452+
"ddtrace.vendor.*",
453+
]
380454

381-
# Generate final assembly job that combines all components
455+
f.write(
456+
f"""
457+
build_base_ddtrace:
458+
extends: .testrunner
459+
stage: setup
460+
needs:
461+
- job: build_riot_dependencies
462+
artifacts: true
463+
parallel:
464+
matrix:
465+
- PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
466+
variables:
467+
CMAKE_BUILD_PARALLEL_LEVEL: '12'
468+
DD_USE_SCCACHE: '1'
469+
PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
470+
SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
471+
DD_BUILD_EXT_INCLUDES: '{",".join(base_patterns)}'
472+
DD_FAST_BUILD: '1'
473+
rules:
474+
- if: '$CI_COMMIT_REF_NAME == "main"'
475+
variables:
476+
DD_FAST_BUILD: '0'
477+
- when: always
478+
script: |
479+
set -e -o pipefail
480+
echo "Building base ddtrace components (hash: {base_hash[:16]})"
481+
apt update && apt install -y build-essential
482+
483+
# Check if we can use cached artifacts
484+
if python3 scripts/build_cache.py check cython_extensions 2>/dev/null; then
485+
echo "Using cached artifacts for base ddtrace components"
486+
python3 scripts/build_cache.py restore cython_extensions || echo "Cache restore failed, will rebuild"
487+
python3 scripts/build_cache.py restore c_extensions || echo "Cache restore failed, will rebuild"
488+
python3 scripts/build_cache.py restore vendor_extensions || echo "Cache restore failed, will rebuild"
489+
fi
490+
491+
# Build base ddtrace extensions
492+
pip install riot==0.20.1
493+
riot -P -v generate --python=$PYTHON_VERSION
494+
495+
echo "Base ddtrace components built successfully"
496+
cache:
497+
# Cache based on base ddtrace source + dependency hash
498+
- key: v1-base_ddtrace-${{PYTHON_VERSION}}-{base_hash[:16]}
499+
paths:
500+
- .cache
501+
- .build_cache/py${{PYTHON_VERSION}}/cython
502+
- .build_cache/py${{PYTHON_VERSION}}/c_extensions
503+
- .build_cache/py${{PYTHON_VERSION}}/vendor
504+
artifacts:
505+
name: base_ddtrace_$PYTHON_VERSION
506+
paths:
507+
- .build_cache/py${{PYTHON_VERSION}}/cython
508+
- .build_cache/py${{PYTHON_VERSION}}/c_extensions
509+
- .build_cache/py${{PYTHON_VERSION}}/vendor
510+
- ddtrace/internal/_rand*.so*
511+
- ddtrace/internal/_tagset*.so*
512+
- ddtrace/internal/_encoding*.so*
513+
- ddtrace/profiling/collector/stack*.so*
514+
- ddtrace/profiling/collector/_traceback*.so*
515+
- ddtrace/profiling/_threading*.so*
516+
- ddtrace/profiling/collector/_task*.so*
517+
- ddtrace/profiling/collector/_memalloc*.so*
518+
- ddtrace/internal/_threads*.so*
519+
- ddtrace/appsec/_iast/_stacktrace*.so*
520+
- ddtrace/appsec/_iast/_ast/iastpatch*.so*
521+
- ddtrace/vendor/**/*.so*
522+
expire_in: 24 hours
523+
"""
524+
)
525+
526+
# Generate final assembly job that combines all 4 caches
382527
f.write(
383528
f"""
384529
build_complete_venv:
385530
extends: .testrunner
386531
stage: setup
387532
needs:
388-
- job: build_base_venv
533+
- job: build_riot_dependencies
534+
artifacts: true
535+
- job: build_profiling_native
536+
artifacts: true
537+
- job: build_iast_native
538+
artifacts: true
539+
- job: build_base_ddtrace
389540
artifacts: true
390-
{chr(10).join(f" - job: build_{component}" + chr(10) + " artifacts: true" for component in all_components)}
391541
parallel:
392542
matrix:
393543
- PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
394544
script: |
395545
set -e -o pipefail
396-
echo "Assembling complete virtual environment from components"
546+
echo "Assembling complete virtual environment from 4 build caches"
397547
398548
# All artifacts are already in place from dependencies
399549
# Run smoke test to verify everything works
@@ -409,6 +559,7 @@ def gen_build_base_venvs() -> None:
409559
- ddtrace/**/*.so*
410560
- ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
411561
- ddtrace/internal/datadog/profiling/test/test_*
562+
- target/
412563
expire_in: 1 day
413564
"""
414565
)
@@ -604,6 +755,14 @@ def _get_dependency_hash() -> str:
604755
return hasher.hexdigest()
605756

606757

758+
def _get_combined_hash(hashes: list) -> str:
759+
"""Combine multiple hashes into a single hash."""
760+
import hashlib
761+
762+
combined = "".join(str(h) for h in hashes)
763+
return hashlib.sha256(combined.encode()).hexdigest()
764+
765+
607766
# -----------------------------------------------------------------------------
608767

609768
# The code below is the boilerplate that makes the script work. There is

0 commit comments

Comments
 (0)