Skip to content

Commit fe42de4

Browse files
lanluo-nvidialanluo-nvidia
committed
test
1 parent 3c6cdbb commit fe42de4

File tree

5 files changed

+236
-162
lines changed

5 files changed

+236
-162
lines changed

‎.github/actions/mlir-tensorrt-build-test/action.yml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
name: build-cache-test
2+
inputs:
3+
cpm_cache_restore_flag:
4+
required: true
5+
default: false
6+
7+
runs:
8+
using: composite
9+
steps:
10+
# Checkout the repository
11+
- name: Checkout TensorRT-Incubator
12+
uses: actions/checkout@v6
13+
with:
14+
fetch-depth: 5
15+
16+
# Create cache folders
17+
- name: Create Cache Folders
18+
run: |
19+
set -euo pipefail
20+
set -x
21+
export CPM_SOURCE_CACHE=${GITHUB_WORKSPACE}/mlir-tensorrt/.cache.cpm
22+
export CCACHE_DIR=${GITHUB_WORKSPACE}/mlir-tensorrt/ccache
23+
24+
echo "CPM_SOURCE_CACHE=$CPM_SOURCE_CACHE" >> "$GITHUB_ENV"
25+
echo "CCACHE_DIR=$CCACHE_DIR" >> "$GITHUB_ENV"
26+
27+
mkdir -p ${CCACHE_DIR}
28+
mkdir -p ${CPM_SOURCE_CACHE}
29+
30+
# Compute cpm key after checkout
31+
- name: Compute CPM Key
32+
id: cpm-key
33+
run: |
34+
hash=$(sha256sum mlir-tensorrt/DependencyProvider.cmake | cut -d' ' -f1)
35+
echo "key=${{ env.CPM_RESTORE_KEY }}-${hash}" >> $GITHUB_OUTPUT
36+
37+
- name: Restore CPM cache
38+
id: restore-cpm
39+
uses: actions/cache/restore@v4
40+
with:
41+
key: ${{ steps.cpm-key.outputs.key }}
42+
enableCrossOsArchive: true
43+
restore-keys: |
44+
${{ env.CPM_RESTORE_KEY }}
45+
# exclude only works for the relative path pattern
46+
# restore must use the exactly same path defined in the save step, cannot ignore the exclude path
47+
path: |
48+
mlir-tensorrt/.cache.cpm/*
49+
!mlir-tensorrt/.cache.cpm/tensorrt
50+
!mlir-tensorrt/.cache.cpm/tensorrt/**
51+
52+
# Compute ccache key after checkout
53+
- name: Compute CCache Key
54+
id: ccache-key
55+
run: |
56+
hash=$( (find mlir-tensorrt/compiler \
57+
mlir-tensorrt/common \
58+
mlir-tensorrt/kernel \
59+
mlir-tensorrt/tensorrt \
60+
mlir-tensorrt/integrations \
61+
mlir-tensorrt/executor \
62+
-type f \( -name '*.cpp' -o -name '*.h' \) \
63+
-exec sha256sum {} \; ; \
64+
sha256sum mlir-tensorrt/DependencyProvider.cmake \
65+
mlir-tensorrt/CMakeLists.txt) \
66+
| sort | sha256sum | cut -d' ' -f1)
67+
echo "key=${{ env.CCACHE_RESTORE_KEY }}-${hash}" >> $GITHUB_OUTPUT
68+
69+
# Restore cache, if exists.
70+
- name: Restore CCache
71+
id: restore-ccache
72+
uses: actions/cache/restore@v4
73+
with:
74+
key: ${{ steps.ccache-key.outputs.key }}
75+
restore-keys: |
76+
${{ env.CCACHE_RESTORE_KEY }}
77+
path: |
78+
${{ env.CCACHE_DIR }}
79+
80+
# Build the project
81+
- name: Build With CUDA:${{ matrix.cuda }}+TensorRT:${{ matrix.trt }}
82+
id: build
83+
run: |
84+
set -euo pipefail
85+
set -x
86+
cd mlir-tensorrt
87+
# build only, skip tests
88+
./build_tools/scripts/cicd-build.sh --build_only
89+
90+
# save ccache if cache is not hit
91+
- name: Save CCache
92+
uses: actions/cache/save@v4
93+
if: ${{ steps.restore-ccache.outputs.cache-hit != 'true' }}
94+
with:
95+
key: ${{ steps.ccache-key.outputs.key }}
96+
path: |
97+
${{ env.CCACHE_DIR }}
98+
99+
# Save cpm cache
100+
- name: Save CPM Cache
101+
# cpm cache is shared across x86_64 and aarch64, we let only x86_64 to save cpm cache when in cache miss case
102+
# this is to avoid both x86_64 and aarch64 to save cpm cache when in cache miss case
103+
if: ${{ matrix.arch == 'x86_64' && steps.restore-cpm.outputs.cache-hit != 'true' && inputs.cpm_cache_restore_flag == true }}
104+
uses: actions/cache/save@v4
105+
with:
106+
key: ${{ steps.cpm-key.outputs.key }}
107+
enableCrossOsArchive: true
108+
# exclude only works for the relative path pattern
109+
path: |
110+
mlir-tensorrt/.cache.cpm/*
111+
!mlir-tensorrt/.cache.cpm/tensorrt
112+
!mlir-tensorrt/.cache.cpm/tensorrt/**
113+
114+
# Run tests
115+
- name: Test With CUDA:${{ matrix.cuda }}+TensorRT:${{ matrix.trt }}
116+
id: test
117+
run: |
118+
set -euo pipefail
119+
set -x
120+
cd mlir-tensorrt
121+
# run tests
122+
./build_tools/scripts/cicd-build.sh

‎.github/workflows/mlir-tensorrt-build-test.yml‎

Lines changed: 91 additions & 154 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ defaults:
1717
shell: bash
1818

1919
jobs:
20-
mlir-tensorrt-build-test:
21-
name: ${{ matrix.arch }}-cu${{ matrix.cuda }}-trt${{ matrix.trt }}-build-test
20+
mlir-tensorrt-basic-build-test:
21+
name: ${{ matrix.arch }}-cu${{ matrix.cuda }}-trt${{ matrix.trt }}-basic-build-test
2222
strategy:
23-
fail-fast: true
23+
fail-fast: false
2424
max-parallel: 1
2525
matrix: ${{ fromJSON(inputs.build-matrix) }}
2626
env:
@@ -30,8 +30,6 @@ jobs:
3030
CHANNEL: ${{ inputs.channel }}
3131
ARCH: ${{ matrix.arch }}
3232
CMAKE_PRESET: ${{ matrix.cmake_preset }}
33-
CCACHE_DIR: ${{ github.workspace }}/mlir-tensorrt/ccache
34-
CPM_SOURCE_CACHE: ${{ github.workspace }}/mlir-tensorrt/.cache.cpm
3533
CCACHE_RESTORE_KEY: mlir-tensorrt-ccache-v1-${{ matrix.arch }}-${{ matrix.cmake_preset }}
3634
CPM_RESTORE_KEY: mlir-tensorrt-cpm-v1
3735
runs-on: ${{ matrix.github_runner }}
@@ -42,162 +40,101 @@ jobs:
4240
--shm-size=1g
4341
timeout-minutes: 120
4442
steps:
45-
# Checkout the repository
46-
- name: Checkout TensorRT-Incubator
47-
uses: actions/checkout@v6
43+
# basic build and test
44+
- name: Basic Build and Test
45+
uses: ./.github/actions/mlir-tensorrt-build-test
4846
with:
49-
fetch-depth: 5
50-
51-
# Create cache folders
52-
- name: Create Cache Folders
53-
run: |
54-
set -euo pipefail
55-
set -x
56-
mkdir -p ${{ env.CCACHE_DIR }}
57-
mkdir -p ${{ env.CPM_SOURCE_CACHE }}
58-
59-
# Compute ccache key after checkout
60-
- name: Compute CCache Key
61-
id: ccache-key
62-
run: |
63-
hash=$( (find mlir-tensorrt/compiler \
64-
mlir-tensorrt/common \
65-
mlir-tensorrt/kernel \
66-
mlir-tensorrt/tensorrt \
67-
mlir-tensorrt/integrations \
68-
mlir-tensorrt/executor \
69-
-type f \( -name '*.cpp' -o -name '*.h' \) \
70-
-exec sha256sum {} \; ; \
71-
sha256sum mlir-tensorrt/DependencyProvider.cmake \
72-
mlir-tensorrt/CMakeLists.txt) \
73-
| sort | sha256sum | cut -d' ' -f1)
74-
echo "key=${{ env.CCACHE_RESTORE_KEY }}-${hash}" >> $GITHUB_OUTPUT
47+
cpm_cache_restore_flag: true
7548

76-
# Compute cpm key after checkout
77-
- name: Compute CPM Key
78-
id: cpm-key
79-
run: |
80-
hash=$(sha256sum mlir-tensorrt/DependencyProvider.cmake | cut -d' ' -f1)
81-
echo "key=${{ env.CPM_RESTORE_KEY }}-${hash}" >> $GITHUB_OUTPUT
82-
83-
# Restore cache, if exists.
84-
- name: Restore CCache
85-
id: restore-ccache
86-
uses: actions/cache/restore@v4
87-
with:
88-
key: ${{ steps.ccache-key.outputs.key }}
89-
restore-keys: |
90-
${{ env.CCACHE_RESTORE_KEY }}
91-
path: |
92-
${{ env.CCACHE_DIR }}
93-
94-
- name: Restore CPM cache
95-
id: restore-cpm
96-
uses: actions/cache/restore@v4
49+
mlir-tensorrt-asan-build-test:
50+
name: ${{ matrix.arch }}-cu${{ matrix.cuda }}-trt${{ matrix.trt }}-asan-build-test
51+
#if: ${{ inputs.channel == 'nightly' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
52+
strategy:
53+
fail-fast: false
54+
max-parallel: 1
55+
matrix: ${{ fromJSON(inputs.build-matrix) }}
56+
env:
57+
# eg. TENSORRT_VERSION: 10.12 or 10.13
58+
MLIR_TRT_DOWNLOAD_TENSORRT_VERSION: ${{ matrix.trt }}
59+
# eg. CHANNEL: nightly, test or release
60+
CHANNEL: ${{ inputs.channel }}
61+
ARCH: ${{ matrix.arch }}
62+
CMAKE_PRESET: ${{ matrix.cmake_preset }}-asan
63+
CCACHE_RESTORE_KEY: mlir-tensorrt-ccache-v1-${{ matrix.arch }}-${{ matrix.cmake_preset }}-asan
64+
CPM_RESTORE_KEY: mlir-tensorrt-cpm-v1
65+
runs-on: ${{ matrix.github_runner }}
66+
container:
67+
image: ${{ matrix.docker_image }}
68+
options: >-
69+
--gpus all
70+
--shm-size=1g
71+
timeout-minutes: 120
72+
steps:
73+
# asan build and test
74+
- name: ASAN Build and Test
75+
uses: ./.github/actions/mlir-tensorrt-build-test
9776
with:
98-
key: ${{ steps.cpm-key.outputs.key }}
99-
enableCrossOsArchive: true
100-
restore-keys: |
101-
${{ env.CPM_RESTORE_KEY }}
102-
# exclude only works for the relative path pattern
103-
# restore must use the exactly same path defined in the save step, cannot ignore the exclude path
104-
path: |
105-
mlir-tensorrt/.cache.cpm/*
106-
!mlir-tensorrt/.cache.cpm/tensorrt
107-
!mlir-tensorrt/.cache.cpm/tensorrt/**
108-
109-
# Build the project
110-
- name: Build With CUDA:${{ matrix.cuda }} + TensorRT:${{ matrix.trt }}
111-
env:
112-
ENABLE_ASAN: "OFF"
113-
ENABLE_NCCL: "OFF"
114-
run: |
115-
set -euo pipefail
116-
set -x
117-
cd mlir-tensorrt
118-
# build only, skip tests
119-
./build_tools/scripts/cicd-build.sh --build_only
77+
cpm_cache_restore_flag: false
12078

121-
# Save ccache when cache is not hit or cache was a fallback(cache-matched is not the same as the cache key)
122-
- name: Save CCache
123-
uses: actions/cache/save@v4
124-
if: ${{ steps.restore-ccache.outputs.cache-hit != 'true' }}
79+
mlir-tensorrt-long-build-test:
80+
name: ${{ matrix.arch }}-cu${{ matrix.cuda }}-trt${{ matrix.trt }}-build-test
81+
#if: ${{ inputs.channel == 'nightly' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
82+
strategy:
83+
fail-fast: false
84+
max-parallel: 1
85+
matrix: ${{ fromJSON(inputs.build-matrix) }}
86+
env:
87+
# eg. TENSORRT_VERSION: 10.12 or 10.13
88+
MLIR_TRT_DOWNLOAD_TENSORRT_VERSION: ${{ matrix.trt }}
89+
# eg. CHANNEL: nightly, test or release
90+
CHANNEL: ${{ inputs.channel }}
91+
ARCH: ${{ matrix.arch }}
92+
CMAKE_PRESET: ${{ matrix.cmake_preset }}-long
93+
CCACHE_RESTORE_KEY: mlir-tensorrt-ccache-v1-${{ matrix.arch }}-${{ matrix.cmake_preset }}-long
94+
CPM_RESTORE_KEY: mlir-tensorrt-cpm-v1
95+
runs-on: ${{ matrix.github_runner }}
96+
container:
97+
image: ${{ matrix.docker_image }}
98+
options: >-
99+
--gpus all
100+
--shm-size=1g
101+
timeout-minutes: 120
102+
steps:
103+
# basic build and test
104+
- name: Basic Build and Test
105+
uses: ./.github/actions/mlir-tensorrt-build-test
125106
with:
126-
key: ${{ steps.ccache-key.outputs.key }}
127-
path: |
128-
${{ env.CCACHE_DIR }}
107+
cpm_cache_restore_flag: false
129108

130-
# Save cpm cache
131-
- name: Save CPM Cache
132-
# cpm cache is shared across x86_64 and aarch64, we let only x86_64 to save cpm cache when in cache miss case
133-
# this is to avoid both x86_64 and aarch64 to save cpm cache when in cache miss case
134-
if: ${{ matrix.arch == 'x86_64' && steps.restore-cpm.outputs.cache-hit != 'true' }}
135-
uses: actions/cache/save@v4
109+
mlir-tensorrt-nccl-long-build-test:
110+
name: ${{ matrix.arch }}-cu${{ matrix.cuda }}-trt${{ matrix.trt }}-nccl-long-build-test
111+
#if: ${{ inputs.channel == 'nightly' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
112+
strategy:
113+
fail-fast: false
114+
max-parallel: 1
115+
matrix: ${{ fromJSON(inputs.build-matrix) }}
116+
env:
117+
# eg. TENSORRT_VERSION: 10.12 or 10.13
118+
MLIR_TRT_DOWNLOAD_TENSORRT_VERSION: ${{ matrix.trt }}
119+
# eg. CHANNEL: nightly, test or release
120+
CHANNEL: ${{ inputs.channel }}
121+
ARCH: ${{ matrix.arch }}
122+
CMAKE_PRESET: ${{ matrix.cmake_preset }}-nccl-long
123+
CCACHE_RESTORE_KEY: mlir-tensorrt-ccache-v1-${{ matrix.arch }}-${{ matrix.cmake_preset }}-nccl-long
124+
CPM_RESTORE_KEY: mlir-tensorrt-cpm-v1
125+
runs-on: ${{ matrix.github_runner }}
126+
container:
127+
image: ${{ matrix.docker_image }}
128+
options: >-
129+
--gpus all
130+
--shm-size=1g
131+
timeout-minutes: 120
132+
steps:
133+
# nccl long build and test
134+
- name: NCCL Long Build and Test
135+
uses: ./.github/actions/mlir-tensorrt-build-test
136136
with:
137-
key: ${{ steps.cpm-key.outputs.key }}
138-
enableCrossOsArchive: true
139-
# exclude only works for the relative path pattern
140-
path: |
141-
mlir-tensorrt/.cache.cpm/*
142-
!mlir-tensorrt/.cache.cpm/tensorrt
143-
!mlir-tensorrt/.cache.cpm/tensorrt/**
144-
145-
# Run tests
146-
- name: Run Basic Tests With CUDA:${{ matrix.cuda }} + TensorRT:${{ matrix.trt }}
147-
id: basic-tests
148-
env:
149-
ENABLE_ASAN: "OFF"
150-
ENABLE_NCCL: "OFF"
151-
run: |
152-
set -euo pipefail
153-
set -x
154-
cd mlir-tensorrt
155-
# run tests
156-
./build_tools/scripts/cicd-build.sh
157-
158-
# Run ASAN tests
159-
- name: Run ASAN Tests With CUDA:${{ matrix.cuda }} + TensorRT:${{ matrix.trt }}
160-
id: asan-tests
161-
if: ${{ inputs.channel == 'nightly' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
162-
env:
163-
ENABLE_ASAN: "ON"
164-
ENABLE_NCCL: "OFF"
165-
run: |
166-
set -euo pipefail
167-
set -x
168-
cd mlir-tensorrt
169-
# run tests
170-
./build_tools/scripts/cicd-build.sh
171-
172-
# Run long tests
173-
- name: Run Long Tests With CUDA:${{ matrix.cuda }} + TensorRT:${{ matrix.trt }}
174-
id: long-tests
175-
if: ${{ inputs.channel == 'nightly' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
176-
env:
177-
LONG_TESTS: "ON"
178-
ENABLE_ASAN: "OFF"
179-
ENABLE_NCCL: "OFF"
180-
run: |
181-
set -euo pipefail
182-
set -x
183-
cd mlir-tensorrt
184-
# run tests
185-
./build_tools/scripts/cicd-build.sh
186-
187-
# Run NCCL long tests
188-
- name: Run Long Tests With CUDA:${{ matrix.cuda }} + TensorRT:${{ matrix.trt }}
189-
id: nccl-long-tests
190-
if: ${{ inputs.channel == 'nightly' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
191-
env:
192-
LONG_TESTS: "ON"
193-
ENABLE_ASAN: "OFF"
194-
ENABLE_NCCL: "ON"
195-
run: |
196-
set -euo pipefail
197-
set -x
198-
cd mlir-tensorrt
199-
# run tests
200-
./build_tools/scripts/cicd-build.sh
137+
cpm_cache_restore_flag: false
201138

202139
concurrency:
203140
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-mlir-tensorrt-${{ inputs.channel }}

0 commit comments

Comments
 (0)