Skip to content

feat(extractor): Update grammars and CodeQL #139

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Sep 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/action/dist/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13530,7 +13530,7 @@ const toolcache = __importStar(__nccwpck_require__(7784));
const github = __importStar(__nccwpck_require__(5438));
const toolrunner = __importStar(__nccwpck_require__(8159));
exports.EXTRACTOR_REPOSITORY = "advanced-security/codeql-extractor-iac";
exports.EXTRACTOR_VERSION = "v0.0.4"; // stable version
exports.EXTRACTOR_VERSION = "v0.4.0"; // stable version
async function newCodeQL() {
var version = core.getInput("extractor-version");
if (version === "") {
Expand Down
16 changes: 8 additions & 8 deletions .github/action/src/codeql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import * as github from "@actions/github";
import * as toolrunner from "@actions/exec/lib/toolrunner";

export const EXTRACTOR_REPOSITORY = "advanced-security/codeql-extractor-iac";
export const EXTRACTOR_VERSION = "v0.0.4"; // stable version
export const EXTRACTOR_VERSION = "v0.4.0"; // stable version

export interface CodeQLConfig {
// The path to the codeql bundle.
Expand Down Expand Up @@ -48,7 +48,7 @@ export async function newCodeQL(): Promise<CodeQLConfig> {

export async function runCommand(
config: CodeQLConfig,
args: string[]
args: string[],
): Promise<any> {
var bin = path.join(config.path, "codeql");
let output = "";
Expand All @@ -68,7 +68,7 @@ export async function runCommand(

export async function runCommandJson(
config: CodeQLConfig,
args: string[]
args: string[],
): Promise<object> {
return JSON.parse(await runCommand(config, args));
}
Expand Down Expand Up @@ -130,12 +130,12 @@ export async function downloadExtractor(config: CodeQLConfig): Promise<string> {
}
// we assume there is only one tar.gz asset
const assets = release.data.assets.filter((asset) =>
asset.browser_download_url.endsWith(".tar.gz")
asset.browser_download_url.endsWith(".tar.gz"),
);

if (assets.length !== 1) {
throw new Error(
`Expected 1 asset to be found, but found ${assets.length} instead.`
`Expected 1 asset to be found, but found ${assets.length} instead.`,
);
}
var asset = assets[0];
Expand All @@ -148,7 +148,7 @@ export async function downloadExtractor(config: CodeQLConfig): Promise<string> {
`token ${core.getInput("token")}`,
{
accept: "application/octet-stream",
}
},
);
core.debug(`Extractor downloaded to ${extractorPath}`);

Expand All @@ -170,7 +170,7 @@ export async function downloadPack(codeql: CodeQLConfig): Promise<boolean> {
}

export async function codeqlDatabaseCreate(
codeql: CodeQLConfig
codeql: CodeQLConfig,
): Promise<string> {
// get runner temp directory for database
var temp = process.env["RUNNER_TEMP"];
Expand All @@ -196,7 +196,7 @@ export async function codeqlDatabaseCreate(

export async function codeqlDatabaseAnalyze(
codeql: CodeQLConfig,
database_path: string
database_path: string,
): Promise<string> {
var codeql_output = codeql.output || "codeql-iac.sarif";

Expand Down
63 changes: 33 additions & 30 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
@@ -1,47 +1,50 @@

docs:
- changed-files:
- any-glob-to-any-file: 'docs/*'
- changed-files:
- any-glob-to-any-file: "docs/*"

action:
- changed-files:
- any-glob-to-any-file: '.github/action/*'
- changed-files:
- any-glob-to-any-file: ".github/action/*"

extractor:
- changed-files:
- any-glob-to-any-file:
- 'extractor/*'
- 'tools/*'
- changed-files:
- any-glob-to-any-file:
- "extractor/*"
- "tools/*"

version:
- changed-files:
- any-glob-to-any-file: ".release.yml"

ql-library:
- changed-files:
- any-glob-to-any-file:
- 'ql/lib/*'
- changed-files:
- any-glob-to-any-file:
- "ql/lib/*"

ql-queries:
- changed-files:
- any-glob-to-any-file:
- 'ql/src/*'
- changed-files:
- any-glob-to-any-file:
- "ql/src/*"

ql-tests:
- changed-files:
- any-glob-to-any-file:
- 'ql/test/*'
- changed-files:
- any-glob-to-any-file:
- "ql/test/*"

# Languages

bicep:
- changed-files:
- any-glob-to-any-file:
- ql/lib/codeql/bicep
- ql/src/security/Bicep
- test/library-tests/bicep
- test/queries-tests/Bicep
- changed-files:
- any-glob-to-any-file:
- ql/lib/codeql/bicep
- ql/src/security/Bicep
- test/library-tests/bicep
- test/queries-tests/Bicep

terraform:
- changed-files:
- any-glob-to-any-file:
- ql/lib/codeql/hcl
- ql/src/security/Terraform
- test/library-tests/hcl
- test/queries-tests/Terraform
- changed-files:
- any-glob-to-any-file:
- ql/lib/codeql/hcl
- ql/src/security/Terraform
- test/library-tests/hcl
- test/queries-tests/Terraform
69 changes: 21 additions & 48 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,38 +11,27 @@ permissions:
pull-requests: read

jobs:
extractor:
tests:
runs-on: ubuntu-latest

strategy:
matrix:
test-folders: ["library-tests", "queries-tests"]
steps:
- uses: actions/checkout@v4
- name: "Checkout"
uses: actions/checkout@v4

- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: changes
- name: "Check for changes"
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: extractor-changes
with:
filters: |
src:
- 'extractor/**'
- 'rust-toolchain.toml'
- 'Cargo.*'

- uses: dtolnay/rust-toolchain@nightly
if: steps.changes.outputs.src == 'true'

- name: "Build Extractor"
if: steps.changes.outputs.src == 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
gh extensions install github/gh-codeql
gh codeql set-version latest

./scripts/create-extractor-pack.sh

gh codeql resolve languages --format=json --search-path ./extractor-pack

- name: "Download Extracter"
if: steps.changes.outputs.src == 'false'
if: steps.extractor-changes.outputs.src == 'false'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand All @@ -55,38 +44,22 @@ jobs:

tar -zxf extractor-iac.tar.gz

# cache the extractor pack
- name: Cache Extractor Pack
uses: actions/cache@v4
with:
path: extractor-pack
key: extractor-pack
- uses: dtolnay/rust-toolchain@nightly
if: steps.extractor-changes.outputs.src == 'true'

tests:
runs-on: ubuntu-latest
needs: [extractor]
strategy:
matrix:
test-folders: ["library-tests", "queries-tests"]
steps:
- uses: actions/checkout@v4
- name: "Build Extractor"
if: steps.extractor-changes.outputs.src == 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
gh extensions install github/gh-codeql
gh codeql set-version latest

# download from cache
- name: "Download from cache"
uses: actions/cache@v4
with:
path: extractor-pack
key: extractor-pack
./scripts/create-extractor-pack.sh

- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: changes
with:
filters: |
src:
- 'ql/**'
gh codeql resolve languages --format=json --search-path ./extractor-pack

- name: "Run Tests"
if: steps.changes.outputs.src == 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand Down
37 changes: 36 additions & 1 deletion .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,40 @@ permissions:
packages: write

jobs:
release-check:
runs-on: ubuntu-latest
outputs:
release: ${{ steps.get_version.outputs.release }}
version: ${{ steps.get_version.outputs.version }}
steps:
- uses: actions/checkout@v4

- name: "Check release version"
id: get_version
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -e

pip install yq
current_version=$(cat .release.yml | yq -r ".version")

released_version=$(gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/:owner/:repo/releases/latest | jq -r ".tag_name")

if [[ "$current_version" == "NA" || "$current_version" == "$released_version" ]]; then
echo "No new release found"
echo "release=false" >> "$GITHUB_OUTPUT"
else
echo "New release found"
echo "version=$current_version" >> "$GITHUB_OUTPUT"
echo "release=true" >> "$GITHUB_OUTPUT"
fi


queries:
runs-on: ubuntu-latest
needs: [release-check]
if: ${{ needs.release-check.outputs.release == 'true' }}

permissions:
contents: read
Expand All @@ -23,7 +55,8 @@ jobs:
packs: ["lib", "src"]

steps:
- uses: actions/checkout@v4
- name: "Checkout"
uses: actions/checkout@v4

- name: "Check and Publish CodeQL Packs"
env:
Expand All @@ -45,6 +78,8 @@ jobs:

compile:
runs-on: ubuntu-latest
needs: [release-check]
if: ${{ needs.release-check.outputs.release == 'true' }}

steps:
- uses: actions/checkout@v4
Expand Down
54 changes: 54 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# GitHub Releasing Workflow
name: GitHub - Release

on:
workflow_dispatch:
inputs:
bump:
type: choice
description: "The type of version bump to perform"
options:
- patch
- minor
- major


permissions:
contents: write

jobs:
release-next:
runs-on: ubuntu-latest
steps:
- name: "Checkout"
uses: actions/checkout@v4

- name: "Patch Release Me"
uses: 42ByteLabs/[email protected]
with:
mode: ${{ github.event.inputs.bump }}

- name: "Bundle"
run: |
set -e
cd .github/action
npm i && npm run bundle

- name: "Get Version"
id: get_version
run: |
set -e
pip install yq
echo "version=$(cat .release.yml | yq -r ".version")" >> "$GITHUB_ENV"
echo "release=true" >> "$GITHUB_ENV"

- name: "Create Release"
uses: peter-evans/create-pull-request@v6
with:
token: ${{ github.token }}
commit-message: "[chore]: Create release for ${{ steps.get_version.outcome.version }}"
title: "[chore]: Create release for ${{ steps.get_version.outcome.version }}"
branch: chore-release-${{ steps.get_version.outcome.version }}
labels: version
body: |
This is an automated PR to create a new release. The release will be created once this PR is merged.
Loading
Loading