Skip to content

chore: refactor ingestion lambda to use loader function from @jsii/spec #920

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jul 20, 2022
Merged

chore: refactor ingestion lambda to use loader function from @jsii/spec #920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
214f4b8
replace .jsii with SPEC_FILE_NAME
kaizencc Jun 28, 2022
fc69956
example usage of loadAssemblyFromBuffer
kaizencc Jun 30, 2022
a243309
update loadAssemblyFromBuffer
kaizencc Jul 1, 2022
e346d56
Merge branch 'main' into conroy/utils
kaizencc Jul 14, 2022
8c6aaa4
update loadassemblyfrombuffer function
kaizencc Jul 14, 2022
07787fa
merge
kaizencc Jul 18, 2022
dc23200
chore: self mutation
invalid-email-address Jul 18, 2022
bb578eb
Merge branch 'main' into conroy/utils
mergify[bot] Jul 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/__tests__/__snapshots__/construct-hub.test.ts.snap
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 12 additions & 11 deletions src/__tests__/backend/ingestion/ingestion.lambda.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
SchemaVersion,
Stability,
TypeKind,
SPEC_FILE_NAME,
} from '@jsii/spec';
import type { metricScope, MetricsLogger } from 'aws-embedded-metrics';
import { Context, SQSEvent } from 'aws-lambda';
Expand Down Expand Up @@ -147,7 +148,7 @@ test('basic happy case', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
name: packageName,
Expand Down Expand Up @@ -374,7 +375,7 @@ test('basic happy case with license file', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/LICENSE.md': fakeLicense,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
Expand Down Expand Up @@ -617,7 +618,7 @@ test('basic happy case with custom package links', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
name: packageName,
Expand Down Expand Up @@ -887,7 +888,7 @@ test('basic happy case with custom tags', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
name: packageName,
Expand Down Expand Up @@ -1123,7 +1124,7 @@ for (const [frameworkName, frameworkPackage] of [
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
name: packageName,
Expand Down Expand Up @@ -1342,7 +1343,7 @@ for (const [frameworkName, frameworkPackage] of [
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
name: frameworkPackage,
Expand Down Expand Up @@ -1569,7 +1570,7 @@ for (const [frameworkName, frameworkPackage] of [
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
name: packageName,
Expand Down Expand Up @@ -1790,7 +1791,7 @@ test('mismatched package name', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/LICENSE.md': fakeLicense,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
Expand Down Expand Up @@ -1928,7 +1929,7 @@ test('mismatched package version', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/LICENSE.md': fakeLicense,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
Expand Down Expand Up @@ -2066,7 +2067,7 @@ test('mismatched package license', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/LICENSE.md': fakeLicense,
'package/index.js': '// Ignore me!',
'package/package.json': JSON.stringify({
Expand Down Expand Up @@ -2332,7 +2333,7 @@ test('missing package.json file', async () => {
mockExtract.mockImplementation(
() =>
new FakeExtract(fakeTar, {
'package/.jsii': fakeDotJsii,
[`package/${SPEC_FILE_NAME}`]: fakeDotJsii,
'package/LICENSE.md': fakeLicense,
'package/index.js': '// Ignore me!',
}) as any
Expand Down
2 changes: 1 addition & 1 deletion src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions src/__tests__/package-sources/__snapshots__/code-artifact.test.ts.snap
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 13 additions & 4 deletions src/__tests__/package-sources/codeartifact/code-artifact-forwarder.lambda.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { pseudoRandomBytes } from 'crypto';
import { SPEC_FILE_NAME } from '@jsii/spec';
import type { Context } from 'aws-lambda';
import * as AWS from 'aws-sdk';
import * as AWSMock from 'aws-sdk-mock';
Expand Down Expand Up @@ -142,7 +143,9 @@ test('happy path', async () => {
expect(tgz).toEqual(
Buffer.from(mockGetPackageVersionAssetResult.asset! as any)
);
expect(selector).toHaveProperty('assemblyJson', { path: 'package/.jsii' });
expect(selector).toHaveProperty('assemblyJson', {
path: `package/${SPEC_FILE_NAME}`,
});
expect(selector).toHaveProperty('packageJson', {
path: 'package/package.json',
required: true,
Expand Down Expand Up @@ -295,7 +298,9 @@ test('no license (i.e: UNLICENSED)', async () => {
expect(tgz).toEqual(
Buffer.from(mockGetPackageVersionAssetResult.asset! as any)
);
expect(selector).toHaveProperty('assemblyJson', { path: 'package/.jsii' });
expect(selector).toHaveProperty('assemblyJson', {
path: `package/${SPEC_FILE_NAME}`,
});
expect(selector).toHaveProperty('packageJson', {
path: 'package/package.json',
required: true,
Expand Down Expand Up @@ -370,7 +375,9 @@ test('ineligible license', async () => {
expect(tgz).toEqual(
Buffer.from(mockGetPackageVersionAssetResult.asset! as any)
);
expect(selector).toHaveProperty('assemblyJson', { path: 'package/.jsii' });
expect(selector).toHaveProperty('assemblyJson', {
path: `package/${SPEC_FILE_NAME}`,
});
expect(selector).toHaveProperty('packageJson', {
path: 'package/package.json',
required: true,
Expand Down Expand Up @@ -444,7 +451,9 @@ test('not a jsii package', async () => {
expect(tgz).toEqual(
Buffer.from(mockGetPackageVersionAssetResult.asset! as any)
);
expect(selector).toHaveProperty('assemblyJson', { path: 'package/.jsii' });
expect(selector).toHaveProperty('assemblyJson', {
path: `package/${SPEC_FILE_NAME}`,
});
expect(selector).toHaveProperty('packageJson', {
path: 'package/package.json',
required: true,
Expand Down
2 changes: 1 addition & 1 deletion src/backend/ingestion/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ export class Ingestion extends Construct implements IGrantable {
/**
* This metrics is the total count of packages that were rejected due to
* mismatched identity (name, version, license) between the `package.json`
* file and te `.jsii` attribute.
* file and the `.jsii` attribute.
*/
public metricMismatchedIdentityRejections(opts?: MetricOptions): Metric {
return new Metric({
Expand Down
34 changes: 25 additions & 9 deletions src/backend/ingestion/ingestion.lambda.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@ import { createHash } from 'crypto';
import { basename, extname } from 'path';
import { URL } from 'url';

import { Assembly, validateAssembly } from '@jsii/spec';
import {
SPEC_FILE_NAME,
Assembly,
loadAssemblyFromBuffer,
SPEC_FILE_NAME_COMPRESSED,
} from '@jsii/spec';
import { metricScope, Configuration, Unit } from 'aws-embedded-metrics';
import type { Context, SQSEvent } from 'aws-lambda';
import { CacheStrategy } from '../../caching';
Expand Down Expand Up @@ -87,18 +92,21 @@ export const handler = metricScope(
);
}

const dotJsiiFile = `package/${SPEC_FILE_NAME}`;
const compDotJsiiFile = `package/${SPEC_FILE_NAME_COMPRESSED}`;

let dotJsii: Buffer;
let compDotJsii: Buffer | undefined;
let packageJson: Buffer;
let licenseText: Buffer | undefined;
try {
({ dotJsii, packageJson, licenseText } = await extractObjects(
Buffer.from(tarball.Body! as any),
{
dotJsii: { path: 'package/.jsii', required: true },
({ dotJsii, compDotJsii, packageJson, licenseText } =
await extractObjects(Buffer.from(tarball.Body! as any), {
dotJsii: { path: dotJsiiFile, required: true },
compDotJsii: { path: compDotJsiiFile },
packageJson: { path: 'package/package.json', required: true },
licenseText: { filter: isLicenseFile },
}
));
}));
} catch (err) {
console.error(`Invalid tarball content: ${err}`);
metrics.putMetric(MetricName.INVALID_TARBALL, 1, Unit.Count);
Expand All @@ -112,8 +120,16 @@ export const handler = metricScope(
let packageVersion: string;
let packageReadme: string;
try {
parsedAssembly = validateAssembly(
JSON.parse(dotJsii.toString('utf-8'))
parsedAssembly = loadAssemblyFromBuffer(
dotJsii,
compDotJsii
? (filename: string) => {
if (filename !== compDotJsiiFile) {
throw new Error('');
}
return compDotJsii!;
}
: undefined
);

// needs `dependencyClosure`
Expand Down
5 changes: 3 additions & 2 deletions src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { SPEC_FILE_NAME } from '@jsii/spec';
import { metricScope, Unit } from 'aws-embedded-metrics';
import type { Context, EventBridgeEvent } from 'aws-lambda';

Expand Down Expand Up @@ -75,7 +76,7 @@ export const handler = metricScope(
const tarball = Buffer.from(asset! as any);

const { assemblyJson, packageJson } = await extractObjects(tarball, {
assemblyJson: { path: 'package/.jsii' },
assemblyJson: { path: `package/${SPEC_FILE_NAME}` },
packageJson: { path: 'package/package.json', required: true },
});
metrics.putMetric(
Expand All @@ -85,7 +86,7 @@ export const handler = metricScope(
);
if (assemblyJson == null) {
console.log(
`Package "${packageName}@${event.detail.packageVersion}" does not contain a .jsii assembly`
`Package "${packageName}@${event.detail.packageVersion}" does not contain a ${SPEC_FILE_NAME} assembly`
);
return;
}
Expand Down