diff --git a/base_folder/src/core/types.ts b/base_folder/src/core/types.ts new file mode 100644 index 0000000..228c5d0 --- /dev/null +++ b/base_folder/src/core/types.ts @@ -0,0 +1,52 @@ +/** + * Type definitions for DevRev function inputs and related types + */ + +export type Context = { + // ID of the dev org for which the function is being invoked. + dev_oid: string; + // ID of the automation/command/snap-kit Action/Event Source for which the function is being invoked. + source_id: string; + // ID of the snap-in as part of which the function is being invoked. + snap_in_id: string; + // ID of the snap-in Version as part of which the function is being invoked. + snap_in_version_id: string; + // ID of the service account. + service_account_id: string; + // This secrets map would contain some secrets which platform would provide to the snap-in. + // `service_account_token`: This is the token of the service account which belongs to this snap-in. This can be used to make API calls to DevRev. + // `actor_session_token`: For commands, and snap-kits, where the user is performing some action, this is the token of the user who is performing the action. + secrets: Record; + }; + + export type ExecutionMetadata = { + // A unique id for the function invocation. Can be used to filter logs for a particular invocation. + request_id: string; + // Function name as defined in the manifest being invoked. + function_name: string; + // Type of event that triggered the function invocation as defined in manifest. + event_type: string; + // DevRev endpoint to which the function can make API calls. + // Example : "https://api.devrev.ai/" + devrev_endpoint: string; + }; + + export type InputData = { + // Map of organization inputs and their corresponding values stored in snap-in. + // The values are passed as string and typing need to be handled by the function + global_values: Record; + // Map of event sources and their corresponding ids stored in snap-in. + // These could be used to schedule events on a schedule based event source. + event_sources: Record; + }; + + // Event sent to our app. + export type FunctionInput = { + // Actual payload of the event. + payload: Record; + // Context of the function invocation. + context: Context; + // Metadata of the function invocation. + execution_metadata: ExecutionMetadata; + input_data: InputData; + }; \ No newline at end of file diff --git a/base_folder/src/core/utils.ts b/base_folder/src/core/utils.ts new file mode 100644 index 0000000..b9a376d --- /dev/null +++ b/base_folder/src/core/utils.ts @@ -0,0 +1,30 @@ +import { AirdropEvent, AirdropMessage } from '@devrev/ts-adaas'; +import { FunctionInput } from './types'; + +export function convertToAirdropEvent(fi: FunctionInput): AirdropEvent { + // Create a properly structured AirdropMessage + const airdropMessage: AirdropMessage = { + connection_data: fi.payload.connection_data, + event_context: { + ...fi.payload.event_context, + ...fi.context, + request_id: fi.execution_metadata?.request_id, + }, + event_type: fi.payload.event_type, + event_data: fi.payload.event_data || {}, + }; + + return { + context: { + ...fi.context, + secrets: { + service_account_token: fi.context.secrets?.service_account_token || '', + ...fi.context.secrets, + }, + }, + payload: airdropMessage, + execution_metadata: fi.execution_metadata, + input_data: fi.input_data, + }; + } + \ No newline at end of file diff --git a/devrev-snapin-template.plain b/devrev-snapin-template.plain index d89b68d..75c3ad9 100644 --- a/devrev-snapin-template.plain +++ b/devrev-snapin-template.plain @@ -4,6 +4,7 @@ - The Function implements part of The Snap-In functionality. - Details on how The Function must be invoked are provided in the resource [Function Invocation](docs/function_invocation.mdx). + - The Function should only take the first event from the array of events and process it. The Function can be invoked multiple times for multiple events. - The Airdrop SDK Library for TypeScript helps developers build snap-ins that integrate with DevRev’s Airdrop platform. - The documentation for the The Airdrop SDK Library is provided in the resource [airdrop_sdk_library_documentation.md](docs/airdrop_sdk_library_documentation.md). @@ -13,6 +14,12 @@ - The Initial Domain Mapping is a JSON object that defines how the logical domain model described by The External Domain Metadata maps to the target domain model in DevRev. It specifies the correspondence between each external record type and DevRev leaf type, as well as the mapping of individual fields, including transformation logic, value conversions, and any fixed or default values. +- The External Sync Unit Item Count is a numeric field representing the total number of items (such as cards) contained within a given external sync unit. It is accessible as "item_count" field of the external sync unit. + +- The Normalization Function is a function that is responsible for normalizing the data from The API to the data expected by The External Domain Metadata JSON object's record type. + +- The Extraction Function is The Function named "extraction". It is responsible for pushing data to the DevRev servers (The DevRev Servers) based on various event types. + - "event_type" equals "EXTRACTION_DATA_START" or "event_type" equals "EXTRACTION_DATA_CONTINUE" and we get a timeout, The Worker Thread should emit event "EXTRACTION_DATA_PROGRESS". ***Non-Functional Requirements:*** @@ -21,8 +28,6 @@ - The Snap-In will run on Node.js as the runtime environment. - The Snap-In should use The Airdrop SDK Library version "1.5.1" for handling data extraction and loading, pushing data, event-driven actions, state management, and artifact handling. - - Please note that 'processTask' function **only** executes task if 'processTask' function is called from a worker thread. Therefore 'processTask' function should be called from a worker thread spawned using 'Spawn' class. - - If the worker is written in TypeScript, it must be referenced with a .ts extension, not .js. - Use snake_case for JSON keys. @@ -32,16 +37,18 @@ - The shared code (including libraries, utilities, adapters, and other common code) used by multiple The Function should be placed in the `src/core` folder. The code in this folder should not depend on any specific The Function implementation. +- During the data extraction phase, The Normalization Function's "data" field for specific record type should map precisely the fields that are present in The External Domain Metadata JSON object for that record type. The Normalization Function should map no additional fields other than those specified in The External Domain Metadata JSON object, and also no object fields from The External Domain Metadata JSON should be omitted in normalization. + ***Test Requirements:*** -- The Conformance Tests of The Snap-In should be implemented in TypeScript. +- The Conformance Tests of The Snap-In should be implemented in TypeScript with target language version es2017. The Conformance Tests should utilize the TypeScript target language version es2017 with configuration settings for module interoperability, strict type checking, declaration file generation, and JSON module support. - The Conformance Tests must be in the top folder. - The Conformance Tests will run on Node.js as the runtime environment. -- The Conformance Tests will be executed using the "npm test" command. +- The Conformance Tests will be executed using the "npm test" command. The command "npm test" should be accessible in the `package.json` file of The Conformance Tests. - The Conformance Tests must complete within 120 seconds (The Conformance Tests Timeout). If The Conformance Tests Timeout elapses, the process executing The Conformance Tests will be terminated. @@ -55,28 +62,36 @@ - The Worker Data Server is part of The DevRev Server and is accessible at http://localhost:8003/external-worker, so it must not be mocked. +- The `jest.setup.js` file is managed externally and cannot be modified within The Conformance Tests. -# The TypeScript Node.js application boilerplate +- All test files from The Conformance Tests must use the `.test.ts` suffix to be discoverable by Jest's default test pattern matching. -***Functional Requirements:*** +- The Conformance Tests should send a single event to the Snap-In Server in the request body without wrapping it in an array. -- Implement The Function that only checks if The Function can be invoked. -- Implement The Function that only checks if the data extraction workflow can be invoked. +# The TypeScript Node.js application boilerplate - ***Acceptance Tests:*** +***Non-Functional Requirements:*** + +- Logic of pushing data to The DevRev Servers must be implemented in the worker thread and invoked with the "spawn" method from The Airdrop SDK Library. - - Test the data extraction workflow invocation using the resource [extraction_health_check.json](test_data/extraction_health_check.json). +- The Function being implemented *MUST not* be called "extraction" or "loading". -- Implement The Function that checks if pushing data works. The URL where data is to be pushed is provided in the "callback_url" field of the "event_context" dictionary which is part of "payload" dictionary. +***Functional Requirements:*** + +- Implement The Function that only checks if The Function can be invoked. -- Implement The Function that provides a test of the 'external sync units' part of the extraction workflow. The name of The Function must be 'extraction_external_sync_unit_check'. +- Implement The Function called "test_external_sync_units" that provides a test of the 'external sync units' part of the extraction workflow as described in the resource [external-sync-units-extraction.mdx](docs/external-sync-units-extraction.mdx). Only check if the extraction can be triggered and that it successfully completes. Disregard all other details. ***Acceptance Tests:*** - Test The Function using the resource [external_sync_unit_check.json](test_data/external_sync_unit_check.json). Test is successful if The Callback Server receives from DevRev an event with "event_type" that equals "EXTRACTION_EXTERNAL_SYNC_UNITS_DONE". The test must not send the event directly to The Callback Server. -- Implement The Function that provides a test of the 'data extraction' part of the extraction workflow as described in the resource [data-extraction.md](docs/data-extraction.md). Only check if data extraction can be triggered and that it successfully completes. Disregard all other details. The name of The Function must be 'data_extraction_check'. +- Implement The Function called "data_extraction_check" (The Data Extraction Check Function) that provides a test of the 'data extraction' part of the extraction workflow as described in the resource [data-extraction.md](docs/data-extraction.md). The Data Extraction Check Function should: + - Initialize a single repo "users" + - normalize the users data using The Normalization Function + - push the normalized users data to The DevRev Servers + - Note: The Data Extraction Check Function should be simplified and should disregard all other details. ***Acceptance Tests:*** diff --git a/docs/airdrop_sdk_library_documentation.md b/docs/airdrop_sdk_library_documentation.md index 776d150..d026bac 100644 --- a/docs/airdrop_sdk_library_documentation.md +++ b/docs/airdrop_sdk_library_documentation.md @@ -363,9 +363,8 @@ Defines the structure of events sent to external extractors from Airdrop platfor Required. An object containing: - _secrets_: An object containing: - - _service_account_token_: Required. A **string** representing the DevRev authentication token for Airdrop platform - - _snap_in_version_id_: Required. A **string** representing the version ID of the snap-in - - _snap_in_id_: Required. A **string** representing the snap-in ID. + - _service_account_token_: A **string** representing the DevRev authentication token for Airdrop platform + - _snap_in_version_id_: A **string** representing the version ID of the snap-in - _payload_ @@ -426,7 +425,7 @@ This function initializes a new worker thread and oversees its lifecycle. It sho #### Usage ```typescript -spawn({ event, initialState, workerPath, options }); +spawn({ event, initialState, workerPath, options, initialDomainMapping }); ``` #### Parameters @@ -443,6 +442,10 @@ spawn({ event, initialState, workerPath, options }); Required. A **string** that represents the path to the worker file. +- _initialDomainMapping_ + + Optional. An **object** containing the initial domain mapping. + - _options_ Optional. An object of type **WorkerAdapterOptions**, which will be passed to the newly created worker. This worker will then initialize a `WorkerAdapter` by invoking the `processTask` function. The options include: @@ -458,10 +461,6 @@ spawn({ event, initialState, workerPath, options }); - `batchSize` A **number** that determines the maximum number of items to be processed and saved to an intermediary file before being sent to the Airdrop platform. The default batch size is 2,000. - - - `initialDomainMapping` - - Optional. A **string** containing the initial domain mapping. #### Return value diff --git a/docs/data-extraction-rules-from-emitting-events.md b/docs/data-extraction-rules-from-emitting-events.md new file mode 100644 index 0000000..c5f00e3 --- /dev/null +++ b/docs/data-extraction-rules-from-emitting-events.md @@ -0,0 +1,8 @@ +Here are the rules when `EXTRACTION_DATA_START` or `EXTRACTION_DATA_CONTINUE` event is received by the snap-in: + +- If there's a timeout ("onTimeout" method in "spawn" method), the snap-in should emit an event with type `EXTRACTION_DATA_PROGRESS` (no "data" payload). +- If the extraction is rate-limited by the external system, the snap-in should emit an event with type `EXTRACTION_DATA_DELAY` ("data" payload should contain attribute "delay" - delay in seconds as an integer). +- If the extraction is successful and all the resources have been extracted, the snap-in should emit an event with type `EXTRACTION_DATA_DONE` (no "data" payload). +- In case of unexpected error, the snap-in should emit an event with type `EXTRACTION_DATA_ERROR` ("data" payload should contain attribute "error" - error message as a string). + +NOTE: In all cases, only a single event should be emitted. \ No newline at end of file diff --git a/docs/data-extraction.md b/docs/data-extraction.md index 6d4308a..10de0e9 100644 --- a/docs/data-extraction.md +++ b/docs/data-extraction.md @@ -112,7 +112,6 @@ Behind the scenes, the SDK library stores items pushed to the repository and upl ### Data normalization Extracted data must be normalized to fit the domain metadata defined in the `external-domain-metadata.json` file. -More details on this process are provided in the [Metadata extraction](/public/snapin-development/adaas/metadata-extraction) section. Normalization rules: @@ -124,6 +123,7 @@ Normalization rules: - References: references must be strings, not numbers or objects. - Number fields must be valid JSON numbers (not strings). - Multiselect fields must be provided as an array (not CSV). +- Rich text: Strings from 3rd party services should be converted to the rich text by splitting the string by newlines, filtering out empty lines, and returning an array of strings. Extracted items are automatically normalized when pushed to the `repo` if a normalization function is provided under the `normalize` key in the repo object. diff --git a/docs/external-sync-units-extraction.mdx b/docs/external-sync-units-extraction.mdx new file mode 100644 index 0000000..26e060b --- /dev/null +++ b/docs/external-sync-units-extraction.mdx @@ -0,0 +1,63 @@ +An _external sync unit_ refers to a single unit in the external system that is being airdropped to DevRev. +In some systems, this is a project; in some it is a repository; in support systems it could be +called a brand or an organization. +What a unit of data is called and what it represents depends on the external system's domain model. +It usually combines contacts, users, work-like items, and comments into a unit of domain objects. + +In the external sync unit extraction phase, the snap-in is expected to obtain a list of external +sync units that it can extract from the external system API and send it to Airdrop in its response. + +External sync unit extraction is executed only during the initial import. + +## Triggering event + +Airdrop starts the external sync unit extraction by sending a message with the event type `EXTRACTION_EXTERNAL_SYNC_UNITS_START`. + +The snap-in must reply to Airdrop with an `EXTRACTION_EXTERNAL_SYNC_UNITS_DONE` message when finished, +or `EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR` if an error occurs. + +## Implementation + + +The snap-in should emit the list of external sync units in the given format: + +```typescript +const externalSyncUnits: ExternalSyncUnit[] = [ + { + id: "devrev", + name: "devrev", + description: "Demo external sync unit", + item_count: 100, + }, +]; +``` + +- `id`: The unique identifier in the external system. +- `name`: The human-readable name in the external system. +- `description`: The short description if the external system provides it. +- `item_count`: The number of items (issues, tickets, comments or others) in the external system. + Item count should be provided if it can be obtained in a lightweight manner, such as by calling an API endpoint. + If there is no such way to get it (for example, if the items would need to be extracted to count them), + then the item count should be `-1` to avoid blocking the import with long-running queries. + +The snap-in must respond to Airdrop with a message, which contains a list of external sync units as a payload: + +```typescript +await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, { + external_sync_units: externalSyncUnits, +}); +``` + +or an error: + +```typescript +await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsError, { + error: { + message: "Failed to extract external sync units. Lambda timeout.", + }, +}); +``` + +**The snap-in must always emit a single message.** + +To test your changes, start a new airdrop in the DevRev App. If external sync units extraction is successful, you should be prompted to choose an external sync unit from the list. diff --git a/docs/function_invocation.mdx b/docs/function_invocation.mdx index 87468a7..991c18e 100644 --- a/docs/function_invocation.mdx +++ b/docs/function_invocation.mdx @@ -1,9 +1,9 @@ -The function is invoked by the runner.ts file. - -runner.ts processes events of type "any", which are basically events of type "AirdropEvent" with some additional fields, and passes the events to the function. You can ignore the additional fields. +A function can be invoked synchronously or asynchronously. You need to implement the run method in your function. The run method is called when the function is invoked. The run method signature is defined below: + ```typescript -async function run(events: []AirdropEvent): void; +async function run(events: []FunctionInput): any; ``` -As of now, it's safe to assume that only one event is passed to the function at a time. The function can be invoked multiple times for multiple events. \ No newline at end of file + +The value returned from the `run` method is passed back in synchronous execution modes, such as commands, snap kit actions, and event source synchronous execution. In asynchronous execution modes, such as automation execution, the return value is ignored. diff --git a/docs/incremental_mode.md b/docs/incremental_mode.md new file mode 100644 index 0000000..cfa0294 --- /dev/null +++ b/docs/incremental_mode.md @@ -0,0 +1,24 @@ +### Implementing Incremental Data Sync + +Incremental data synchronization retrieves only records that have been created or updated since the last successful sync. This process requires persistently tracking the timestamp of the last successful data extraction. On subsequent runs, this timestamp is used to query the source system for changes. + +Incremental mode should only be handled if the "event_type" is `EXTRACTION_DATA_START`. + +To check if we're in incremental mode, you should check if the value of `adapter.event.payload.event_context.mode` is `SyncMode.INCREMENTAL`. + +#### How implement incremental mode + +If we're in incremental mode, you should reset The Extraction State, indicating the sync hasn't been completed yet for all data types that we support incremental mode. + +Value of `adapter.state.lastSuccessfulSyncStarted` (of format ISO 8601 Extended Format with timezone) represents you the information since when you should query resources from the 3rd party API. + +To retrieve only the resources from the API that have to be updated, filtering on The API should be implemented. + +Note: +- `adapter.state.lastSuccessfulSyncStarted` and `adapter.state.lastSyncStarted` are internal properties of the ts-adaas library, so no need to define it. This should be a read-only property. + +#### Remember the last successful sync time + +If the sync is successful, update the state object with the current time. + +Note: No need to modify any time-related properties in adapter.state object. \ No newline at end of file diff --git a/docs/metadata-extraction.mdx b/docs/metadata-extraction.mdx deleted file mode 100644 index 82676a1..0000000 --- a/docs/metadata-extraction.mdx +++ /dev/null @@ -1,88 +0,0 @@ - ### Configure state transitions - - If an external record type has some concept of states, between which only certain transitions are - possible (for example, to move to the `resolved` status, an issue first has to be `in_testing`), you - can declare these in the metadata too. - - This allows creation of a matching *stage diagram* (a collection of stages and their permitted - transitions) in DevRev, which enables a much simpler import and a closer preservation of the external - data than mapping to DevRev's built-in stages. - - To declare this in the metadata, make sure the status is represented as an enum field, and then - declare the allowed transitions (which you might have to retrieve from an API at runtime, if they - are also customized): - - ```json - { - "fields": { - "status": { - "name": "Status", - "is_required": true, - "type": "enum", - "enum": { - "values": [ - { - "key": "detected", - "name": "Detected" - }, - { - "key": "mitigated", - "name": "Mitigated" - }, - { - "key": "rca_ready", - "name": "RCA Ready" - }, - { - "key": "archived", - "name": "Archived" - } - ] - } - } - }, - "stage_diagram": { - "controlling_field": "status", - "starting_stage": "detected", - "all_transitions_allowed": false, - "stages": { - "detected": { - "transitions_to": ["mitigated", "archived", "rca_ready"], - "state": "new" - }, - "mitigated": { - "transitions_to": ["archived", "detected"], - "state": "work_in_progress" - }, - "rca_ready": { - "transitions_to": ["archived"], - "state": "work_in_progress" - }, - "archived": { - "transitions_to": [], - "state": "completed" - } - }, - "states": { - "new": { - "name": "New" - }, - "work_in_progress": { - "name": "Work in Progress" - }, - "completed": { - "name": "Completed", - "is_end_state": true - } - } - } - } - ``` - - In the above example: - - The status field is the controlling field of the stage diagram. - - If a status field has no explicit transitions but you still want a stage diagram, set `all_transitions_allowed` to `true`, which creates a diagram where all the defined stages can transition to each other. - - External systems may categorize statuses (like Jira's status categories), which can be included in the diagram metadata (`states` in the example). - - The `starting_stage` defines the initial stage for new object instances. This data should always be provided if available, otherwise the starting stage is selected alphabetically. - - The order and human-readable name are taken from the enum values defined on the controlling field. - - If the `states` field is not provided, default DevRev states are used: `open`, `in_progress`, and `closed`. \ No newline at end of file diff --git a/mock_devrev_server.py b/mock_devrev_server.py index 45726ea..3cd52e9 100644 --- a/mock_devrev_server.py +++ b/mock_devrev_server.py @@ -26,9 +26,9 @@ class ExternalWorkerResponse(BaseModel): class AirdropArtifactResponse(BaseModel): artifact_id: str upload_url: str - form_data: List[FormDataField] + form_data: Dict[str, str] -@app.post("/upload/{artifact_id}") +@app.post("/upload/{artifact_id:path}") async def upload_artifact( artifact_id: str, request: Request, @@ -125,11 +125,11 @@ async def airdrop_artifacts_upload_url( upload_url = f"http://localhost:8003/upload/{artifact_id}" # Create form data fields that would typically be required for S3 upload - form_data = [ - FormDataField(key="key", value=f"airdrop-artifacts/{artifact_id}/{file_name}"), - FormDataField(key="Content-Type", value=file_type), - FormDataField(key="x-amz-meta-artifact-id", value=artifact_id), - ] + form_data = { + "key": f"airdrop-artifacts/{artifact_id}/{file_name}", + "Content-Type": file_type, + "x-amz-meta-artifact-id": artifact_id, + } return AirdropArtifactResponse( artifact_id=artifact_id, @@ -137,6 +137,17 @@ async def airdrop_artifacts_upload_url( form_data=form_data ) +@app.post("/internal/airdrop.artifacts.confirm-upload") +async def confirm_upload(request: Request): + try: + body = await request.json() + print("Received /internal/airdrop.artifacts.confirm-upload POST body:") + print(json.dumps(body, indent=2)) + except Exception as e: + print("Could not parse JSON from /internal/airdrop.artifacts.confirm-upload request body:", e) + + return {"status": "success"} + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="localhost", port=8003) diff --git a/resources/chef_cli_output_schema.json b/resources/chef_cli_output_schema.json new file mode 100644 index 0000000..36c8247 --- /dev/null +++ b/resources/chef_cli_output_schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "IfUserDecides": { + "type": "object", + "description": "Object containing user decision parameters" + }, + "Outcome": { + "type": "string", + "description": "Description of the operation outcome" + }, + "Warnings": { + "type": ["array", "null", "string"], + "items": {}, + "description": "Array of warning messages, or null if no warnings" + }, + "RemainingDeficiencies": { + "type": ["array", "null"], + "items": {}, + "description": "Array of any type, or null if none" + } + }, + "additionalProperties": false + }, + "description": "Loosened schema for chef CLI output containing operation results" +} \ No newline at end of file diff --git a/run_devrev_snapin_conformance_tests.sh b/run_devrev_snapin_conformance_tests.sh index 8fc23d5..30f8032 100755 --- a/run_devrev_snapin_conformance_tests.sh +++ b/run_devrev_snapin_conformance_tests.sh @@ -5,14 +5,44 @@ NPM_INSTALL_OUTPUT_FILTER="up to date in|added [0-9]* packages, removed [0-9]* p # ANSI escape code pattern to remove color codes and formatting from output ANSI_ESCAPE_PATTERN="s/\x1b\[[0-9;]*[mK]//g" +# Maximum number of characters to display from log files +SNAP_IN_LOG_MAX_CHARS=8000 +DEVREV_SERVER_LOG_MAX_CHARS=4000 + +# Function to print a log file, truncating it if it's too large +print_log_file() { + local file_path="$1" + local max_chars="$2" + if [ ! -f "$file_path" ]; then + printf "Log file not found: %s\n" "$file_path" + return + fi + + local total_chars=$(wc -c < "$file_path") + + if [ "$total_chars" -le "$max_chars" ]; then + cat "$file_path" + else + # Truncate the file, showing 85% from the start and 15% from the end + local start_chars=$((max_chars * 85 / 100)) + local end_chars=$((max_chars * 15 / 100)) + local omitted_chars=$((total_chars - start_chars - end_chars)) + + head -c "$start_chars" "$file_path" + printf "\n\n... [%s characters omitted] ...\n\n" "$omitted_chars" + tail -c "$end_chars" "$file_path" + fi +} + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Get the directory from where the script is being executed EXEC_DIR="$(pwd)" +MOCK_DEVREV_SERVER_LOG="$EXEC_DIR/devrev_server.log" # Source environment variables from .env file - look in execution directory if [ ! -f "$EXEC_DIR/.env" ]; then printf "Error: .env file not found in $EXEC_DIR. Please ensure .env file exists with required environment variables.\n" - exit 1 + exit 69 # EXIT_SERVICE_UNAVAILABLE fi set -a # automatically export all variables @@ -52,8 +82,7 @@ get_children() { # Function to start the mock DevRev server start_mock_devrev_server() { - printf "Starting mock DevRev server...\n" - python3 "$SCRIPT_DIR/mock_devrev_server.py" 2>&1 & + python3 "$SCRIPT_DIR/mock_devrev_server.py" > "$MOCK_DEVREV_SERVER_LOG" 2>&1 & MOCK_SERVER_PID=$! sleep 2 # Give the server time to start printf "\n" @@ -96,6 +125,7 @@ cleanup() { # Remove temporary files if they exist [ -f "$build_output" ] && rm "$build_output" 2>/dev/null + [ -f "$MOCK_DEVREV_SERVER_LOG" ] && rm "$MOCK_DEVREV_SERVER_LOG" 2>/dev/null } # Set up trap to call cleanup function on script exit, interrupt, or termination @@ -120,6 +150,12 @@ if [ -z "$CHEF_CLI_PATH" ] || [ ! -f "$CHEF_CLI_PATH" ] || [ ! -x "$CHEF_CLI_PAT exit 69 # EXIT_SERVICE_UNAVAILABLE fi +if [ -z "$EXTRACTED_FILES_FOLDER_PATH" ]; then + echo "Error: extracted files folder not found at EXTRACTED_FILES_FOLDER_PATH. Please ensure EXTRACTED_FILES_FOLDER_PATH is set." + exit 69 # EXIT_SERVICE_UNAVAILABLE +fi + + # Check if build folder name is provided if [ -z "$1" ]; then printf "Error: No build folder name provided.\n" @@ -272,20 +308,21 @@ if [ $? -ne 0 ]; then exit 2 fi -if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then - printf "Running conformance tests...\n" -fi +printf "\n#### Running conformance tests...\n" npm test -- --runInBand --setupFilesAfterEnv="$SCRIPT_DIR/jest.setup.js" --detectOpenHandles 2>&1 | sed -E "$ANSI_ESCAPE_PATTERN" conformance_tests_result=$? +printf "\n#### Output of the DevRev server log file:\n\n" +print_log_file "$MOCK_DEVREV_SERVER_LOG" "$DEVREV_SERVER_LOG_MAX_CHARS" +printf "\n#### Output of The Snap-In log file:\n" +print_log_file "$NODE_SUBFOLDER/app.log" "$SNAP_IN_LOG_MAX_CHARS" +printf "\n" +printf "\n" + if [ $conformance_tests_result -ne 0 ]; then if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then printf "Error: Conformance tests have failed.\n" fi - printf "\n### Output of the The Snap-In log file:\n" - cat "$current_dir/$NODE_SUBFOLDER/app.log" - printf "\n" - exit 2 fi diff --git a/run_unittests_jest.sh b/run_unittests_jest.sh index 1ba97fd..d6da5f2 100755 --- a/run_unittests_jest.sh +++ b/run_unittests_jest.sh @@ -75,7 +75,7 @@ rm "$build_output" printf "### Step 2: Running unittests in $FOLDER_NAME...\n" -npm test -- --runInBand --setupFilesAfterEnv="$SCRIPT_DIR/jest.setup.js" --detectOpenHandles 2>&1 | sed -E "$ANSI_ESCAPE_PATTERN" +npm test -- --runInBand --setupFilesAfterEnv="$SCRIPT_DIR/jest.setup.js" --detectOpenHandles --testPathIgnorePatterns="dist" 2>&1 | sed -E "$ANSI_ESCAPE_PATTERN" TEST_EXIT_CODE=$? # Check if tests failed diff --git a/templates/chef_cli_normalization_validation.plain b/templates/chef_cli_normalization_validation.plain new file mode 100644 index 0000000..19a8cb8 --- /dev/null +++ b/templates/chef_cli_normalization_validation.plain @@ -0,0 +1,10 @@ + - Then, validate The Normalization Function with the chef-cli tool (The Chef CLI) using the command "chef-cli validate-data -m -r {{ record_type }}": + - To invoke the function that returns The External Domain Metadata JSON object, send the event [external_domain_metadata_event_payload.json](test_data/external_domain_metadata_event_payload.json) to The Snap-In server. + - The External Domain Metadata JSON object should then be stored in temporary file and provided as path to the validation of The Normalization Function. + - The Chef CLI reads the extracted file (The Extracted File) from stdin. + - The Extracted File is inside folder provided in the environment variable "EXTRACTED_FILES_FOLDER_PATH". To retrieve the relevant file name, command `ls EXTRACTED_FILES_FOLDER_PATH | grep extractor_{{ record_type }} | sort -r | head -n 1` should be executed. The test should always fail if The Extracted File is not present. + - Please note that the expected behaviour is that only after the invocation of The Extraction Function, the folder provided in the environment variable "EXTRACTED_FILES_FOLDER_PATH" should exist. The test should always fail if before the execution of The Chef CLI, the folder "EXTRACTED_FILES_FOLDER_PATH" does not exist or if The Extracted File is not found. + - The path to The Chef CLI executable is provided in the environment variable `CHEF_CLI_PATH`. + - For the validation of The Normalization Function to be successful The Chef CLI must return an empty output. The implementation of The Extraction Function directly infuences the output of The Chef CLI. + - The stdout and stderr of The Chef CLI *must* be printed to the console. + - The test should always fail if The Chef CLI is not available. \ No newline at end of file diff --git a/templates/external_domain_metadata_boilerplate.plain b/templates/external_domain_metadata_boilerplate.plain new file mode 100644 index 0000000..d80e336 --- /dev/null +++ b/templates/external_domain_metadata_boilerplate.plain @@ -0,0 +1,26 @@ +***Definitions:*** + +- The structure of The External Domain Metadata JSON object is specified by the JSON schema defined in the resource [external_domain_metadata_schema.json](external_domain_metadata_schema.json). + - Please note that all refers_to keys in reference and typed_reference fields must use the format "#record:", where matches a key in the top-level record_types. This ensures references are unambiguous and valid. Use {} as the value to refer by default identifier (id), or specify { "by_field": "field_name" } to use a custom key. + +- A “display name” is the human-readable label of a field within a given record type. + +***Non-Functional Requirements:*** + +- Store The External Domain Metadata JSON object as a separate JSON file. + +***Test Requirements:*** + +- Validate generated The External Domain Metadata JSON object with the chef-cli tool (The Chef CLI) using the command "chef-cli validate-metadata": + - This command reads JSON object from stdin. + - The path to The Chef CLI executable is provided in the environment variables `CHEF_CLI_PATH`. + - For the validation of The External Domain Metadata JSON object to be successful The Chef CLI must return an empty output. + - The test should always fail if The Chef CLI is not available. + +- The External Domain Metadata JSON object should always be read from The Implementation Code. + +- If The Conformance Tests test The External Domain Metadata JSON object, test it only against The Chef CLI. Don't test it against the literal JSON contents of The External Domain Metadata JSON object. + +- The stdout and stderr of The Chef CLI *must* be printed to the console. + +- The Conformance Tests should not modify nor do any transformations on The External Domain Metadata JSON object at all. If The Conformance Tests indicates a problem with The External Domain Metadata JSON object, The External Domain Metadata JSON object in The Implementation Code should be adjusted to fix the problem. The tests should only validate the JSON object in its original form. \ No newline at end of file diff --git a/templates/initial_domain_mapping_boilerplate.plain b/templates/initial_domain_mapping_boilerplate.plain new file mode 100644 index 0000000..d1e4043 --- /dev/null +++ b/templates/initial_domain_mapping_boilerplate.plain @@ -0,0 +1,45 @@ +***Definitions:*** + +- The structure of The Initial Domain Mapping JSON object is specified by the JSON schema defined in the resource [initial_mappings_schema.yaml](initial_mappings_schema.yaml). + - For a complete list of supported DevRev object types and their fields, see resource [Supported DevRev object types for Airdrop](docs/supported-object-types.md). + - For information about transformation methods, see resource [Mapping Reasons](docs/mapping-reasons.mdx). + - When working with devrev_leaf_type, be aware that the schema expects different formats depending on context. In most places, it should be passed as an object with object_category and object_type. However, in the airdrop-recipe-create-possible-record-type-mapping context, it must be a string representing only the object_type. + - Please note that mappings are split into separate ‘shards’ - one for each record type - for easier manipulation and storage. + - Please note that a leaf type or a concrete id has to be selected for use_devrev_record, but not both. + +- The Stock Field Mapping Field is a configuration object within the Initial Domain Mapping JSON structure that defines how external system data fields are mapped to DevRev's built-in (stock) fields. It is located within the "stock_field_mappings" object inside a blueprint shard configuration. + +- The Fixed Transformation Method is a transformation method that is used to set a fixed value to a field. To apply The Fixed Transformation Method to The Stock Field Mapping Field, set the Metadata extraction transformation method type to "use_fixed_value". If using The Fixed Transformation Method, the value of the field "transformation_method_for_set" inside The Stock Field Mapping Field should be `{"enum": "", "transformation_method": "use_fixed_value", "value": "enum_value"}`, where `` is the fixed value. + +- The External Transformation Method is a transformation method that is used to map an external field to a DevRev stock field. To apply The External Transformation Method to The Stock Field Mapping Field, set the Metadata extraction transformation method type to "use_directly", "use_as_array_value", "use_rich_text", depending on the context. Field "primary_external_field" should be set to the name of the external field to map. + +- The DevRev Record Transformation Method is a transformation method that is used enable use of a fixed reference to something in DevRev. To apply The DevRev Record Transformation Method to The Stock Field Mapping Field, set the Metadata extraction transformation method type to "use_devrev_record". + +- The Shard Mode Type is an option in The Initial Domain Mapping JSON object through which you can specify shard type for the mapping for the given record type. + - You can either create a fresh shard to use for this mapping, assign an existing one. + +- For every record type in The Initial Domain Mapping JSON object, The Shard Mode Type should be set to "create_shard". + +***Non-Functional Requirements:*** + +- Store The Initial Domain Mapping JSON object as a separate JSON file. + +- The External Domain Metadata JSON object should not be modified. If there are discrepancies between The External Domain Metadata JSON object and The Initial Domain Mapping JSON object, assume The External Domain Metadata JSON object is correct and The Initial Domain Mapping JSON object needs to be adjusted. + +***Test Requirements:*** + +- Validate generated The Initial Domain Mapping JSON object with the chef-cli tool (The Chef CLI) using the command "chef-cli initial-mapping check -m ": + - The Chef CLI reads The Initial Domain Mapping JSON object from stdin. + - For return type of The Chef CLI to stdout, see resource [chef_cli_output_schema.json](resources/chef_cli_output_schema.json). Fields "RemainingDeficiencies" and "Warnings" contains information about the deficiencies that need to be resolved for the validation to pass. + - The path to The Chef CLI executable is provided in the environment variables `CHEF_CLI_PATH`. + - The test should always fail if we get an empty output. + - The test must fail unless both of the fields "RemainingDeficiencies" and "Warnings" of the first object in the Chef CLI output array are present and both of their values are exactly null. + - The test should always fail if The Chef CLI is not available. + +- The Initial Domain Mapping JSON object should always be read from The Implementation Code. + +- The stdout and stderr of The Chef CLI *must* be printed to the console. + +- If The Conformance Tests test The Initial Domain Mapping JSON object, test it only against The Chef CLI. Don't test it against the literal JSON contents of The Initial Domain Mapping JSON object. + +- The Conformance Tests should not modify nor do any transformations on The Initial Domain Mapping JSON object at all. If The Conformance Tests indicates a problem with The Initial Domain Mapping JSON object, The Initial Domain Mapping JSON object in The Implementation Code should be adjusted to fix the problem. The tests should only validate the JSON object in its original form. diff --git a/templates/spawn_method_instructions.plain b/templates/spawn_method_instructions.plain new file mode 100644 index 0000000..37fa47c --- /dev/null +++ b/templates/spawn_method_instructions.plain @@ -0,0 +1,8 @@ +- All operations accessing external services or data providers must be implemented in worker thread (The Worker Thread). The Worker Thread should be initialized with the method "spawn" (The Spawn Method). The Spawn Method should be initialized with the following parameters: + - The Extraction Function should start The Worker Thread with timeout of 10 minutes. + - "isLocalDevelopment" *must not* be set. + - "initialState" should be The Extraction State Object with all "completed" fields set to false, and all the rest of the fields in The Extraction State Object set to undefined, false, 0, or empty string, depending on the type of the field. + - Use The Initial Domain Mapping JSON object for initialDomainMapping parameter when spawning a new worker. + - Note: The Initial Domain Mapping JSON object should be read directly from the JSON file. + - Note: No additional fields should be added to The Extraction State Object. + - Note: The Worker Thread must be implemented in TypeScript and it should be referenced with a .ts extension, not .js. diff --git a/test_data/extraction_health_check.json b/test_data/extraction_health_check.json deleted file mode 100644 index d63226e..0000000 --- a/test_data/extraction_health_check.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "execution_metadata": { - "function_name": "extraction_health_check", - "devrev_endpoint": "http://localhost:8003" - }, - "payload" : { - "event_type": "EXTRACTION_EXTERNAL_SYNC_UNITS_START", - "event_context": { - "callback_url": "http://localhost:8002/callback", - "dev_org": "test-dev-org", - "external_sync_unit_id": "test-external-sync-unit", - "sync_unit_id": "test-sync-unit", - "worker_data_url": "http://localhost:8003/external-worker" - }, - "connection_data": { - "org_id": "test-org-id", - "key": "key=test-key&token=test-token" - } - }, - "context": { - "secrets": { - "service_account_token": "test-token" - } - } -} \ No newline at end of file