feat streaming & custom code validator support (#308)

Author: chavda-bhavik

apps/api/package.json

@@ -33,7 +33,8 @@
     "ajv": "^8.11.0",
     "ajv-formats": "^2.1.1",
     "ajv-keywords": "^5.1.0",
-    "axios": "^0.26.1",
+    "async-mutex": "^0.4.0",
+    "axios": "1.4.0",
     "bcryptjs": "^2.4.3",
     "body-parser": "^1.20.0",
     "class-transformer": "^0.5.1",
@@ -47,6 +48,7 @@
     "hat": "^0.0.3",
     "jsonwebtoken": "^9.0.0",
     "nodemailer": "^6.9.3",
+    "papaparse": "^5.4.1",
     "passport": "^0.6.0",
     "passport-github2": "^0.1.12",
     "passport-jwt": "^4.0.1",
@@ -69,6 +71,7 @@
     "@types/multer": "^1.4.7",
     "@types/node": "^18.7.18",
     "@types/nodemailer": "^6.4.8",
+    "@types/papaparse": "^5.3.7",
     "@types/passport": "^1.0.11",
     "@types/passport-github2": "^1.2.5",
     "@types/passport-jwt": "^3.0.8",
@@ -86,5 +89,8 @@
     "*.{js,jsx,ts,tsx}": [
       "eslint --fix"
     ]
-  }
+  },
+  "main": ".eslintrc.js",
+  "keywords": [],
+  "description": ""
 }

apps/api/src/app/column/dtos/column-request.dto.ts

@@ -1,6 +1,5 @@
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 import {
-  ArrayMinSize,
   IsArray,
   IsBoolean,
   IsOptional,
@@ -12,7 +11,7 @@ import {
   Validate,
 } from 'class-validator';
 import { Type } from 'class-transformer';
-import { ColumnTypesEnum, Defaults } from '@impler/shared';
+import { ColumnTypesEnum } from '@impler/shared';
 import { IsValidRegex } from '@shared/framework/is-valid-regex.validator';
 
 export class ColumnRequestDto {
@@ -81,10 +80,8 @@ export class ColumnRequestDto {
     description: 'List of possible values for column if type is Select',
   })
   @ValidateIf((object) => object.type === ColumnTypesEnum.SELECT)
-  @IsArray()
-  @ArrayMinSize(Defaults.ONE)
   @Type(() => Array<string>)
-  selectValues: string[];
+  selectValues: string[] = [];
 
   @ApiProperty({
     description: 'Sequence of column',

apps/api/src/app/mapping/mapping.controller.ts

@@ -4,19 +4,21 @@ import { ACCESS_KEY_NAME, Defaults, UploadStatusEnum } from '@impler/shared';
 import { MappingEntity } from '@impler/dal';
 
 import { JwtAuthGuard } from '@shared/framework/auth.gaurd';
+import { validateNotFound } from '@shared/helpers/common.helper';
+import { validateUploadStatus } from '@shared/helpers/upload.helpers';
+import { GetUpload } from '@shared/usecases/get-upload/get-upload.usecase';
 import { ValidateMongoId } from '@shared/validations/valid-mongo-id.validation';
 import { GetUploadCommand } from '@shared/usecases/get-upload/get-upload.command';
+
+import { UpdateMappingDto } from './dtos/update-columns.dto';
 import { DoMapping } from './usecases/do-mapping/do-mapping.usecase';
-import { DoMappingCommand } from './usecases/do-mapping/do-mapping.command';
-import { GetUpload } from '@shared/usecases/get-upload/get-upload.usecase';
 import { GetMappings } from './usecases/get-mappings/get-mappings.usecase';
-import { UpdateMappingCommand } from './usecases/update-mappings/update-mappings.command';
+import { DoMappingCommand } from './usecases/do-mapping/do-mapping.command';
 import { UpdateMappings } from './usecases/update-mappings/update-mappings.usecase';
 import { FinalizeUpload } from './usecases/finalize-upload/finalize-upload.usecase';
-import { UpdateMappingDto } from './dtos/update-columns.dto';
 import { ValidateMapping } from './usecases/validate-mapping/validate-mapping.usecase';
-import { validateUploadStatus } from '@shared/helpers/upload.helpers';
-import { validateNotFound } from '@shared/helpers/common.helper';
+import { UpdateMappingCommand } from './usecases/update-mappings/update-mappings.command';
+import { ReanameFileHeadings } from './usecases/rename-file-headings/rename-file-headings.usecase';
 
 @Controller('/mapping')
 @ApiTags('Mappings')
@@ -29,7 +31,8 @@ export class MappingController {
     private getMappings: GetMappings,
     private updateMappings: UpdateMappings,
     private finalizeUpload: FinalizeUpload,
-    private validateMapping: ValidateMapping
+    private validateMapping: ValidateMapping,
+    private renameFileHeadings: ReanameFileHeadings
   ) {}
 
   @Get(':uploadId')
@@ -93,7 +96,7 @@ export class MappingController {
 
     // save mapping
     if (Array.isArray(body) && body.length > Defaults.ZERO) {
-      this.updateMappings.execute(
+      await this.updateMappings.execute(
         body
           .filter((columnDataItem) => !!columnDataItem.columnHeading)
           .map((updateColumnData) =>
@@ -107,7 +110,9 @@ export class MappingController {
       );
     }
 
+    const { headings } = await this.renameFileHeadings.execute(_uploadId);
+
     // update mapping status
-    return this.finalizeUpload.execute(_uploadId);
+    return this.finalizeUpload.execute(_uploadId, headings);
   }
 }

apps/api/src/app/mapping/usecases/finalize-upload/finalize-upload.usecase.ts

@@ -6,7 +6,10 @@ import { UploadStatusEnum } from '@impler/shared';
 export class FinalizeUpload {
   constructor(private uploadRepository: UploadRepository) {}
 
-  async execute(_uploadId: string) {
-    return await this.uploadRepository.findOneAndUpdate({ _id: _uploadId }, { status: UploadStatusEnum.MAPPED });
+  async execute(_uploadId: string, headings: string[]) {
+    return await this.uploadRepository.findOneAndUpdate(
+      { _id: _uploadId },
+      { status: UploadStatusEnum.MAPPED, headings }
+    );
   }
 }

apps/api/src/app/mapping/usecases/index.ts

@@ -3,6 +3,7 @@ import { GetMappings } from './get-mappings/get-mappings.usecase';
 import { UpdateMappings } from './update-mappings/update-mappings.usecase';
 import { FinalizeUpload } from './finalize-upload/finalize-upload.usecase';
 import { ValidateMapping } from './validate-mapping/validate-mapping.usecase';
+import { ReanameFileHeadings } from './rename-file-headings/rename-file-headings.usecase';
 import { GetUpload } from '@shared/usecases/get-upload/get-upload.usecase';
 
 export const USE_CASES = [
@@ -11,6 +12,7 @@ export const USE_CASES = [
   UpdateMappings,
   FinalizeUpload,
   ValidateMapping,
+  ReanameFileHeadings,
   GetUpload,
   //
 ];

apps/api/src/app/mapping/usecases/rename-file-headings/rename-file-headings.usecase.ts

@@ -0,0 +1,28 @@
+import { Injectable } from '@nestjs/common';
+import { MappingRepository, UploadRepository } from '@impler/dal';
+
+@Injectable()
+export class ReanameFileHeadings {
+  constructor(private uploadRepository: UploadRepository, private mappingRepository: MappingRepository) {}
+
+  async execute(_uploadId: string): Promise<{ headings: string[] }> {
+    return new Promise(async (resolve, reject) => {
+      try {
+        const uploadInfo = await this.uploadRepository.findById(_uploadId, 'headings _uploadedFileId');
+        const mappingInfo = await this.mappingRepository.getMappingWithColumnInfo(_uploadId);
+
+        const newHeadings = uploadInfo.headings.reduce((headingsArr, heading) => {
+          const foundMapping = mappingInfo.find((mapping) => mapping.columnHeading === heading);
+          if (foundMapping) headingsArr.push(foundMapping.column.key);
+          else headingsArr.push(heading);
+
+          return headingsArr;
+        }, []);
+
+        return resolve({ headings: newHeadings });
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+}

apps/api/src/app/review/review.controller.ts

@@ -1,5 +1,6 @@
+import { Response } from 'express';
 import { ApiOperation, ApiTags, ApiSecurity, ApiQuery, ApiOkResponse } from '@nestjs/swagger';
-import { BadRequestException, Body, Controller, Get, Param, Post, Query, UseGuards } from '@nestjs/common';
+import { BadRequestException, Body, Controller, Get, Param, Post, Query, Res, UseGuards } from '@nestjs/common';
 
 import { APIMessages } from '@shared/constants';
 import { FileEntity, UploadEntity } from '@impler/dal';
@@ -11,10 +12,8 @@ import {
   DoReview,
   GetUpload,
   StartProcess,
-  SaveReviewData,
   UpdateImportCount,
   GetFileInvalidData,
-  ReanameFileHeadings,
   StartProcessCommand,
   GetUploadInvalidData,
   UpdateImportCountCommand,
@@ -29,14 +28,11 @@ import { PaginationResponseDto } from '@shared/dtos/pagination-response.dto';
 @Controller('/review')
 @ApiTags('Review')
 @ApiSecurity(ACCESS_KEY_NAME)
-@UseGuards(JwtAuthGuard)
 export class ReviewController {
   constructor(
     private doReview: DoReview,
     private getUpload: GetUpload,
     private startProcess: StartProcess,
-    private renameFileHeadings: ReanameFileHeadings,
-    private saveReviewData: SaveReviewData,
     private updateImportCount: UpdateImportCount,
     private getFileInvalidData: GetFileInvalidData,
     private getUploadInvalidData: GetUploadInvalidData
@@ -63,36 +59,39 @@ export class ReviewController {
     type: PaginationResponseDto,
   })
   async getReview(
+    @Res() res: Response,
     @Param('uploadId') _uploadId: string,
     @Query('page') page = Defaults.ONE,
     @Query('limit') limit = Defaults.PAGE_LIMIT
-  ): Promise<PaginationResponseDto> {
+  ) {
     const uploadData = await this.getUploadInvalidData.execute(_uploadId);
     if (!uploadData) throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND);
 
-    // Only Mapped & Reviewing status are allowed
-    validateUploadStatus(uploadData.status as UploadStatusEnum, [UploadStatusEnum.MAPPED, UploadStatusEnum.REVIEWING]);
+    res.setHeader('Content-Type', 'text/event-stream');
+    res.setHeader('Access-Control-Allow-Origin', '*');
+    res.setHeader('Access-Control-Allow-Credentials', 'true');
+    res.setHeader('Connection', 'keep-alive');
+    res.setHeader('Cache-Control', 'no-cache');
 
-    // Get Invalid Data either from Validation or Validation Result
-    let invalidData = [];
+    let invalidDataFilePath: string;
     if (uploadData.status === UploadStatusEnum.MAPPED) {
-      // uploaded file is mapped, do review
-      const reviewData = await this.doReview.execute(_uploadId);
-      // save invalid data to storage
-      await this.saveReviewData.execute(_uploadId, reviewData.invalid, reviewData.valid);
-
-      invalidData = reviewData.invalid;
+      invalidDataFilePath = await this.doReview.execute(_uploadId);
     } else {
-      // Uploaded file is already reviewed, return reviewed data
-      invalidData = await this.getFileInvalidData.execute(
-        (uploadData._invalidDataFileId as unknown as FileEntity).path
-      );
+      invalidDataFilePath = (uploadData._invalidDataFileId as unknown as FileEntity).path;
     }
 
-    return paginateRecords(invalidData, page, limit);
+    // Uploaded file is already reviewed, return reviewed data
+    const invalidData = await this.getFileInvalidData.execute(invalidDataFilePath);
+    const { data, ...rest } = paginateRecords(invalidData, page, limit);
+    for (const item of data) {
+      res.write(`data: ${JSON.stringify(item)}\n\n`);
+    }
+    res.write(`data: ${JSON.stringify(rest)}\n\n`);
+    res.end();
   }
 
   @Post(':uploadId/confirm')
+  @UseGuards(JwtAuthGuard)
   @ApiOperation({
     summary: 'Confirm review data for uploaded file',
   })
@@ -113,13 +112,6 @@ export class ReviewController {
     // upload files with status reviewing can only be confirmed
     validateUploadStatus(uploadInformation.status as UploadStatusEnum, [UploadStatusEnum.REVIEWING]);
 
-    // rename file headings
-    await this.renameFileHeadings.execute(
-      _uploadId,
-      uploadInformation._validDataFileId,
-      uploadInformation._invalidDataFileId
-    );
-
     await this.updateImportCount.execute(
       uploadInformation._templateId,
       UpdateImportCountCommand.create({

apps/api/src/app/review/review.module.ts

@@ -3,11 +3,12 @@ import { USE_CASES } from './usecases';
 import { ReviewController } from './review.controller';
 import { SharedModule } from '@shared/shared.module';
 import { AJVService } from './service/AJV.service';
+import { Sandbox, SManager } from './service/Sandbox';
 import { QueueService } from '@shared/services/queue.service';
 
 @Module({
   imports: [SharedModule],
-  providers: [...USE_CASES, AJVService, QueueService],
+  providers: [...USE_CASES, AJVService, QueueService, SManager, Sandbox],
   controllers: [ReviewController],
 })
 export class ReviewModule {}