Merge pull request diffblue#573 from diffblue/cleanup/gitlab-wrapper

Cleanup the gitlab wrapper [SEC-626]

Author: NathanJPhillips

gitlab-integration/.gitignore

@@ -1,4 +1,3 @@
-*.js
-*.js.map
+built
+dist
 node_modules
-

gitlab-integration/analyse-project.ts

@@ -1,9 +1,3 @@
 #!/bin/bash
 
-WORKDIR=$(readlink -f $(pwd))
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-cd $DIR
-
-node analyse-project.js $CI_REPOSITORY_URL $CI_COMMIT_REF_NAME $WORKDIR/gl-sast-report.json
-exit 0
+npm start --prefix "$( dirname "${BASH_SOURCE[0]}" )"

gitlab-integration/gitlab-wrapper.sh

@@ -12,14 +12,19 @@
     "John Bergqvist <[email protected]>",
     "Nathan Phillips <[email protected]>"
   ],
-  "main": "analyse-project.js",
+  "main": "./built/analyse-project.js",
+  "bin": {
+    "gitlab-integration": "./built/analyse-project.js"
+  },
   "scripts": {
-    "build": "tsc"
+    "build": "tsc",
+    "lint": "tslint src/**/*.ts",
+    "fix-lint": "tslint src/**/*.ts --fix",
+    "gen-docs": "typedoc --out doc",
+    "start": "node built/analyse-project.js"
   },
   "dependencies": {
-    "@types/moment-duration-format": "^2.2.1",
     "blow-http-statuses": "0.2.4",
-    "express": "4.16.3",
     "form-data": "2.3.2",
     "fs-extra": "6.0.1",
     "js-yaml": "3.12.0",
@@ -28,19 +33,20 @@
     "moment-duration-format": "2.2.2",
     "node-fetch": "2.1.2",
     "node-ssh": "5.1.2",
-    "util.promisify": "1.0.0",
     "winston": "2.4.3",
     "winston-papertrail": "1.0.5"
   },
   "devDependencies": {
-    "@types/es6-shim": "0.31.37",
-    "@types/express": "4.16.0",
     "@types/form-data": "2.2.1",
     "@types/fs-extra": "5.0.2",
     "@types/lodash": "4.14.110",
+    "@types/moment-duration-format": "2.2.2",
     "@types/node": "8.0.47",
-    "@types/node-fetch": "2.1.1",
+    "@types/node-fetch": "2.1.2",
     "@types/winston": "2.3.9",
+    "tslint": "^5.11.0",
+    "tslint-language-service": "^0.9.9",
+    "typedoc": "^0.12.0",
     "typescript": "2.9.2"
   }
 }

gitlab-integration/package.json

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+
+import * as fs from "fs-extra";
+import * as path from "path";
+import * as platform from "./api";
+import { TwoColumnLogger } from "./logger";
+import { wait } from "./misc";
+import * as models from "./models";
+
+async function securityAnalysis() {
+  if (process.env.CI_REPOSITORY_URL === undefined || process.env.CI_COMMIT_REF_NAME === undefined) {
+    console.error("This utility must be run inside a Gitlab runner");
+    process.exit(1);
+    return;
+  }
+  if (process.env.INIT_CWD === undefined) {
+    console.error("This utility must be run inside npm from the folder where the output file should be written");
+    process.exit(1);
+    return;
+  }
+
+  const log = new TwoColumnLogger("Diffblue Gitlab Integration");
+  try {
+    const projectName = `gitlab-${Date.now()}`;
+    let localMachine: models.Instance = {
+      name: "local-executor",
+      ip: "localhost",
+      log: log,
+      project: {
+        name: projectName,
+        fullName: "not-needed",
+        uri: process.env.CI_REPOSITORY_URL,
+        branch: process.env.CI_COMMIT_REF_NAME,
+        manualVerify: { verifyBranch: "x", innerDir: "y", jacocoFile: "z" },
+      },
+      status: "USER_CREATED",
+      type: "notype",
+      zones: [],
+      zone: "nozone",
+      zoneIndex: 0,
+      diskImg: { name: "nodiskimgname", link: "nodiskimglink" },
+      username: "diffblue",
+      duration: { start: "nodurationstart" },
+    };
+
+    const startTime = Date.now();
+
+    localMachine = await platform.createProject(localMachine);
+    localMachine = await platform.initialiseProject(localMachine);
+    localMachine = await platform.startAnalysis(localMachine);
+
+    do {
+      localMachine = await platform.getAnalysisProgress(localMachine);
+      if (localMachine.analysisStatus === "RUNNING" || localMachine.analysisStatus === "WAITING") {
+        const elapsed = (Date.now() - startTime) / 1000;
+        const nextWait = Math.ceil(elapsed / 10);   // Wait for 10% of time elapsed so far
+        log.debug(`Waiting ${nextWait} seconds before checking again whether Diffblue platform is ready`);
+        await wait(nextWait);
+      }
+    } while (localMachine.analysisStatus === "RUNNING" || localMachine.analysisStatus === "WAITING");
+
+    const issues = await platform.getIssues(localMachine);
+
+    log.debug(`Identified ${issues.length} issues`);
+    log.debug(JSON.stringify(issues));
+
+    const report = issues.map(
+      function (issue: any) {
+
+        // TODO: query the platform's public-facing URL
+        const testUrl = `http://localhost/${projectName}/${localMachine.analysisNum}/tests/${issue.id}/trace`;
+
+        return {
+          file: `${issue.testClassDir}/${issue.testClass}`,
+          message: issue.testName,
+          priority: "High",
+          tool: "Diffblue-security",
+          url: testUrl,
+          cve: `Click here ${testUrl}`,
+          // cve: testUrl // Mandatory key
+        };
+      }
+    );
+
+    const reportFileName = path.join(process.env.INIT_CWD, "gl-sast-report.json");
+    log.debug(`Writing issue report to ${reportFileName}`);
+    await fs.writeFile(reportFileName, JSON.stringify(report));
+
+    process.exit(issues.length === 0 ? 0 : 2);
+  } catch (err) {
+    if (err.message) {
+      log.debug(err.message);
+      console.log(err.message);
+    } else {
+      log.debug(err);
+      console.dir(err);
+    }
+    process.exit(3);
+  }
+}
+
+securityAnalysis();

gitlab-integration/src/analyse-project.ts

@@ -3,13 +3,11 @@ import * as FormData from "form-data";
 import * as fs from "fs-extra";
 import * as moment from "moment";
 import fetch, { Response } from "node-fetch";
-import * as models from "./models";
 import { hasExceeded, wait } from "./misc";
+import * as models from "./models";
 
 const nodeSSH = require("node-ssh");
 const yaml = require("js-yaml");
-const util = require("util");
-require("util.promisify").shim();
 
 export async function tweak(vm: models.Instance): Promise<models.Instance> {
   vm.ssh = new nodeSSH();
@@ -68,7 +66,7 @@ async function updateYMLFile(vm: models.Instance): Promise<void> {
     vm.log.warn(`Unable to retrieve /opt/diffblue/docker-compose.yml: ${error}`);
     throw error;
   }
-  const dockerCompose = yaml.safeLoad(await util.promisify(fs.readFile)(`./${vm.name}.yml`, "utf8"));
+  const dockerCompose = yaml.safeLoad(await fs.readFile(`./${vm.name}.yml`, "utf8"));
   dockerCompose.services["application-server"].environment.DEBUG = "true";
   // dockerCompose.services["job-runner"].environment.DEBUG = "true";
   // dockerCompose.services["job-runner"].environment.LOGLEVEL = "debug";
@@ -163,7 +161,7 @@ async function signIn(vm: models.Instance, secsToWait = 30, secsWaiting = 0): Pr
   let response;
   try {
     response = await fetch(`http://${vm.ip}/api/session`, {
-      method: "POST", body: JSON.stringify({ userName: vm.username, password: "password" }), headers: { "Content-Type": "application/json" },
+      method: "POST", body: JSON.stringify({ userName: vm.username, password: "password123" }), headers: { "Content-Type": "application/json" },
     });
     if (response.status !== HttpStatus.OK && response.status !== HttpStatus.Unauthorized)
       throw new Error(`received ${response.status} (${response.statusText}) response`);
@@ -522,7 +520,7 @@ export async function getIssues(vm: models.Instance, secsToWait = 600, secsWaiti
   let response;
   try {
     response = await fetch(`http://${vm.ip}/api/users/${vm.username}/projects/${vm.project.name}/analyses/${vm.analysisNum}/tests`, {
-      headers: vm.cookie ? { Cookie: vm.cookie } : {}
+      headers: vm.cookie ? { Cookie: vm.cookie } : {},
     });
     if (response.status !== HttpStatus.OK && response.status !== HttpStatus.NotModified && response.status !== HttpStatus.Unauthorized)
       throw new Error(`received ${response.status} (${response.statusText}) response`);
@@ -532,7 +530,7 @@ export async function getIssues(vm: models.Instance, secsToWait = 600, secsWaiti
     await wait(secsToWait);
     return await getIssues(vm, secsToWait * 2, secsWaiting + secsToWait);
   }
-  let body = await parseBody(response);
+  const body = await parseBody(response);
   const fetchErrPrefix = `${errPrefix}: Received ${response.status} (${response.statusText}) response`;
   if (response.status === HttpStatus.Unauthorized && body.message === "Invalid or missing authorization token") {
     vm.log.debug(`${fetchErrPrefix}. Signing in`);

gitlab-integration/api.ts gitlab-integration/src/api.tsgitlab-integration/api.ts renamed to gitlab-integration/src/api.ts

@@ -1,18 +1,13 @@
-// Taken from diffblue/dashboard/src/api/utils/logger.ts
-
 // Copyright 2017-2018 Diffblue Limited. All Rights Reserved.
 
-import * as express from "express";
 import * as _ from "lodash";
 import * as winston from "winston";
 require("winston-papertrail").Papertrail; // tslint:disable-line:no-unused-expression
 
-const app = express();
-
 const maxLevelLength = 18; // "Verbose" in colour
 
 let transport;
-if (app.get("env") === "development" || !process.env.papertrailHost || !process.env.papertrailPort)
+if (process.env.papertrailHost === undefined || process.env.papertrailPort === undefined)
   transport =
     new winston.transports.Console({
       level: "debug",
@@ -39,7 +34,7 @@ else
       handleExceptions: true,
       host: process.env.papertrailHost,
       port: process.env.papertrailPort,
-      hostname: app.get("env"),
+      hostname: "gitlab-integration",
       program: "Dashboard",
       logFormat: (level: string, message: string) => { return padString(level, maxLevelLength) + message; },
     });

gitlab-integration/logger.ts gitlab-integration/src/logger.tsgitlab-integration/logger.ts renamed to gitlab-integration/src/logger.ts

@@ -1,19 +1,23 @@
 {
   "compilerOptions": {
-    "noImplicitAny": true,
+    "rootDir": "src",
+    "outDir": "built",
+    "target": "es6",
+    "lib": [ "es6" ],
     "module": "commonjs",
+    "moduleResolution": "node",
     "noEmitOnError": true,
-    "removeComments": false,
-    "sourceMap": true,
-    "target": "es5",
-    "lib": [ "es5" ],
+    "noImplicitAny": true,
     "strictNullChecks": true,
     "noUnusedLocals": true,
     "noUnusedParameters": true,
-    "typeRoots": [ "./node_modules/@types" ]
+    "sourceMap": true,
+    "removeComments": true
   },
   "exclude": [
-    "./node_modules/**/*"
+    "node_modules",
+    "built",
+    "dist"
   ],
   "plugins": [
     { "name": "tslint-language-service" }