Merge pull request #8 from RedisGraph/redisgraph-cross-validation

Redisgraph cross validation

Author: filipecosta90

.circleci/config.yml

@@ -5,15 +5,11 @@ workflows:
   version: 2
   build:
     jobs:
-      - test:
-          filters:
-            branches:
-              ignore:
-                - stable
+      - test
 
 jobs:
   test:
-    resource_class: large
+    resource_class: xlarge
     machine:
       image: ubuntu-2004:202008-01
     steps:
@@ -24,41 +20,114 @@ jobs:
             export DEBIAN_FRONTEND=noninteractive
             sudo apt update
             # install dependencies
-            sudo apt install -y curl git wget unzip maven netcat
-            # driver
-            git clone --depth 1 --branch dev https://github.com/ldbc/ldbc_snb_driver && cd ldbc_snb_driver && mvn install -DskipTests && cd ..
-            # Cypher
+            sudo apt install -y git
+            # clone, build, and install driver
+            git clone --depth 1 --branch 0.3.5 https://github.com/ldbc/ldbc_snb_driver && cd ldbc_snb_driver && mvn install -DskipTests && cd ..
+            # PostgreSQL
+            pip3 install --user --progress-bar off psycopg2-binary
+      - run:
+          name: Build project
+          command: ./build.sh
+      - run:
+          name: Tiny data set - Load Cypher/Neo4j database (1st time)
+          command: |
             cd cypher
-            scripts/install-dependencies.sh
+            . scripts/environment-variables-default.sh
+            scripts/load-in-one-step.sh
             cd ..
-            # PostgreSQL
-            # TODO
       - run:
-          name: Download data
+          name: Tiny data set - Create validation parameters with Neo4j/Cypher project
           command: |
-            mkdir data/
-            cd data
-            wget -q https://ldbc.github.io/ldbc_snb_data_converter/csv-composite-projected-fk.zip
-            unzip csv-composite-projected-fk.zip
+            cd cypher
+            driver/create-validation-parameters.sh
+            mv validation_params.csv validation_params_cypher_tiny.csv
             cd ..
       - run:
-          name: Load
+          name: Tiny data set - Load PostgreSQL database (1st time)
+          command: |
+            cd postgres
+            export POSTGRES_CSV_DIR=`pwd`/test-data/
+            scripts/start.sh
+            scripts/create-db.sh
+            scripts/load.sh
+            cd ..
+      - run:
+          name: Tiny data set - Create validation parameters with the PostgreSQL project
+          command: |
+            cd postgres
+            driver/create-validation-parameters.sh
+            mv validation_params.csv validation_params_postgres_tiny.csv
+            cd ..
+      - run:
+          name: Tiny data set - Load Cypher/Neo4j database (2nd time)
           command: |
-            # PostgreSQL
-            # TODO
-            # Cypher
             cd cypher
             . scripts/environment-variables-default.sh
-            export NEO4J_CSV_DIR=`pwd`/../data/csv-composite-projected-fk
-            export NEO4J_CSV_POSTFIX=.csv
             scripts/load-in-one-step.sh
             cd ..
       - run:
-          name: Test
-          command: mvn test -B
+          name: Tiny data set - Cross-validate the Neo4j/Cypher project based on the results from Postgres
+          command: |
+            mv postgres/validation_params_postgres_tiny.csv cypher/validation_params.csv
+            cd cypher
+            driver/validate.sh | tee validation-log.txt
+            grep 'Validation Result: PASS' validation-log.txt
+            cd ..
       - run:
-          name: Run Cypher Python script
+          name: Tiny data set - Load PostgreSQL database (2nd time)
+          command: |
+            cd postgres
+            export POSTGRES_CSV_DIR=`pwd`/test-data/
+            scripts/start.sh
+            scripts/create-db.sh
+            scripts/load.sh
+            cd ..
+      - run:
+          name: Tiny data set - Cross-validate the Postgres project based on the results from Neo4j/Cypher
+          command: |
+            mv cypher/validation_params_cypher_tiny.csv postgres/validation_params.csv
+            cd postgres
+            driver/validate.sh | tee validation-log.txt
+            grep 'Validation Result: PASS' validation-log.txt
+            cd ..
+      - run:
+          name: Download SF1 data set
+          command: |
+            wget https://surfdrive.surf.nl/files/index.php/s/Ar7kacLw3LVzQaf/download --quiet --output-document=sf1.zip
+            unzip sf1.zip
+            # substitution parameters and updateStreams for Neo4j/Cypher
+            cp ./sf1/substitution_parameters/* ./cypher/test-data/substitution_parameters/
+            cp ./sf1/social_network-csv_composite/updateStream* ./cypher/test-data/update_streams/
+      - run:
+          name: SF1 data set - Load Cypher/Neo4j database
           command: |
             cd cypher
-            python3 bi.py
+            . scripts/environment-variables-default.sh
+            export NEO4J_VANILLA_CSV_DIR=`pwd`/../sf1/social_network-csv_composite
+            scripts/load-in-one-step.sh
+            cd ..
+      - run:
+          name: SF1 data set - Create validation parameters with the Neo4j/Cypher
+          command: |
+            cd cypher
+            driver/create-validation-parameters.sh
+            mv validation_params.csv validation_params_cypher_sf1.csv
+            cd ..
+      - run:
+          name: SF1 data set - Load PostgreSQL database
+          command: |
+            cd postgres
+            export POSTGRES_CSV_DIR=`pwd`/../sf1/social_network-csv_merge_foreign
+            scripts/start.sh
+            scripts/create-db.sh
+            scripts/load.sh
+            cd ..
+      - run:
+          name: SF1 data set - Cross-validate the Postgres project based on the results from Neo4j/Cypher
+          command: |
+            mv cypher/validation_params_cypher_sf1.csv postgres/validation_params.csv
+            cd postgres
+            driver/validate.sh | tee validation-log.txt
+            grep 'Validation Result: PASS' validation-log.txt
+            cd ..
       - slack/status

.gitignore

@@ -10,6 +10,5 @@ validation_params.csv
 *.swp
 *-actual.json
 *-expected.json
-results-*/
+results/
 *.DS_Store
-*.report