Add support for S3 as a remote filesystem (#2676)

Author: zackgalbreath

.env.dev

@@ -19,3 +19,6 @@ SLOW_PAGE_TIME=1000000
 MAIL_MAILER=smtp
 MAIL_HOST=mailpit
 MAIL_PORT=1025
+
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin

.env.example

@@ -18,6 +18,8 @@ DB_PASSWORD=secret
 #DB_PORT=
 #DB_USERNAME=
 
+# cdash.php
+
 # How long since the last submission before considering a project inactive.
 # Set to 0 to always show all projects on viewProjects.php.
 #ACTIVE_PROJECT_DAYS=7
@@ -77,6 +79,12 @@ DB_PASSWORD=secret
 # something other than an email address in LDAP.
 #LOGIN_FIELD=Email
 
+# The maximum visibility level for user-created projects on this instance.
+# Instance admins are able to override this setting and set project visibility
+# to anything.  Thus, this setting is only meaningful if USER_CREATE_PROJECTS=true.
+# Options: PUBLIC, PROTECTED, PRIVATE
+# MAX_PROJECT_VISIBILITY=PUBLIC
+
 # Maximum per-project upload quota, in GB
 #MAX_UPLOAD_QUOTA=10
 
@@ -148,6 +156,9 @@ DB_PASSWORD=secret
 # VCS (eg. GitHub) API endpoints.
 #USE_VCS_API=true
 
+# Should normal users be allowed to create projects
+# USER_CREATE_PROJECTS = false
+
 # logging.php
 #LOG_CHANNEL=stack
 
@@ -172,6 +183,33 @@ QUEUE_CONNECTION=database
 # Number of minutes before and idle session is allowed to expire.
 #SESSION_LIFETIME=120
 
+# filesystem.php
+
+# Default filesystem driver for CDash to use.
+# Supported options are 'local' and 's3'.
+#FILESYSTEM_DRIVER=local
+
+# The following env vars are only relevant for S3 support.
+# The name of the bucket that CDash where will store files.
+# AWS_BUCKET=cdash
+
+# The AWS region where this S3 bucket is stored.
+# Otherwise set this to 'local' if you're using MinIO.
+# AWS_REGION=
+
+# Credentials for access to this S3 bucket.
+#AWS_ACCESS_KEY_ID=
+#AWS_SECRET_ACCESS_KEY=
+
+# Set this to true if you're using MinIO
+#AWS_USE_PATH_STYLE_ENDPOINT=false
+
+# URL of your MinIO server (if you're using MinIO). Leave blank otherwise.
+#AWS_ENDPOINT=
+
+# URL of the bucket on your MinIO server (if you're using MinIO). Leave blank otherwise.
+#AWS_URL=
+
 # mail.php
 #MAIL_MAILER=smtp
 #MAIL_HOST=smtp.mailgun.org
@@ -258,12 +296,3 @@ MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"
 
 # Whether or not to automatically register new users upon first login
 #SAML2_AUTO_REGISTER_NEW_USERS=false
-
-# Should normal users be allowed to create projects
-# USER_CREATE_PROJECTS = false
-
-# The maximum visibility level for user-created projects on this instance.
-# Instance admins are able to override this setting and set project visibility
-# to anything.  Thus, this setting is only meaningful if USER_CREATE_PROJECTS=true.
-# Options: PUBLIC, PROTECTED, PRIVATE
-# MAX_PROJECT_VISIBILITY=PUBLIC

.github/workflows/ci.yml

@@ -14,22 +14,32 @@ jobs:
     env:
       SITENAME: GitHub Actions
       BASE_IMAGE: ${{matrix.base-image}}
+      STORAGE_TYPE: ${{matrix.storage}}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         database: ['mysql', 'postgres']
         base-image: ['debian', 'ubi']
+        storage: ['local', 'minio']
+        exclude:
+          - storage: minio
+            base-image: ubi
+          - storage: minio
+            database: mysql
     steps:
       - uses: actions/checkout@v4
       - uses: docker/setup-buildx-action@v3
       - name: Build images
         shell: bash
         run: |
+          if [ "${{matrix.storage}}" == "minio" ]; then
+            extra_args="-f docker/docker-compose.minio.yml"
+          fi
           docker compose \
               -f docker/docker-compose.yml \
               -f docker/docker-compose.dev.yml \
-              -f "docker/docker-compose.${{matrix.database}}.yml" \
+              -f "docker/docker-compose.${{matrix.database}}.yml" ${extra_args} \
               --env-file .env.dev up -d \
               --build \
               --wait

.github/workflows/ctest_driver_script.cmake

@@ -18,6 +18,7 @@ ctest_empty_binary_directory("${CTEST_BINARY_DIRECTORY}")
 set(cfg_options
   "-DCDASH_DIR_NAME="
   "-DCDASH_SERVER=localhost:8080"
+  "-DCDASH_STORAGE_TYPE=${STORAGE_TYPE}"
 )
 
 # Backup .env file

.github/workflows/submit.sh

@@ -14,6 +14,8 @@ submit_type="${submit_type:-Experimental}"
 
 site="${SITENAME:-$(hostname)}"
 
+storage_type="${STORAGE_TYPE:-local}"
+
 echo "site=$site"
 echo "database=$database"
 echo "ctest_driver=$ctest_driver"
@@ -40,6 +42,7 @@ docker exec cdash bash -c "\
     --schedule-random \
     -DSITENAME=\"${site}\" \
     -DDATABASE=\"${database}\" \
+    -DSTORAGE_TYPE=\"${storage_type}\" \
     -DSUBMIT_TYPE=\"${submit_type}\" \
     -S \"${ctest_driver}\" \
 "

CMakeLists.txt

@@ -10,6 +10,9 @@ configure_file(
 # to configure the testing install
 set(CDASH_SERVER localhost CACHE STRING "CDash testing server")
 set(CDASH_IMAGE "$ENV{BASE_IMAGE}" CACHE STRING "Docker image name")
+if(NOT DEFINED CDASH_STORAGE_TYPE)
+  set(CDASH_STORAGE_TYPE "local")
+endif()
 
 get_filename_component(CDASH_DIR_NAME_DEFAULT ${CDash_SOURCE_DIR} NAME)
 set(CDASH_DIR_NAME "${CDASH_DIR_NAME_DEFAULT}" CACHE STRING "URL suffix. Ie 'http://<CDASH_SERVER>/<CDASH_DIR_NAME>'")

app/Console/Commands/ValidateXml.php

@@ -6,6 +6,8 @@
 use App\Utils\SubmissionUtils;
 use BadMethodCallException;
 use Illuminate\Console\Command;
+use League\Flysystem\UnableToReadFile;
+use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
 
 class ValidateXml extends Command
 {
@@ -69,6 +71,9 @@ public function handle(): int
                 $this->warn("WARNING: Skipped input file '{$input_xml_file}' as validation"
                     . ' of this file format is currently not supported.');
                 $has_skipped = true;
+            } catch (FileNotFoundException|UnableToReadFile $e) {
+                $this->error($e->getMessage());
+                $has_skipped = true;
             }
         }
 

app/Http/Controllers/BuildController.php

@@ -29,7 +29,7 @@
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\View\View;
-use Symfony\Component\HttpFoundation\BinaryFileResponse;
+use Symfony\Component\HttpFoundation\StreamedResponse;
 
 require_once 'include/api_common.php';
 
@@ -829,7 +829,7 @@ public function files(int $build_id): View
             ->with('urls', $urls);
     }
 
-    public function build_file(int $build_id, int $file_id): BinaryFileResponse
+    public function build_file(int $build_id, int $file_id): StreamedResponse
     {
         $this->setBuildById($build_id);
 
@@ -841,10 +841,26 @@ public function build_file(int $build_id, int $file_id): BinaryFileResponse
         $uploadFile = new UploadFile();
         $uploadFile->Id = $file_id;
         $uploadFile->Fill();
-        return response()->file(Storage::path("upload/{$uploadFile->Sha1Sum}"), [
+
+        // The code below satisfies the following requirements:
+        // 1) Render text and images in browser (as opposed to forcing a download).
+        // 2) Download other files to the proper filename (not a numeric identifier).
+        // 3) Support downloading files that are larger than the PHP memory_limit.
+        $fp = Storage::readStream("upload/{$uploadFile->Sha1Sum}");
+        if ($fp === null) {
+            abort(404, 'File not found');
+        }
+        $filename = $uploadFile->Filename;
+        $headers = [
             'Content-Type' => 'text/plain',
             'Content-Disposition' => "inline/attachment; filename={$uploadFile->Filename}",
-        ]);
+        ];
+        return response()->streamDownload(function () use ($fp) {
+            while (!feof($fp)) {
+                echo fread($fp, 1024);
+            }
+            fclose($fp);
+        }, $filename, $headers, 'inline');
     }
 
     public function ajaxBuildNote(): View

app/Http/Controllers/RemoteProcessingController.php

@@ -99,6 +99,7 @@ public function requeueSubmissionFile(): Response
         $filename = request()->string('filename');
         $buildid = request()->integer('buildid');
         $projectid = request()->integer('projectid');
+        $md5 = request()->string('md5');
         if (!Storage::exists("inprogress/{$filename}")) {
             return response('File not found', Response::HTTP_NOT_FOUND);
         }
@@ -112,7 +113,7 @@ public function requeueSubmissionFile(): Response
         // Requeue the file with exponential backoff.
         PendingSubmissions::IncrementForBuildId($buildid);
         $delay = ((int) config('cdash.retry_base')) ** $retry_handler->Retries;
-        ProcessSubmission::dispatch($filename, $projectid, $buildid, md5_file(Storage::path("inbox/{$filename}")))->delay(now()->addSeconds($delay));
+        ProcessSubmission::dispatch($filename, $projectid, $buildid, $md5)->delay(now()->addSeconds($delay));
         return response('OK', Response::HTTP_OK);
     }
 }

app/Http/Controllers/SubmissionController.php

@@ -21,6 +21,8 @@
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Str;
+use League\Flysystem\UnableToReadFile;
+use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
 use Symfony\Component\HttpKernel\Exception\HttpException;
 
 final class SubmissionController extends AbstractProjectController
@@ -85,23 +87,22 @@ private function submitProcess(): Response
         $authtoken = AuthTokenUtil::getBearerToken();
         $authtoken_hash = $authtoken === null || $authtoken === '' ? '' : AuthTokenUtil::hashToken($authtoken);
 
-        // Save the incoming file in the inbox directory.
-        $filename = "{$projectname}_-_{$authtoken_hash}_-_" . Str::uuid()->toString() . "_-_{$expected_md5}.xml";
-        $fp = request()->getContent(true);
-        if (!Storage::put("inbox/{$filename}", $fp)) {
-            Log::error("Failed to save submission to inbox for $projectname (md5=$expected_md5)");
-            abort(Response::HTTP_INTERNAL_SERVER_ERROR, 'Failed to save submission file.');
-        }
-
         // Check that the md5sum of the file matches what we were told to expect.
+        $fp = request()->getContent(true);
         if (strlen($expected_md5) > 0) {
-            $md5sum = md5_file(Storage::path("inbox/{$filename}"));
+            $md5sum = SubmissionUtils::hashFileHandle($fp, 'md5');
             if ($md5sum != $expected_md5) {
-                Storage::delete("inbox/{$filename}");
                 abort(Response::HTTP_BAD_REQUEST, "md5 mismatch. expected: {$expected_md5}, received: {$md5sum}");
             }
         }
 
+        // Save the incoming file in the inbox directory.
+        $filename = "{$projectname}_-_{$authtoken_hash}_-_" . Str::uuid()->toString() . "_-_{$expected_md5}.xml";
+        if (!Storage::put("inbox/{$filename}", $fp)) {
+            Log::error("Failed to save submission to inbox for $projectname (md5=$expected_md5)");
+            abort(Response::HTTP_INTERNAL_SERVER_ERROR, 'Failed to save submission file.');
+        }
+
         // Check if we can connect to the database before proceeding any further.
         try {
             DB::connection()->getPdo();
@@ -138,7 +139,7 @@ private function submitProcess(): Response
         $stored_filename = 'inbox/' . $filename;
         $xml_info = [];
         try {
-            $xml_info = SubmissionUtils::get_xml_type(fopen(Storage::path($stored_filename), 'r'), $stored_filename);
+            $xml_info = SubmissionUtils::get_xml_type(Storage::readStream($stored_filename), $stored_filename);
         } catch (BadSubmissionException $e) {
             $xml_info['xml_handler'] = '';
             $message = "Could not determine submission file type for: '{$stored_filename}'";
@@ -149,7 +150,15 @@ private function submitProcess(): Response
         }
         if ($xml_info['xml_handler'] !== '') {
             // If validation is enabled and if this file has a corresponding schema, validate it
-            $validation_errors = $xml_info['xml_handler']::validate(storage_path('app/' . $stored_filename));
+            $validation_errors = [];
+            try {
+                $validation_errors = $xml_info['xml_handler']::validate($stored_filename);
+            } catch (FileNotFoundException|UnableToReadFile $e) {
+                Log::warning($e->getMessage());
+                if ((bool) config('cdash.validate_xml_submissions') === true) {
+                    abort(400, "XML validation failed for $filename:" . PHP_EOL . $e->getMessage());
+                }
+            }
             if (count($validation_errors) > 0) {
                 $error_string = implode(PHP_EOL, $validation_errors);
 
@@ -233,18 +242,24 @@ public function storeUploadedFile(Request $request): Response
         }
 
         try {
-            $sha1sum = decrypt($request->input('sha1sum'));
+            $expected_sha1sum = decrypt($request->input('sha1sum'));
         } catch (DecryptException $e) {
             return response('This feature is disabled', Response::HTTP_CONFLICT);
         }
 
         $uploaded_file = array_values(request()->allFiles())[0];
-        $stored_path = $uploaded_file->storeAs('upload', $sha1sum);
+        $stored_path = $uploaded_file->storeAs('upload', $expected_sha1sum);
         if ($stored_path === false) {
             abort(Response::HTTP_INTERNAL_SERVER_ERROR, 'Failed to store uploaded file');
         }
 
-        if (sha1_file(Storage::path($stored_path)) !== $sha1sum) {
+        $fp = Storage::readStream($stored_path);
+        if ($fp === null) {
+            abort(Response::HTTP_INTERNAL_SERVER_ERROR, 'Failed to store uploaded file');
+        }
+
+        $found_sha1sum = SubmissionUtils::hashFileHandle($fp, 'sha1');
+        if ($found_sha1sum !== $expected_sha1sum) {
             Storage::delete($stored_path);
             return response('Uploaded file does not match expected sha1sum', Response::HTTP_BAD_REQUEST);
         }