From 095dc60629e35e2f91bc5aa0545eccf221a45626 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 2 Apr 2023 21:40:06 +0100 Subject: [PATCH 1/8] Basics: add support for `.tar.gz` archives `swift experimental-destination install` subcommand fails when pointed to a destination bundle compressed with `.tar.gz` extension, compressed with tar and gzip. This archival format is vastly superior to zip in the context of cross-compilation destination bundles. In our typical scenario we see that `.tar.gz` archives are at least 4x better than `.zip`. --- Sources/Basics/Archiver+Tar.swift | 147 +++++++++++++++++ Sources/Basics/CMakeLists.txt | 1 + Tests/BasicsTests/Inputs/archive.tar.gz | Bin 0 -> 346 bytes .../BasicsTests/Inputs/invalid_archive.tar.gz | 1 + Tests/BasicsTests/TarArchiverTests.swift | 154 ++++++++++++++++++ Tests/BasicsTests/ZipArchiverTests.swift | 6 +- 6 files changed, 306 insertions(+), 3 deletions(-) create mode 100644 Sources/Basics/Archiver+Tar.swift create mode 100644 Tests/BasicsTests/Inputs/archive.tar.gz create mode 100644 Tests/BasicsTests/Inputs/invalid_archive.tar.gz create mode 100644 Tests/BasicsTests/TarArchiverTests.swift diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift new file mode 100644 index 00000000000..8914991e9ed --- /dev/null +++ b/Sources/Basics/Archiver+Tar.swift @@ -0,0 +1,147 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import _Concurrency + +import class Dispatch.DispatchQueue +import class TSCBasic.Process +import protocol TSCBasic.FileSystem +import struct Dispatch.DispatchTime +import struct TSCBasic.AbsolutePath +import struct TSCBasic.FileSystemError + +/// An `Archiver` that handles Tar archives using the command-line `tar` tool. +public struct TarArchiver: Archiver { + public var supportedExtensions: Set { ["tar", "tar.gz"] } + + /// The file-system implementation used for various file-system operations and checks. + private let fileSystem: FileSystem + + /// Helper for cancelling in-flight requests + private let cancellator: Cancellator + + /// The underlying command + private let tarCommand: String + + /// Creates a `TarArchiver`. + /// + /// - Parameters: + /// - fileSystem: The file system to used by the `TarArchiver`. + /// - cancellator: Cancellation handler + public init(fileSystem: FileSystem, cancellator: Cancellator? = .none) { + self.fileSystem = fileSystem + self.cancellator = cancellator ?? Cancellator(observabilityScope: .none) + +#if os(Windows) + self.tarCommand = "tar.exe" +#else + self.tarCommand = "tar" +#endif + } + + public func extract( + from archivePath: AbsolutePath, + to destinationPath: AbsolutePath, + completion: @escaping (Result) -> Void + ) { + do { + guard self.fileSystem.exists(archivePath) else { + throw FileSystemError(.noEntry, archivePath) + } + + guard self.fileSystem.isDirectory(destinationPath) else { + throw FileSystemError(.notDirectory, destinationPath) + } + + let process = TSCBasic.Process(arguments: [self.tarCommand, "xzf", archivePath.pathString, "-C", destinationPath.pathString]) + + guard let registrationKey = self.cancellator.register(process) else { + throw StringError("cancellation") + } + + DispatchQueue.sharedConcurrent.async { + defer { self.cancellator.deregister(registrationKey) } + completion(.init(catching: { + try process.launch() + let processResult = try process.waitUntilExit() + guard processResult.exitStatus == .terminated(code: 0) else { + throw try StringError(processResult.utf8stderrOutput()) + } + })) + } + } catch { + return completion(.failure(error)) + } + } + + public func compress( + directory: AbsolutePath, + to destinationPath: AbsolutePath, + completion: @escaping (Result) -> Void + ) { + do { + guard self.fileSystem.isDirectory(directory) else { + throw FileSystemError(.notDirectory, directory) + } + + let process = TSCBasic.Process( + arguments: [self.tarCommand, "acf", destinationPath.pathString, directory.basename], + workingDirectory: directory.parentDirectory + ) + + guard let registrationKey = self.cancellator.register(process) else { + throw StringError("Failed to register cancellation for Archiver") + } + + DispatchQueue.sharedConcurrent.async { + defer { self.cancellator.deregister(registrationKey) } + completion(.init(catching: { + try process.launch() + let processResult = try process.waitUntilExit() + guard processResult.exitStatus == .terminated(code: 0) else { + throw try StringError(processResult.utf8stderrOutput()) + } + })) + } + } catch { + return completion(.failure(error)) + } + } + + public func validate(path: AbsolutePath, completion: @escaping (Result) -> Void) { + do { + guard self.fileSystem.exists(path) else { + throw FileSystemError(.noEntry, path) + } + + let process = TSCBasic.Process(arguments: [self.tarCommand, "tf", path.pathString]) + guard let registrationKey = self.cancellator.register(process) else { + throw StringError("cancellation") + } + + DispatchQueue.sharedConcurrent.async { + defer { self.cancellator.deregister(registrationKey) } + completion(.init(catching: { + try process.launch() + let processResult = try process.waitUntilExit() + return processResult.exitStatus == .terminated(code: 0) + })) + } + } catch { + return completion(.failure(error)) + } + } + + public func cancel(deadline: DispatchTime) throws { + try self.cancellator.cancel(deadline: deadline) + } +} diff --git a/Sources/Basics/CMakeLists.txt b/Sources/Basics/CMakeLists.txt index f137a67d623..3a9d6293e62 100644 --- a/Sources/Basics/CMakeLists.txt +++ b/Sources/Basics/CMakeLists.txt @@ -8,6 +8,7 @@ add_library(Basics Archiver.swift + Archiver+Tar.swift Archiver+Zip.swift AuthorizationProvider.swift ByteString+Extensions.swift diff --git a/Tests/BasicsTests/Inputs/archive.tar.gz b/Tests/BasicsTests/Inputs/archive.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..24a5d221fb5c262ed5eea8ff12a0398505ecde1e GIT binary patch literal 346 zcmV-g0j2&QiwFQ`<|$+V1JjF7%gjk-peZmgFfcPQQ2^2AW~N};zzD(z17l+o1w&&~ zb2BqzLvuqT1p`APBNGz_1q0eT3j#`uOA?EKPRUKINWrHLs4OiFk37UH3UGR;09G+} z1||ju-^64E|6qj(1tnI05fHBeVi5lhaYj0ZgakpEARSYH*aJ-oBSM@%IX_n~ zv7jI)RWCO&IS1q<20@*d8;-Acy)oa~KVqH!lSazDISR0YfW!)q)Wnq3B7NGG0|fKG zsTo@SH!&~(=Kl)XI0qGs@ceIRY^>np>EsyX?BN;as#lR%Qc|Rcnj>wU0v&^V{Bz4I z%q-oEye!N^3^Pl8w1b@-sRXzT4NMh+fe!GA#O?rU6$o6^4ZGn0XkYsi&i}ytj$Z#8 s7#WS$|MUPJsX00M3gP)hIVp+*?Uqq63P!;w7zOkL0Jf&$Q2-DC06+qYr~m)} literal 0 HcmV?d00001 diff --git a/Tests/BasicsTests/Inputs/invalid_archive.tar.gz b/Tests/BasicsTests/Inputs/invalid_archive.tar.gz new file mode 100644 index 00000000000..06001991658 --- /dev/null +++ b/Tests/BasicsTests/Inputs/invalid_archive.tar.gz @@ -0,0 +1 @@ +not an archive \ No newline at end of file diff --git a/Tests/BasicsTests/TarArchiverTests.swift b/Tests/BasicsTests/TarArchiverTests.swift new file mode 100644 index 00000000000..258d672eb09 --- /dev/null +++ b/Tests/BasicsTests/TarArchiverTests.swift @@ -0,0 +1,154 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Basics +import TSCBasic +import TSCTestSupport +import XCTest +import TSCclibc // for SPM_posix_spawn_file_actions_addchdir_np_supported + +final class TarArchiverTests: XCTestCase { + func testTarArchiverSuccess() throws { + try testWithTemporaryDirectory { tmpdir in + let archiver = TarArchiver(fileSystem: localFileSystem) + let inputArchivePath = AbsolutePath(path: #file).parentDirectory.appending(components: "Inputs", "archive.tar.gz") + try archiver.extract(from: inputArchivePath, to: tmpdir) + let content = tmpdir.appending("file") + XCTAssert(localFileSystem.exists(content)) + XCTAssertEqual((try? localFileSystem.readFileContents(content))?.cString, "Hello World!") + } + } + + func testTarArchiverArchiveDoesntExist() { + let fileSystem = InMemoryFileSystem() + let archiver = TarArchiver(fileSystem: fileSystem) + let archive = AbsolutePath("/archive.tar.gz") + XCTAssertThrowsError(try archiver.extract(from: archive, to: "/")) { error in + XCTAssertEqual(error as? FileSystemError, FileSystemError(.noEntry, archive)) + } + } + + func testTarArchiverDestinationDoesntExist() throws { + let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz") + let archiver = TarArchiver(fileSystem: fileSystem) + let destination = AbsolutePath("/destination") + XCTAssertThrowsError(try archiver.extract(from: "/archive.tar.gz", to: destination)) { error in + XCTAssertEqual(error as? FileSystemError, FileSystemError(.notDirectory, destination)) + } + } + + func testTarArchiverDestinationIsFile() throws { + let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz", "/destination") + let archiver = TarArchiver(fileSystem: fileSystem) + let destination = AbsolutePath("/destination") + XCTAssertThrowsError(try archiver.extract(from: "/archive.tar.gz", to: destination)) { error in + XCTAssertEqual(error as? FileSystemError, FileSystemError(.notDirectory, destination)) + } + } + + func testTarArchiverInvalidArchive() throws { + try testWithTemporaryDirectory { tmpdir in + let archiver = TarArchiver(fileSystem: localFileSystem) + let inputArchivePath = AbsolutePath(path: #file).parentDirectory + .appending(components: "Inputs", "invalid_archive.tar.gz") + XCTAssertThrowsError(try archiver.extract(from: inputArchivePath, to: tmpdir)) { error in + XCTAssertMatch((error as? StringError)?.description, .contains("Unrecognized archive format")) + } + } + } + + func testValidation() throws { + // valid + try testWithTemporaryDirectory { tmpdir in + let archiver = TarArchiver(fileSystem: localFileSystem) + let path = AbsolutePath(path: #file).parentDirectory + .appending(components: "Inputs", "archive.tar.gz") + XCTAssertTrue(try archiver.validate(path: path)) + } + // invalid + try testWithTemporaryDirectory { tmpdir in + let archiver = TarArchiver(fileSystem: localFileSystem) + let path = AbsolutePath(path: #file).parentDirectory + .appending(components: "Inputs", "invalid_archive.tar.gz") + XCTAssertFalse(try archiver.validate(path: path)) + } + // error + try testWithTemporaryDirectory { tmpdir in + let archiver = TarArchiver(fileSystem: localFileSystem) + let path = AbsolutePath.root.appending("does_not_exist.tar.gz") + XCTAssertThrowsError(try archiver.validate(path: path)) { error in + XCTAssertEqual(error as? FileSystemError, FileSystemError(.noEntry, path)) + } + } + } + + func testCompress() throws { + #if os(Linux) + guard SPM_posix_spawn_file_actions_addchdir_np_supported() else { + throw XCTSkip("working directory not supported on this platform") + } + #endif + + try testWithTemporaryDirectory { tmpdir in + let archiver = TarArchiver(fileSystem: localFileSystem) + + let rootDir = tmpdir.appending(component: UUID().uuidString) + try localFileSystem.createDirectory(rootDir) + try localFileSystem.writeFileContents(rootDir.appending("file1.txt"), string: "Hello World!") + + let dir1 = rootDir.appending("dir1") + try localFileSystem.createDirectory(dir1) + try localFileSystem.writeFileContents(dir1.appending("file2.txt"), string: "Hello World 2!") + + let dir2 = dir1.appending("dir2") + try localFileSystem.createDirectory(dir2) + try localFileSystem.writeFileContents(dir2.appending("file3.txt"), string: "Hello World 3!") + try localFileSystem.writeFileContents(dir2.appending("file4.txt"), string: "Hello World 4!") + + let archivePath = tmpdir.appending(component: UUID().uuidString + ".tar.gz") + try archiver.compress(directory: rootDir, to: archivePath) + XCTAssertFileExists(archivePath) + + let extractRootDir = tmpdir.appending(component: UUID().uuidString) + try localFileSystem.createDirectory(extractRootDir) + try archiver.extract(from: archivePath, to: extractRootDir) + try localFileSystem.stripFirstLevel(of: extractRootDir) + + XCTAssertFileExists(extractRootDir.appending("file1.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractRootDir.appending("file1.txt")), + "Hello World!" + ) + + let extractedDir1 = extractRootDir.appending("dir1") + XCTAssertDirectoryExists(extractedDir1) + XCTAssertFileExists(extractedDir1.appending("file2.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractedDir1.appending("file2.txt")), + "Hello World 2!" + ) + + let extractedDir2 = extractedDir1.appending("dir2") + XCTAssertDirectoryExists(extractedDir2) + XCTAssertFileExists(extractedDir2.appending("file3.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractedDir2.appending("file3.txt")), + "Hello World 3!" + ) + XCTAssertFileExists(extractedDir2.appending("file4.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractedDir2.appending("file4.txt")), + "Hello World 4!" + ) + } + } +} diff --git a/Tests/BasicsTests/ZipArchiverTests.swift b/Tests/BasicsTests/ZipArchiverTests.swift index 8374d145b6f..6facf338777 100644 --- a/Tests/BasicsTests/ZipArchiverTests.swift +++ b/Tests/BasicsTests/ZipArchiverTests.swift @@ -254,17 +254,17 @@ class ArchiverTests: XCTestCase { } extension Archiver { - fileprivate func extract(from: AbsolutePath, to: AbsolutePath) throws { + func extract(from: AbsolutePath, to: AbsolutePath) throws { try tsc_await { self.extract(from: from, to: to, completion: $0) } } - fileprivate func compress(directory: AbsolutePath, to: AbsolutePath) throws { + func compress(directory: AbsolutePath, to: AbsolutePath) throws { try tsc_await { self.compress(directory: directory, to: to, completion: $0) } } - fileprivate func validate(path: AbsolutePath) throws -> Bool { + func validate(path: AbsolutePath) throws -> Bool { try tsc_await { self.validate(path: path, completion: $0) } From 2d5f5425521053612f240f28d3e4305573ffb00b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 2 Apr 2023 21:41:45 +0100 Subject: [PATCH 2/8] `TarArchiver` is a reference type --- Sources/Basics/Archiver+Tar.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift index 8914991e9ed..9b92d2b2961 100644 --- a/Sources/Basics/Archiver+Tar.swift +++ b/Sources/Basics/Archiver+Tar.swift @@ -20,7 +20,7 @@ import struct TSCBasic.AbsolutePath import struct TSCBasic.FileSystemError /// An `Archiver` that handles Tar archives using the command-line `tar` tool. -public struct TarArchiver: Archiver { +public final class TarArchiver: Archiver { public var supportedExtensions: Set { ["tar", "tar.gz"] } /// The file-system implementation used for various file-system operations and checks. From 32914256412accc07a10edb9264aa429646e4ca9 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 3 Apr 2023 10:58:04 +0100 Subject: [PATCH 3/8] Fix Linux test failure --- Sources/Basics/Archiver+Tar.swift | 14 +-- Tests/BasicsTests/TarArchiverTests.swift | 131 ++++++++++++----------- 2 files changed, 76 insertions(+), 69 deletions(-) diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift index 9b92d2b2961..a0880ef7af4 100644 --- a/Sources/Basics/Archiver+Tar.swift +++ b/Sources/Basics/Archiver+Tar.swift @@ -13,11 +13,11 @@ import _Concurrency import class Dispatch.DispatchQueue -import class TSCBasic.Process -import protocol TSCBasic.FileSystem import struct Dispatch.DispatchTime import struct TSCBasic.AbsolutePath +import protocol TSCBasic.FileSystem import struct TSCBasic.FileSystemError +import class TSCBasic.Process /// An `Archiver` that handles Tar archives using the command-line `tar` tool. public final class TarArchiver: Archiver { @@ -41,11 +41,11 @@ public final class TarArchiver: Archiver { self.fileSystem = fileSystem self.cancellator = cancellator ?? Cancellator(observabilityScope: .none) -#if os(Windows) + #if os(Windows) self.tarCommand = "tar.exe" -#else + #else self.tarCommand = "tar" -#endif + #endif } public func extract( @@ -62,7 +62,9 @@ public final class TarArchiver: Archiver { throw FileSystemError(.notDirectory, destinationPath) } - let process = TSCBasic.Process(arguments: [self.tarCommand, "xzf", archivePath.pathString, "-C", destinationPath.pathString]) + let process = TSCBasic.Process( + arguments: [self.tarCommand, "xzf", archivePath.pathString, "-C", destinationPath.pathString] + ) guard let registrationKey = self.cancellator.register(process) else { throw StringError("cancellation") diff --git a/Tests/BasicsTests/TarArchiverTests.swift b/Tests/BasicsTests/TarArchiverTests.swift index 258d672eb09..f5d7859a2d4 100644 --- a/Tests/BasicsTests/TarArchiverTests.swift +++ b/Tests/BasicsTests/TarArchiverTests.swift @@ -12,15 +12,16 @@ import Basics import TSCBasic +import TSCclibc // for SPM_posix_spawn_file_actions_addchdir_np_supported import TSCTestSupport import XCTest -import TSCclibc // for SPM_posix_spawn_file_actions_addchdir_np_supported final class TarArchiverTests: XCTestCase { - func testTarArchiverSuccess() throws { + func testSuccess() throws { try testWithTemporaryDirectory { tmpdir in let archiver = TarArchiver(fileSystem: localFileSystem) - let inputArchivePath = AbsolutePath(path: #file).parentDirectory.appending(components: "Inputs", "archive.tar.gz") + let inputArchivePath = AbsolutePath(path: #file).parentDirectory + .appending(components: "Inputs", "archive.tar.gz") try archiver.extract(from: inputArchivePath, to: tmpdir) let content = tmpdir.appending("file") XCTAssert(localFileSystem.exists(content)) @@ -28,7 +29,7 @@ final class TarArchiverTests: XCTestCase { } } - func testTarArchiverArchiveDoesntExist() { + func testArchiveDoesntExist() { let fileSystem = InMemoryFileSystem() let archiver = TarArchiver(fileSystem: fileSystem) let archive = AbsolutePath("/archive.tar.gz") @@ -37,7 +38,7 @@ final class TarArchiverTests: XCTestCase { } } - func testTarArchiverDestinationDoesntExist() throws { + func testDestinationDoesntExist() throws { let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz") let archiver = TarArchiver(fileSystem: fileSystem) let destination = AbsolutePath("/destination") @@ -46,7 +47,7 @@ final class TarArchiverTests: XCTestCase { } } - func testTarArchiverDestinationIsFile() throws { + func testDestinationIsFile() throws { let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz", "/destination") let archiver = TarArchiver(fileSystem: fileSystem) let destination = AbsolutePath("/destination") @@ -55,34 +56,38 @@ final class TarArchiverTests: XCTestCase { } } - func testTarArchiverInvalidArchive() throws { + func testInvalidArchive() throws { try testWithTemporaryDirectory { tmpdir in let archiver = TarArchiver(fileSystem: localFileSystem) let inputArchivePath = AbsolutePath(path: #file).parentDirectory .appending(components: "Inputs", "invalid_archive.tar.gz") XCTAssertThrowsError(try archiver.extract(from: inputArchivePath, to: tmpdir)) { error in + #if os(Linux) + XCTAssertMatch((error as? StringError)?.description, .contains("not in gzip format")) + #else XCTAssertMatch((error as? StringError)?.description, .contains("Unrecognized archive format")) + #endif } } } func testValidation() throws { // valid - try testWithTemporaryDirectory { tmpdir in + try testWithTemporaryDirectory { _ in let archiver = TarArchiver(fileSystem: localFileSystem) let path = AbsolutePath(path: #file).parentDirectory .appending(components: "Inputs", "archive.tar.gz") XCTAssertTrue(try archiver.validate(path: path)) } // invalid - try testWithTemporaryDirectory { tmpdir in + try testWithTemporaryDirectory { _ in let archiver = TarArchiver(fileSystem: localFileSystem) let path = AbsolutePath(path: #file).parentDirectory .appending(components: "Inputs", "invalid_archive.tar.gz") XCTAssertFalse(try archiver.validate(path: path)) } // error - try testWithTemporaryDirectory { tmpdir in + try testWithTemporaryDirectory { _ in let archiver = TarArchiver(fileSystem: localFileSystem) let path = AbsolutePath.root.appending("does_not_exist.tar.gz") XCTAssertThrowsError(try archiver.validate(path: path)) { error in @@ -98,57 +103,57 @@ final class TarArchiverTests: XCTestCase { } #endif - try testWithTemporaryDirectory { tmpdir in - let archiver = TarArchiver(fileSystem: localFileSystem) - - let rootDir = tmpdir.appending(component: UUID().uuidString) - try localFileSystem.createDirectory(rootDir) - try localFileSystem.writeFileContents(rootDir.appending("file1.txt"), string: "Hello World!") - - let dir1 = rootDir.appending("dir1") - try localFileSystem.createDirectory(dir1) - try localFileSystem.writeFileContents(dir1.appending("file2.txt"), string: "Hello World 2!") - - let dir2 = dir1.appending("dir2") - try localFileSystem.createDirectory(dir2) - try localFileSystem.writeFileContents(dir2.appending("file3.txt"), string: "Hello World 3!") - try localFileSystem.writeFileContents(dir2.appending("file4.txt"), string: "Hello World 4!") - - let archivePath = tmpdir.appending(component: UUID().uuidString + ".tar.gz") - try archiver.compress(directory: rootDir, to: archivePath) - XCTAssertFileExists(archivePath) - - let extractRootDir = tmpdir.appending(component: UUID().uuidString) - try localFileSystem.createDirectory(extractRootDir) - try archiver.extract(from: archivePath, to: extractRootDir) - try localFileSystem.stripFirstLevel(of: extractRootDir) - - XCTAssertFileExists(extractRootDir.appending("file1.txt")) - XCTAssertEqual( - try? localFileSystem.readFileContents(extractRootDir.appending("file1.txt")), - "Hello World!" - ) - - let extractedDir1 = extractRootDir.appending("dir1") - XCTAssertDirectoryExists(extractedDir1) - XCTAssertFileExists(extractedDir1.appending("file2.txt")) - XCTAssertEqual( - try? localFileSystem.readFileContents(extractedDir1.appending("file2.txt")), - "Hello World 2!" - ) - - let extractedDir2 = extractedDir1.appending("dir2") - XCTAssertDirectoryExists(extractedDir2) - XCTAssertFileExists(extractedDir2.appending("file3.txt")) - XCTAssertEqual( - try? localFileSystem.readFileContents(extractedDir2.appending("file3.txt")), - "Hello World 3!" - ) - XCTAssertFileExists(extractedDir2.appending("file4.txt")) - XCTAssertEqual( - try? localFileSystem.readFileContents(extractedDir2.appending("file4.txt")), - "Hello World 4!" - ) - } - } + try testWithTemporaryDirectory { tmpdir in + let archiver = TarArchiver(fileSystem: localFileSystem) + + let rootDir = tmpdir.appending(component: UUID().uuidString) + try localFileSystem.createDirectory(rootDir) + try localFileSystem.writeFileContents(rootDir.appending("file1.txt"), string: "Hello World!") + + let dir1 = rootDir.appending("dir1") + try localFileSystem.createDirectory(dir1) + try localFileSystem.writeFileContents(dir1.appending("file2.txt"), string: "Hello World 2!") + + let dir2 = dir1.appending("dir2") + try localFileSystem.createDirectory(dir2) + try localFileSystem.writeFileContents(dir2.appending("file3.txt"), string: "Hello World 3!") + try localFileSystem.writeFileContents(dir2.appending("file4.txt"), string: "Hello World 4!") + + let archivePath = tmpdir.appending(component: UUID().uuidString + ".tar.gz") + try archiver.compress(directory: rootDir, to: archivePath) + XCTAssertFileExists(archivePath) + + let extractRootDir = tmpdir.appending(component: UUID().uuidString) + try localFileSystem.createDirectory(extractRootDir) + try archiver.extract(from: archivePath, to: extractRootDir) + try localFileSystem.stripFirstLevel(of: extractRootDir) + + XCTAssertFileExists(extractRootDir.appending("file1.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractRootDir.appending("file1.txt")), + "Hello World!" + ) + + let extractedDir1 = extractRootDir.appending("dir1") + XCTAssertDirectoryExists(extractedDir1) + XCTAssertFileExists(extractedDir1.appending("file2.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractedDir1.appending("file2.txt")), + "Hello World 2!" + ) + + let extractedDir2 = extractedDir1.appending("dir2") + XCTAssertDirectoryExists(extractedDir2) + XCTAssertFileExists(extractedDir2.appending("file3.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractedDir2.appending("file3.txt")), + "Hello World 3!" + ) + XCTAssertFileExists(extractedDir2.appending("file4.txt")) + XCTAssertEqual( + try? localFileSystem.readFileContents(extractedDir2.appending("file4.txt")), + "Hello World 4!" + ) + } + } } From c6809a3481df730f265f1c303c2f2d2603a94f8c Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 3 Apr 2023 11:50:04 +0100 Subject: [PATCH 4/8] Remove unused import --- Sources/Basics/Archiver+Tar.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift index a0880ef7af4..aa5b96384a0 100644 --- a/Sources/Basics/Archiver+Tar.swift +++ b/Sources/Basics/Archiver+Tar.swift @@ -10,8 +10,6 @@ // //===----------------------------------------------------------------------===// -import _Concurrency - import class Dispatch.DispatchQueue import struct Dispatch.DispatchTime import struct TSCBasic.AbsolutePath From cca0f60359feb726d2d48bfc760c688fdd551382 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 4 Apr 2023 12:39:26 +0100 Subject: [PATCH 5/8] Clean up cancellation errors --- Package.swift | 6 ++++-- Sources/Basics/Archiver+Tar.swift | 6 +++--- Sources/Basics/Archiver+Zip.swift | 6 +++--- Sources/Basics/Cancellator.swift | 19 +++++++++++++++++-- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Package.swift b/Package.swift index 9d23a3b9357..828bcaed236 100644 --- a/Package.swift +++ b/Package.swift @@ -554,8 +554,10 @@ let package = Package( name: "BasicsTests", dependencies: ["Basics", "SPMTestSupport", "tsan_utils"], exclude: [ - "Inputs/archive.zip", - "Inputs/invalid_archive.zip", + "Archiver/Inputs/archive.tar.gz", + "Archiver/Inputs/archive.zip", + "Archiver/Inputs/invalid_archive.tar.gz", + "Archiver/Inputs/invalid_archive.zip", ] ), .testTarget( diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift index aa5b96384a0..500df32cbcc 100644 --- a/Sources/Basics/Archiver+Tar.swift +++ b/Sources/Basics/Archiver+Tar.swift @@ -65,7 +65,7 @@ public final class TarArchiver: Archiver { ) guard let registrationKey = self.cancellator.register(process) else { - throw StringError("cancellation") + throw CancellationError.failedToRegisterProcess(process) } DispatchQueue.sharedConcurrent.async { @@ -99,7 +99,7 @@ public final class TarArchiver: Archiver { ) guard let registrationKey = self.cancellator.register(process) else { - throw StringError("Failed to register cancellation for Archiver") + throw CancellationError.failedToRegisterProcess(process) } DispatchQueue.sharedConcurrent.async { @@ -125,7 +125,7 @@ public final class TarArchiver: Archiver { let process = TSCBasic.Process(arguments: [self.tarCommand, "tf", path.pathString]) guard let registrationKey = self.cancellator.register(process) else { - throw StringError("cancellation") + throw CancellationError.failedToRegisterProcess(process) } DispatchQueue.sharedConcurrent.async { diff --git a/Sources/Basics/Archiver+Zip.swift b/Sources/Basics/Archiver+Zip.swift index 9ead40d6de7..ff2a727a36a 100644 --- a/Sources/Basics/Archiver+Zip.swift +++ b/Sources/Basics/Archiver+Zip.swift @@ -53,7 +53,7 @@ public struct ZipArchiver: Archiver, Cancellable { let process = TSCBasic.Process(arguments: ["unzip", archivePath.pathString, "-d", destinationPath.pathString]) #endif guard let registrationKey = self.cancellator.register(process) else { - throw StringError("cancellation") + throw CancellationError.failedToRegisterProcess(process) } DispatchQueue.sharedConcurrent.async { @@ -95,7 +95,7 @@ public struct ZipArchiver: Archiver, Cancellable { #endif guard let registrationKey = self.cancellator.register(process) else { - throw StringError("Failed to register cancellation for Archiver") + throw CancellationError.failedToRegisterProcess(process) } DispatchQueue.sharedConcurrent.async { @@ -125,7 +125,7 @@ public struct ZipArchiver: Archiver, Cancellable { let process = TSCBasic.Process(arguments: ["unzip", "-t", path.pathString]) #endif guard let registrationKey = self.cancellator.register(process) else { - throw StringError("cancellation") + throw CancellationError.failedToRegisterProcess(process) } DispatchQueue.sharedConcurrent.async { diff --git a/Sources/Basics/Cancellator.swift b/Sources/Basics/Cancellator.swift index 3a2387b1d03..f1a86dc2a8b 100644 --- a/Sources/Basics/Cancellator.swift +++ b/Sources/Basics/Cancellator.swift @@ -112,9 +112,24 @@ public protocol Cancellable { } public struct CancellationError: Error, CustomStringConvertible { - public let description = "Operation cancelled" + public let description: String - public init() {} + public init() { + self.init(description: "Operation cancelled") + } + + private init(description: String) { + self.description = description + } + + static func failedToRegisterProcess(_ process: TSCBasic.Process) -> Self { + Self(description: """ + failed to register a cancellation handler for this process invocation `\( + process.arguments.joined(separator: " ") + )` + """ + ) + } } extension TSCBasic.Process { From 2efc0b99aa6c50c9af168b20e0d5b2b57574f0df Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 4 Apr 2023 12:40:32 +0100 Subject: [PATCH 6/8] Update tar archiver process arguments --- Sources/Basics/Archiver+Tar.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift index 500df32cbcc..352017bbc49 100644 --- a/Sources/Basics/Archiver+Tar.swift +++ b/Sources/Basics/Archiver+Tar.swift @@ -61,7 +61,7 @@ public final class TarArchiver: Archiver { } let process = TSCBasic.Process( - arguments: [self.tarCommand, "xzf", archivePath.pathString, "-C", destinationPath.pathString] + arguments: [self.tarCommand, "zxf", archivePath.pathString, "-C", destinationPath.pathString] ) guard let registrationKey = self.cancellator.register(process) else { From c00497ea7081113307d6400eb7dbd9e80b7f9a1e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 4 Apr 2023 18:01:06 +0100 Subject: [PATCH 7/8] Address PR feedback --- Sources/Basics/Archiver+Tar.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift index 352017bbc49..6476212c9fb 100644 --- a/Sources/Basics/Archiver+Tar.swift +++ b/Sources/Basics/Archiver+Tar.swift @@ -18,8 +18,8 @@ import struct TSCBasic.FileSystemError import class TSCBasic.Process /// An `Archiver` that handles Tar archives using the command-line `tar` tool. -public final class TarArchiver: Archiver { - public var supportedExtensions: Set { ["tar", "tar.gz"] } +public struct TarArchiver: Archiver { + public let supportedExtensions: Set = ["tar", "tar.gz"] /// The file-system implementation used for various file-system operations and checks. private let fileSystem: FileSystem From 2cb673bfe1c8a18f4e12d6268c6d9d4b6ff5d782 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 4 Apr 2023 18:03:59 +0100 Subject: [PATCH 8/8] Formatting fix --- Sources/Basics/Archiver+Tar.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Basics/Archiver+Tar.swift b/Sources/Basics/Archiver+Tar.swift index 6476212c9fb..5d4a210a3f2 100644 --- a/Sources/Basics/Archiver+Tar.swift +++ b/Sources/Basics/Archiver+Tar.swift @@ -19,7 +19,7 @@ import class TSCBasic.Process /// An `Archiver` that handles Tar archives using the command-line `tar` tool. public struct TarArchiver: Archiver { - public let supportedExtensions: Set = ["tar", "tar.gz"] + public let supportedExtensions: Set = ["tar", "tar.gz"] /// The file-system implementation used for various file-system operations and checks. private let fileSystem: FileSystem