From 784cfcb6f616f31a138666de78f0d28e8373b7d8 Mon Sep 17 00:00:00 2001 From: Fernando Rojo Date: Mon, 13 May 2024 16:05:56 -0700 Subject: [PATCH 1/4] Update RustCli Parsing to process pkgId, and allow manual override to fallback with DisableRustCliScan --- .../rust/RustCliDetector.cs | 100 ++++++++++++++++-- 1 file changed, 92 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs index d5463cdae..c8d9da527 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs @@ -13,6 +13,7 @@ namespace Microsoft.ComponentDetection.Detectors.Rust; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Rust.Contracts; using Microsoft.Extensions.Logging; +using MoreLinq.Extensions; using Newtonsoft.Json; using Tomlyn; @@ -21,8 +22,11 @@ namespace Microsoft.ComponentDetection.Detectors.Rust; /// public class RustCliDetector : FileComponentDetector { - //// PkgName[ Version][ (Source)] - private static readonly Regex DependencyFormatRegex = new Regex( + private static readonly Regex DependencyFormatRegexPkgId = new Regex( + @"^([^#@]+)?(?:[@#]?([^@]*))(?:@(.+))?$", + RegexOptions.Compiled); + + private static readonly Regex DependencyFormatRegexCargoLock = new Regex( @"^(?[^ ]+)(?: (?[^ ]+))?(?: \((?[^()]*)\))?$", RegexOptions.Compiled); @@ -33,22 +37,27 @@ public class RustCliDetector : FileComponentDetector private readonly ICommandLineInvocationService cliService; + private readonly IEnvironmentVariableService envVarService; + /// /// Initializes a new instance of the class. /// /// The component stream enumerable factory. /// The walker factory. /// The command line invocation service. + /// The environment variable reader service. /// The logger. public RustCliDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, ICommandLineInvocationService cliService, + IEnvironmentVariableService envVarService, ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; this.cliService = cliService; + this.envVarService = envVarService; this.Logger = logger; } @@ -77,7 +86,14 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID try { - if (!await this.cliService.CanCommandBeLocatedAsync("cargo", null)) + if (this.IsRustCliManuallyDisabled()) + { + this.Logger.LogWarning("Rust Cli has been manually disabled, fallback strategy performed."); + record.DidRustCliCommandFail = false; + record.WasRustFallbackStrategyUsed = true; + record.FallbackReason = "Manually Disabled"; + } + else if (!await this.cliService.CanCommandBeLocatedAsync("cargo", null)) { this.Logger.LogWarning("Could not locate cargo command. Skipping Rust CLI detection"); record.DidRustCliCommandFail = true; @@ -187,9 +203,71 @@ private static bool ShouldFallbackFromError(string error) return true; } - private static bool ParseDependency(string dependency, out string packageName, out string version, out string source) + private static bool ParseDependencyMetadata(string dependency, out string packageName, out string version, out string source) + { + // There are a few different formats for pkgids: https://doc.rust-lang.org/cargo/commands/cargo-pkgid.html#description + // 1. name => packageName + // 2. name@version packageName@1.0.4 + // 3. url => https://github.com/rust-lang/cargo + // 4. url#version => https://github.com/rust-lang/cargo#0.33.0 + // 5. url#name => https://github.com/rust-lang/crates.io-index#packageName + // 6. url#name@version => https://github.com/rust-lang/cargo#crates-io@0.21.0 + var match = DependencyFormatRegexPkgId.Match(dependency); + packageName = null; + version = null; + source = null; + if (!match.Success) + { + return false; + } + + var firstGroup = match.Groups[1]; + var secondGroup = match.Groups[2]; + var thirdGroup = match.Groups[3]; + + // cases 3-6 + if (Uri.IsWellFormedUriString(dependency, UriKind.Absolute)) + { + // in this case, first group is guaranteed to be the source. + // packageName is also set here for case 3 + source = firstGroup.Success ? firstGroup.Value : null; + packageName = source; + + // if there is a third group, then the second must be packageName, third is version. + if (thirdGroup.Success) + { + packageName = secondGroup.Value; + version = thirdGroup.Value; + } + + // if there is no third group, but there is a second, the second group could be either the name or the version, check if the value starts with a number (not allowed) + else if (secondGroup.Success) + { + var nameOrVersion = secondGroup.Value; + if (char.IsDigit(nameOrVersion[0])) + { + version = nameOrVersion; + } + else + { + packageName = nameOrVersion; + } + } + } + + // cases 1 and 2 + else + { + packageName = firstGroup.Success ? firstGroup.Value : null; + version = secondGroup.Success ? secondGroup.Value : null; + } + + return match.Success; + } + + private static bool ParseDependencyCargoLock(string dependency, out string packageName, out string version, out string source) { - var match = DependencyFormatRegex.Match(dependency); + var match = DependencyFormatRegexCargoLock.Match(dependency); var packageNameMatch = match.Groups["packageName"]; var versionMatch = match.Groups["version"]; var sourceMatch = match.Groups["source"]; @@ -206,6 +284,11 @@ private static bool ParseDependency(string dependency, out string packageName, o return match.Success; } + private bool IsRustCliManuallyDisabled() + { + return this.envVarService.IsEnvironmentVariableValueTrue("DisableRustCliScan"); + } + private void TraverseAndRecordComponents( ISingleFileComponentRecorder recorder, string location, @@ -221,7 +304,8 @@ private void TraverseAndRecordComponents( try { var isDevelopmentDependency = depInfo?.DepKinds.Any(x => x.Kind is Kind.Dev) ?? false; - if (!ParseDependency(id, out var name, out var version, out var source)) + + if (!ParseDependencyMetadata(id, out var name, out var version, out var source)) { // Could not parse the dependency string this.Logger.LogWarning("Failed to parse dependency '{Id}'", id); @@ -298,7 +382,7 @@ private async Task ProcessCargoLockFallbackAsync(IComponentStream cargoTomlFile, var cargoLockFileStream = this.FindCorrespondingCargoLock(cargoTomlFile, singleFileComponentRecorder); if (cargoLockFileStream == null) { - this.Logger.LogWarning("Could not find Cargo.lock file for {CargoTomlLocation}, skipping processing", cargoTomlFile.Location); + this.Logger.LogWarning("Fallback failed, could not find Cargo.lock file for {CargoTomlLocation}, skipping processing", cargoTomlFile.Location); record.FallbackCargoLockFound = false; return; } @@ -402,7 +486,7 @@ private void ProcessDependency( try { // Extract the information from the dependency (name with optional version and source) - if (!ParseDependency(dependency, out var childName, out var childVersion, out var childSource)) + if (!ParseDependencyCargoLock(dependency, out var childName, out var childVersion, out var childSource)) { // Could not parse the dependency string throw new FormatException($"Failed to parse dependency '{dependency}'"); From 2712c44b910384711681a2ed673523db17e64826 Mon Sep 17 00:00:00 2001 From: Fernando Rojo Date: Mon, 13 May 2024 19:52:34 -0700 Subject: [PATCH 2/4] add tests --- .../rust/RustCliDetector.cs | 10 + .../RustCliDetectorTests.cs | 311 +++++++++++++++++- 2 files changed, 303 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs index c8d9da527..c2d6ad0a3 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs @@ -212,6 +212,16 @@ private static bool ParseDependencyMetadata(string dependency, out string packag // 4. url#version => https://github.com/rust-lang/cargo#0.33.0 // 5. url#name => https://github.com/rust-lang/crates.io-index#packageName // 6. url#name@version => https://github.com/rust-lang/cargo#crates-io@0.21.0 + + // First, try parsing using the old format in cases where a version of rust older than 1.77 is being used. + if (ParseDependencyCargoLock(dependency, out packageName, out version, out source)) + { + if (!(string.IsNullOrEmpty(packageName) || string.IsNullOrEmpty(version))) + { + return true; + } + } + var match = DependencyFormatRegexPkgId.Match(dependency); packageName = null; version = null; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs index 882411c90..550ad6d12 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs @@ -193,6 +193,180 @@ public class RustCliDetectorTests : BaseDetectorTest ""metadata"": null }"; + private readonly string mockMetadataV1pkgId = @" +{ + ""packages"": [ + { + ""name"": ""registry-package-1"", + ""version"": ""1.0.1"", + ""id"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", + ""license"": null, + ""license_file"": null, + ""description"": ""test registry package 1"", + ""source"": ""registry+https://test.com/registry-package-1"", + ""dependencies"": [ + { + ""name"": ""inner-dependency-1"", + ""source"": ""registry+registry+https://test.com/inner-dependency-1"", + ""req"": ""^0.3.0"", + ""kind"": ""dev"", + ""rename"": null, + ""optional"": false, + ""uses_default_features"": true, + ""features"": [], + ""target"": null, + ""registry"": null + } + ] + }, + { + ""name"": ""rust-test"", + ""version"": ""0.1.0"", + ""id"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"", + ""license"": null, + ""license_file"": null, + ""description"": null, + ""source"": null, + ""dependencies"": [ + { + ""name"": ""registry-package-1"", + ""source"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", + ""req"": ""^1.0.1"", + ""kind"": null, + ""rename"": null, + ""optional"": false, + ""uses_default_features"": true, + ""features"": [], + ""target"": null, + ""registry"": null + }, + { + ""name"": ""rust-test-inner"", + ""source"": ""(path+file:///C:/test/rust-test-inner)"", + ""req"": ""*"", + ""kind"": null, + ""rename"": null, + ""optional"": false, + ""uses_default_features"": true, + ""features"": [], + ""target"": null, + ""registry"": null, + ""path"": ""C:\\test\\rust-test-inner"" + }, + { + ""name"": ""dev-dependency-1"", + ""source"": ""registry+https://test.com/dev-dependency-1"", + ""req"": ""^0.4.0"", + ""kind"": ""dev"", + ""rename"": null, + ""optional"": false, + ""uses_default_features"": true, + ""features"": [], + ""target"": null, + ""registry"": null + } + ] + }, + { + ""name"": ""rust-test-inner"", + ""version"": ""0.1.0"", + ""id"": ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", + ""license"": null, + ""license_file"": null, + ""description"": null, + ""source"": null, + ""dependencies"": [] + }, + { + ""name"": ""dev-dependency-1"", + ""version"": ""0.4.0"", + ""id"": ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"", + ""license"": null, + ""license_file"": null, + ""description"": ""test dev dependency"", + ""source"": ""registry+https://github.com/rust-lang/crates.io-index"", + ""dependencies"": [] + } + ], + ""workspace_members"": [ + ""path+file:///C:/test#rust-test@0.1.0"" + ], + ""workspace_default_members"": [ + ""path+file:///C:/test#rust-test@0.1.0"" + ], + ""resolve"": { + ""nodes"": [ + { + ""id"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", + ""dependencies"": [], + ""deps"": [], + ""features"": [ + ""default"", + ""std"" + ] + }, + { + ""id"": ""path+file:///C:/test#rust-test@0.1.0"", + ""dependencies"": [ + ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", + ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", + ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"" + ], + ""deps"": [ + { + ""name"": ""registry-package-1"", + ""pkg"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", + ""dep_kinds"": [ + { + ""kind"": null, + ""target"": null + } + ] + }, + { + ""name"": ""cargo"", + ""pkg"": ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", + ""dep_kinds"": [ + { + ""kind"": null, + ""target"": null + } + ] + }, + { + ""name"": ""dev-dependency-1"", + ""pkg"": ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"", + ""dep_kinds"": [ + { + ""kind"": ""dev"", + ""target"": null + } + ] + } + ], + ""features"": [] + }, + { + ""id"": ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", + ""dependencies"": [], + ""deps"": [], + ""features"": [] + }, + { + ""id"": ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"", + ""dependencies"": [], + ""deps"": [], + ""features"": [] + } + ], + ""root"": ""path+file:///C:/test#rust-test@0.1.0"" + }, + ""target_directory"": ""C:\\test"", + ""version"": 1, + ""workspace_root"": ""C:\\test"", + ""metadata"": null +}"; + private readonly string mockMetadataWithLicenses = @" { ""packages"": [ @@ -504,15 +678,20 @@ public class RustCliDetectorTests : BaseDetectorTest }"; private Mock mockCliService; + + private Mock mockEnvVarService; + private Mock mockComponentStreamEnumerableFactory; [TestInitialize] - public void InitCliMock() + public void InitMocks() { this.mockCliService = new Mock(); this.DetectorTestUtility.AddServiceMock(this.mockCliService); this.mockComponentStreamEnumerableFactory = new Mock(); this.DetectorTestUtility.AddServiceMock(this.mockComponentStreamEnumerableFactory); + this.mockEnvVarService = new Mock(); + this.DetectorTestUtility.AddServiceMock(this.mockEnvVarService); } [TestMethod] @@ -548,6 +727,80 @@ public async Task RustCliDetector_FailExecutingCommandSuccessAsync() componentRecorder.GetDetectedComponents().Should().BeEmpty(); } + [TestMethod] + public async Task RustCliDetector_RespectsFallBackVariableAsync() + { + var testCargoLockString = @" +[[package]] +name = ""my_dependency"" +version = ""1.0.0"" +source = ""registry+https://github.com/rust-lang/crates.io-index"" +dependencies = [ + ""same_package 1.0.0"" +] + +[[package]] +name = ""other_dependency"" +version = ""0.4.0"" +source = ""registry+https://github.com/rust-lang/crates.io-index"" +dependencies = [ + ""other_dependency_dependency"", +] + +[[package]] +name = ""other_dependency_dependency"" +version = ""0.1.12-alpha.6"" +source = ""registry+https://github.com/rust-lang/crates.io-index"" + +[[package]] +name = ""my_dev_dependency"" +version = ""1.0.0"" +source = ""registry+https://github.com/rust-lang/crates.io-index"" +dependencies = [ + ""other_dependency_dependency 0.1.12-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)"", + ""dev_dependency_dependency 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)"", +]"; + this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("cargo", It.IsAny>())).ReturnsAsync(true); + this.mockCliService.Setup(x => x.ExecuteCommandAsync(It.IsAny(), It.IsAny>(), It.IsAny())) + .Throws(new InvalidOperationException()); + this.mockEnvVarService + .Setup(x => x.IsEnvironmentVariableValueTrue("DisableRustCliScan")) + .Returns(true); + using var stream = new MemoryStream(); + using var writer = new StreamWriter(stream); + await writer.WriteAsync(testCargoLockString); + await writer.FlushAsync(); + stream.Position = 0; + this.mockComponentStreamEnumerableFactory.Setup(x => x.GetComponentStreams(It.IsAny(), new List { "Cargo.lock" }, It.IsAny(), false)) + .Returns(new[] { new ComponentStream() { Location = "Cargo.toml", Stream = stream } }); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Cargo.toml", string.Empty) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().HaveCount(4); + + componentRecorder + .GetDetectedComponents() + .Select(x => x.Component.Id) + .Should() + .BeEquivalentTo("other_dependency_dependency 0.1.12-alpha.6 - Cargo", "my_dev_dependency 1.0.0 - Cargo", "my_dependency 1.0.0 - Cargo", "other_dependency 0.4.0 - Cargo"); + + var components = componentRecorder.GetDetectedComponents(); + + foreach (var component in components) + { + if (component.Component is CargoComponent cargoComponent) + { + cargoComponent.Author.Should().Be(null); + cargoComponent.License.Should().Be(null); + } + } + + return; + } + [TestMethod] public async Task RustCliDetector_HandlesNonZeroExitCodeAsync() { @@ -586,6 +839,28 @@ public async Task RustCliDetector_RegistersCorrectRootDepsAsync() .BeEquivalentTo("registry-package-1 1.0.1 - Cargo", "dev-dependency-1 0.4.0 - Cargo"); } + [TestMethod] + public async Task RustCliDetector_RegistersCorrectRootDepsPkgIdAsync() + { + var cargoMetadata = this.mockMetadataV1pkgId; + this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("cargo", It.IsAny>())).ReturnsAsync(true); + this.mockCliService.Setup(x => x.ExecuteCommandAsync(It.IsAny(), It.IsAny>(), It.IsAny())) + .ReturnsAsync(new CommandLineExecutionResult { StdOut = cargoMetadata }); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Cargo.toml", string.Empty) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().HaveCount(2); + + componentRecorder + .GetDetectedComponents() + .Select(x => x.Component.Id) + .Should() + .BeEquivalentTo("registry-package-1 1.0.1 - Cargo", "dev-dependency-1 0.4.0 - Cargo"); + } + [TestMethod] public async Task RustCliDetector_NotInGraphAsync() { @@ -593,19 +868,19 @@ public async Task RustCliDetector_NotInGraphAsync() { ""packages"": [], ""workspace_members"": [ - ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"" + ""path+file:///home/justin/rust-test#rust-test@0.1.0"" ], ""resolve"": { ""nodes"": [ { - ""id"": ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"", + ""id"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"", ""dependencies"": [ - ""libc 0.2.147 (registry+https://github.com/rust-lang/crates.io-index)"" + ""registry+https://github.com/rust-lang/crates.io-index#libc@0.2.147"" ], ""deps"": [ { ""name"": ""libc"", - ""pkg"": ""libc 0.2.147 (registry+https://github.com/rust-lang/crates.io-index)"", + ""pkg"": ""registry+https://github.com/rust-lang/crates.io-index#libc@0.2.147"", ""dep_kinds"": [ { ""kind"": null, @@ -617,7 +892,7 @@ public async Task RustCliDetector_NotInGraphAsync() ""features"": [] } ], - ""root"": ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"" + ""root"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"" }, ""target_directory"": ""/home/justin/rust-test/target"", ""version"": 1, @@ -643,14 +918,14 @@ public async Task RustCliDetector_InvalidNameAsync() { ""packages"": [], ""workspace_members"": [ - ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"" + ""path+file:///home/justin/rust-test#rust-test@0.1.0"" ], ""resolve"": { ""nodes"": [ { - ""id"": ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"", + ""id"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"", ""dependencies"": [ - ""libc 0.2.147 (registry+https://github.com/rust-lang/crates.io-index)"" + ""registry+https://github.com/rust-lang/crates.io-index#libc@0.2.147"" ], ""deps"": [ { @@ -667,7 +942,7 @@ public async Task RustCliDetector_InvalidNameAsync() ""features"": [] } ], - ""root"": ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"" + ""root"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"" }, ""target_directory"": ""/home/justin/rust-test/target"", ""version"": 1, @@ -759,12 +1034,12 @@ public async Task RustCliDetector_AuthorAndLicenseEmptyStringAsync() { ""packages"": [], ""workspace_members"": [ - ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"" + ""path+file:///home/justin/rust-test#rust-test@0.1.0"" ], ""resolve"": { ""nodes"": [ { - ""id"": ""libc 0.2.147 (registry+https://github.com/rust-lang/crates.io-index)"", + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#libc@0.2.147"", ""dependencies"": [], ""deps"": [], ""features"": [ @@ -773,14 +1048,14 @@ public async Task RustCliDetector_AuthorAndLicenseEmptyStringAsync() ] }, { - ""id"": ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"", + ""id"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"", ""dependencies"": [ - ""libc 0.2.147 (registry+https://github.com/rust-lang/crates.io-index)"" + ""registry+https://github.com/rust-lang/crates.io-index#libc@0.2.147"" ], ""deps"": [ { ""name"": ""libc"", - ""pkg"": ""libc 0.2.147 (registry+https://github.com/rust-lang/crates.io-index)"", + ""pkg"": ""registry+https://github.com/rust-lang/crates.io-index#libc@0.2.147"", ""dep_kinds"": [ { ""kind"": null, @@ -792,13 +1067,13 @@ public async Task RustCliDetector_AuthorAndLicenseEmptyStringAsync() ""features"": [] } ], - ""root"": ""rust-test 0.1.0 (path+file:///home/justin/rust-test)"" + ""root"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"" }, ""packages"": [ { ""name"": ""libc"", ""version"": ""0.2.147"", - ""id"": ""libc 0.2.147 (registry+https://github.com/rust-lang/crates.io-index)"", + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#libc@0.2.147"", ""license"": """", ""license_file"": null, ""description"": """", @@ -826,7 +1101,7 @@ public async Task RustCliDetector_AuthorAndLicenseEmptyStringAsync() { ""name"": ""rust-test"", ""version"": ""0.1.0"", - ""id"": ""rust-test (registry+https://github.com/rust-lang/crates.io-index)"", + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#rust-test"", ""license"": """", ""license_file"": null, ""description"": ""A non-cryptographic hash function using AES-NI for high performance"", From ea2ef827ce4e1961a845625b632a618a3e00563c Mon Sep 17 00:00:00 2001 From: Fernando Rojo Date: Mon, 13 May 2024 20:25:34 -0700 Subject: [PATCH 3/4] Update detector version --- .../rust/RustCliDetector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs index c2d6ad0a3..7b5572529 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs @@ -71,7 +71,7 @@ public RustCliDetector( public override IEnumerable SupportedComponentTypes => new[] { ComponentType.Cargo }; /// - public override int Version => 3; + public override int Version => 4; /// public override IList SearchPatterns { get; } = new[] { "Cargo.toml" }; From e24a67823060e9fe655c28cc968cde63024cdde7 Mon Sep 17 00:00:00 2001 From: Fernando Rojo Date: Wed, 15 May 2024 18:21:20 -0700 Subject: [PATCH 4/4] Update cli detector to use manifest packages instead of manually parsing --- .../TypedComponent/CargoComponent.cs | 6 +- .../rust/RustCliDetector.cs | 99 +-------- .../RustCliDetectorTests.cs | 199 +----------------- 3 files changed, 17 insertions(+), 287 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index ee76ad126..7ac040943 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -10,12 +10,13 @@ private CargoComponent() // reserved for deserialization } - public CargoComponent(string name, string version, string author = null, string license = null) + public CargoComponent(string name, string version, string author = null, string license = null, string source = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Cargo)); this.Version = this.ValidateRequiredInput(version, nameof(this.Version), nameof(ComponentType.Cargo)); this.Author = author; this.License = license; + this.Source = source; } public string Name { get; set; } @@ -28,6 +29,9 @@ public CargoComponent(string name, string version, string author = null, string [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] public string? License { get; set; } + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? Source { get; set; } #nullable disable public override ComponentType Type => ComponentType.Cargo; diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs index 7b5572529..5aa4408de 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs @@ -22,10 +22,6 @@ namespace Microsoft.ComponentDetection.Detectors.Rust; /// public class RustCliDetector : FileComponentDetector { - private static readonly Regex DependencyFormatRegexPkgId = new Regex( - @"^([^#@]+)?(?:[@#]?([^@]*))(?:@(.+))?$", - RegexOptions.Compiled); - private static readonly Regex DependencyFormatRegexCargoLock = new Regex( @"^(?[^ ]+)(?: (?[^ ]+))?(?: \((?[^()]*)\))?$", RegexOptions.Compiled); @@ -128,10 +124,13 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var graph = BuildGraph(metadata); var packages = metadata.Packages.ToDictionary( - x => $"{x.Name} {x.Version}", - x => ( + x => $"{x.Id}", + x => new CargoComponent( + x.Name, + x.Version, (x.Authors == null || x.Authors.Any(a => string.IsNullOrWhiteSpace(a)) || !x.Authors.Any()) ? null : string.Join(", ", x.Authors), - string.IsNullOrWhiteSpace(x.License) ? null : x.License)); + string.IsNullOrWhiteSpace(x.License) ? null : x.License, + x.Source)); var root = metadata.Resolve.Root; HashSet visitedDependencies = new(); @@ -203,78 +202,6 @@ private static bool ShouldFallbackFromError(string error) return true; } - private static bool ParseDependencyMetadata(string dependency, out string packageName, out string version, out string source) - { - // There are a few different formats for pkgids: https://doc.rust-lang.org/cargo/commands/cargo-pkgid.html#description - // 1. name => packageName - // 2. name@version packageName@1.0.4 - // 3. url => https://github.com/rust-lang/cargo - // 4. url#version => https://github.com/rust-lang/cargo#0.33.0 - // 5. url#name => https://github.com/rust-lang/crates.io-index#packageName - // 6. url#name@version => https://github.com/rust-lang/cargo#crates-io@0.21.0 - - // First, try parsing using the old format in cases where a version of rust older than 1.77 is being used. - if (ParseDependencyCargoLock(dependency, out packageName, out version, out source)) - { - if (!(string.IsNullOrEmpty(packageName) || string.IsNullOrEmpty(version))) - { - return true; - } - } - - var match = DependencyFormatRegexPkgId.Match(dependency); - packageName = null; - version = null; - source = null; - if (!match.Success) - { - return false; - } - - var firstGroup = match.Groups[1]; - var secondGroup = match.Groups[2]; - var thirdGroup = match.Groups[3]; - - // cases 3-6 - if (Uri.IsWellFormedUriString(dependency, UriKind.Absolute)) - { - // in this case, first group is guaranteed to be the source. - // packageName is also set here for case 3 - source = firstGroup.Success ? firstGroup.Value : null; - packageName = source; - - // if there is a third group, then the second must be packageName, third is version. - if (thirdGroup.Success) - { - packageName = secondGroup.Value; - version = thirdGroup.Value; - } - - // if there is no third group, but there is a second, the second group could be either the name or the version, check if the value starts with a number (not allowed) - else if (secondGroup.Success) - { - var nameOrVersion = secondGroup.Value; - if (char.IsDigit(nameOrVersion[0])) - { - version = nameOrVersion; - } - else - { - packageName = nameOrVersion; - } - } - } - - // cases 1 and 2 - else - { - packageName = firstGroup.Success ? firstGroup.Value : null; - version = secondGroup.Success ? secondGroup.Value : null; - } - - return match.Success; - } - private static bool ParseDependencyCargoLock(string dependency, out string packageName, out string version, out string source) { var match = DependencyFormatRegexCargoLock.Match(dependency); @@ -306,7 +233,7 @@ private void TraverseAndRecordComponents( string id, DetectedComponent parent, Dep depInfo, - IReadOnlyDictionary packagesMetadata, + IReadOnlyDictionary packagesMetadata, ISet visitedDependencies, bool explicitlyReferencedDependency = false, bool isTomlRoot = false) @@ -315,18 +242,14 @@ private void TraverseAndRecordComponents( { var isDevelopmentDependency = depInfo?.DepKinds.Any(x => x.Kind is Kind.Dev) ?? false; - if (!ParseDependencyMetadata(id, out var name, out var version, out var source)) + if (!packagesMetadata.TryGetValue($"{id}", out var cargoComponent)) { // Could not parse the dependency string - this.Logger.LogWarning("Failed to parse dependency '{Id}'", id); + this.Logger.LogWarning("Did not find dependency '{Id}' in Manifest.packages, skipping", id); return; } - var (authors, license) = packagesMetadata.TryGetValue($"{name} {version}", out var package) - ? package - : (null, null); - - var detectedComponent = new DetectedComponent(new CargoComponent(name, version, authors, license)); + var detectedComponent = new DetectedComponent(cargoComponent); if (!graph.TryGetValue(id, out var node)) { @@ -334,7 +257,7 @@ private void TraverseAndRecordComponents( return; } - var shouldRegister = !isTomlRoot && !source.StartsWith("path+file"); + var shouldRegister = !isTomlRoot && cargoComponent.Source != null; if (shouldRegister) { recorder.RegisterUsage( diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs index 550ad6d12..7932f90c7 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliDetectorTests.cs @@ -193,180 +193,6 @@ public class RustCliDetectorTests : BaseDetectorTest ""metadata"": null }"; - private readonly string mockMetadataV1pkgId = @" -{ - ""packages"": [ - { - ""name"": ""registry-package-1"", - ""version"": ""1.0.1"", - ""id"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", - ""license"": null, - ""license_file"": null, - ""description"": ""test registry package 1"", - ""source"": ""registry+https://test.com/registry-package-1"", - ""dependencies"": [ - { - ""name"": ""inner-dependency-1"", - ""source"": ""registry+registry+https://test.com/inner-dependency-1"", - ""req"": ""^0.3.0"", - ""kind"": ""dev"", - ""rename"": null, - ""optional"": false, - ""uses_default_features"": true, - ""features"": [], - ""target"": null, - ""registry"": null - } - ] - }, - { - ""name"": ""rust-test"", - ""version"": ""0.1.0"", - ""id"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"", - ""license"": null, - ""license_file"": null, - ""description"": null, - ""source"": null, - ""dependencies"": [ - { - ""name"": ""registry-package-1"", - ""source"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", - ""req"": ""^1.0.1"", - ""kind"": null, - ""rename"": null, - ""optional"": false, - ""uses_default_features"": true, - ""features"": [], - ""target"": null, - ""registry"": null - }, - { - ""name"": ""rust-test-inner"", - ""source"": ""(path+file:///C:/test/rust-test-inner)"", - ""req"": ""*"", - ""kind"": null, - ""rename"": null, - ""optional"": false, - ""uses_default_features"": true, - ""features"": [], - ""target"": null, - ""registry"": null, - ""path"": ""C:\\test\\rust-test-inner"" - }, - { - ""name"": ""dev-dependency-1"", - ""source"": ""registry+https://test.com/dev-dependency-1"", - ""req"": ""^0.4.0"", - ""kind"": ""dev"", - ""rename"": null, - ""optional"": false, - ""uses_default_features"": true, - ""features"": [], - ""target"": null, - ""registry"": null - } - ] - }, - { - ""name"": ""rust-test-inner"", - ""version"": ""0.1.0"", - ""id"": ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", - ""license"": null, - ""license_file"": null, - ""description"": null, - ""source"": null, - ""dependencies"": [] - }, - { - ""name"": ""dev-dependency-1"", - ""version"": ""0.4.0"", - ""id"": ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"", - ""license"": null, - ""license_file"": null, - ""description"": ""test dev dependency"", - ""source"": ""registry+https://github.com/rust-lang/crates.io-index"", - ""dependencies"": [] - } - ], - ""workspace_members"": [ - ""path+file:///C:/test#rust-test@0.1.0"" - ], - ""workspace_default_members"": [ - ""path+file:///C:/test#rust-test@0.1.0"" - ], - ""resolve"": { - ""nodes"": [ - { - ""id"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", - ""dependencies"": [], - ""deps"": [], - ""features"": [ - ""default"", - ""std"" - ] - }, - { - ""id"": ""path+file:///C:/test#rust-test@0.1.0"", - ""dependencies"": [ - ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", - ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", - ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"" - ], - ""deps"": [ - { - ""name"": ""registry-package-1"", - ""pkg"": ""registry+https://test.com/registry-package-1#registry-package-1@1.0.1"", - ""dep_kinds"": [ - { - ""kind"": null, - ""target"": null - } - ] - }, - { - ""name"": ""cargo"", - ""pkg"": ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", - ""dep_kinds"": [ - { - ""kind"": null, - ""target"": null - } - ] - }, - { - ""name"": ""dev-dependency-1"", - ""pkg"": ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"", - ""dep_kinds"": [ - { - ""kind"": ""dev"", - ""target"": null - } - ] - } - ], - ""features"": [] - }, - { - ""id"": ""path+file:///C:/test/rust-test-inner#rust-test-inner@0.1.0"", - ""dependencies"": [], - ""deps"": [], - ""features"": [] - }, - { - ""id"": ""registry+https://test.com/dev-dependency-1#dev-dependency-1@0.4.0"", - ""dependencies"": [], - ""deps"": [], - ""features"": [] - } - ], - ""root"": ""path+file:///C:/test#rust-test@0.1.0"" - }, - ""target_directory"": ""C:\\test"", - ""version"": 1, - ""workspace_root"": ""C:\\test"", - ""metadata"": null -}"; - private readonly string mockMetadataWithLicenses = @" { ""packages"": [ @@ -839,28 +665,6 @@ public async Task RustCliDetector_RegistersCorrectRootDepsAsync() .BeEquivalentTo("registry-package-1 1.0.1 - Cargo", "dev-dependency-1 0.4.0 - Cargo"); } - [TestMethod] - public async Task RustCliDetector_RegistersCorrectRootDepsPkgIdAsync() - { - var cargoMetadata = this.mockMetadataV1pkgId; - this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("cargo", It.IsAny>())).ReturnsAsync(true); - this.mockCliService.Setup(x => x.ExecuteCommandAsync(It.IsAny(), It.IsAny>(), It.IsAny())) - .ReturnsAsync(new CommandLineExecutionResult { StdOut = cargoMetadata }); - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("Cargo.toml", string.Empty) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - componentRecorder.GetDetectedComponents().Should().HaveCount(2); - - componentRecorder - .GetDetectedComponents() - .Select(x => x.Component.Id) - .Should() - .BeEquivalentTo("registry-package-1 1.0.1 - Cargo", "dev-dependency-1 0.4.0 - Cargo"); - } - [TestMethod] public async Task RustCliDetector_NotInGraphAsync() { @@ -1032,7 +836,6 @@ public async Task RustCliDetector_AuthorAndLicenseEmptyStringAsync() { var cargoMetadata = @" { - ""packages"": [], ""workspace_members"": [ ""path+file:///home/justin/rust-test#rust-test@0.1.0"" ], @@ -1101,7 +904,7 @@ public async Task RustCliDetector_AuthorAndLicenseEmptyStringAsync() { ""name"": ""rust-test"", ""version"": ""0.1.0"", - ""id"": ""registry+https://github.com/rust-lang/crates.io-index#rust-test"", + ""id"": ""path+file:///home/justin/rust-test#rust-test@0.1.0"", ""license"": """", ""license_file"": null, ""description"": ""A non-cryptographic hash function using AES-NI for high performance"",