From b3ce58296cff9805dc8fedb02ffb26f6903203d5 Mon Sep 17 00:00:00 2001 From: Sebastian Gomez <69322674+sebasgomez238@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:09:25 -0500 Subject: [PATCH] Extract package info from local package cache for NuGet and Maven (#441) * Initial commit for package details factory. * fix indentation. * PackageDetailsFactory improvements * Remove unnecesary if statement * Add telemetry for packageDetails entries. * Minor improvements * Distinguish between organization and person (Maven) * Update tests. * Address Feedback * Address feedback. * Remove NoWarn * Rename to ExtendedScannedResult --------- Co-authored-by: Sebastian Gomez --- Directory.Packages.props | 1 + .../CargoComponentExtensions.cs | 8 +- ...ltWithLicense.cs => ExtendedScanResult.cs} | 4 +- ...License.cs => ExtendedScannedComponent.cs} | 24 ++- .../MavenComponentExtensions.cs | 8 +- .../NpmComponentExtensions.cs | 8 +- .../NuGetComponentExtensions.cs | 11 +- .../PipComponentExtensions.cs | 8 +- .../PodComponentExtensions.cs | 8 +- .../RubyGemsComponentExtensions.cs | 8 +- .../ScannedComponentExtensions.cs | 21 +- .../ComponentDetectionToSBOMPackageAdapter.cs | 2 +- .../Config/Args/GenerationArgs.cs | 7 + .../PackageMetadataParsingException.cs | 18 ++ .../Executors/ComponentDetectionBaseWalker.cs | 36 +++- .../ComponentToPackageInfoConverter.cs | 4 +- .../Executors/LicenseInformationFetcher.cs | 7 +- .../Executors/PackagesWalker.cs | 5 +- .../Executors/SBOMComponentsWalker.cs | 5 +- .../Microsoft.Sbom.Api.csproj | 1 + .../Telemetry/Entities/SBOMTelemetry.cs | 10 + .../Output/Telemetry/IRecorder.cs | 15 +- .../Output/Telemetry/TelemetryRecorder.cs | 36 +++- .../IPackageManagerUtils.cs | 24 +++ .../ComponentDetailsUtils/MavenUtils.cs | 188 ++++++++++++++++++ .../ComponentDetailsUtils/NugetUtils.cs | 115 +++++++++++ .../PackageDetails/IPackageDetailsFactory.cs | 17 ++ .../PackageDetails/PackageDetails.cs | 11 + .../PackageDetails/PackageDetailsFactory.cs | 106 ++++++++++ .../ParsedPackageInformation.cs | 12 ++ .../CGScannedPackagesProvider.cs | 5 +- .../CommonPackagesProvider.cs | 2 + .../PackagesProviders/SBOMPackagesProvider.cs | 4 +- .../Config/Configuration.cs | 9 + .../Config/IConfiguration.cs | 5 + .../Config/InputConfiguration.cs | 3 + src/Microsoft.Sbom.Common/FileSystemUtils.cs | 3 + src/Microsoft.Sbom.Common/IFileSystemUtils.cs | 7 + .../ServiceCollectionExtensions.cs | 6 +- ...onentDetectionToSBOMPackageAdapterTests.cs | 38 +--- .../ComponentToPackageInfoConverterTests.cs | 65 +++--- .../Executors/PackagesWalkerTests.cs | 39 ++-- .../Executors/SBOMComponentsWalkerTests.cs | 22 +- .../PackageDetails/MavenUtilsTests.cs | 156 +++++++++++++++ .../PackageDetails/NugetUtilsTests.cs | 128 ++++++++++++ .../PackageDetails/SampleMetadataFiles.cs | 134 +++++++++++++ .../ManifestGenerationWorkflowTests.cs | 5 +- 47 files changed, 1204 insertions(+), 155 deletions(-) rename src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/{ScanResultWithLicense.cs => ExtendedScanResult.cs} (82%) rename src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/{ScannedComponentWithLicense.cs => ExtendedScannedComponent.cs} (62%) create mode 100644 src/Microsoft.Sbom.Api/Exceptions/PackageMetadataParsingException.cs create mode 100644 src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/IPackageManagerUtils.cs create mode 100644 src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/MavenUtils.cs create mode 100644 src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/NugetUtils.cs create mode 100644 src/Microsoft.Sbom.Api/PackageDetails/IPackageDetailsFactory.cs create mode 100644 src/Microsoft.Sbom.Api/PackageDetails/PackageDetails.cs create mode 100644 src/Microsoft.Sbom.Api/PackageDetails/PackageDetailsFactory.cs create mode 100644 src/Microsoft.Sbom.Api/PackageDetails/ParsedPackageInformation.cs create mode 100644 test/Microsoft.Sbom.Api.Tests/PackageDetails/MavenUtilsTests.cs create mode 100644 test/Microsoft.Sbom.Api.Tests/PackageDetails/NugetUtilsTests.cs create mode 100644 test/Microsoft.Sbom.Api.Tests/PackageDetails/SampleMetadataFiles.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 6a669117..68acb1a7 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -31,6 +31,7 @@ + diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/CargoComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/CargoComponentExtensions.cs index 8662f268..cf442b0a 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/CargoComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/CargoComponentExtensions.cs @@ -15,17 +15,17 @@ internal static class CargoComponentExtensions /// Converts a to an . /// /// The to convert. - /// The license to use for the . + /// The version of the CargoComponent /// The converted . - public static SbomPackage ToSbomPackage(this CargoComponent cargoComponent, string? license = null) => new() + public static SbomPackage ToSbomPackage(this CargoComponent cargoComponent, ExtendedScannedComponent component) => new() { Id = cargoComponent.Id, PackageUrl = cargoComponent.PackageUrl?.ToString(), PackageName = cargoComponent.Name, PackageVersion = cargoComponent.Version, - LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo + LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo { - Concluded = license, + Concluded = component.LicenseConcluded, }, FilesAnalyzed = false, Type = "cargo", diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScanResultWithLicense.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScanResult.cs similarity index 82% rename from src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScanResultWithLicense.cs rename to src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScanResult.cs index e8a9afe6..281907ac 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScanResultWithLicense.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScanResult.cs @@ -12,10 +12,10 @@ namespace Microsoft.Sbom.Adapters.ComponentDetection; /// A with license information. /// [JsonObject(MemberSerialization.OptOut, NamingStrategyType = typeof(CamelCaseNamingStrategy))] -public sealed class ScanResultWithLicense : ScanResult +public sealed class ExtendedScanResult : ScanResult { /// /// Gets or sets the scanned components with license information. /// - public new IEnumerable? ComponentsFound { get; init; } + public new IEnumerable? ComponentsFound { get; init; } } diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentWithLicense.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScannedComponent.cs similarity index 62% rename from src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentWithLicense.cs rename to src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScannedComponent.cs index 141f5e82..9c93e425 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentWithLicense.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScannedComponent.cs @@ -8,15 +8,15 @@ namespace Microsoft.Sbom.Adapters.ComponentDetection; using Microsoft.Sbom.Contracts; /// -/// A with license information. +/// A with additional properties extracted from package metadata files. /// -public class ScannedComponentWithLicense : ScannedComponent +public class ExtendedScannedComponent : ScannedComponent { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The to copy properties from. - public ScannedComponentWithLicense(ScannedComponent? other = null) + public ExtendedScannedComponent(ScannedComponent? other = null) { if (other == null) { @@ -35,12 +35,22 @@ public ScannedComponentWithLicense(ScannedComponent? other = null) } /// - /// Gets or sets the license. + /// Gets or sets the license concluded which is retrieved from the ClearlyDefined API. /// - public string? License { get; set; } + public string? LicenseConcluded { get; set; } /// - /// Converts a to an . + /// Gets or sets the license declared which is found directly in the package metadata. + /// + public string? LicenseDeclared { get; set; } + + /// + /// Gets or sets the supplier. + /// + public string? Supplier { get; set; } + + /// + /// Converts a to an . /// /// The to use. /// The converted . diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/MavenComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/MavenComponentExtensions.cs index 19fa6740..590b7a6e 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/MavenComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/MavenComponentExtensions.cs @@ -15,14 +15,20 @@ internal static class MavenComponentExtensions /// Converts a to an . /// /// The to convert. + /// The version of the MavenComponent /// The converted . - public static SbomPackage? ToSbomPackage(this MavenComponent mavenComponent) => new() + public static SbomPackage? ToSbomPackage(this MavenComponent mavenComponent, ExtendedScannedComponent component) => new() { Id = mavenComponent.Id, PackageName = $"{mavenComponent.GroupId}.{mavenComponent.ArtifactId}", PackageUrl = mavenComponent.PackageUrl?.ToString(), PackageVersion = mavenComponent.Version, FilesAnalyzed = false, + Supplier = string.IsNullOrEmpty(component.Supplier) ? null : component.Supplier, + LicenseInfo = string.IsNullOrEmpty(component.LicenseDeclared) ? null : new LicenseInfo + { + Declared = component.LicenseDeclared, + }, Type = "maven", }; } diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NpmComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NpmComponentExtensions.cs index 5f7084f0..b00616e8 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NpmComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NpmComponentExtensions.cs @@ -17,9 +17,9 @@ internal static class NpmComponentExtensions /// Converts a to an . /// /// The to convert. - /// License information for the package that component that is being converted. + /// The version of the NpmComponent /// The converted . - public static SbomPackage ToSbomPackage(this NpmComponent npmComponent, string? license = null) => new() + public static SbomPackage ToSbomPackage(this NpmComponent npmComponent, ExtendedScannedComponent component) => new() { Id = npmComponent.Id, PackageUrl = npmComponent.PackageUrl?.ToString(), @@ -33,9 +33,9 @@ internal static class NpmComponentExtensions }, }, Supplier = npmComponent.Author?.AsSupplier(), - LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo + LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo { - Concluded = license, + Concluded = component.LicenseConcluded, }, FilesAnalyzed = false, Type = "npm", diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NuGetComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NuGetComponentExtensions.cs index 9ff92e30..d70062eb 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NuGetComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NuGetComponentExtensions.cs @@ -16,18 +16,19 @@ internal static class NuGetComponentExtensions /// Converts a to an . /// /// The to convert. - /// License information for the package that component that is being converted. + /// The version of the NuGetComponent /// The converted . - public static SbomPackage ToSbomPackage(this NuGetComponent nuGetComponent, string? license = null) => new() + public static SbomPackage ToSbomPackage(this NuGetComponent nuGetComponent, ExtendedScannedComponent component) => new() { Id = nuGetComponent.Id, PackageUrl = nuGetComponent.PackageUrl?.ToString(), PackageName = nuGetComponent.Name, PackageVersion = nuGetComponent.Version, - Supplier = nuGetComponent.Authors?.Any() == true ? $"Organization: {nuGetComponent.Authors.First()}" : null, - LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo + Supplier = nuGetComponent.Authors?.Any() == true ? $"Organization: {nuGetComponent.Authors.First()}" : component.Supplier, + LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) && string.IsNullOrEmpty(component.LicenseDeclared) ? null : new LicenseInfo { - Concluded = license, + Concluded = component.LicenseConcluded, + Declared = component.LicenseDeclared, }, FilesAnalyzed = false, Type = "nuget", diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PipComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PipComponentExtensions.cs index 930a1249..257fafdd 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PipComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PipComponentExtensions.cs @@ -15,17 +15,17 @@ internal static class PipComponentExtensions /// Converts a to an . /// /// The to convert. - /// The license to use. + /// The version of the PipComponent /// The converted . - public static SbomPackage ToSbomPackage(this PipComponent pipComponent, string? license = null) => new() + public static SbomPackage ToSbomPackage(this PipComponent pipComponent, ExtendedScannedComponent component) => new() { Id = pipComponent.Id, PackageUrl = pipComponent.PackageUrl?.ToString(), PackageName = pipComponent.Name, PackageVersion = pipComponent.Version, - LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo + LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo { - Concluded = license, + Concluded = component.LicenseConcluded, }, FilesAnalyzed = false, Type = "python", diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PodComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PodComponentExtensions.cs index 944994fc..29b4bd69 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PodComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PodComponentExtensions.cs @@ -15,18 +15,18 @@ internal static class PodComponentExtensions /// Converts a to an . /// /// The to convert. - /// The license to use. + /// The version of the PodComponent /// The converted . - public static SbomPackage? ToSbomPackage(this PodComponent podComponent, string? license = null) => new() + public static SbomPackage? ToSbomPackage(this PodComponent podComponent, ExtendedScannedComponent component) => new() { Id = podComponent.Id, PackageUrl = podComponent.PackageUrl?.ToString(), PackageName = podComponent.Name, PackageVersion = podComponent.Version, PackageSource = podComponent.SpecRepo, - LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo + LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo { - Concluded = license, + Concluded = component.LicenseConcluded, }, FilesAnalyzed = false, Type = "pod", diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/RubyGemsComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/RubyGemsComponentExtensions.cs index 0bbcb140..ba6c40ca 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/RubyGemsComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/RubyGemsComponentExtensions.cs @@ -15,18 +15,18 @@ internal static class RubyGemsComponentExtensions /// Converts a to an . /// /// The to convert. - /// The license to use. + /// The version of the RubyGemsComponent /// The converted . - public static SbomPackage ToSbomPackage(this RubyGemsComponent rubyGemsComponent, string? license = null) => new() + public static SbomPackage ToSbomPackage(this RubyGemsComponent rubyGemsComponent, ExtendedScannedComponent component) => new() { Id = rubyGemsComponent.Id, PackageUrl = rubyGemsComponent.PackageUrl?.ToString(), PackageName = rubyGemsComponent.Name, PackageVersion = rubyGemsComponent.Version, PackageSource = rubyGemsComponent.Source, - LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo + LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo { - Concluded = license, + Concluded = component.LicenseConcluded, }, FilesAnalyzed = false, Type = "ruby", diff --git a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentExtensions.cs b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentExtensions.cs index 8f63edb5..eb758e38 100644 --- a/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentExtensions.cs +++ b/src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentExtensions.cs @@ -10,31 +10,30 @@ namespace Microsoft.Sbom.Adapters.ComponentDetection; /// -/// Extensions methods for . +/// Extensions methods for . /// public static class ScannedComponentExtensions { /// - /// Converts a to an . + /// Converts a to an . /// - public static SbomPackage? ToSbomPackage(this ScannedComponentWithLicense component, AdapterReport report) + public static SbomPackage? ToSbomPackage(this ExtendedScannedComponent component, AdapterReport report) { return component.Component switch { - CargoComponent cargoComponent => cargoComponent.ToSbomPackage(component?.License), - ConanComponent conanComponent => conanComponent.ToSbomPackage(), + CargoComponent cargoComponent => cargoComponent.ToSbomPackage(component), CondaComponent condaComponent => condaComponent.ToSbomPackage(), DockerImageComponent dockerImageComponent => dockerImageComponent.ToSbomPackage(), GitComponent gitComponent => gitComponent.ToSbomPackage(), GoComponent goComponent => goComponent.ToSbomPackage(), LinuxComponent linuxComponent => linuxComponent.ToSbomPackage(), - MavenComponent mavenComponent => mavenComponent.ToSbomPackage(), - NpmComponent npmComponent => npmComponent.ToSbomPackage(component?.License), - NuGetComponent nuGetComponent => nuGetComponent.ToSbomPackage(component?.License), + MavenComponent mavenComponent => mavenComponent.ToSbomPackage(component), + NpmComponent npmComponent => npmComponent.ToSbomPackage(component), + NuGetComponent nuGetComponent => nuGetComponent.ToSbomPackage(component), OtherComponent otherComponent => otherComponent.ToSbomPackage(), - PipComponent pipComponent => pipComponent.ToSbomPackage(component?.License), - PodComponent podComponent => podComponent.ToSbomPackage(component?.License), - RubyGemsComponent rubyGemsComponent => rubyGemsComponent.ToSbomPackage(component?.License), + PipComponent pipComponent => pipComponent.ToSbomPackage(component), + PodComponent podComponent => podComponent.ToSbomPackage(component), + RubyGemsComponent rubyGemsComponent => rubyGemsComponent.ToSbomPackage(component), null => Error(report => report.LogNullComponent(nameof(ToSbomPackage))), _ => Error(report => report.LogNoConversionFound(component.Component.GetType(), component.Component)) }; diff --git a/src/Microsoft.Sbom.Adapters/ComponentDetectionToSBOMPackageAdapter.cs b/src/Microsoft.Sbom.Adapters/ComponentDetectionToSBOMPackageAdapter.cs index 42c9a1f1..5843e15d 100644 --- a/src/Microsoft.Sbom.Adapters/ComponentDetectionToSBOMPackageAdapter.cs +++ b/src/Microsoft.Sbom.Adapters/ComponentDetectionToSBOMPackageAdapter.cs @@ -34,7 +34,7 @@ public class ComponentDetectionToSBOMPackageAdapter try { - var componentDetectionScanResult = JsonConvert.DeserializeObject(File.ReadAllText(bcdeOutputPath)); + var componentDetectionScanResult = JsonConvert.DeserializeObject(File.ReadAllText(bcdeOutputPath)); if (componentDetectionScanResult == null) { diff --git a/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs b/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs index 6fe3fc3b..5917c16f 100644 --- a/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs +++ b/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs @@ -115,4 +115,11 @@ public class GenerationArgs : CommonArgs [ArgShortcut("li")] [ArgDescription("If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi.")] public bool? FetchLicenseInformation { get; set; } + + /// + /// If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi. + /// + [ArgShortcut("pm")] + [ArgDescription("If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi.")] + public bool? EnablePackageMetadataParsing { get; set; } } diff --git a/src/Microsoft.Sbom.Api/Exceptions/PackageMetadataParsingException.cs b/src/Microsoft.Sbom.Api/Exceptions/PackageMetadataParsingException.cs new file mode 100644 index 00000000..fb5f97a9 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/PackageMetadataParsingException.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.Sbom.Api.Exceptions; + +/// +/// Exception thrown while parsing a response from ClearlyDefined. +/// +[Serializable] +public class PackageMetadataParsingException : Exception +{ + public PackageMetadataParsingException(string message) + : base(message) + { + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs index f2905d80..8a4171f5 100644 --- a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs +++ b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs @@ -10,20 +10,19 @@ using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Adapters.ComponentDetection; using Microsoft.Sbom.Api.Config.Extensions; using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; -using Serilog.Events; using Constants = Microsoft.Sbom.Api.Utils.Constants; using ILogger = Serilog.ILogger; namespace Microsoft.Sbom.Api.Executors; -using Microsoft.Sbom.Adapters.ComponentDetection; - /// /// Abstract class that runs component detection tool in the given folder. /// @@ -35,6 +34,7 @@ public abstract class ComponentDetectionBaseWalker private readonly ISbomConfigProvider sbomConfigs; private readonly IFileSystemUtils fileSystemUtils; private readonly ILicenseInformationFetcher licenseInformationFetcher; + private readonly IPackageDetailsFactory packageDetailsFactory; public ConcurrentDictionary LicenseDictionary = new ConcurrentDictionary(); private bool licenseInformationRetrieved = false; @@ -47,6 +47,7 @@ public ComponentDetectionBaseWalker( IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, + IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher) { this.log = log ?? throw new ArgumentNullException(nameof(log)); @@ -54,6 +55,7 @@ public ComponentDetectionBaseWalker( this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.packageDetailsFactory = packageDetailsFactory ?? throw new ArgumentNullException(nameof(packageDetailsFactory)); this.licenseInformationFetcher = licenseInformationFetcher ?? throw new ArgumentNullException(nameof(licenseInformationFetcher)); } @@ -97,6 +99,8 @@ public ComponentDetectionBaseWalker( async Task Scan(string path) { + IDictionary<(string Name, string Version), PackageDetails.PackageDetails> packageDetailsDictionary = new ConcurrentDictionary<(string, string), PackageDetails.PackageDetails>(); + cliArgumentBuilder.SourceDirectory(buildComponentDirPath); var cmdLineParams = configuration.ToComponentDetectorCommandLineParams(cliArgumentBuilder); @@ -113,6 +117,14 @@ async Task Scan(string path) var uniqueComponents = FilterScannedComponents(scanResult); + if (configuration.EnablePackageMetadataParsing?.Value == true) + { + if (uniqueComponents.Any()) + { + packageDetailsDictionary = packageDetailsFactory.GetPackageDetailsDictionary(uniqueComponents); + } + } + // Check if the configuration is set to fetch license information. if (configuration.FetchLicenseInformation?.Value == true) { @@ -147,21 +159,27 @@ async Task Scan(string path) var componentName = scannedComponent.Component.PackageUrl?.Name; var componentVersion = scannedComponent.Component.PackageUrl?.Version; - ScannedComponentWithLicense extendedComponent; + ExtendedScannedComponent extendedComponent; - if (scannedComponent is ScannedComponentWithLicense existingExtendedComponent) + if (scannedComponent is ExtendedScannedComponent existingExtendedScannedComponent) { - extendedComponent = existingExtendedComponent; + extendedComponent = existingExtendedScannedComponent; } else { - // Use copy constructor to pass over all the properties to the ScanndedComponentWithLicense. - extendedComponent = new ScannedComponentWithLicense(scannedComponent); + // Use copy constructor to pass over all the properties to the ExtendedScannedComponent. + extendedComponent = new ExtendedScannedComponent(scannedComponent); } if (LicenseDictionary != null && LicenseDictionary.ContainsKey($"{componentName}@{componentVersion}")) { - extendedComponent.License = LicenseDictionary[$"{componentName}@{componentVersion}"]; + extendedComponent.LicenseConcluded = LicenseDictionary[$"{componentName}@{componentVersion}"]; + } + + if (packageDetailsDictionary != null && packageDetailsDictionary.ContainsKey((componentName, componentVersion))) + { + extendedComponent.Supplier = string.IsNullOrEmpty(packageDetailsDictionary[(componentName, componentVersion)].Supplier) ? null : packageDetailsDictionary[(componentName, componentVersion)].Supplier; + extendedComponent.LicenseDeclared = string.IsNullOrEmpty(packageDetailsDictionary[(componentName, componentVersion)].License) ? null : packageDetailsDictionary[(componentName, componentVersion)].License; } await output.Writer.WriteAsync(extendedComponent); diff --git a/src/Microsoft.Sbom.Api/Executors/ComponentToPackageInfoConverter.cs b/src/Microsoft.Sbom.Api/Executors/ComponentToPackageInfoConverter.cs index f990da29..8aa02c03 100644 --- a/src/Microsoft.Sbom.Api/Executors/ComponentToPackageInfoConverter.cs +++ b/src/Microsoft.Sbom.Api/Executors/ComponentToPackageInfoConverter.cs @@ -40,7 +40,7 @@ public virtual (ChannelReader output, ChannelReader { var report = new AdapterReport(); - await foreach (ScannedComponentWithLicense scannedComponent in componentReader.ReadAllAsync()) + await foreach (ExtendedScannedComponent scannedComponent in componentReader.ReadAllAsync()) { await ConvertComponentToPackage(scannedComponent, output, errors); } @@ -48,7 +48,7 @@ public virtual (ChannelReader output, ChannelReader output, Channel errors) + async Task ConvertComponentToPackage(ExtendedScannedComponent scannedComponent, Channel output, Channel errors) { try { diff --git a/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs b/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs index b2281b7e..47dc6f7d 100644 --- a/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs +++ b/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs @@ -120,7 +120,7 @@ public Dictionary ConvertClearlyDefinedApiResponseToList(string } // Filter out undefined licenses. - foreach (var kvp in extractedLicenses.Where(kvp => kvp.Value.ToLower() == "noassertion" || kvp.Value.ToLower() == "unlicense" || kvp.Value.ToLower() == "other").ToList()) + foreach (var kvp in extractedLicenses.Where(kvp => IsUndefinedLicense(kvp.Value)).ToList()) { extractedLicenses.Remove(kvp.Key); } @@ -161,4 +161,9 @@ public string GetFromLicenseDictionary(string key) return value; } + + private bool IsUndefinedLicense(string license) + { + return license.Equals("noassertion", StringComparison.InvariantCultureIgnoreCase) || license.Equals("unlicense", StringComparison.InvariantCultureIgnoreCase) || license.Equals("other", StringComparison.InvariantCultureIgnoreCase); + } } diff --git a/src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs b/src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs index 95d1070e..0e7e6346 100644 --- a/src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs +++ b/src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs @@ -5,6 +5,7 @@ using System.Linq; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; @@ -18,8 +19,8 @@ namespace Microsoft.Sbom.Api.Executors; /// public class PackagesWalker : ComponentDetectionBaseWalker { - public PackagesWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, ILicenseInformationFetcher licenseInformationFetcher) - : base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, licenseInformationFetcher) + public PackagesWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher) + : base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, packageDetailsFactory, licenseInformationFetcher) { } diff --git a/src/Microsoft.Sbom.Api/Executors/SBOMComponentsWalker.cs b/src/Microsoft.Sbom.Api/Executors/SBOMComponentsWalker.cs index 53d833ee..5f434ada 100644 --- a/src/Microsoft.Sbom.Api/Executors/SBOMComponentsWalker.cs +++ b/src/Microsoft.Sbom.Api/Executors/SBOMComponentsWalker.cs @@ -5,6 +5,7 @@ using System.Linq; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; @@ -18,8 +19,8 @@ namespace Microsoft.Sbom.Api.Executors; /// public class SBOMComponentsWalker : ComponentDetectionBaseWalker { - public SBOMComponentsWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, ILicenseInformationFetcher licenseInformationFetcher) - : base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, licenseInformationFetcher) + public SBOMComponentsWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher) + : base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, packageDetailsFactory, licenseInformationFetcher) { } diff --git a/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj index 45a3f681..76d696a5 100644 --- a/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj +++ b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj @@ -17,6 +17,7 @@ + diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMTelemetry.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMTelemetry.cs index 75d8d47f..f8775f27 100644 --- a/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMTelemetry.cs +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMTelemetry.cs @@ -61,8 +61,18 @@ public class SBOMTelemetry /// public IDictionary APIExceptions { get; set; } + /// + /// Gets or sets if any exceptions during detection/parsing of package metadata files was thrown. + /// + public IDictionary MetadataExceptions { get; set; } + /// /// Gets or sets the total number of licenses detected in the SBOM. /// public int TotalLicensesDetected { get; set; } + + /// + /// Gets or sets the total number of PackageDetails entries created during the execution of the tool. + /// + public int PackageDetailsEntries { get; set; } } diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs index 99affd5b..4c1d5613 100644 --- a/src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs @@ -31,9 +31,15 @@ public interface IRecorder /// /// Record the total number of unique packages that were detected during the execution of the SBOM tool. /// - /// Total number of packages encountered while validating the SBOM. + /// Total number of packages encountered while validating the SBOM. public void RecordTotalNumberOfPackages(int count); + /// + /// Adds onto the total number of packageDetail entries found by the PackageDetailsFactory. + /// + /// The total packageDetails count after execution of the PackageDetailsFactory. + public void AddToTotalNumberOfPackageDetailsEntries(int count); + /// /// Adds onto the total count of licenses that were retrieved from the API. /// @@ -71,6 +77,13 @@ public interface IRecorder /// If the exception is null. public void RecordAPIException(Exception exception); + /// + /// Record any exception that was encountered during the detection or parsing of individual package metadata files. + /// + /// The exception that was encountered. + /// If the exception is null. + public void RecordMetadataException(Exception exception); + /// /// Finalize the recorder, and log the telemetry. /// diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs index 3d9081fa..0a857266 100644 --- a/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs @@ -19,6 +19,7 @@ using PowerArgs; using Serilog; using Serilog.Core; +using Spectre.Console; using Constants = Microsoft.Sbom.Api.Utils.Constants; namespace Microsoft.Sbom.Api.Output.Telemetry; @@ -33,11 +34,14 @@ public class TelemetryRecorder : IRecorder private readonly IDictionary switches = new Dictionary(); private readonly IList exceptions = new List(); private readonly IList apiExceptions = new List(); - private int totalNumberOfPackages = 0; - private int totalNumberOfLicenses = 0; + private readonly IList metadataExceptions = new List(); private IList errors = new List(); private Result result = Result.Success; + private int totalNumberOfPackages = 0; + private int totalNumberOfLicenses = 0; + private int packageDetailsEntries = 0; + public IFileSystemUtils FileSystemUtils { get; } public IConfiguration Configuration { get; } @@ -212,6 +216,21 @@ public void RecordAPIException(Exception apiException) this.apiExceptions.Add(apiException); } + /// + /// Record any exception that was encountered during the detection or parsing of individual package metadata files. + /// + /// The exception that was encountered. + /// If the exception is null. + public void RecordMetadataException(Exception metadataException) + { + if (metadataException is null) + { + throw new ArgumentNullException(); + } + + this.metadataExceptions.Add(metadataException); + } + /// /// Record the total number of packages that were processed during the execution of the SBOM tool. /// @@ -221,6 +240,15 @@ public void RecordTotalNumberOfPackages(int packageCount) this.totalNumberOfPackages = packageCount; } + /// + /// Adds onto the total number of packageDetail entries found by the PackageDetailsFactory. + /// + /// The total packageDetails count after execution of the PackageDetailsFactory. + public void AddToTotalNumberOfPackageDetailsEntries(int packageDetailsCount) + { + Interlocked.Add(ref this.packageDetailsEntries, packageDetailsCount); + } + /// /// Adds onto the total count of licenses that were retrieved from the API. /// @@ -292,7 +320,9 @@ public async Task FinalizeAndLogTelemetryAsync() Switches = this.switches, Exceptions = this.exceptions.GroupBy(e => e.GetType().ToString()).ToDictionary(group => group.Key, group => group.First().Message), APIExceptions = this.apiExceptions.GroupBy(e => e.GetType().ToString()).ToDictionary(group => group.Key, group => group.First().Message), - TotalLicensesDetected = this.totalNumberOfLicenses + MetadataExceptions = this.metadataExceptions.GroupBy(e => e.GetType().ToString()).ToDictionary(g => g.Key, g => g.First().Message), + TotalLicensesDetected = this.totalNumberOfLicenses, + PackageDetailsEntries = this.packageDetailsEntries }; // Log to logger. diff --git a/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/IPackageManagerUtils.cs b/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/IPackageManagerUtils.cs new file mode 100644 index 00000000..c99fe96a --- /dev/null +++ b/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/IPackageManagerUtils.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; + +namespace Microsoft.Sbom.Api.PackageDetails; + +public interface IPackageManagerUtils + where T : IPackageManagerUtils +{ + /// + /// Takes in a ScannedComponent object and attempts to find the corresponding .pom file. + /// + /// A single from a component detection scan. + /// + public string GetMetadataLocation(ScannedComponent scannedComponent); + + /// + /// Takes in the path to a package metadata file (ex: .nuspec, .pom) file and returns a tuple consisting of the package name, version, and details such as its license and supplier. + /// + /// Path to a package metadata file. + /// A tuple containing the name, version, and of the specified metadata file. + public ParsedPackageInformation ParseMetadata(string pomLocation); +} diff --git a/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/MavenUtils.cs b/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/MavenUtils.cs new file mode 100644 index 00000000..dfbf038a --- /dev/null +++ b/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/MavenUtils.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#nullable enable +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Xml; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Common; +using Serilog; + +namespace Microsoft.Sbom.Api.PackageDetails; + +/// +/// Utilities for retrieving information from maven packages that may not be present on the buildDropPath. +/// +public class MavenUtils : IPackageManagerUtils +{ + private readonly IFileSystemUtils fileSystemUtils; + private readonly ILogger log; + private readonly IRecorder recorder; + + private static readonly string EnvHomePath = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "HOMEPATH" : "HOME"; + private static readonly string? HomePath = Environment.GetEnvironmentVariable(EnvHomePath); + private static readonly string MavenPackagesPath = Path.Join(HomePath, ".m2/repository"); + private readonly string userDefinedLocalRepositoryPath; + + private bool MavenPackagesPathHasReadPermissions => fileSystemUtils.DirectoryHasReadPermissions(MavenPackagesPath); + + public MavenUtils(IFileSystemUtils fileSystemUtils, ILogger log, IRecorder recorder) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); + + this.userDefinedLocalRepositoryPath = GetLocalRepositoryPath() ?? string.Empty; + } + + // Takes in a scanned component and attempts to find the associated pom file. If it is not found then it returns null. + public string? GetMetadataLocation(ScannedComponent scannedComponent) + { + var pomLocation = MavenPackagesPath; + + // Check if a user-defined local repository exists. If not, continue with the default value. + if (!string.IsNullOrEmpty(userDefinedLocalRepositoryPath)) + { + pomLocation = userDefinedLocalRepositoryPath; + } + + var componentName = scannedComponent.Component.PackageUrl?.Name.ToLower(); + var componentNamespace = scannedComponent.Component.PackageUrl?.Namespace?.ToLower(); + var componentVersion = scannedComponent.Component.PackageUrl?.Version; + + // Take the component namespace and split it by "." in order to get the correct path to the .pom + if (!string.IsNullOrEmpty(componentNamespace)) + { + var componentNamespacePath = componentNamespace.Replace('.', '/'); // Example: "org/apache/commons/commons-lang3" + + var relativePomPath = Path.Join(componentNamespacePath, $"/{componentName}/{componentVersion}/{componentName}-{componentVersion}.pom"); + + pomLocation = Path.Join(pomLocation, relativePomPath); + } + + pomLocation = Path.GetFullPath(pomLocation); + + // Check for file permissions on the .m2 directory before attempting to check if the file exists + if (MavenPackagesPathHasReadPermissions) + { + if (fileSystemUtils.FileExists(pomLocation)) + { + return pomLocation; + } + else + { + log.Verbose($"Pom location could not be found at: {pomLocation}"); + } + } + + return null; + } + + public ParsedPackageInformation? ParseMetadata(string pomLocation) + { + var supplierField = string.Empty; + var licenseField = string.Empty; + + try + { + var pomBytes = fileSystemUtils.ReadAllBytes(pomLocation); + using var pomStream = new MemoryStream(pomBytes, false); + + var doc = new XmlDocument(); + doc.Load(pomStream); + + XmlNode? projectNode = doc["project"]; + XmlNode? developersNode = projectNode?["developers"]; + XmlNode? licensesNode = projectNode?["licenses"]; + XmlNode? organizationNode = projectNode?["organization"]; + + var name = projectNode?["artifactId"]?.InnerText; + var version = projectNode?["version"]?.InnerText; + + if (organizationNode != null) + { + var organizationName = organizationNode["name"]?.InnerText; + if (!string.IsNullOrEmpty(organizationName)) + { + supplierField = $"Organization: {organizationName}"; + } + } + else if (developersNode != null) + { + // Take the first developer name and use it as the supplier when there is no organization listed. + var developerName = developersNode["developer"]?["name"]?.InnerText; + if (!string.IsNullOrEmpty(developerName)) + { + supplierField = $"Person: {developerName}"; + } + } + + if (licensesNode != null) + { + foreach (XmlNode licenseNode in licensesNode.ChildNodes) + { + var licenseName = licenseNode["name"]?.InnerText; + + if (!string.IsNullOrEmpty(licenseName)) + { + licenseField = licenseName; + } + } + } + + return new ParsedPackageInformation(name, version, new PackageDetails(licenseField, supplierField)); + } + catch (PackageMetadataParsingException e) + { + log.Error("Error encountered while extracting supplier info from pom file. Supplier information may be incomplete.", e); + recorder.RecordMetadataException(e); + + return null; + } + } + + /// + /// Gets the local repository path for the Maven protocol. Returns null if a settings.xml is not found. + /// + /// The path to the local repository path defined in the settings.xml + private static string? GetLocalRepositoryPath() + { + var m2Path = $"{HomePath}/.m2"; + + var userSettingsXmlPath = $"{m2Path}/settings.xml"; + var backupSettingsXmlPath = $"{m2Path}/_settings.xml"; + + if (File.Exists(userSettingsXmlPath)) + { + return GetRepositoryPathFromXml(userSettingsXmlPath); + } + else if (File.Exists(backupSettingsXmlPath)) + { + return GetRepositoryPathFromXml(backupSettingsXmlPath); + } + + return null; + } + + private static string? GetRepositoryPathFromXml(string settingsXmlFilePath) + { + var settingsXmlBytes = File.ReadAllBytes(settingsXmlFilePath); + using var xmlStream = new MemoryStream(settingsXmlBytes, false); + + var doc = new XmlDocument(); + doc.Load(xmlStream); + + var localRepositoryNode = doc["settings"]?["localRepository"]; + + if (localRepositoryNode != null) + { + return localRepositoryNode.InnerText; + } + + return null; + } +} diff --git a/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/NugetUtils.cs b/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/NugetUtils.cs new file mode 100644 index 00000000..7c5e5890 --- /dev/null +++ b/src/Microsoft.Sbom.Api/PackageDetails/ComponentDetailsUtils/NugetUtils.cs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.Linq; +using System.Xml; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Common; +using NuGet.Configuration; +using Serilog; + +namespace Microsoft.Sbom.Api.PackageDetails; + +/// +/// Utilities for retrieving information from maven packages that may not be present on the buildDropPath +/// +public class NugetUtils : IPackageManagerUtils +{ + private readonly IFileSystemUtils fileSystemUtils; + private readonly ILogger log; + private readonly IRecorder recorder; + + private static readonly string NugetPackagesPath = SettingsUtility.GetGlobalPackagesFolder(new NullSettings()); + + public NugetUtils(IFileSystemUtils fileSystemUtils, ILogger log, IRecorder recorder) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); + } + + // Takes in a scanned component and attempts to find the associated nuspec file. If it is not found then it returns null. + public string GetMetadataLocation(ScannedComponent scannedComponent) + { + var nuspecLocation = string.Empty; + + var componentName = scannedComponent.Component.PackageUrl?.Name.ToLower(); + var componentVersion = scannedComponent.Component.PackageUrl?.Version; + var componentType = scannedComponent.Component.PackageUrl?.Type.ToLower(); + + if (componentType == "nuget") + { + nuspecLocation = Path.Join(NugetPackagesPath, $"{componentName}/{componentVersion}/{componentName}.nuspec"); + } + + // Check for file permissions on the .nuget directory before attempting to check if every file exists + if (fileSystemUtils.DirectoryHasReadPermissions(NugetPackagesPath)) + { + if (fileSystemUtils.FileExists(nuspecLocation)) + { + return nuspecLocation; + } + else + { + log.Verbose($"Nuspec file could not be found at: {nuspecLocation}"); + } + } + + return null; + } + + public ParsedPackageInformation ParseMetadata(string nuspecPath) + { + var supplierField = string.Empty; + var licenseField = string.Empty; + + try + { + var nuspecBytes = fileSystemUtils.ReadAllBytes(nuspecPath); + using var nuspecStream = new MemoryStream(nuspecBytes, false); + + var doc = new XmlDocument(); + doc.Load(nuspecStream); + + XmlNode packageNode = doc["package"]; + XmlNode metadataNode = packageNode["metadata"]; + + var name = metadataNode["id"]?.InnerText; + var version = metadataNode["version"]?.InnerText; + var authors = metadataNode["authors"]?.InnerText; + var license = metadataNode["license"]; + + if (license != null && license.Attributes?["type"].Value != "file") + { + licenseField = license.InnerText; + } + + if (!string.IsNullOrEmpty(authors)) + { + // If authors contains a comma, then split it and put it back together with a comma and space. + if (authors.Contains(',')) + { + var authorsArray = authors.Split(','); + supplierField = $"Organization: {authorsArray.First()}"; + } + else + { + supplierField = $"Organization: {authors}"; + } + } + + return new ParsedPackageInformation(name, version, new PackageDetails(licenseField, supplierField)); + } + catch (PackageMetadataParsingException e) + { + log.Error("Error encountered while extracting supplier info from nuspec file. Supplier information may be incomplete.", e); + recorder.RecordMetadataException(e); + + return null; + } + } +} diff --git a/src/Microsoft.Sbom.Api/PackageDetails/IPackageDetailsFactory.cs b/src/Microsoft.Sbom.Api/PackageDetails/IPackageDetailsFactory.cs new file mode 100644 index 00000000..e4f57eb4 --- /dev/null +++ b/src/Microsoft.Sbom.Api/PackageDetails/IPackageDetailsFactory.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using Microsoft.ComponentDetection.Contracts.BcdeModels; + +namespace Microsoft.Sbom.Api.PackageDetails; + +public interface IPackageDetailsFactory +{ + /// + /// Takes in a list of ScannedComponents and returns a dictionary where the key is the component name and version and the value is PackageDetails record which is made up of information found in the package files. + /// + /// An IEnumerable of ScannedComponents which is the output of a component-detection scan. + /// + IDictionary<(string Name, string Version), PackageDetails> GetPackageDetailsDictionary(IEnumerable scannedComponents); +} diff --git a/src/Microsoft.Sbom.Api/PackageDetails/PackageDetails.cs b/src/Microsoft.Sbom.Api/PackageDetails/PackageDetails.cs new file mode 100644 index 00000000..e234662c --- /dev/null +++ b/src/Microsoft.Sbom.Api/PackageDetails/PackageDetails.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.PackageDetails; + +/// +/// Object used to define the information extracted from package metadata files. +/// +/// The license declared by the package in its own metadata file. +/// The people/company who are listed in the package as the author or supplier. +public record PackageDetails(string License, string Supplier); diff --git a/src/Microsoft.Sbom.Api/PackageDetails/PackageDetailsFactory.cs b/src/Microsoft.Sbom.Api/PackageDetails/PackageDetailsFactory.cs new file mode 100644 index 00000000..fa39d8f4 --- /dev/null +++ b/src/Microsoft.Sbom.Api/PackageDetails/PackageDetailsFactory.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.Output.Telemetry; +using Serilog; + +namespace Microsoft.Sbom.Api.PackageDetails; + +/// +/// Class responsible for taking the output of a component-detection scan and extracting additional information about the package based on its protocol. +/// +public class PackageDetailsFactory : IPackageDetailsFactory +{ + private readonly ILogger log; + private readonly IRecorder recorder; + private readonly IPackageManagerUtils mavenUtils; + private readonly IPackageManagerUtils nugetUtils; + + public PackageDetailsFactory(ILogger log, IRecorder recorder, IPackageManagerUtils mavenUtils, IPackageManagerUtils nugetUtils) + { + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); + this.mavenUtils = mavenUtils ?? throw new ArgumentNullException(nameof(mavenUtils)); + this.nugetUtils = nugetUtils ?? throw new ArgumentNullException(nameof(nugetUtils)); + } + + public IDictionary<(string Name, string Version), PackageDetails> GetPackageDetailsDictionary(IEnumerable scannedComponents) + { + var packageDetailsLocations = GetPackageDetailsLocations(scannedComponents); + + return ExtractPackageDetailsFromFiles(packageDetailsLocations); + } + + private List GetPackageDetailsLocations(IEnumerable scannedComponents) + { + var packageDetailsConfirmedLocations = new List(); + + foreach (var scannedComponent in scannedComponents) + { + var componentType = scannedComponent.Component.Type; + + switch (componentType) + { + case ComponentType.NuGet: + packageDetailsConfirmedLocations.Add(nugetUtils.GetMetadataLocation(scannedComponent)); + break; + case ComponentType.Maven: + packageDetailsConfirmedLocations.Add(mavenUtils.GetMetadataLocation(scannedComponent)); + break; + default: + break; + } + } + + return packageDetailsConfirmedLocations; + } + + private IDictionary<(string Name, string Version), PackageDetails> ExtractPackageDetailsFromFiles(List packageDetailsPaths) + { + var packageDetailsDictionary = new ConcurrentDictionary<(string, string), PackageDetails>(); + + foreach (var path in packageDetailsPaths) + { + if (!string.IsNullOrEmpty(path)) + { + switch (Path.GetExtension(path)?.ToLowerInvariant()) + { + case ".nuspec": + var nuspecDetails = nugetUtils.ParseMetadata(path); + if (!string.IsNullOrEmpty(nuspecDetails.PackageDetails.License) || !string.IsNullOrEmpty(nuspecDetails.PackageDetails.Supplier)) + { + packageDetailsDictionary.TryAdd((nuspecDetails.Name, nuspecDetails.Version), nuspecDetails.PackageDetails); + } + + break; + case ".pom": + var pomDetails = mavenUtils.ParseMetadata(path); + if (!string.IsNullOrEmpty(pomDetails.PackageDetails.License) || !string.IsNullOrEmpty(pomDetails.PackageDetails.Supplier)) + { + packageDetailsDictionary.TryAdd((pomDetails.Name, pomDetails.Version), pomDetails.PackageDetails); + } + + break; + default: + log.Verbose($"File extension {Path.GetExtension(path)} is not supported for extracting supplier info."); + break; + } + } + } + + if (packageDetailsPaths.Count > 0) + { + log.Information($"Found additional information for {packageDetailsDictionary.Count} components out of {packageDetailsPaths.Count} supported components."); + } + + recorder.AddToTotalNumberOfPackageDetailsEntries(packageDetailsDictionary.Count); + + return packageDetailsDictionary; + } +} diff --git a/src/Microsoft.Sbom.Api/PackageDetails/ParsedPackageInformation.cs b/src/Microsoft.Sbom.Api/PackageDetails/ParsedPackageInformation.cs new file mode 100644 index 00000000..21cc4c89 --- /dev/null +++ b/src/Microsoft.Sbom.Api/PackageDetails/ParsedPackageInformation.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.PackageDetails; + +/// +/// Object used to define the information extracted from package metadata files. +/// +/// The name declared by the package in its own metadata file. +/// The version of the package being described by the metadata file. +/// The additional package details extracted from the metadata file. +public record ParsedPackageInformation(string Name, string Version, PackageDetails PackageDetails); diff --git a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CGScannedPackagesProvider.cs b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CGScannedPackagesProvider.cs index fc3af2c5..c1512efd 100644 --- a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CGScannedPackagesProvider.cs +++ b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CGScannedPackagesProvider.cs @@ -7,8 +7,8 @@ using System.Threading.Channels; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.Sbom.Api.Entities; -using Microsoft.Sbom.Api.Exceptions; using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; using Serilog; @@ -34,8 +34,9 @@ public CGScannedPackagesProvider( PackageInfoJsonWriter packageInfoJsonWriter, ComponentToPackageInfoConverter packageInfoConverter, PackagesWalker packagesWalker, + IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher) - : base(configuration, channelUtils, logger, sbomConfigs, packageInfoJsonWriter, licenseInformationFetcher) + : base(configuration, channelUtils, logger, sbomConfigs, packageInfoJsonWriter, packageDetailsFactory, licenseInformationFetcher) { this.packageInfoConverter = packageInfoConverter ?? throw new ArgumentNullException(nameof(packageInfoConverter)); this.packagesWalker = packagesWalker ?? throw new ArgumentNullException(nameof(packagesWalker)); diff --git a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CommonPackagesProvider.cs b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CommonPackagesProvider.cs index 64e72a15..ec9d3c9a 100644 --- a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CommonPackagesProvider.cs +++ b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CommonPackagesProvider.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Extensions; @@ -31,6 +32,7 @@ protected CommonPackagesProvider( ILogger logger, ISbomConfigProvider sbomConfigs, PackageInfoJsonWriter packageInfoJsonWriter, + IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher) : base(configuration, channelUtils, logger) { diff --git a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/SBOMPackagesProvider.cs b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/SBOMPackagesProvider.cs index 94b3123d..c24fcef1 100644 --- a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/SBOMPackagesProvider.cs +++ b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/SBOMPackagesProvider.cs @@ -6,6 +6,7 @@ using System.Threading.Channels; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Extensions; @@ -24,8 +25,9 @@ public SBOMPackagesProvider( ILogger logger, ISbomConfigProvider sbomConfigs, PackageInfoJsonWriter packageInfoJsonWriter, + IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher) - : base(configuration, channelUtils, logger, sbomConfigs, packageInfoJsonWriter, licenseInformationFetcher) + : base(configuration, channelUtils, logger, sbomConfigs, packageInfoJsonWriter, packageDetailsFactory, licenseInformationFetcher) { } diff --git a/src/Microsoft.Sbom.Common/Config/Configuration.cs b/src/Microsoft.Sbom.Common/Config/Configuration.cs index e6453319..c653ad67 100644 --- a/src/Microsoft.Sbom.Common/Config/Configuration.cs +++ b/src/Microsoft.Sbom.Common/Config/Configuration.cs @@ -47,6 +47,7 @@ public class Configuration : IConfiguration private static readonly AsyncLocal> generationTimestamp = new(); private static readonly AsyncLocal> followSymlinks = new(); private static readonly AsyncLocal> fetchLicenseInformation = new(); + private static readonly AsyncLocal> enablePackageMetadataParsing = new(); private static readonly AsyncLocal> deleteManifestDirIfPresent = new(); private static readonly AsyncLocal> failIfNoPackages = new(); private static readonly AsyncLocal> verbosity = new(); @@ -305,4 +306,12 @@ public ConfigurationSetting FetchLicenseInformation get => fetchLicenseInformation.Value; set => fetchLicenseInformation.Value = value; } + + /// + [DefaultValue(false)] + public ConfigurationSetting EnablePackageMetadataParsing + { + get => enablePackageMetadataParsing.Value; + set => enablePackageMetadataParsing.Value = value; + } } diff --git a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs index fd359c2b..51abad7b 100644 --- a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs +++ b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs @@ -193,4 +193,9 @@ public interface IConfiguration /// If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi. /// ConfigurationSetting FetchLicenseInformation { get; set; } + + /// + /// If set to true, we will attempt to locate and parse package metadata files for additional informtion to include in the SBOM such as .nuspec/.pom files in the local package cache. + /// + ConfigurationSetting EnablePackageMetadataParsing { get; set; } } diff --git a/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs b/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs index 528f8d82..17794959 100644 --- a/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs +++ b/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs @@ -140,4 +140,7 @@ public class InputConfiguration : IConfiguration /// [DefaultValue(false)] public ConfigurationSetting FetchLicenseInformation { get; set; } + + [DefaultValue(false)] + public ConfigurationSetting EnablePackageMetadataParsing { get; set; } } diff --git a/src/Microsoft.Sbom.Common/FileSystemUtils.cs b/src/Microsoft.Sbom.Common/FileSystemUtils.cs index 34fdca34..905b190d 100644 --- a/src/Microsoft.Sbom.Common/FileSystemUtils.cs +++ b/src/Microsoft.Sbom.Common/FileSystemUtils.cs @@ -103,4 +103,7 @@ public bool IsDirectoryEmpty(string directoryPath) => /// public DirectoryInfo GetParentDirectory(string path) => Directory.GetParent(path); + + /// + public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path); } diff --git a/src/Microsoft.Sbom.Common/IFileSystemUtils.cs b/src/Microsoft.Sbom.Common/IFileSystemUtils.cs index 738c1c0a..8ddc2b83 100644 --- a/src/Microsoft.Sbom.Common/IFileSystemUtils.cs +++ b/src/Microsoft.Sbom.Common/IFileSystemUtils.cs @@ -170,4 +170,11 @@ public interface IFileSystemUtils /// The absolute or relative path of the file or directory. /// The parent directory. DirectoryInfo GetParentDirectory(string path); + + /// + /// Read all the content of the specified file as an array of bytes. + /// + /// The absolute relative path of a file. + /// Byte array content of the file. + byte[] ReadAllBytes(string path); } diff --git a/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs b/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs index 8f06071f..8384ba3f 100644 --- a/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs @@ -44,6 +44,7 @@ using Microsoft.Sbom.Api.Manifest.FileHashes; using Microsoft.Sbom.Api.Output; using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Api.Providers; using Microsoft.Sbom.Api.SignValidator; using Microsoft.Sbom.Api.Utils; @@ -150,6 +151,9 @@ public static IServiceCollection AddSbomTool(this IServiceCollection services, L .AddTransient() .AddTransient() .AddTransient() + .AddSingleton() + .AddSingleton, NugetUtils>() + .AddSingleton, MavenUtils>() .AddSingleton() .AddSingleton() .AddSingleton() @@ -195,7 +199,7 @@ public static IServiceCollection AddSbomTool(this IServiceCollection services, L var manifestData = new ManifestData(); - if (!configuration.ManifestInfo.Value.Contains(Api.Utils.Constants.SPDX22ManifestInfo)) + if (!configuration.ManifestInfo.Value.Contains(Constants.SPDX22ManifestInfo)) { var sbomConfig = sbomConfigs.Get(configuration.ManifestInfo?.Value?.FirstOrDefault()); var parserProvider = x.GetRequiredService(); diff --git a/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs b/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs index df9fc422..55a1561b 100644 --- a/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs +++ b/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs @@ -118,7 +118,7 @@ public void BadInput_ThrowsException() public void CargoComponent_ToSbomPackage() { var cargoComponent = new CargoComponent("name", "version"); - var scannedComponent = new ScannedComponentWithLicense() { Component = cargoComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = cargoComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -128,31 +128,11 @@ public void CargoComponent_ToSbomPackage() Assert.AreEqual(cargoComponent.Version, sbomPackage.PackageVersion); } - [TestMethod] - public void ConanComponent_ToSbomPackage() - { - var md5 = Guid.NewGuid().ToString(); - var sha1Hash = Guid.NewGuid().ToString(); - - var conanComponent = new ConanComponent("name", "version", md5, sha1Hash); - var scannedComponent = new ScannedComponentWithLicense() { Component = conanComponent }; - - var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); - - Assert.IsNotNull(sbomPackage.Id); - Assert.IsNotNull(sbomPackage.PackageUrl); - Assert.AreEqual(conanComponent.Name, sbomPackage.PackageName); - Assert.AreEqual(conanComponent.Version, sbomPackage.PackageVersion); - Assert.IsNotNull(sbomPackage.Checksum.First(x => x.ChecksumValue == conanComponent.Md5Hash)); - Assert.IsNotNull(sbomPackage.Checksum.First(x => x.ChecksumValue == conanComponent.Sha1Hash)); - Assert.AreEqual(conanComponent.PackageSourceURL, sbomPackage.PackageSource); - } - [TestMethod] public void CondaComponent_ToSbomPackage() { var condaComponent = new CondaComponent("name", "version", "build", "channel", "subdir", "namespace", "http://microsoft.com", "md5"); - var scannedComponent = new ScannedComponentWithLicense() { Component = condaComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = condaComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -168,7 +148,7 @@ public void CondaComponent_ToSbomPackage() public void DockerImageComponent_ToSbomPackage() { var dockerImageComponent = new DockerImageComponent("name", "version", "tag") { Digest = "digest" }; - var scannedComponent = new ScannedComponentWithLicense() { Component = dockerImageComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = dockerImageComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -183,7 +163,7 @@ public void DockerImageComponent_ToSbomPackage() public void NpmComponent_ToSbomPackage() { var npmComponent = new NpmComponent("name", "verison", author: new NpmAuthor("name", "email@contoso.com")); - var scannedComponent = new ScannedComponentWithLicense() { Component = npmComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = npmComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -198,7 +178,7 @@ public void NpmComponent_ToSbomPackage() public void NpmComponent_ToSbomPackage_NoAuthor() { var npmComponent = new NpmComponent("name", "verison"); - var scannedComponent = new ScannedComponentWithLicense() { Component = npmComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = npmComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -213,7 +193,7 @@ public void NpmComponent_ToSbomPackage_NoAuthor() public void NuGetComponent_ToSbomPackage() { var nuGetComponent = new NuGetComponent("name", "version", new string[] { "Author Name1, Another Author" }); - var scannedComponent = new ScannedComponentWithLicense() { Component = nuGetComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = nuGetComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -228,7 +208,7 @@ public void NuGetComponent_ToSbomPackage() public void NuGetComponent_ToSbomPackage_NoAuthor() { var nuGetComponent = new NuGetComponent("name", "version"); - var scannedComponent = new ScannedComponentWithLicense() { Component = nuGetComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = nuGetComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -243,7 +223,7 @@ public void NuGetComponent_ToSbomPackage_NoAuthor() public void PipComponent_ToSbomPackage() { var pipComponent = new PipComponent("name", "version"); - var scannedComponent = new ScannedComponentWithLicense() { Component = pipComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = pipComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); @@ -258,7 +238,7 @@ public void GitComponent_ToSbomPackage() { var uri = new Uri("https://microsoft.com"); var gitComponent = new GitComponent(uri, "version"); - var scannedComponent = new ScannedComponentWithLicense() { Component = gitComponent }; + var scannedComponent = new ExtendedScannedComponent() { Component = gitComponent }; var sbomPackage = scannedComponent.ToSbomPackage(new AdapterReport()); diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs index 300ca5a7..63ec764f 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs @@ -50,24 +50,24 @@ public ComponentToPackageInfoConverterTests() [TestMethod] public async Task ConvertTestAsync() { - var scannedComponents = new List() + var scannedComponents = new List() { - new ScannedComponentWithLicense + new ExtendedScannedComponent { LocationsFoundAt = "test".Split(), Component = new NuGetComponent("nugetpackage", "1.0.0") }, - new ScannedComponentWithLicense + new ExtendedScannedComponent { LocationsFoundAt = "test".Split(), Component = new NuGetComponent("nugetpackage2", "1.0.0") }, - new ScannedComponentWithLicense + new ExtendedScannedComponent { LocationsFoundAt = "test".Split(), Component = new GitComponent(new Uri("http://test.uri"), "hash") }, - new ScannedComponentWithLicense + new ExtendedScannedComponent { LocationsFoundAt = "test".Split(), Component = new MavenComponent("groupId", "artifactId", "1.0.0") @@ -89,7 +89,7 @@ public async Task ConvertTestAsync() [TestMethod] public async Task ConvertNuGet_AuthorPopulated() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NuGetComponent("nugetpackage", "1.0.0") { @@ -105,7 +105,7 @@ public async Task ConvertNuGet_AuthorPopulated() [TestMethod] public async Task ConvertNuGet_AuthorNotPopulated() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NuGetComponent("nugetpackage", "1.0.0") { Authors = null } }; @@ -116,36 +116,53 @@ public async Task ConvertNuGet_AuthorNotPopulated() } [TestMethod] - public async Task ConvertNuGet_LicensePopulated() + public async Task ConvertNuGet_LicenseConcludedPopulated() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NuGetComponent("nugetpackage", "1.0.0") { Authors = null }, - License = "MIT" + LicenseConcluded = "MIT" }; var packageInfo = await ConvertScannedComponent(scannedComponent); Assert.AreEqual("MIT", packageInfo.LicenseInfo.Concluded); + Assert.IsNull(packageInfo.LicenseInfo?.Declared); } [TestMethod] - public async Task ConvertNuGet_LicenseNotPopulated() + public async Task ConvertNuGet_LicenseDeclaredPopulated() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NuGetComponent("nugetpackage", "1.0.0") { Authors = null }, + LicenseDeclared = "MIT" }; var packageInfo = await ConvertScannedComponent(scannedComponent); + Assert.AreEqual("MIT", packageInfo.LicenseInfo.Declared); Assert.IsNull(packageInfo.LicenseInfo?.Concluded); } + [TestMethod] + public async Task ConvertNuGet_LicensesNotPopulated() + { + var scannedComponent = new ExtendedScannedComponent + { + Component = new NuGetComponent("nugetpackage", "1.0.0") { Authors = null }, + }; + + var packageInfo = await ConvertScannedComponent(scannedComponent); + + Assert.IsNull(packageInfo.LicenseInfo?.Concluded); + Assert.IsNull(packageInfo.LicenseInfo?.Declared); + } + [TestMethod] public async Task ConvertNpm_AuthorPopulated_Name() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("nugetpackage", "1.0.0", author: new NpmAuthor("Suzy Author")) }; @@ -158,7 +175,7 @@ public async Task ConvertNpm_AuthorPopulated_Name() [TestMethod] public async Task ConvertNpm_AuthorPopulated_NameAndEmail() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("nugetpackage", "1.0.0", author: new NpmAuthor("Suzy Author", "suzya@contoso.com")) }; @@ -171,7 +188,7 @@ public async Task ConvertNpm_AuthorPopulated_NameAndEmail() [TestMethod] public async Task ConvertNpm_AuthorNotPopulated() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("npmpackage", "1.0.0") { Author = null } }; @@ -184,10 +201,10 @@ public async Task ConvertNpm_AuthorNotPopulated() [TestMethod] public async Task ConvertNpm_LicensePopulated() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("npmpackage", "1.0.0") { Author = null }, - License = "MIT" + LicenseConcluded = "MIT" }; var packageInfo = await ConvertScannedComponent(scannedComponent); @@ -198,7 +215,7 @@ public async Task ConvertNpm_LicensePopulated() [TestMethod] public async Task ConvertNpm_LicenseNotPopulated() { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("npmpackage", "1.0.0") { Author = null }, }; @@ -211,21 +228,21 @@ public async Task ConvertNpm_LicenseNotPopulated() [TestMethod] public async Task ConvertWorksWithBuildComponentPathNull() { - var scannedComponents = new List() + var scannedComponents = new List() { - new ScannedComponentWithLicense + new ExtendedScannedComponent { Component = new NuGetComponent("nugetpackage", "1.0.0") }, - new ScannedComponentWithLicense + new ExtendedScannedComponent { Component = new NuGetComponent("nugetpackage2", "1.0.0") }, - new ScannedComponentWithLicense + new ExtendedScannedComponent { Component = new GitComponent(new Uri("http://test.uri"), "hash") }, - new ScannedComponentWithLicense + new ExtendedScannedComponent { Component = new MavenComponent("groupId", "artifactId", "1.0.0") } @@ -243,7 +260,7 @@ public async Task ConvertWorksWithBuildComponentPathNull() Assert.IsFalse(errors?.Any()); } - private async Task ConvertScannedComponent(ScannedComponentWithLicense scannedComponent) + private async Task ConvertScannedComponent(ExtendedScannedComponent scannedComponent) { var componentsChannel = Channel.CreateUnbounded(); await componentsChannel.Writer.WriteAsync(scannedComponent); diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/PackagesWalkerTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/PackagesWalkerTests.cs index 8364e0e0..f29b022a 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/PackagesWalkerTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/PackagesWalkerTests.cs @@ -8,8 +8,11 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Orchestrator.Commands; +using Microsoft.Sbom.Adapters.ComponentDetection; using Microsoft.Sbom.Api.Exceptions; using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; @@ -23,9 +26,6 @@ namespace Microsoft.Sbom.Api.Executors.Tests; -using Microsoft.ComponentDetection.Orchestrator.Commands; -using Microsoft.Sbom.Adapters.ComponentDetection; - [TestClass] public class PackagesWalkerTests { @@ -34,6 +34,7 @@ public class PackagesWalkerTests private readonly Mock mockSbomConfigs = new Mock(); private readonly Mock mockFileSystemUtils = new Mock(); private readonly Mock mockLicenseInformationFetcher = new Mock(); + private readonly Mock mockPackageDetailsFactory = new Mock(); public PackagesWalkerTests() { @@ -48,10 +49,10 @@ public PackagesWalkerTests() [TestMethod] public async Task ScanSuccessTestAsync() { - var scannedComponents = new List(); + var scannedComponents = new List(); for (var i = 1; i < 4; i++) { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("componentName", $"{i}") }; @@ -59,7 +60,7 @@ public async Task ScanSuccessTestAsync() scannedComponents.Add(scannedComponent); } - var scannedComponentOther = new ScannedComponentWithLicense + var scannedComponentOther = new ExtendedScannedComponent { Component = new NpmComponent("componentName", "3") }; @@ -75,12 +76,12 @@ public async Task ScanSuccessTestAsync() }; mockDetector.Setup(o => o.ScanAsync(It.IsAny())).Returns(Task.FromResult(scanResult)); - var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockLicenseInformationFetcher.Object); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockPackageDetailsFactory.Object, mockLicenseInformationFetcher.Object); var packagesChannelReader = walker.GetComponents("root"); var countDistinctComponents = 0; - await foreach (ScannedComponentWithLicense package in packagesChannelReader.output.ReadAllAsync()) + await foreach (ExtendedScannedComponent package in packagesChannelReader.output.ReadAllAsync()) { countDistinctComponents++; Assert.IsTrue(scannedComponents.Remove(package)); @@ -99,10 +100,10 @@ public async Task ScanSuccessTestAsync() [TestMethod] public async Task ScanCombinePackagesWithSameNameDifferentCase() { - var scannedComponents = new List(); + var scannedComponents = new List(); for (var i = 1; i < 4; i++) { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("componentName", $"{i}") }; @@ -110,7 +111,7 @@ public async Task ScanCombinePackagesWithSameNameDifferentCase() scannedComponents.Add(scannedComponent); } - var scannedComponentOther = new ScannedComponentWithLicense + var scannedComponentOther = new ExtendedScannedComponent { // Component with changed case. should also match 'componentName' and // thus only 3 components should be detected. @@ -128,12 +129,12 @@ public async Task ScanCombinePackagesWithSameNameDifferentCase() }; mockDetector.Setup(o => o.ScanAsync(It.IsAny())).Returns(Task.FromResult(scanResult)); - var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockLicenseInformationFetcher.Object); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockPackageDetailsFactory.Object, mockLicenseInformationFetcher.Object); var packagesChannelReader = walker.GetComponents("root"); var countDistinctComponents = 0; - await foreach (ScannedComponentWithLicense package in packagesChannelReader.output.ReadAllAsync()) + await foreach (ExtendedScannedComponent package in packagesChannelReader.output.ReadAllAsync()) { countDistinctComponents++; Assert.IsTrue(scannedComponents.Remove(package)); @@ -154,7 +155,7 @@ public void ScanWithNullOrEmptyPathSuccessTest() { var mockDetector = new Mock(new Mock().Object, new Mock().Object); - var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockLicenseInformationFetcher.Object); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockPackageDetailsFactory.Object, mockLicenseInformationFetcher.Object); walker.GetComponents(null); walker.GetComponents(string.Empty); @@ -173,7 +174,7 @@ public async Task ScanFailureTestAsync() }; mockDetector.Setup(o => o.ScanAsync(It.IsAny())).Returns(Task.FromResult(scanResult)); - var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockLicenseInformationFetcher.Object); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockPackageDetailsFactory.Object, mockLicenseInformationFetcher.Object); var packagesChannelReader = walker.GetComponents("root"); ComponentDetectorException actualError = null; @@ -194,10 +195,10 @@ public async Task ScanFailureTestAsync() [TestMethod] public async Task ScanIgnoreSbomComponents() { - var scannedComponents = new List(); + var scannedComponents = new List(); for (var i = 1; i < 4; i++) { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new NpmComponent("componentName", $"{i}") }; @@ -205,7 +206,7 @@ public async Task ScanIgnoreSbomComponents() scannedComponents.Add(scannedComponent); } - var scannedComponentOther = new ScannedComponentWithLicense + var scannedComponentOther = new ExtendedScannedComponent { Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.com"), "componentName", "123", "abcdf", "path1") }; @@ -221,7 +222,7 @@ public async Task ScanIgnoreSbomComponents() }; mockDetector.Setup(o => o.ScanAsync(It.IsAny())).Returns(Task.FromResult(scanResult)); - var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockLicenseInformationFetcher.Object); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystemUtils.Object, mockPackageDetailsFactory.Object, mockLicenseInformationFetcher.Object); var packagesChannelReader = walker.GetComponents("root"); var discoveredComponents = await packagesChannelReader.output.ReadAllAsync().ToListAsync(); diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/SBOMComponentsWalkerTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/SBOMComponentsWalkerTests.cs index 7952eda4..5d776eba 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/SBOMComponentsWalkerTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/SBOMComponentsWalkerTests.cs @@ -8,8 +8,10 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; -using Microsoft.Sbom.Api.Exceptions; +using Microsoft.ComponentDetection.Orchestrator.Commands; +using Microsoft.Sbom.Adapters.ComponentDetection; using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; @@ -23,9 +25,6 @@ namespace Microsoft.Sbom.Api.Executors.Tests; -using Microsoft.ComponentDetection.Orchestrator.Commands; -using Microsoft.Sbom.Adapters.ComponentDetection; - [TestClass] public class SBOMComponentsWalkerTests { @@ -33,6 +32,7 @@ public class SBOMComponentsWalkerTests private readonly Mock mockConfiguration = new Mock(); private readonly Mock mockSbomConfigs = new Mock(); private readonly Mock mockFileSystem = new Mock(); + private readonly Mock mockPackageDetailsFactory = new Mock(); private readonly Mock mockLicenseInformationFetcher = new Mock(); public SBOMComponentsWalkerTests() @@ -48,10 +48,10 @@ public SBOMComponentsWalkerTests() [TestMethod] public async Task GetComponents() { - var scannedComponents = new List(); + var scannedComponents = new List(); for (var i = 1; i < 4; i++) { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), "componentName", $"123{i}", "abcdef", $"path{i}"), DetectorId = "SPDX22SBOM" @@ -69,7 +69,7 @@ public async Task GetComponents() }; mockDetector.Setup(o => o.ScanAsync(It.IsAny())).Returns(Task.FromResult(scanResult)); - var walker = new SBOMComponentsWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystem.Object, mockLicenseInformationFetcher.Object); + var walker = new SBOMComponentsWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystem.Object, mockPackageDetailsFactory.Object, mockLicenseInformationFetcher.Object); var packagesChannelReader = walker.GetComponents("root"); var discoveredComponents = await packagesChannelReader.output.ReadAllAsync().ToListAsync(); @@ -86,10 +86,10 @@ public async Task GetComponents() [TestMethod] public async Task GetComponentsWithFiltering() { - var scannedComponents = new List(); + var scannedComponents = new List(); for (var i = 1; i < 4; i++) { - var scannedComponent = new ScannedComponentWithLicense + var scannedComponent = new ExtendedScannedComponent { Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), "componentName", $"123{i}", "abcdef", $"path{i}"), DetectorId = "SPDX22SBOM" @@ -98,7 +98,7 @@ public async Task GetComponentsWithFiltering() scannedComponents.Add(scannedComponent); } - var nonSbomComponent = new ScannedComponentWithLicense + var nonSbomComponent = new ExtendedScannedComponent { Component = new NpmComponent("componentName", "123"), DetectorId = "notSPDX22SBOM" @@ -114,7 +114,7 @@ public async Task GetComponentsWithFiltering() }; mockDetector.Setup(o => o.ScanAsync(It.IsAny())).Returns(Task.FromResult(scanResult)); - var walker = new SBOMComponentsWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystem.Object, mockLicenseInformationFetcher.Object); + var walker = new SBOMComponentsWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object, mockFileSystem.Object, mockPackageDetailsFactory.Object, mockLicenseInformationFetcher.Object); var packagesChannelReader = walker.GetComponents("root"); var discoveredComponents = await packagesChannelReader.output.ReadAllAsync().ToListAsync(); diff --git a/test/Microsoft.Sbom.Api.Tests/PackageDetails/MavenUtilsTests.cs b/test/Microsoft.Sbom.Api.Tests/PackageDetails/MavenUtilsTests.cs new file mode 100644 index 00000000..d4179e6c --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/PackageDetails/MavenUtilsTests.cs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.PackageDetails; +using Microsoft.Sbom.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; + +namespace Microsoft.Sbom.Api.Tests.PackageDetails; + +[TestClass] +public class MavenUtilsTests +{ + private readonly Mock mockFileSystemUtils = new Mock(); + private readonly Mock mockLogger = new Mock(); + private readonly Mock mockRecorder = new Mock(); + + private static readonly string EnvHomePath = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "HOMEPATH" : "HOME"; + private static readonly string HomePath = Environment.GetEnvironmentVariable(EnvHomePath); + private static readonly string MavenPackagesPath = Path.Join(HomePath, ".m2/repository"); + + [TestMethod] + public void GetPomLocation_WhenPomExists_ShouldReturnPath() + { + var mavenUtils = new MavenUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var scannedComponent = new ScannedComponent + { + Component = new MavenComponent("testGroupId", "testArtifactId", "1.0.0") + }; + + var pathToPom = Path.Join(MavenPackagesPath, "testgroupid/testartifactid/1.0.0/testartifactid-1.0.0.pom"); + + var expectedPath = Path.GetFullPath(pathToPom); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + + var result = mavenUtils.GetMetadataLocation(scannedComponent); + + Assert.AreEqual(expectedPath, result); + } + + [TestMethod] + public void GetPomLocation_WhenPomDoesNotExist_ShouldReturnNull() + { + var mavenUtils = new MavenUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var scannedComponent = new ScannedComponent + { + Component = new MavenComponent("testGroupId", "testArtifactId", "1.0.0") + }; + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(false); + + var result = mavenUtils.GetMetadataLocation(scannedComponent); + + Assert.IsNull(result); + } + + [TestMethod] + public void ParsePom_WhenPomIsValid() + { + var mavenUtils = new MavenUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var pomContent = SampleMetadataFiles.PomWithLicensesAndDevelopers; + + // Convert pomContent to an array of bytes + var pomBytes = Encoding.UTF8.GetBytes(pomContent); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.ReadAllBytes(It.IsAny())).Returns(pomBytes); + + var parsedInfo = mavenUtils.ParseMetadata(pomContent); + + Assert.AreEqual("test-package", parsedInfo.Name); + Assert.AreEqual("1.3", parsedInfo.Version); + Assert.AreEqual("New BSD License", parsedInfo.PackageDetails.License); + Assert.AreEqual("Person: Sample Name", parsedInfo.PackageDetails.Supplier); + } + + [TestMethod] + public void ParsePom_WithoutDeveloperSection() + { + var mavenUtils = new MavenUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var pomContent = SampleMetadataFiles.PomWithoutDevelopersSection; + + // Convert pomContent to an array of bytes + var pomBytes = Encoding.UTF8.GetBytes(pomContent); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.ReadAllBytes(It.IsAny())).Returns(pomBytes); + + var parsedInfo = mavenUtils.ParseMetadata(pomContent); + + Assert.AreEqual("test-package", parsedInfo.Name); + Assert.AreEqual("1.3", parsedInfo.Version); + Assert.AreEqual("New BSD License", parsedInfo.PackageDetails.License); + Assert.IsTrue(string.IsNullOrEmpty(parsedInfo.PackageDetails.Supplier)); + } + + [TestMethod] + public void ParsePom_WithoutLicense() + { + var mavenUtils = new MavenUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var pomContent = SampleMetadataFiles.PomWithoutLicense; + + // Convert pomContent to an array of bytes + var pomBytes = Encoding.UTF8.GetBytes(pomContent); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.ReadAllBytes(It.IsAny())).Returns(pomBytes); + + var parsedInfo = mavenUtils.ParseMetadata(pomContent); + + Assert.AreEqual("test-package", parsedInfo.Name); + Assert.AreEqual("1.3", parsedInfo.Version); + Assert.IsTrue(string.IsNullOrEmpty(parsedInfo.PackageDetails.License)); + Assert.AreEqual("Person: Sample Name", parsedInfo.PackageDetails.Supplier); + } + + public void ParsePom_WithOrganizationAndDevelopers_PopulatesAsOrganization() + { + var mavenUtils = new MavenUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var pomContent = SampleMetadataFiles.PomWithDevelopersAndOrganization; + + // Convert pomContent to an array of bytes + var pomBytes = Encoding.UTF8.GetBytes(pomContent); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.ReadAllBytes(It.IsAny())).Returns(pomBytes); + + var parsedInfo = mavenUtils.ParseMetadata(pomContent); + + Assert.AreEqual("test-package", parsedInfo.Name); + Assert.AreEqual("1.3", parsedInfo.Version); + Assert.IsTrue(string.IsNullOrEmpty(parsedInfo.PackageDetails.License)); + Assert.AreEqual("Person: Sample Name", parsedInfo.PackageDetails.Supplier); + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/PackageDetails/NugetUtilsTests.cs b/test/Microsoft.Sbom.Api.Tests/PackageDetails/NugetUtilsTests.cs new file mode 100644 index 00000000..1bc11af2 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/PackageDetails/NugetUtilsTests.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.PackageDetails; +using Microsoft.Sbom.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using NuGet.Configuration; +using Serilog; + +namespace Microsoft.Sbom.Api.Tests.PackageDetails; + +[TestClass] +public class NugetUtilsTests +{ + private readonly Mock mockFileSystemUtils = new Mock(); + private readonly Mock mockLogger = new Mock(); + private readonly Mock mockRecorder = new Mock(); + + private static readonly string NugetPackagesPath = SettingsUtility.GetGlobalPackagesFolder(new NullSettings()); + + [TestMethod] + public void GetNuspecLocation_WhenNuspecExists_ShouldReturnPath() + { + var nugetUtils = new NugetUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var scannedComponent = new ScannedComponent + { + Component = new NuGetComponent("testName", "1.0.0") + }; + + var nuspecPath = $"{NugetPackagesPath}{((NuGetComponent)scannedComponent.Component).Name.ToLower()}/{((NuGetComponent)scannedComponent.Component).Version}/{((NuGetComponent)scannedComponent.Component).Name.ToLower()}.nuspec"; + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + + var result = nugetUtils.GetMetadataLocation(scannedComponent); + + Assert.AreEqual(nuspecPath, result); + } + + [TestMethod] + public void GetNuspecLocation_WhenNuspecDoesNotExist_ShouldReturnNull() + { + var nugetUtils = new NugetUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var scannedComponent = new ScannedComponent + { + Component = new NuGetComponent("testName", "1.0.0") + }; + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(false); + + var result = nugetUtils.GetMetadataLocation(scannedComponent); + + Assert.IsNull(result); + } + + [TestMethod] + public void ParseNuspec_WhenNuspecIsValid() + { + var nugetUtils = new NugetUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var nuspecContent = SampleMetadataFiles.NuspecWithValidLicenseAndAuthors; + + // Convert nuspecContent to an array of bytes + var nuspecBytes = Encoding.UTF8.GetBytes(nuspecContent); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.ReadAllBytes(It.IsAny())).Returns(nuspecBytes); + + var parsedPackageInfo = nugetUtils.ParseMetadata(nuspecContent); + + Assert.AreEqual("FakePackageName", parsedPackageInfo.Name); + Assert.AreEqual("1.0", parsedPackageInfo.Version); + Assert.AreEqual("FakeLicense", parsedPackageInfo.PackageDetails.License); + Assert.AreEqual("Organization: FakeAuthor", parsedPackageInfo.PackageDetails.Supplier); + } + + [TestMethod] + public void ParseNuspec_LicenseIsFile_SupplierSucceeds() + { + var nugetUtils = new NugetUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var nuspecContent = SampleMetadataFiles.NuspecWithInvalidLicense; + + // Convert nuspecContent to an array of bytes + var nuspecBytes = Encoding.UTF8.GetBytes(nuspecContent); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.ReadAllBytes(It.IsAny())).Returns(nuspecBytes); + + var parsedPackageInfo = nugetUtils.ParseMetadata(nuspecContent); + + Assert.AreEqual("FakePackageName", parsedPackageInfo.Name); + Assert.AreEqual("1.0", parsedPackageInfo.Version); + Assert.AreEqual("Organization: FakeAuthor", parsedPackageInfo.PackageDetails.Supplier); + Assert.IsTrue(string.IsNullOrEmpty(parsedPackageInfo.PackageDetails.License)); + } + + [TestMethod] + public void ParseNuspec_NoAuthorFound_DoesNotFail() + { + var nugetUtils = new NugetUtils(mockFileSystemUtils.Object, mockLogger.Object, mockRecorder.Object); + + var nuspecContent = SampleMetadataFiles.NuspecWithoutAuthor; + + // Convert nuspecContent to an array of bytes + var nuspecBytes = Encoding.UTF8.GetBytes(nuspecContent); + + mockFileSystemUtils.Setup(fs => fs.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.FileExists(It.IsAny())).Returns(true); + mockFileSystemUtils.Setup(fs => fs.ReadAllBytes(It.IsAny())).Returns(nuspecBytes); + + var parsedPackageInfo = nugetUtils.ParseMetadata(nuspecContent); + + Assert.AreEqual("FakePackageName", parsedPackageInfo.Name); + Assert.AreEqual("1.0", parsedPackageInfo.Version); + Assert.IsTrue(string.IsNullOrEmpty(parsedPackageInfo.PackageDetails.Supplier)); + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/PackageDetails/SampleMetadataFiles.cs b/test/Microsoft.Sbom.Api.Tests/PackageDetails/SampleMetadataFiles.cs new file mode 100644 index 00000000..79220856 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/PackageDetails/SampleMetadataFiles.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.Tests.PackageDetails; + +public static class SampleMetadataFiles +{ + public const string PomWithLicensesAndDevelopers = @" + + 4.0.0 + + org.test + test-package + 1.3 + pom + + + + New BSD License + http://www.opensource.org/licenses/bsd-license.php + repo + + + + + + SAMPLE + Sample Name + + Developer + + + + "; + + public const string PomWithoutDevelopersSection = @" + + 4.0.0 + + org.test + test-package + 1.3 + pom + + + + New BSD License + http://www.opensource.org/licenses/bsd-license.php + repo + + + "; + + public const string PomWithDevelopersAndOrganization = @" + + 4.0.0 + + org.test + test-package + 1.3 + pom + + + + SAMPLE + Sample Name + + Developer + + + + + + Microsoft + + + "; + + public const string PomWithoutLicense = @" + + 4.0.0 + + org.test + test-package + 1.3 + pom + + + + SAMPLE + Sample Name + + Developer + + + + SAMPLE + Sample Name2 + + Developer + + + + "; + + public const string NuspecWithValidLicenseAndAuthors = @" + + FakePackageName + 1.0 + FakeAuthor + FakeLicense + - + "; + + public const string NuspecWithInvalidLicense = @" + + FakePackageName + 1.0 + FakeAuthor + FakeLicense + - + "; + + public const string NuspecWithoutAuthor = @" + + FakePackageName + 1.0 + - + "; +} diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index 22bf1b2e..cbae545d 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -21,6 +21,7 @@ using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output; using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.PackageDetails; using Microsoft.Sbom.Api.Providers; using Microsoft.Sbom.Api.Providers.ExternalDocumentReferenceProviders; using Microsoft.Sbom.Api.Providers.FilesProviders; @@ -64,6 +65,7 @@ public class ManifestGenerationWorkflowTests private readonly Mock sBOMReaderForExternalDocumentReferenceMock = new Mock(); private readonly Mock fileSystemUtilsExtensionMock = new Mock(); private readonly Mock licenseInformationFetcherMock = new Mock(); + private readonly Mock mockPackageDetailsFactory = new Mock(); [TestInitialize] public void Setup() @@ -278,7 +280,8 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef manifestGeneratorProvider, mockLogger.Object), packageInfoConverterMock.Object, - new PackagesWalker(mockLogger.Object, mockDetector.Object, configurationMock.Object, sbomConfigs, fileSystemMock.Object, licenseInformationFetcherMock.Object), + new PackagesWalker(mockLogger.Object, mockDetector.Object, configurationMock.Object, sbomConfigs, fileSystemMock.Object, mockPackageDetailsFactory.Object, licenseInformationFetcherMock.Object), + mockPackageDetailsFactory.Object, licenseInformationFetcherMock.Object); var externalDocumentReferenceProvider = new ExternalDocumentReferenceProvider(