Extract package info from local package cache for NuGet and Maven (#441)

* Initial commit for package details factory.

* fix indentation.

* PackageDetailsFactory improvements

* Remove unnecesary if statement

* Add telemetry for packageDetails entries.

* Minor improvements

* Distinguish between organization and person (Maven)

* Update tests.

* Address Feedback

* Address feedback.

* Remove NoWarn

* Rename to ExtendedScannedResult

---------

Co-authored-by: Sebastian Gomez <segomez@microsoft.com>

Directory.Packages.props

@@ -31,6 +31,7 @@
         <PackageVersion Include="Mono.Posix.NETStandard" Version="1.0.0" Condition="'$(TargetFramework)' == 'net6.0'"/>
         <PackageVersion Include="Moq" Version="4.17.2" />
         <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
+        <PackageVersion Include="NuGet.Configuration" Version="6.7.0" />
         <PackageVersion Include="NuGet.Frameworks" Version="6.7.0" />
         <PackageVersion Include="packageurl-dotnet" Version="1.1.0" />
         <PackageVersion Include="PowerArgs" Version="3.6.0" />

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/CargoComponentExtensions.cs

@@ -15,17 +15,17 @@ internal static class CargoComponentExtensions
     /// Converts a <see cref="CargoComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="cargoComponent">The <see cref="CargoComponent" /> to convert.</param>
-    /// <param name="license">The license to use for the <see cref="SbomPackage" />.</param>
+    /// <param name="component">The <see cref="ExtendedScannedComponent"/> version of the CargoComponent</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>
-    public static SbomPackage ToSbomPackage(this CargoComponent cargoComponent, string? license = null) => new()
+    public static SbomPackage ToSbomPackage(this CargoComponent cargoComponent, ExtendedScannedComponent component) => new()
     {
         Id = cargoComponent.Id,
         PackageUrl = cargoComponent.PackageUrl?.ToString(),
         PackageName = cargoComponent.Name,
         PackageVersion = cargoComponent.Version,
-        LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo
+        LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo
         {
-            Concluded = license,
+            Concluded = component.LicenseConcluded,
         },
         FilesAnalyzed = false,
         Type = "cargo",

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScanResultWithLicense.cs → src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScanResult.cs

@@ -12,10 +12,10 @@ namespace Microsoft.Sbom.Adapters.ComponentDetection;
 /// A <see cref="ScanResult" /> with license information.
 /// </summary>
 [JsonObject(MemberSerialization.OptOut, NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-public sealed class ScanResultWithLicense : ScanResult
+public sealed class ExtendedScanResult : ScanResult
 {
     /// <summary>
     /// Gets or sets the scanned components with license information.
     /// </summary>
-    public new IEnumerable<ScannedComponentWithLicense>? ComponentsFound { get; init; }
+    public new IEnumerable<ExtendedScannedComponent>? ComponentsFound { get; init; }
 }

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentWithLicense.cs → src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ExtendedScannedComponent.cs

@@ -8,15 +8,15 @@ namespace Microsoft.Sbom.Adapters.ComponentDetection;
 using Microsoft.Sbom.Contracts;
 
 /// <summary>
-/// A <see cref="ScannedComponent" /> with license information.
+/// A <see cref="ScannedComponent" /> with additional properties extracted from package metadata files.
 /// </summary>
-public class ScannedComponentWithLicense : ScannedComponent
+public class ExtendedScannedComponent : ScannedComponent
 {
     /// <summary>
-    /// Initializes a new instance of the <see cref="ScannedComponentWithLicense" /> class.
+    /// Initializes a new instance of the <see cref="ExtendedScannedComponent" /> class.
     /// </summary>
     /// <param name="other">The <see cref="ScannedComponent" /> to copy properties from.</param>
-    public ScannedComponentWithLicense(ScannedComponent? other = null)
+    public ExtendedScannedComponent(ScannedComponent? other = null)
     {
         if (other == null)
         {
@@ -35,12 +35,22 @@ public ScannedComponentWithLicense(ScannedComponent? other = null)
     }
 
     /// <summary>
-    /// Gets or sets the license.
+    /// Gets or sets the license concluded which is retrieved from the ClearlyDefined API.
     /// </summary>
-    public string? License { get; set; }
+    public string? LicenseConcluded { get; set; }
 
     /// <summary>
-    /// Converts a <see cref="ScannedComponentWithLicense" /> to an <see cref="SbomPackage" />.
+    /// Gets or sets the license declared which is found directly in the package metadata.
+    /// </summary>
+    public string? LicenseDeclared { get; set; }
+
+    /// <summary>
+    /// Gets or sets the supplier.
+    /// </summary>
+    public string? Supplier { get; set; }
+
+    /// <summary>
+    /// Converts a <see cref="ExtendedScannedComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="report">The <see cref="AdapterReport" /> to use.</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/MavenComponentExtensions.cs

@@ -15,14 +15,20 @@ internal static class MavenComponentExtensions
     /// Converts a <see cref="MavenComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="mavenComponent">The <see cref="MavenComponent" /> to convert.</param>
+    /// <param name="component">The <see cref="ExtendedScannedComponent"/> version of the MavenComponent</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>
-    public static SbomPackage? ToSbomPackage(this MavenComponent mavenComponent) => new()
+    public static SbomPackage? ToSbomPackage(this MavenComponent mavenComponent, ExtendedScannedComponent component) => new()
     {
         Id = mavenComponent.Id,
         PackageName = $"{mavenComponent.GroupId}.{mavenComponent.ArtifactId}",
         PackageUrl = mavenComponent.PackageUrl?.ToString(),
         PackageVersion = mavenComponent.Version,
         FilesAnalyzed = false,
+        Supplier = string.IsNullOrEmpty(component.Supplier) ? null : component.Supplier,
+        LicenseInfo = string.IsNullOrEmpty(component.LicenseDeclared) ? null : new LicenseInfo
+        {
+            Declared = component.LicenseDeclared,
+        },
         Type = "maven",
     };
 }

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NpmComponentExtensions.cs

@@ -17,9 +17,9 @@ internal static class NpmComponentExtensions
     /// Converts a <see cref="NpmComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="npmComponent">The <see cref="NpmComponent" /> to convert.</param>
-    /// <param name="license"> License information for the package that component that is being converted.</param>
+    /// <param name="component">The <see cref="ExtendedScannedComponent"/> version of the NpmComponent</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>
-    public static SbomPackage ToSbomPackage(this NpmComponent npmComponent, string? license = null) => new()
+    public static SbomPackage ToSbomPackage(this NpmComponent npmComponent, ExtendedScannedComponent component) => new()
     {
         Id = npmComponent.Id,
         PackageUrl = npmComponent.PackageUrl?.ToString(),
@@ -33,9 +33,9 @@ internal static class NpmComponentExtensions
             },
         },
         Supplier = npmComponent.Author?.AsSupplier(),
-        LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo
+        LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo
         {
-            Concluded = license,
+            Concluded = component.LicenseConcluded,
         },
         FilesAnalyzed = false,
         Type = "npm",

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/NuGetComponentExtensions.cs

@@ -16,18 +16,19 @@ internal static class NuGetComponentExtensions
     /// Converts a <see cref="NuGetComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="nuGetComponent">The <see cref="NuGetComponent" /> to convert.</param>
-    /// <param name="license"> License information for the package that component that is being converted.</param>
+    /// <param name="component">The <see cref="ExtendedScannedComponent"/> version of the NuGetComponent</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>
-    public static SbomPackage ToSbomPackage(this NuGetComponent nuGetComponent, string? license = null) => new()
+    public static SbomPackage ToSbomPackage(this NuGetComponent nuGetComponent, ExtendedScannedComponent component) => new()
     {
         Id = nuGetComponent.Id,
         PackageUrl = nuGetComponent.PackageUrl?.ToString(),
         PackageName = nuGetComponent.Name,
         PackageVersion = nuGetComponent.Version,
-        Supplier = nuGetComponent.Authors?.Any() == true ? $"Organization: {nuGetComponent.Authors.First()}" : null,
-        LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo
+        Supplier = nuGetComponent.Authors?.Any() == true ? $"Organization: {nuGetComponent.Authors.First()}" : component.Supplier,
+        LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) && string.IsNullOrEmpty(component.LicenseDeclared) ? null : new LicenseInfo
         {
-            Concluded = license,
+            Concluded = component.LicenseConcluded,
+            Declared = component.LicenseDeclared,
         },
         FilesAnalyzed = false,
         Type = "nuget",

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PipComponentExtensions.cs

@@ -15,17 +15,17 @@ internal static class PipComponentExtensions
     /// Converts a <see cref="PipComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="pipComponent">The <see cref="PipComponent" /> to convert.</param>
-    /// <param name="license">The license to use.</param>
+    /// <param name="component">The <see cref="ExtendedScannedComponent"/> version of the PipComponent</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>
-    public static SbomPackage ToSbomPackage(this PipComponent pipComponent, string? license = null) => new()
+    public static SbomPackage ToSbomPackage(this PipComponent pipComponent, ExtendedScannedComponent component) => new()
     {
         Id = pipComponent.Id,
         PackageUrl = pipComponent.PackageUrl?.ToString(),
         PackageName = pipComponent.Name,
         PackageVersion = pipComponent.Version,
-        LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo
+        LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo
         {
-            Concluded = license,
+            Concluded = component.LicenseConcluded,
         },
         FilesAnalyzed = false,
         Type = "python",

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/PodComponentExtensions.cs

@@ -15,18 +15,18 @@ internal static class PodComponentExtensions
     /// Converts a <see cref="PodComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="podComponent">The <see cref="PodComponent" /> to convert.</param>
-    /// <param name="license">The license to use.</param>
+    /// <param name="component">The <see cref="ExtendedScannedComponent"/> version of the PodComponent</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>
-    public static SbomPackage? ToSbomPackage(this PodComponent podComponent, string? license = null) => new()
+    public static SbomPackage? ToSbomPackage(this PodComponent podComponent, ExtendedScannedComponent component) => new()
     {
         Id = podComponent.Id,
         PackageUrl = podComponent.PackageUrl?.ToString(),
         PackageName = podComponent.Name,
         PackageVersion = podComponent.Version,
         PackageSource = podComponent.SpecRepo,
-        LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo
+        LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo
         {
-            Concluded = license,
+            Concluded = component.LicenseConcluded,
         },
         FilesAnalyzed = false,
         Type = "pod",

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/RubyGemsComponentExtensions.cs

@@ -15,18 +15,18 @@ internal static class RubyGemsComponentExtensions
     /// Converts a <see cref="RubyGemsComponent" /> to an <see cref="SbomPackage" />.
     /// </summary>
     /// <param name="rubyGemsComponent">The <see cref="RubyGemsComponent" /> to convert.</param>
-    /// <param name="license">The license to use.</param>
+    /// <param name="component">The <see cref="ExtendedScannedComponent"/> version of the RubyGemsComponent</param>
     /// <returns>The converted <see cref="SbomPackage" />.</returns>
-    public static SbomPackage ToSbomPackage(this RubyGemsComponent rubyGemsComponent, string? license = null) => new()
+    public static SbomPackage ToSbomPackage(this RubyGemsComponent rubyGemsComponent, ExtendedScannedComponent component) => new()
     {
         Id = rubyGemsComponent.Id,
         PackageUrl = rubyGemsComponent.PackageUrl?.ToString(),
         PackageName = rubyGemsComponent.Name,
         PackageVersion = rubyGemsComponent.Version,
         PackageSource = rubyGemsComponent.Source,
-        LicenseInfo = string.IsNullOrWhiteSpace(license) ? null : new LicenseInfo
+        LicenseInfo = string.IsNullOrWhiteSpace(component.LicenseConcluded) ? null : new LicenseInfo
         {
-            Concluded = license,
+            Concluded = component.LicenseConcluded,
         },
         FilesAnalyzed = false,
         Type = "ruby",

src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentExtensions.cs

@@ -10,31 +10,30 @@
 namespace Microsoft.Sbom.Adapters.ComponentDetection;
 
 /// <summary>
-/// Extensions methods for <see cref="ScannedComponentWithLicense"/>.
+/// Extensions methods for <see cref="ExtendedScannedComponent"/>.
 /// </summary>
 public static class ScannedComponentExtensions
 {
     /// <summary>
-    /// Converts a <see cref="ScannedComponentWithLicense"/> to an <see cref="SbomPackage"/>.
+    /// Converts a <see cref="ExtendedScannedComponent"/> to an <see cref="SbomPackage"/>.
     /// </summary>
-    public static SbomPackage? ToSbomPackage(this ScannedComponentWithLicense component, AdapterReport report)
+    public static SbomPackage? ToSbomPackage(this ExtendedScannedComponent component, AdapterReport report)
     {
         return component.Component switch
         {
-            CargoComponent cargoComponent => cargoComponent.ToSbomPackage(component?.License),
-            ConanComponent conanComponent => conanComponent.ToSbomPackage(),
+            CargoComponent cargoComponent => cargoComponent.ToSbomPackage(component),
             CondaComponent condaComponent => condaComponent.ToSbomPackage(),
             DockerImageComponent dockerImageComponent => dockerImageComponent.ToSbomPackage(),
             GitComponent gitComponent => gitComponent.ToSbomPackage(),
             GoComponent goComponent => goComponent.ToSbomPackage(),
             LinuxComponent linuxComponent => linuxComponent.ToSbomPackage(),
-            MavenComponent mavenComponent => mavenComponent.ToSbomPackage(),
-            NpmComponent npmComponent => npmComponent.ToSbomPackage(component?.License),
-            NuGetComponent nuGetComponent => nuGetComponent.ToSbomPackage(component?.License),
+            MavenComponent mavenComponent => mavenComponent.ToSbomPackage(component),
+            NpmComponent npmComponent => npmComponent.ToSbomPackage(component),
+            NuGetComponent nuGetComponent => nuGetComponent.ToSbomPackage(component),
             OtherComponent otherComponent => otherComponent.ToSbomPackage(),
-            PipComponent pipComponent => pipComponent.ToSbomPackage(component?.License),
-            PodComponent podComponent => podComponent.ToSbomPackage(component?.License),
-            RubyGemsComponent rubyGemsComponent => rubyGemsComponent.ToSbomPackage(component?.License),
+            PipComponent pipComponent => pipComponent.ToSbomPackage(component),
+            PodComponent podComponent => podComponent.ToSbomPackage(component),
+            RubyGemsComponent rubyGemsComponent => rubyGemsComponent.ToSbomPackage(component),
             null => Error(report => report.LogNullComponent(nameof(ToSbomPackage))),
             _ => Error(report => report.LogNoConversionFound(component.Component.GetType(), component.Component))
         };

src/Microsoft.Sbom.Adapters/ComponentDetectionToSBOMPackageAdapter.cs

@@ -34,7 +34,7 @@ public class ComponentDetectionToSBOMPackageAdapter
 
         try
         {
-            var componentDetectionScanResult = JsonConvert.DeserializeObject<ScanResultWithLicense>(File.ReadAllText(bcdeOutputPath));
+            var componentDetectionScanResult = JsonConvert.DeserializeObject<ExtendedScanResult>(File.ReadAllText(bcdeOutputPath));
 
             if (componentDetectionScanResult == null)
             {

src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs

@@ -115,4 +115,11 @@ public class GenerationArgs : CommonArgs
     [ArgShortcut("li")]
     [ArgDescription("If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi.")]
     public bool? FetchLicenseInformation { get; set; }
+
+    /// <summary>
+    /// If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi.
+    /// </summary>
+    [ArgShortcut("pm")]
+    [ArgDescription("If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi.")]
+    public bool? EnablePackageMetadataParsing { get; set; }
 }

src/Microsoft.Sbom.Api/Exceptions/PackageMetadataParsingException.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace Microsoft.Sbom.Api.Exceptions;
+
+/// <summary>
+/// Exception thrown while parsing a response from ClearlyDefined.
+/// </summary>
+[Serializable]
+public class PackageMetadataParsingException : Exception
+{
+    public PackageMetadataParsingException(string message)
+        : base(message)
+    {
+    }
+}