Skip to content

Commit

Permalink
Update to latest version of Component-Detection Libraries. (#436)
Browse files Browse the repository at this point in the history 
* Update to latest version of Component-Detection Libraries.

* Fix formatting.

* Remove accidental import.

* Fix cd version.

* Rename method and add xml doc.

* Add Conan support.

* Use same logger template for all logs.

* Fix DirectoryExclusionList bug and remove pre-release spectre version

* Ensure duplicates aren't passed to DirectoryExclusionList

* Rename lst to checksums.

---------

Co-authored-by: Sebastian Gomez <segomez@microsoft.com>
  • Loading branch information
@sebasgomez238
sebasgomez238 and sebasgomez238 authored Oct 25, 2023
1 parent 7bb13d7 commit fe1e2e0
Show file tree
Hide file tree
Showing 19 changed files with 298 additions and 167 deletions.
6 changes: 5 additions & 1 deletion Directory.Packages.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
</PackageVersion>
</ItemDefinitionGroup>
<PropertyGroup>
<ComponentDetectionPackageVersion>3.6.4</ComponentDetectionPackageVersion>
<ComponentDetectionPackageVersion>4.0.6</ComponentDetectionPackageVersion>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="AutoMapper" Version="10.1.1" />
Expand Down Expand Up @@ -36,7 +36,11 @@
<PackageVersion Include="PowerArgs" Version="3.6.0" />
<PackageVersion Include="Scrutor" Version="4.2.0" />
<PackageVersion Include="Serilog.Extensions.Hosting" Version="7.0.0" />
<PackageVersion Include="Serilog.Sinks.Async" Version="1.5.0" />
<PackageVersion Include="Serilog.Sinks.Console" Version="4.1.0" />
<PackageVersion Include="Serilog.Sinks.File" Version="5.0.0" />
<PackageVersion Include="Serilog.Sinks.Map" Version="1.0.2" />
<PackageVersion Include="Spectre.Console.Cli" Version="0.47.0" />
<PackageVersion Include="StyleCop.Analyzers" Version="1.2.0-beta.507" />
<PackageVersion Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
Expand Down
45 changes: 45 additions & 0 deletions src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ConanComponentExtensions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Generic;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.Sbom.Contracts;

namespace Microsoft.Sbom.Adapters.ComponentDetection;

internal static class ConanComponentExtensions
{
public static SbomPackage? ToSbomPackage(this ConanComponent conanComponent)
{
var checksums = new List<Checksum>();
if (!string.IsNullOrEmpty(conanComponent.Md5Hash))
{
checksums.Add(new Checksum
{
Algorithm = Contracts.Enums.AlgorithmName.MD5,
ChecksumValue = conanComponent.Md5Hash
});
}

if (!string.IsNullOrEmpty(conanComponent.Sha1Hash))
{
checksums.Add(new Checksum
{
Algorithm = Contracts.Enums.AlgorithmName.SHA1,
ChecksumValue = conanComponent.Sha1Hash
});
}

return new()
{
Id = conanComponent.Id,
PackageUrl = conanComponent.PackageUrl?.ToString(),
PackageName = conanComponent.Name,
PackageVersion = conanComponent.Version,
PackageSource = conanComponent.PackageSourceURL,
FilesAnalyzed = false,
Checksum = checksums,
Type = "conan"
};
}
}
1 change: 1 addition & 0 deletions src/Microsoft.Sbom.Adapters/Adapters/ComponentDetection/ScannedComponentExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public static class ScannedComponentExtensions
return component.Component switch
{
CargoComponent cargoComponent => cargoComponent.ToSbomPackage(component?.License),
ConanComponent conanComponent => conanComponent.ToSbomPackage(),
CondaComponent condaComponent => condaComponent.ToSbomPackage(),
DockerImageComponent dockerImageComponent => dockerImageComponent.ToSbomPackage(),
GitComponent gitComponent => gitComponent.ToSbomPackage(),
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.Sbom.Api/Config/Extensions/ConfigurationExtensions.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Generic;
Expand Down
13 changes: 5 additions & 8 deletions src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,7 @@ public ComponentDetectionBaseWalker(
Directory.CreateDirectory(buildComponentDirPath);
}

var verbosity = configuration.Verbosity.Value switch
{
LogEventLevel.Verbose => VerbosityMode.Verbose,
_ => VerbosityMode.Normal,
};

cliArgumentBuilder = new ComponentDetectionCliArgumentBuilder().Scan().Verbosity(verbosity);
cliArgumentBuilder = new ComponentDetectionCliArgumentBuilder();

// Enable SPDX22 detector which is disabled by default.
cliArgumentBuilder.AddDetectorArg("SPDX22SBOM", "EnableIfDefaultOff");
Expand All @@ -104,9 +98,12 @@ public ComponentDetectionBaseWalker(
async Task Scan(string path)
{
cliArgumentBuilder.SourceDirectory(buildComponentDirPath);

var cmdLineParams = configuration.ToComponentDetectorCommandLineParams(cliArgumentBuilder);

var scanResult = await componentDetector.ScanAsync(cmdLineParams);
var scanSettings = cliArgumentBuilder.BuildScanSettingsFromParsedArgs(cmdLineParams);

var scanResult = await componentDetector.ScanAsync(scanSettings);

if (scanResult.ResultCode != ProcessingResultCode.Success)
{
Expand Down
3 changes: 2 additions & 1 deletion src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<AssemblyName>Microsoft.Sbom.Api</AssemblyName>
Expand All @@ -21,6 +21,7 @@
<PackageReference Include="PowerArgs" />
<PackageReference Include="Serilog.Extensions.Hosting" />
<PackageReference Include="Serilog.Sinks.Console" />
<PackageReference Include="Spectre.Console.Cli" />
<PackageReference Include="System.IO.FileSystem.AccessControl" />
<PackageReference Include="System.Private.Uri" />
<PackageReference Include="System.Threading.Channels" />
Expand Down
100 changes: 68 additions & 32 deletions src/Microsoft.Sbom.Api/Utils/ComponentDetectionCliArgumentBuilder.cs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Orchestrator.Commands;
using PowerArgs;

namespace Microsoft.Sbom.Api.Utils;
Expand All @@ -14,8 +15,6 @@ namespace Microsoft.Sbom.Api.Utils;
/// </summary>
public class ComponentDetectionCliArgumentBuilder
{
private string action;
private VerbosityMode verbosity = VerbosityMode.Quiet;
private string sourceDirectory;
private Dictionary<string, string> detectorArgs = new Dictionary<string, string>() {
{ TimeoutArgsParamName, TimeoutDefaultSeconds.ToString() }
Expand All @@ -24,25 +23,18 @@ public class ComponentDetectionCliArgumentBuilder
private Dictionary<string, string> keyValueArgs = new Dictionary<string, string>();
private List<string> keyArgs = new List<string>();

private const string VerbosityParamName = "Verbosity";
private const string SourceDirectoryParamName = "SourceDirectory";
private const string DirectoryExclusionListParamName = "DirectoryExclusionList";
private const string DetectorArgsParamName = "DetectorArgs";
private const string TimeoutArgsParamName = "Timeout";
private const int TimeoutDefaultSeconds = 15 * 60; // 15 minutes

private const string ScanAction = "scan";

public ComponentDetectionCliArgumentBuilder()
{
}

private void Validate()
{
if (string.IsNullOrEmpty(action))
{
throw new ArgumentNullException("Action should be specified.");
}

if (string.IsNullOrEmpty(sourceDirectory))
{
throw new ArgumentNullException("Source directory should be specified.");
Expand All @@ -53,7 +45,7 @@ public string[] Build()
{
Validate();

var command = $"{action} --{VerbosityParamName} {AsArgumentValue(verbosity.ToString())} --{SourceDirectoryParamName} {AsArgumentValue(sourceDirectory)}";
var command = $"--{SourceDirectoryParamName} {AsArgumentValue(sourceDirectory)}";

if (detectorArgs.Any())
{
Expand Down Expand Up @@ -81,10 +73,63 @@ public string[] Build()
return Args.Convert(command.Trim());
}

public ComponentDetectionCliArgumentBuilder Scan()
/// <summary>
/// Takes a set of parsed arguments for Component Detection and converts them into a ScanSettings object.
/// </summary>
/// <param name="args">Set of arguments in the proper format for component detection.</param>
/// <returns></returns>
public ScanSettings BuildScanSettingsFromParsedArgs(string[] args)
{
action = ScanAction;
return this;
Validate();

// Create a new instance of ScanSettings
var scanSettings = new ScanSettings();

for (var i = 0; i < args.Length - 1; i++)
{
var argumentName = args[i];
var argumentValue = args[i + 1];

// Map the argument to the corresponding property in ScanSettings
switch (argumentName)
{
case "--DirectoryExclusionList":
scanSettings.DirectoryExclusionList = argumentValue.Split(';');
break;
case "--SourceDirectory":
scanSettings.SourceDirectory = new DirectoryInfo(argumentValue);
break;
case "--SourceFileRoot":
scanSettings.SourceFileRoot = new DirectoryInfo(argumentValue);
break;
case "--DetectorArgs":
var keyValuePairs = argumentValue.Split(',');
foreach (var keyValue in keyValuePairs)
{
var pair = keyValue.Split('=');
if (pair.Length == 2)
{
scanSettings.DetectorArgs[pair[0]] = pair[1];
}
}

break;
case "--DetectorCategories":
scanSettings.DetectorCategories = argumentValue.Split(",");
break;
case "--ManifestFile":
scanSettings.ManifestFile = new FileInfo(argumentValue);
break;
case "--PrintManifest":
scanSettings.PrintManifest = bool.Parse(argumentValue);
break;
case "--DockerImagesToScan":
scanSettings.DockerImagesToScan = argumentValue.Split(",");
break;
}
}

return scanSettings;
}

public ComponentDetectionCliArgumentBuilder AddDetectorArg(string name, string value)
Expand All @@ -93,12 +138,6 @@ public ComponentDetectionCliArgumentBuilder AddDetectorArg(string name, string v
return this;
}

public ComponentDetectionCliArgumentBuilder Verbosity(VerbosityMode verbosity)
{
this.verbosity = verbosity;
return this;
}

public ComponentDetectionCliArgumentBuilder SourceDirectory(string directory)
{
sourceDirectory = directory;
Expand Down Expand Up @@ -129,16 +168,6 @@ public ComponentDetectionCliArgumentBuilder AddArg(string name, string value)
return SourceDirectory(value);
}

if (name.Equals(VerbosityParamName, StringComparison.OrdinalIgnoreCase))
{
if (!Enum.TryParse(value, out VerbosityMode verbosity))
{
throw new ArgumentException($"Invalid verbosity value provided - {value}.");
}

return Verbosity(verbosity);
}

if (name.Equals(DetectorArgsParamName, StringComparison.OrdinalIgnoreCase))
{
var detectorArgs = value.Split(",").Select(arg => arg.Trim()).Select(arg => arg.Split("="));
Expand All @@ -156,6 +185,13 @@ public ComponentDetectionCliArgumentBuilder AddArg(string name, string value)
return this;
}

// Check if a key already exists for the --DirectoryExclusionList, if so, check that the value isn't a duplicate. If these conditions are true then append the new value delimited by a semicolon.
if (keyValueArgs.ContainsKey(name) && !keyValueArgs.ContainsValue(value) && name.Equals(DirectoryExclusionListParamName, StringComparison.OrdinalIgnoreCase))
{
keyValueArgs[name] = $"{keyValueArgs[name]};{value}";
return this;
}

keyValueArgs[name] = value;
return this;
}
Expand Down
49 changes: 24 additions & 25 deletions src/Microsoft.Sbom.Api/Utils/ComponentDetector.cs
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.ComponentDetection.Common;
using Microsoft.ComponentDetection.Contracts.BcdeModels;
using Microsoft.ComponentDetection.Orchestrator;
using Microsoft.ComponentDetection.Orchestrator.Commands;
using Microsoft.ComponentDetection.Orchestrator.Services;
using Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation;
using Microsoft.Extensions.Logging;

namespace Microsoft.Sbom.Api.Utils;
Expand All @@ -17,35 +16,35 @@ namespace Microsoft.Sbom.Api.Utils;
/// </summary>
public class ComponentDetector : IComponentDetector
{
private readonly IServiceProvider serviceProvider;
private readonly IEnumerable<IArgumentHandlingService> argumentHandlers;
private readonly IFileWritingService fileWritingService;
private readonly IArgumentHelper argumentHelper;
private readonly ILogger<Orchestrator> logger;
private readonly IEnumerable<ComponentDetection.Contracts.IComponentDetector> detectors;
private readonly IDetectorProcessingService detectorProcessingService;
private readonly IDetectorRestrictionService detectorRestrictionService;
private readonly IGraphTranslationService graphTranslationService;
private readonly ILogger<ScanExecutionService> logger;

public ComponentDetector(
IServiceProvider serviceProvider,
IEnumerable<IArgumentHandlingService> argumentHandlers,
IFileWritingService fileWritingService,
IArgumentHelper argumentHelper,
ILogger<Orchestrator> logger)
IEnumerable<ComponentDetection.Contracts.IComponentDetector> detectors,
IDetectorProcessingService detectorProcessingService,
IDetectorRestrictionService detectorRestrictionService,
IGraphTranslationService graphTranslationService,
ILogger<ScanExecutionService> logger)
{
this.serviceProvider = serviceProvider;
this.argumentHandlers = argumentHandlers;
this.fileWritingService = fileWritingService;
this.argumentHelper = argumentHelper;
this.detectors = detectors;
this.detectorProcessingService = detectorProcessingService;
this.detectorRestrictionService = detectorRestrictionService;
this.graphTranslationService = graphTranslationService;
this.logger = logger;
}

public virtual async Task<ScanResult> ScanAsync(string[] args)
public virtual async Task<ScanResult> ScanAsync(ScanSettings args)
{
var orchestrator = new Orchestrator(
serviceProvider,
argumentHandlers,
fileWritingService,
argumentHelper,
var executionService = new ScanExecutionService(
detectors,
detectorProcessingService,
detectorRestrictionService,
graphTranslationService,
logger);

return await orchestrator.LoadAsync(args);
return await executionService.ExecuteScanAsync(args);
}
}
Loading

0 comments on commit fe1e2e0

Please sign in to comment.