Skip to content

Commit

Permalink
Defining and generating spdx 3.0 json elements (#830)
Browse files Browse the repository at this point in the history
* spdx 3.0 generator changes + unit tests

* address PR review comments

* fix bug in UT + address PR comments

* remove empty constructors

* add debug line

* fix UT with regex

* remove extra debug line

---------

Co-authored-by: ppandrate <ppandrate@microsoft.com>
  • Loading branch information
pragnya17 and ppandrate authored Dec 16, 2024
1 parent fd23d63 commit 43e316d
Show file tree
Hide file tree
Showing 43 changed files with 2,636 additions and 71 deletions.
12 changes: 12 additions & 0 deletions Microsoft.Sbom.sln
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Tool.Tests",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Sbom.Targets.E2E.Tests", "test\Microsoft.Sbom.Targets.E2E.Tests\Microsoft.Sbom.Targets.E2E.Tests.csproj", "{3FDE7800-F61F-4C45-93AB-648A4C7979C7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Sbom.Parsers.Spdx30SbomParser", "src\Microsoft.Sbom.Parsers.Spdx30SbomParser\Microsoft.Sbom.Parsers.Spdx30SbomParser.csproj", "{476B9C87-293F-4BF7-B39A-EB732083409F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests", "test\Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests\Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests.csproj", "{E3FE33BB-FAB2-4F60-B930-BEB736AACE25}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -131,6 +135,14 @@ Global
{3FDE7800-F61F-4C45-93AB-648A4C7979C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3FDE7800-F61F-4C45-93AB-648A4C7979C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3FDE7800-F61F-4C45-93AB-648A4C7979C7}.Release|Any CPU.Build.0 = Release|Any CPU
{476B9C87-293F-4BF7-B39A-EB732083409F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{476B9C87-293F-4BF7-B39A-EB732083409F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{476B9C87-293F-4BF7-B39A-EB732083409F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{476B9C87-293F-4BF7-B39A-EB732083409F}.Release|Any CPU.Build.0 = Release|Any CPU
{E3FE33BB-FAB2-4F60-B930-BEB736AACE25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E3FE33BB-FAB2-4F60-B930-BEB736AACE25}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E3FE33BB-FAB2-4F60-B930-BEB736AACE25}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E3FE33BB-FAB2-4F60-B930-BEB736AACE25}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
40 changes: 40 additions & 0 deletions src/Microsoft.Sbom.Common/GeneratorUtils.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.Sbom.Common;

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Sbom.Contracts;
using Microsoft.Sbom.Contracts.Enums;
using Microsoft.Sbom.Extensions.Exceptions;

/// <summary>
/// A class for methods that are used by SPDX generators, regardless of which SPDX version is being used.
/// </summary>
public class GeneratorUtils
{
// Throws a <see cref="MissingHashValueException"/> if the filehashes are missing
// any of the required hashes
public static void EnsureRequiredHashesPresent(Checksum[] fileHashes, AlgorithmName[] requiredHashAlgorithms)
{
foreach (var hashAlgorithmName in from hashAlgorithmName in requiredHashAlgorithms
where !fileHashes.Select(fh => fh.Algorithm).Contains(hashAlgorithmName)
select hashAlgorithmName)
{
throw new MissingHashValueException($"The hash value for algorithm {hashAlgorithmName} is missing from {nameof(fileHashes)}");
}
}

public static string EnsureRelativePathStartsWithDot(string path)
{
if (!path.StartsWith(".", StringComparison.Ordinal))
{
return "." + path;
}

return path;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@
using System.Text;
using Microsoft.Sbom.Extensions;
using Microsoft.Sbom.Extensions.Entities;
using Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities;
using HashAlgorithmName = Microsoft.Sbom.Contracts.Enums.AlgorithmName;

namespace Microsoft.Sbom.Parsers.Spdx22SbomParser.Utils;
namespace Microsoft.Sbom.Common;

/// <summary>
/// Provides helper functions to generate identity strings for SPDX.
Expand Down Expand Up @@ -51,59 +50,19 @@ public static string GetPackageName(this IInternalMetadataProvider internalMetad
$"Please provide the package name in the 'PackageName' parameter.");
}

/// <summary>
/// Generates the package verification code for a given package using the SPDX 2.2 specification.
///
/// Algorithm defined here https://spdx.github.io/spdx-spec/v2.2.2/package-information/#79-package-verification-code-field.
/// </summary>
/// <param name="internalMetadataProvider"></param>
/// <returns></returns>
public static PackageVerificationCode GetPackageVerificationCode(this IInternalMetadataProvider internalMetadataProvider)
{
if (internalMetadataProvider is null)
{
throw new ArgumentNullException(nameof(internalMetadataProvider));
}

// Get a list of SHA1 checksums
IList<string> sha1Checksums = new List<string>();
foreach (var checksumArray in internalMetadataProvider.GetGenerationData(Constants.Spdx22ManifestInfo).Checksums)
{
sha1Checksums.Add(checksumArray
.Where(c => c.Algorithm == HashAlgorithmName.SHA1)
.Select(c => c.ChecksumValue)
.FirstOrDefault());
}

var packageChecksumString = string.Join(string.Empty, sha1Checksums.OrderBy(s => s));
#pragma warning disable CA5350 // Suppress Do Not Use Weak Cryptographic Algorithms as we use SHA1 intentionally
var sha1Hasher = SHA1.Create();
#pragma warning restore CA5350
var hashByteArray = sha1Hasher.ComputeHash(Encoding.Default.GetBytes(packageChecksumString));

return new PackageVerificationCode
{
PackageVerificationCodeValue = BitConverter
.ToString(hashByteArray)
.Replace("-", string.Empty)
.ToLowerInvariant(),
PackageVerificationCodeExcludedFiles = null // We currently don't ignore any files.
};
}

/// <summary>
/// Gets a list of file ids that are included in this package.
/// </summary>
/// <param name="internalMetadataProvider"></param>
/// <returns></returns>
public static List<string> GetPackageFilesList(this IInternalMetadataProvider internalMetadataProvider)
public static List<string> GetPackageFilesList(this IInternalMetadataProvider internalMetadataProvider, ManifestInfo manifestInfo)
{
if (internalMetadataProvider is null)
{
throw new ArgumentNullException(nameof(internalMetadataProvider));
}

return internalMetadataProvider.GetGenerationData(Constants.Spdx22ManifestInfo).FileIds.ToList();
return internalMetadataProvider.GetGenerationData(manifestInfo).FileIds.ToList();
}

/// <summary>
Expand Down
1 change: 0 additions & 1 deletion src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,4 @@
<ItemGroup>
<ProjectReference Include="..\Microsoft.Sbom.Extensions\Microsoft.Sbom.Extensions.csproj" />
</ItemGroup>

</Project>
73 changes: 47 additions & 26 deletions src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Generator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Sbom.Common;
using Microsoft.Sbom.Contracts;
using Microsoft.Sbom.Contracts.Enums;
using Microsoft.Sbom.Extensions;
Expand Down Expand Up @@ -68,15 +71,15 @@ private SPDXFile ConvertSbomFileToSpdxFile(InternalSbomFileInfo fileInfo)
throw new ArgumentException(nameof(fileInfo.Path));
}

EnsureRequiredHashesPresent(fileInfo.Checksum.ToArray());
GeneratorUtils.EnsureRequiredHashesPresent(fileInfo.Checksum.ToArray(), RequiredHashAlgorithms);

var spdxFileElement = new SPDXFile
{
FileChecksums = fileInfo.Checksum
.Where(c => RequiredHashAlgorithms.Contains(c.Algorithm))
.Select(fh => new Entities.Checksum { Algorithm = fh.Algorithm.ToString(), ChecksumValue = fh.ChecksumValue.ToLower() })
.ToList(),
FileName = EnsureRelativePathStartsWithDot(fileInfo.Path),
FileName = GeneratorUtils.EnsureRelativePathStartsWithDot(fileInfo.Path),
FileCopyrightText = fileInfo.FileCopyrightText ?? Constants.NoAssertionValue,
LicenseConcluded = fileInfo.LicenseConcluded ?? Constants.NoAssertionValue,
LicenseInfoInFiles = fileInfo.LicenseInfoInFiles ?? Constants.NoAssertionListValue,
Expand All @@ -87,18 +90,6 @@ private SPDXFile ConvertSbomFileToSpdxFile(InternalSbomFileInfo fileInfo)
return spdxFileElement;
}

// Throws a <see cref="MissingHashValueException"/> if the filehashes are missing
// any of the required hashes
private void EnsureRequiredHashesPresent(Contracts.Checksum[] fileHashes)
{
foreach (var hashAlgorithmName in from hashAlgorithmName in RequiredHashAlgorithms
where !fileHashes.Select(fh => fh.Algorithm).Contains(hashAlgorithmName)
select hashAlgorithmName)
{
throw new MissingHashValueException($"The hash value for algorithm {hashAlgorithmName} is missing from {nameof(fileHashes)}");
}
}

public ManifestInfo RegisterManifest() => Constants.Spdx22ManifestInfo;

public IDictionary<string, object> GetMetadataDictionary(IInternalMetadataProvider internalMetadataProvider)
Expand Down Expand Up @@ -184,16 +175,6 @@ public GenerationResult GenerateJsonDocument(SbomPackage packageInfo)
};
}

private string EnsureRelativePathStartsWithDot(string path)
{
if (!path.StartsWith(".", StringComparison.Ordinal))
{
return "." + path;
}

return path;
}

public GenerationResult GenerateRootPackage(
IInternalMetadataProvider internalMetadataProvider)
{
Expand Down Expand Up @@ -223,9 +204,9 @@ public GenerationResult GenerateRootPackage(
LicenseDeclared = Constants.NoAssertionValue,
LicenseInfoFromFiles = Constants.NoAssertionListValue,
FilesAnalyzed = true,
PackageVerificationCode = internalMetadataProvider.GetPackageVerificationCode(),
PackageVerificationCode = GetPackageVerificationCode(internalMetadataProvider),
Supplier = string.Format(Constants.PackageSupplierFormatString, internalMetadataProvider.GetPackageSupplier()),
HasFiles = internalMetadataProvider.GetPackageFilesList()
HasFiles = internalMetadataProvider.GetPackageFilesList(Constants.Spdx22ManifestInfo)
};

return new GenerationResult
Expand Down Expand Up @@ -331,4 +312,44 @@ public GenerationResult GenerateJsonDocument(ExternalDocumentReferenceInfo exter
}
};
}

/// <summary>
/// Generates the package verification code for a given package using the SPDX 2.2 specification.
///
/// Algorithm defined here https://spdx.github.io/spdx-spec/v2.2.2/package-information/#79-package-verification-code-field.
/// </summary>
/// <param name="internalMetadataProvider"></param>
/// <returns></returns>
private PackageVerificationCode GetPackageVerificationCode(IInternalMetadataProvider internalMetadataProvider)
{
if (internalMetadataProvider is null)
{
throw new ArgumentNullException(nameof(internalMetadataProvider));
}

// Get a list of SHA1 checksums
IList<string> sha1Checksums = new List<string>();
foreach (var checksumArray in internalMetadataProvider.GetGenerationData(Constants.Spdx22ManifestInfo).Checksums)
{
sha1Checksums.Add(checksumArray
.Where(c => c.Algorithm == AlgorithmName.SHA1)
.Select(c => c.ChecksumValue)
.FirstOrDefault());
}

var packageChecksumString = string.Join(string.Empty, sha1Checksums.OrderBy(s => s));
#pragma warning disable CA5350 // Suppress Do Not Use Weak Cryptographic Algorithms as we use SHA1 intentionally
var sha1Hasher = SHA1.Create();
#pragma warning restore CA5350
var hashByteArray = sha1Hasher.ComputeHash(Encoding.Default.GetBytes(packageChecksumString));

return new PackageVerificationCode
{
PackageVerificationCodeValue = BitConverter
.ToString(hashByteArray)
.Replace("-", string.Empty)
.ToLowerInvariant(),
PackageVerificationCodeExcludedFiles = null // We currently don't ignore any files.
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\Microsoft.Sbom.Common\Microsoft.Sbom.Common.csproj" />
<ProjectReference Include="..\Microsoft.Sbom.Contracts\Microsoft.Sbom.Contracts.csproj" />
<ProjectReference Include="..\Microsoft.Sbom.Extensions\Microsoft.Sbom.Extensions.csproj" />
</ItemGroup>
Expand Down
37 changes: 37 additions & 0 deletions src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Generic;
using Microsoft.Sbom.Extensions.Entities;

namespace Microsoft.Sbom.Parsers.Spdx30SbomParser;

internal static class Constants
{
internal const string SPDXName = "SPDX";
internal const string SPDXVersion = "3.0";
internal const string DataLicenceValue = "CC0-1.0";
internal const string SPDXDocumentIdValue = "SPDXRef-DOCUMENT";
internal const string RootPackageIdValue = "SPDXRef-RootPackage";
internal const string SPDXDocumentNameFormatString = "{0} {1}";
internal const string PackageSupplierFormatString = "Organization: {0}";

/// <summary>
/// Use if SPDX creator
/// - made an attempt to retrieve the info but cannot determine correct values.
/// - made no attempt to retrieve the info.
/// - has intentionally provided no information.
/// </summary>
internal const string NoAssertionValue = "NOASSERTION";

/// <summary>
/// The <see cref="NoAssertionValue"/> value as a list with a single item.
/// </summary>
internal static IEnumerable<string> NoAssertionListValue = new List<string> { NoAssertionValue };

internal static ManifestInfo Spdx30ManifestInfo = new ManifestInfo
{
Name = SPDXName,
Version = SPDXVersion
};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Generic;
using System.Text.Json.Serialization;

namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities;

/// <summary>
/// Class defintion is based on: https://spdx.github.io/spdx-spec/v3.0.1/model/SimpleLicensing/Classes/AnyLicenseInfo/
/// </summary>
public class AnyLicenseInfo : Element
{
public AnyLicenseInfo()
{
SpdxId = "SPDXRef-AnyLicenseInfo";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Generic;
using System.Text.Json.Serialization;

namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities;

/// <summary>
/// A ContentIdentifier is a canonical, unique, immutable identifier of the content of a software artifact, such as a package, a file, or a snippet.
/// It can be used for verifying its identity and integrity.
/// https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/ContentIdentifier/
/// </summary>
public class ContentIdentifier : Software
{
private string contentIdentifierType;

/// <summary>
/// Gets or sets the content identifier type.
/// Allowed types are Git Object ID and Software Hash Identifier (swhid).
/// We will use swhid unless otherwise specified.
/// </summary>
[JsonRequired]
[JsonPropertyName("contentIdentifierType")]
public override string ContentIdentifierType
{
get => this.contentIdentifierType ?? "swhid";
set => this.contentIdentifierType = value;
}
}
Loading

0 comments on commit 43e316d

Please sign in to comment.