Skip to content

Commit

Permalink
Return hasSBOM and hasSLSA IDs from the assembler (#2069)
Browse files Browse the repository at this point in the history 
* update ingestion to return hasSBOM and hasSLSA IDs on ingestion

Signed-off-by: pxp928 <parth.psu@gmail.com>

* update to return ingested IDs but not needed for ingestion cmd

Signed-off-by: pxp928 <parth.psu@gmail.com>

* fix spacing issue on header

Signed-off-by: pxp928 <parth.psu@gmail.com>

---------

Signed-off-by: pxp928 <parth.psu@gmail.com>
  • Loading branch information
@pxp928
pxp928 authored Aug 6, 2024
1 parent b9dc127 commit e4357e5
Show file tree
Hide file tree
Showing 13 changed files with 63 additions and 50 deletions.
2 changes: 1 addition & 1 deletion cmd/guacingest/cmd/ingest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func ingest(cmd *cobra.Command, args []string) {
defer csubClient.Close()

emit := func(d *processor.Document) error {
if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil {
if _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil {
var urlErr *url.Error
if errors.As(err, &urlErr) {
return fmt.Errorf("unable to ingest document due to connection error with graphQL %q : %w", d.SourceInformation.Source, urlErr)
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/annotate_metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ var annotateMetadata = &cobra.Command{
preds.HasMetadata = append(preds.HasMetadata, metadata)
assemblerInputs := []assembler.IngestPredicates{*preds}

err = assemblerFunc(assemblerInputs)
_, err = assemblerFunc(assemblerInputs)
if err != nil {
logger.Fatalf("unable to assemble graphs: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/certify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ var certifyCmd = &cobra.Command{

assemblerInputs := []assembler.IngestPredicates{*preds}

err = assemblerFunc(assemblerInputs)
_, err = assemblerFunc(assemblerInputs)
if err != nil {
logger.Fatalf("unable to assemble graphs: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/deps_dev.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ var depsDevCmd = &cobra.Command{
emit := func(d *processor.Document) error {
totalNum += 1

if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csc, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil {
if _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csc, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil {
gotErr = true
return fmt.Errorf("unable to ingest document: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ var filesCmd = &cobra.Command{

emit := func(d *processor.Document) error {
totalNum += 1
if err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil {
if _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil {
gotErr = true
filesWithErrors = append(filesWithErrors, d.SourceInformation.Source)
return fmt.Errorf("unable to ingest document: %w", err)
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/gcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ var gcsCmd = &cobra.Command{

emit := func(d *processor.Document) error {
totalNum += 1
err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)
_, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)

if err != nil {
gotErr = true
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ var githubCmd = &cobra.Command{
var errFound bool

emit := func(d *processor.Document) error {
err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)
_, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)

if err != nil {
errFound = true
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ var ociCmd = &cobra.Command{
// Set emit function to go through the entire pipeline
emit := func(d *processor.Document) error {
totalNum += 1
err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)
_, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)

if err != nil {
gotErr = true
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll
errFound := false

emit := func(d *processor.Document) error {
err := ingestor.Ingest(ctx, d, s3Opts.graphqlEndpoint, transport, csubClient, s3Opts.queryVulnOnIngestion, s3Opts.queryLicenseOnIngestion)
_, err := ingestor.Ingest(ctx, d, s3Opts.graphqlEndpoint, transport, csubClient, s3Opts.queryVulnOnIngestion, s3Opts.queryLicenseOnIngestion)

if err != nil {
errFound = true
Expand Down
2 changes: 1 addition & 1 deletion cmd/guacone/cmd/scorecard.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ var scorecardCmd = &cobra.Command{
// Set emit function to go through the entire pipeline
emit := func(d *processor.Document) error {
totalNum += 1
err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)
_, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion)

if err != nil {
return fmt.Errorf("unable to ingest document: %v", err)
Expand Down
26 changes: 14 additions & 12 deletions pkg/assembler/backends/ent/backend/slsa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,12 @@ func upsertBulkSLSA(ctx context.Context, tx *ent.Tx, subjects []*model.IDorArtif
for i, slsa := range css {
slsa := slsa
var err error
creates[i], err = generateSLSACreate(ctx, tx, subjects[index], builtFromList[index], builtByList[index], slsa)
var hasSBOMID *uuid.UUID
creates[i], hasSBOMID, err = generateSLSACreate(ctx, tx, subjects[index], builtFromList[index], builtByList[index], slsa)
if err != nil {
return nil, gqlerror.Errorf("generateSLSACreate :: %s", err)
}
ids = append(ids, hasSBOMID.String())
index++
}

Expand All @@ -248,7 +250,7 @@ func setDefaultTime(inputTime *time.Time) time.Time {
}
}

func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArtifactInput, builtFrom []*model.IDorArtifactInput, builtBy *model.IDorBuilderInput, slsa *model.SLSAInputSpec) (*ent.SLSAAttestationCreate, error) {
func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArtifactInput, builtFrom []*model.IDorArtifactInput, builtBy *model.IDorBuilderInput, slsa *model.SLSAInputSpec) (*ent.SLSAAttestationCreate, *uuid.UUID, error) {
slsaCreate := tx.SLSAAttestation.Create()

slsaCreate.
Expand All @@ -262,20 +264,20 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
SetFinishedOn(setDefaultTime(slsa.FinishedOn))

if builtBy == nil {
return nil, fmt.Errorf("builtBy not specified for SLSA")
return nil, nil, fmt.Errorf("builtBy not specified for SLSA")
}
var buildID uuid.UUID
if builtBy.BuilderID != nil {
var err error
builtGlobalID := fromGlobalID(*builtBy.BuilderID)
buildID, err = uuid.Parse(builtGlobalID.id)
if err != nil {
return nil, fmt.Errorf("uuid conversion from BuilderID failed with error: %w", err)
return nil, nil, fmt.Errorf("uuid conversion from BuilderID failed with error: %w", err)
}
} else {
builder, err := tx.Builder.Query().Where(builderInputQueryPredicate(*builtBy.BuilderInput)).Only(ctx)
if err != nil {
return nil, err
return nil, nil, err
}
buildID = builder.ID
}
Expand All @@ -287,12 +289,12 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
artGlobalID := fromGlobalID(*subject.ArtifactID)
subjectArtifactID, err = uuid.Parse(artGlobalID.id)
if err != nil {
return nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
return nil, nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
}
} else {
foundArt, err := tx.Artifact.Query().Where(artifactQueryInputPredicates(*subject.ArtifactInput)).Only(ctx)
if err != nil {
return nil, fmt.Errorf("failed to query for artifact")
return nil, nil, fmt.Errorf("failed to query for artifact")
}
subjectArtifactID = foundArt.ID
}
Expand All @@ -309,7 +311,7 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
} else {
foundArt, err := tx.Artifact.Query().Where(artifactQueryInputPredicates(*bf.ArtifactInput)).Only(ctx)
if err != nil {
return nil, err
return nil, nil, err
}
builtFromIDs = append(builtFromIDs, foundArt.ID.String())
}
Expand All @@ -320,7 +322,7 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
for _, sbfID := range sortedBuildFromIDs {
sbfUUID, err := uuid.Parse(sbfID)
if err != nil {
return nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
return nil, nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
}
slsaCreate.AddBuiltFromIDs(sbfUUID)
}
Expand All @@ -334,17 +336,17 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti

slsaID, err := guacSLSAKey(ptrfrom.String(subjectArtifactID.String()), builtFromHash, ptrfrom.String(buildID.String()), slsa)
if err != nil {
return nil, fmt.Errorf("failed to create slsa uuid with error: %w", err)
return nil, nil, fmt.Errorf("failed to create slsa uuid with error: %w", err)
}

slsaCreate.SetID(*slsaID)

return slsaCreate, nil
return slsaCreate, slsaID, nil
}

func upsertSLSA(ctx context.Context, tx *ent.Tx, subject model.IDorArtifactInput, builtFrom []*model.IDorArtifactInput, builtBy model.IDorBuilderInput, slsa model.SLSAInputSpec) (*string, error) {

slsaCreate, err := generateSLSACreate(ctx, tx, &subject, builtFrom, &builtBy, &slsa)
slsaCreate, _, err := generateSLSACreate(ctx, tx, &subject, builtFrom, &builtBy, &slsa)
if err != nil {
return nil, gqlerror.Errorf("generateSLSACreate :: %s", err)
}
Expand Down
44 changes: 27 additions & 17 deletions pkg/assembler/clients/helpers/bulk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,16 @@ import (
"github.com/guacsec/guac/pkg/assembler/helpers"
)

func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient graphql.Client) func([]assembler.AssemblerInput) error {
return func(preds []assembler.IngestPredicates) error {
type AssemblerIngestedIDs struct {
hasSBOMIDs []string
hasSLSAIDs []string
}

func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient graphql.Client) func([]assembler.AssemblerInput) (*AssemblerIngestedIDs, error) {
return func(preds []assembler.IngestPredicates) (*AssemblerIngestedIDs, error) {
var rvErr error
ingestedIDs := &AssemblerIngestedIDs{}

for _, p := range preds {

// Ingest Packages
Expand All @@ -41,7 +48,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorPkgInputs, err := ingestPackages(ctx, gqlclient, packages)
if err != nil {
return fmt.Errorf("ingestPackages failed with error: %w", err)
return nil, fmt.Errorf("ingestPackages failed with error: %w", err)
}

var pkgVersionIDs []string
Expand All @@ -57,7 +64,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorSrcInputs, err := ingestSources(ctx, gqlclient, sources)
if err != nil {
return fmt.Errorf("ingestSources failed with error: %w", err)
return nil, fmt.Errorf("ingestSources failed with error: %w", err)
}

// Ingest Artifacts
Expand All @@ -67,7 +74,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorArtInputs, err := ingestArtifacts(ctx, gqlclient, artifacts)
if err != nil {
return fmt.Errorf("ingestArtifacts failed with error: %w", err)
return nil, fmt.Errorf("ingestArtifacts failed with error: %w", err)
}
var artIDs []string
for _, artID := range collectedIDorArtInputs {
Expand All @@ -82,7 +89,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorMatInputs, err := ingestArtifacts(ctx, gqlclient, materials)
if err != nil {
return fmt.Errorf("ingestArtifacts failed with error: %w", err)
return nil, fmt.Errorf("ingestArtifacts failed with error: %w", err)
}

// Ingest Builders
Expand All @@ -91,7 +98,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorBuilderInputs, err := ingestBuilders(ctx, gqlclient, builders)
if err != nil {
return fmt.Errorf("ingestBuilders failed with error: %w", err)
return nil, fmt.Errorf("ingestBuilders failed with error: %w", err)
}

// Ingest Vulnerabilities
Expand All @@ -100,7 +107,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorVulnInputs, err := ingestVulnerabilities(ctx, gqlclient, vulns)
if err != nil {
return fmt.Errorf("ingestVulnerabilities failed with error: %w", err)
return nil, fmt.Errorf("ingestVulnerabilities failed with error: %w", err)
}

// Ingest Licenses
Expand All @@ -109,7 +116,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorLicenseInputs, err := ingestLicenses(ctx, gqlclient, licenses)
if err != nil {
return fmt.Errorf("ingestLicenses failed with error: %w", err)
return nil, fmt.Errorf("ingestLicenses failed with error: %w", err)
}

logger.Infof("assembling CertifyScorecard: %v", len(p.CertifyScorecard))
Expand Down Expand Up @@ -137,7 +144,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient
}

logger.Infof("assembling HasSLSA: %v", len(p.HasSlsa))
if err := ingestHasSLSAs(ctx, gqlclient, p.HasSlsa, collectedIDorArtInputs, collectedIDorMatInputs, collectedIDorBuilderInputs); err != nil {
if err := ingestHasSLSAs(ctx, gqlclient, p.HasSlsa, collectedIDorArtInputs, collectedIDorMatInputs, collectedIDorBuilderInputs, ingestedIDs); err != nil {
logger.Errorf("ingestHasSLSAs failed with error: %v", err)
rvErr = err
}
Expand Down Expand Up @@ -196,7 +203,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient
Artifacts: artifactIDs,
Dependencies: isDependenciesIDs,
Occurrences: isOccurrencesIDs,
}, collectedIDorPkgInputs, collectedIDorArtInputs); err != nil {
}, collectedIDorPkgInputs, collectedIDorArtInputs, ingestedIDs); err != nil {
logger.Errorf("ingestHasSBOMs failed with error: %v", err)
rvErr = err
}
Expand Down Expand Up @@ -225,7 +232,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient
rvErr = err
}
}
return rvErr
return ingestedIDs, rvErr
}
}

Expand Down Expand Up @@ -566,7 +573,7 @@ func ingestHasSourceAts(ctx context.Context, client graphql.Client, hs []assembl
}

func ingestHasSLSAs(ctx context.Context, client graphql.Client, hs []assembler.HasSlsaIngest, artInputMap map[string]*model.IDorArtifactInput,
matInputSpec map[string]*model.IDorArtifactInput, builderInputMap map[string]*model.IDorBuilderInput) error {
matInputSpec map[string]*model.IDorArtifactInput, builderInputMap map[string]*model.IDorBuilderInput, ingestedIDs *AssemblerIngestedIDs) error {

var subjectIDs []model.IDorArtifactInput
var slsaAttestations []model.SLSAInputSpec
Expand Down Expand Up @@ -595,10 +602,11 @@ func ingestHasSLSAs(ctx context.Context, client graphql.Client, hs []assembler.H
slsaAttestations = append(slsaAttestations, *ingest.HasSlsa)
}
if len(hs) > 0 {
_, err := model.IngestSLSAForArtifacts(ctx, client, subjectIDs, materialIDs, builderIDs, slsaAttestations)
hasSLSAArtResponse, err := model.IngestSLSAForArtifacts(ctx, client, subjectIDs, materialIDs, builderIDs, slsaAttestations)
if err != nil {
return fmt.Errorf("SLSAForArtifacts failed with error: %w", err)
}
ingestedIDs.hasSLSAIDs = append(ingestedIDs.hasSLSAIDs, hasSLSAArtResponse.IngestSLSAs...)
}
return nil
}
Expand Down Expand Up @@ -710,7 +718,7 @@ func ingestHashEquals(ctx context.Context, client graphql.Client, he []assembler
}

func ingestHasSBOMs(ctx context.Context, client graphql.Client, hs []assembler.HasSBOMIngest, includes model.HasSBOMIncludesInputSpec, packageInputMap map[string]*model.IDorPkgInput,
artInputMap map[string]*model.IDorArtifactInput) error {
artInputMap map[string]*model.IDorArtifactInput, ingestedIDs *AssemblerIngestedIDs) error {

var pkgIDs []model.IDorPkgInput
var artIDs []model.IDorArtifactInput
Expand Down Expand Up @@ -745,16 +753,18 @@ func ingestHasSBOMs(ctx context.Context, client graphql.Client, hs []assembler.H
}
}
if len(artIDs) > 0 {
_, err := model.IngestHasSBOMArtifacts(ctx, client, artIDs, artSBOMs, artIncludes)
hasSBOMArtResponse, err := model.IngestHasSBOMArtifacts(ctx, client, artIDs, artSBOMs, artIncludes)
if err != nil {
return fmt.Errorf("hasSBOMArtifacts failed with error: %w", err)
}
ingestedIDs.hasSBOMIDs = append(ingestedIDs.hasSBOMIDs, hasSBOMArtResponse.IngestHasSBOMs...)
}
if len(pkgIDs) > 0 {
_, err := model.IngestHasSBOMPkgs(ctx, client, pkgIDs, pkgSBOMs, pkgIncludes)
hasSBOMPkgResponse, err := model.IngestHasSBOMPkgs(ctx, client, pkgIDs, pkgSBOMs, pkgIncludes)
if err != nil {
return fmt.Errorf("hasSBOMPkgs failed with error: %w", err)
}
ingestedIDs.hasSBOMIDs = append(ingestedIDs.hasSBOMIDs, hasSBOMPkgResponse.IngestHasSBOMs...)
}
return nil
}
Expand Down
Loading

0 comments on commit e4357e5

Please sign in to comment.