Skip to content

Commit

Permalink
update ingestion to return hasSBOM and hasSLSA IDs on ingestion
Browse files Browse the repository at this point in the history 
Signed-off-by: pxp928 <parth.psu@gmail.com>
  • Loading branch information
@pxp928
pxp928 committed Aug 2, 2024
1 parent 3161b7a commit 0c8d54c
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 39 deletions.
26 changes: 14 additions & 12 deletions pkg/assembler/backends/ent/backend/slsa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,12 @@ func upsertBulkSLSA(ctx context.Context, tx *ent.Tx, subjects []*model.IDorArtif
for i, slsa := range css {
slsa := slsa
var err error
creates[i], err = generateSLSACreate(ctx, tx, subjects[index], builtFromList[index], builtByList[index], slsa)
var hasSBOMID *uuid.UUID
creates[i], hasSBOMID, err = generateSLSACreate(ctx, tx, subjects[index], builtFromList[index], builtByList[index], slsa)
if err != nil {
return nil, gqlerror.Errorf("generateSLSACreate :: %s", err)
}
ids = append(ids, hasSBOMID.String())
index++
}

Expand All @@ -248,7 +250,7 @@ func setDefaultTime(inputTime *time.Time) time.Time {
}
}

func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArtifactInput, builtFrom []*model.IDorArtifactInput, builtBy *model.IDorBuilderInput, slsa *model.SLSAInputSpec) (*ent.SLSAAttestationCreate, error) {
func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArtifactInput, builtFrom []*model.IDorArtifactInput, builtBy *model.IDorBuilderInput, slsa *model.SLSAInputSpec) (*ent.SLSAAttestationCreate, *uuid.UUID, error) {
slsaCreate := tx.SLSAAttestation.Create()

slsaCreate.
Expand All @@ -262,20 +264,20 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
SetFinishedOn(setDefaultTime(slsa.FinishedOn))

if builtBy == nil {
return nil, fmt.Errorf("builtBy not specified for SLSA")
return nil, nil, fmt.Errorf("builtBy not specified for SLSA")
}
var buildID uuid.UUID
if builtBy.BuilderID != nil {
var err error
builtGlobalID := fromGlobalID(*builtBy.BuilderID)
buildID, err = uuid.Parse(builtGlobalID.id)
if err != nil {
return nil, fmt.Errorf("uuid conversion from BuilderID failed with error: %w", err)
return nil, nil, fmt.Errorf("uuid conversion from BuilderID failed with error: %w", err)
}
} else {
builder, err := tx.Builder.Query().Where(builderInputQueryPredicate(*builtBy.BuilderInput)).Only(ctx)
if err != nil {
return nil, err
return nil, nil, err
}
buildID = builder.ID
}
Expand All @@ -287,12 +289,12 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
artGlobalID := fromGlobalID(*subject.ArtifactID)
subjectArtifactID, err = uuid.Parse(artGlobalID.id)
if err != nil {
return nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
return nil, nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
}
} else {
foundArt, err := tx.Artifact.Query().Where(artifactQueryInputPredicates(*subject.ArtifactInput)).Only(ctx)
if err != nil {
return nil, fmt.Errorf("failed to query for artifact")
return nil, nil, fmt.Errorf("failed to query for artifact")
}
subjectArtifactID = foundArt.ID
}
Expand All @@ -309,7 +311,7 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
} else {
foundArt, err := tx.Artifact.Query().Where(artifactQueryInputPredicates(*bf.ArtifactInput)).Only(ctx)
if err != nil {
return nil, err
return nil, nil, err
}
builtFromIDs = append(builtFromIDs, foundArt.ID.String())
}
Expand All @@ -320,7 +322,7 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti
for _, sbfID := range sortedBuildFromIDs {
sbfUUID, err := uuid.Parse(sbfID)
if err != nil {
return nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
return nil, nil, fmt.Errorf("uuid conversion from ArtifactID failed with error: %w", err)
}
slsaCreate.AddBuiltFromIDs(sbfUUID)
}
Expand All @@ -334,17 +336,17 @@ func generateSLSACreate(ctx context.Context, tx *ent.Tx, subject *model.IDorArti

slsaID, err := guacSLSAKey(ptrfrom.String(subjectArtifactID.String()), builtFromHash, ptrfrom.String(buildID.String()), slsa)
if err != nil {
return nil, fmt.Errorf("failed to create slsa uuid with error: %w", err)
return nil, nil, fmt.Errorf("failed to create slsa uuid with error: %w", err)
}

slsaCreate.SetID(*slsaID)

return slsaCreate, nil
return slsaCreate, slsaID, nil
}

func upsertSLSA(ctx context.Context, tx *ent.Tx, subject model.IDorArtifactInput, builtFrom []*model.IDorArtifactInput, builtBy model.IDorBuilderInput, slsa model.SLSAInputSpec) (*string, error) {

slsaCreate, err := generateSLSACreate(ctx, tx, &subject, builtFrom, &builtBy, &slsa)
slsaCreate, _, err := generateSLSACreate(ctx, tx, &subject, builtFrom, &builtBy, &slsa)
if err != nil {
return nil, gqlerror.Errorf("generateSLSACreate :: %s", err)
}
Expand Down
46 changes: 28 additions & 18 deletions pkg/assembler/clients/helpers/bulk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
Expand All @@ -28,9 +28,16 @@ import (
"github.com/guacsec/guac/pkg/assembler/helpers"
)

func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient graphql.Client) func([]assembler.AssemblerInput) error {
return func(preds []assembler.IngestPredicates) error {
type AssemblerIngestedIDs struct {
hasSBOMIDs []string
hasSLSAIDs []string
}

func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient graphql.Client) func([]assembler.AssemblerInput) (*AssemblerIngestedIDs, error) {
return func(preds []assembler.IngestPredicates) (*AssemblerIngestedIDs, error) {
var rvErr error
ingestedIDs := &AssemblerIngestedIDs{}

for _, p := range preds {

// Ingest Packages
Expand All @@ -41,7 +48,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorPkgInputs, err := ingestPackages(ctx, gqlclient, packages)
if err != nil {
return fmt.Errorf("ingestPackages failed with error: %w", err)
return nil, fmt.Errorf("ingestPackages failed with error: %w", err)
}

var pkgVersionIDs []string
Expand All @@ -57,7 +64,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorSrcInputs, err := ingestSources(ctx, gqlclient, sources)
if err != nil {
return fmt.Errorf("ingestSources failed with error: %w", err)
return nil, fmt.Errorf("ingestSources failed with error: %w", err)
}

// Ingest Artifacts
Expand All @@ -67,7 +74,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorArtInputs, err := ingestArtifacts(ctx, gqlclient, artifacts)
if err != nil {
return fmt.Errorf("ingestArtifacts failed with error: %w", err)
return nil, fmt.Errorf("ingestArtifacts failed with error: %w", err)
}
var artIDs []string
for _, artID := range collectedIDorArtInputs {
Expand All @@ -82,7 +89,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorMatInputs, err := ingestArtifacts(ctx, gqlclient, materials)
if err != nil {
return fmt.Errorf("ingestArtifacts failed with error: %w", err)
return nil, fmt.Errorf("ingestArtifacts failed with error: %w", err)
}

// Ingest Builders
Expand All @@ -91,7 +98,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorBuilderInputs, err := ingestBuilders(ctx, gqlclient, builders)
if err != nil {
return fmt.Errorf("ingestBuilders failed with error: %w", err)
return nil, fmt.Errorf("ingestBuilders failed with error: %w", err)
}

// Ingest Vulnerabilities
Expand All @@ -100,7 +107,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorVulnInputs, err := ingestVulnerabilities(ctx, gqlclient, vulns)
if err != nil {
return fmt.Errorf("ingestVulnerabilities failed with error: %w", err)
return nil, fmt.Errorf("ingestVulnerabilities failed with error: %w", err)
}

// Ingest Licenses
Expand All @@ -109,7 +116,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient

collectedIDorLicenseInputs, err := ingestLicenses(ctx, gqlclient, licenses)
if err != nil {
return fmt.Errorf("ingestLicenses failed with error: %w", err)
return nil, fmt.Errorf("ingestLicenses failed with error: %w", err)
}

logger.Infof("assembling CertifyScorecard: %v", len(p.CertifyScorecard))
Expand Down Expand Up @@ -137,7 +144,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient
}

logger.Infof("assembling HasSLSA: %v", len(p.HasSlsa))
if err := ingestHasSLSAs(ctx, gqlclient, p.HasSlsa, collectedIDorArtInputs, collectedIDorMatInputs, collectedIDorBuilderInputs); err != nil {
if err := ingestHasSLSAs(ctx, gqlclient, p.HasSlsa, collectedIDorArtInputs, collectedIDorMatInputs, collectedIDorBuilderInputs, ingestedIDs); err != nil {
logger.Errorf("ingestHasSLSAs failed with error: %v", err)
rvErr = err
}
Expand Down Expand Up @@ -196,7 +203,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient
Artifacts: artifactIDs,
Dependencies: isDependenciesIDs,
Occurrences: isOccurrencesIDs,
}, collectedIDorPkgInputs, collectedIDorArtInputs); err != nil {
}, collectedIDorPkgInputs, collectedIDorArtInputs, ingestedIDs); err != nil {
logger.Errorf("ingestHasSBOMs failed with error: %v", err)
rvErr = err
}
Expand Down Expand Up @@ -225,7 +232,7 @@ func GetBulkAssembler(ctx context.Context, logger *zap.SugaredLogger, gqlclient
rvErr = err
}
}
return rvErr
return ingestedIDs, rvErr
}
}

Expand Down Expand Up @@ -566,7 +573,7 @@ func ingestHasSourceAts(ctx context.Context, client graphql.Client, hs []assembl
}

func ingestHasSLSAs(ctx context.Context, client graphql.Client, hs []assembler.HasSlsaIngest, artInputMap map[string]*model.IDorArtifactInput,
matInputSpec map[string]*model.IDorArtifactInput, builderInputMap map[string]*model.IDorBuilderInput) error {
matInputSpec map[string]*model.IDorArtifactInput, builderInputMap map[string]*model.IDorBuilderInput, ingestedIDs *AssemblerIngestedIDs) error {

var subjectIDs []model.IDorArtifactInput
var slsaAttestations []model.SLSAInputSpec
Expand Down Expand Up @@ -595,10 +602,11 @@ func ingestHasSLSAs(ctx context.Context, client graphql.Client, hs []assembler.H
slsaAttestations = append(slsaAttestations, *ingest.HasSlsa)
}
if len(hs) > 0 {
_, err := model.IngestSLSAForArtifacts(ctx, client, subjectIDs, materialIDs, builderIDs, slsaAttestations)
hasSLSAArtResponse, err := model.IngestSLSAForArtifacts(ctx, client, subjectIDs, materialIDs, builderIDs, slsaAttestations)
if err != nil {
return fmt.Errorf("SLSAForArtifacts failed with error: %w", err)
}
ingestedIDs.hasSLSAIDs = append(ingestedIDs.hasSLSAIDs, hasSLSAArtResponse.IngestSLSAs...)
}
return nil
}
Expand Down Expand Up @@ -710,7 +718,7 @@ func ingestHashEquals(ctx context.Context, client graphql.Client, he []assembler
}

func ingestHasSBOMs(ctx context.Context, client graphql.Client, hs []assembler.HasSBOMIngest, includes model.HasSBOMIncludesInputSpec, packageInputMap map[string]*model.IDorPkgInput,
artInputMap map[string]*model.IDorArtifactInput) error {
artInputMap map[string]*model.IDorArtifactInput, ingestedIDs *AssemblerIngestedIDs) error {

var pkgIDs []model.IDorPkgInput
var artIDs []model.IDorArtifactInput
Expand Down Expand Up @@ -745,16 +753,18 @@ func ingestHasSBOMs(ctx context.Context, client graphql.Client, hs []assembler.H
}
}
if len(artIDs) > 0 {
_, err := model.IngestHasSBOMArtifacts(ctx, client, artIDs, artSBOMs, artIncludes)
hasSBOMArtResponse, err := model.IngestHasSBOMArtifacts(ctx, client, artIDs, artSBOMs, artIncludes)
if err != nil {
return fmt.Errorf("hasSBOMArtifacts failed with error: %w", err)
}
ingestedIDs.hasSBOMIDs = append(ingestedIDs.hasSBOMIDs, hasSBOMArtResponse.IngestHasSBOMs...)
}
if len(pkgIDs) > 0 {
_, err := model.IngestHasSBOMPkgs(ctx, client, pkgIDs, pkgSBOMs, pkgIncludes)
hasSBOMPkgResponse, err := model.IngestHasSBOMPkgs(ctx, client, pkgIDs, pkgSBOMs, pkgIncludes)
if err != nil {
return fmt.Errorf("hasSBOMPkgs failed with error: %w", err)
}
ingestedIDs.hasSBOMIDs = append(ingestedIDs.hasSBOMIDs, hasSBOMPkgResponse.IngestHasSBOMs...)
}
return nil
}
Expand Down
18 changes: 9 additions & 9 deletions pkg/ingestor/ingestor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func Ingest(
csubClient csub_client.Client,
scanForVulns bool,
scanForLicense bool,
) error {
) (*helpers.AssemblerIngestedIDs, error) {
logger := d.ChildLogger
// Get pipeline of components
processorFunc := GetProcessor(ctx)
Expand All @@ -56,26 +56,26 @@ func Ingest(

docTree, err := processorFunc(d)
if err != nil {
return fmt.Errorf("unable to process doc: %v, format: %v, document: %v", err, d.Format, d.Type)
return nil, fmt.Errorf("unable to process doc: %v, format: %v, document: %v", err, d.Format, d.Type)
}

predicates, idstrings, err := ingestorFunc(docTree)
if err != nil {
return fmt.Errorf("unable to ingest doc tree: %v", err)
return nil, fmt.Errorf("unable to ingest doc tree: %v", err)
}

if err := collectSubEmitFunc(idstrings); err != nil {
logger.Infof("unable to create entries in collectsub server, but continuing: %v", err)
}

if err := assemblerFunc(predicates); err != nil {
return fmt.Errorf("error assembling graphs for %q : %w", d.SourceInformation.Source, err)
if ingestedIDs, err := assemblerFunc(predicates); err != nil {
return nil, fmt.Errorf("error assembling graphs for %q : %w", d.SourceInformation.Source, err)
}

t := time.Now()
elapsed := t.Sub(start)
logger.Infof("[%v] completed doc %+v", elapsed, d.SourceInformation)
return nil
return ingestedIDs, nil
}

func MergedIngest(
Expand Down Expand Up @@ -174,11 +174,11 @@ func GetAssembler(
childLogger *zap.SugaredLogger,
graphqlEndpoint string,
transport http.RoundTripper,
) func([]assembler.IngestPredicates) error {
) func([]assembler.IngestPredicates) (*helpers.AssemblerIngestedIDs, error) {
httpClient := http.Client{Transport: transport}
gqlclient := graphql.NewClient(graphqlEndpoint, &httpClient)
f := helpers.GetBulkAssembler(ctx, childLogger, gqlclient)
return f

return helpers.GetBulkAssembler(ctx, childLogger, gqlclient)
}

func GetCollectSubEmit(ctx context.Context, csubClient csub_client.Client) func([]*parser_common.IdentifierStrings) error {
Expand Down

0 comments on commit 0c8d54c

Please sign in to comment.