From b21b412f7c97ec5ddb48f91463eae2b2a33abb11 Mon Sep 17 00:00:00 2001 From: pxp928 Date: Fri, 16 Aug 2024 15:37:18 -0400 Subject: [PATCH] remove daysSinceLastScan as it is redundant with certifier interval Signed-off-by: pxp928 --- cmd/guaccollect/cmd/license.go | 10 +- cmd/guaccollect/cmd/osv.go | 14 +- cmd/guaccollect/cmd/scorecard.go | 9 +- cmd/guacone/cmd/license.go | 2 +- cmd/guacone/cmd/osv.go | 2 +- cmd/guacone/cmd/scorecard.go | 3 +- container_files/guac/guac.yaml | 3 +- guac.yaml | 3 +- internal/testing/cmd/pubsub_test/cmd/osv.go | 2 +- .../components/root_package/root_package.go | 50 +---- .../root_package/root_package_test.go | 202 ++---------------- pkg/certifier/components/source/source.go | 54 +---- .../components/source/source_test.go | 191 ++--------------- pkg/cli/store.go | 2 - 14 files changed, 70 insertions(+), 477 deletions(-) diff --git a/cmd/guaccollect/cmd/license.go b/cmd/guaccollect/cmd/license.go index 82a9d58d9f..60c284796c 100644 --- a/cmd/guaccollect/cmd/license.go +++ b/cmd/guaccollect/cmd/license.go @@ -45,9 +45,6 @@ type cdOptions struct { interval time.Duration // enable/disable message publish to queue publishToQueue bool - // days since the last vulnerability scan was run. - // 0 means only run once - daysSinceLastScan int // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -81,7 +78,6 @@ you have access to read and write to the respective blob store.`, viper.GetString("interval"), viper.GetBool("service-poll"), viper.GetBool("publish-to-queue"), - viper.GetInt("last-scan"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), ) @@ -102,7 +98,7 @@ you have access to read and write to the respective blob store.`, httpClient := http.Client{Transport: transport} gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient) - packageQueryFunc, err := getPackageQuery(gqlclient, opts.daysSinceLastScan, opts.batchSize, opts.addedLatency) + packageQueryFunc, err := getPackageQuery(gqlclient, opts.batchSize, opts.addedLatency) if err != nil { logger.Errorf("error: %v", err) os.Exit(1) @@ -120,7 +116,6 @@ func validateCDFlags( interval string, poll bool, pubToQueue bool, - daysSince int, certifierLatencyStr string, batchSize int) (cdOptions, error) { @@ -138,7 +133,6 @@ func validateCDFlags( return opts, fmt.Errorf("failed to parser duration with error: %w", err) } opts.interval = i - opts.daysSinceLastScan = daysSince if certifierLatencyStr != "" { addedLatency, err := time.ParseDuration(certifierLatencyStr) @@ -157,7 +151,7 @@ func validateCDFlags( func init() { set, err := cli.BuildFlags([]string{"interval", - "last-scan", "header-file", "certifier-latency", + "header-file", "certifier-latency", "certifier-batch-size"}) if err != nil { fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err) diff --git a/cmd/guaccollect/cmd/osv.go b/cmd/guaccollect/cmd/osv.go index 7f59caddec..fd4d0a3903 100644 --- a/cmd/guaccollect/cmd/osv.go +++ b/cmd/guaccollect/cmd/osv.go @@ -54,9 +54,6 @@ type osvOptions struct { interval time.Duration // enable/disable message publish to queue publishToQueue bool - // days since the last vulnerability scan was run. - // 0 means only run once - daysSinceLastScan int // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -90,7 +87,6 @@ you have access to read and write to the respective blob store.`, viper.GetString("interval"), viper.GetBool("service-poll"), viper.GetBool("publish-to-queue"), - viper.GetInt("last-scan"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), ) @@ -111,7 +107,7 @@ you have access to read and write to the respective blob store.`, httpClient := http.Client{Transport: transport} gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient) - packageQueryFunc, err := getPackageQuery(gqlclient, opts.daysSinceLastScan, opts.batchSize, opts.addedLatency) + packageQueryFunc, err := getPackageQuery(gqlclient, opts.batchSize, opts.addedLatency) if err != nil { logger.Errorf("error: %v", err) os.Exit(1) @@ -129,7 +125,6 @@ func validateOSVFlags( interval string, poll bool, pubToQueue bool, - daysSince int, certifierLatencyStr string, batchSize int) (osvOptions, error) { @@ -147,7 +142,6 @@ func validateOSVFlags( return opts, fmt.Errorf("failed to parser duration with error: %w", err) } opts.interval = i - opts.daysSinceLastScan = daysSince if certifierLatencyStr != "" { addedLatency, err := time.ParseDuration(certifierLatencyStr) @@ -170,9 +164,9 @@ func getCertifierPublish(ctx context.Context, blobStore *blob.BlobStore, pubsub }, nil } -func getPackageQuery(client graphql.Client, daysSinceLastScan int, batchSize int, addedLatency *time.Duration) (func() certifier.QueryComponents, error) { +func getPackageQuery(client graphql.Client, batchSize int, addedLatency *time.Duration) (func() certifier.QueryComponents, error) { return func() certifier.QueryComponents { - packageQuery := root_package.NewPackageQuery(client, daysSinceLastScan, batchSize, addedLatency) + packageQuery := root_package.NewPackageQuery(client, batchSize, addedLatency) return packageQuery }, nil } @@ -255,7 +249,7 @@ func initializeNATsandCertifier(ctx context.Context, blobAddr, pubsubAddr string func init() { set, err := cli.BuildFlags([]string{"interval", - "last-scan", "header-file", "certifier-latency", + "header-file", "certifier-latency", "certifier-batch-size"}) if err != nil { fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err) diff --git a/cmd/guaccollect/cmd/scorecard.go b/cmd/guaccollect/cmd/scorecard.go index f8c96417f1..bb419ca20e 100644 --- a/cmd/guaccollect/cmd/scorecard.go +++ b/cmd/guaccollect/cmd/scorecard.go @@ -46,8 +46,6 @@ type scorecardOptions struct { interval time.Duration // enable/disable message publish to queue publishToQueue bool - // setting "daysSinceLastScan" to 0 does not check the timestamp on the scorecard that exist - daysSinceLastScan int // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -81,7 +79,6 @@ you have access to read and write to the respective blob store.`, viper.GetString("interval"), viper.GetBool("service-poll"), viper.GetBool("publish-to-queue"), - viper.GetInt("last-scan"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), ) @@ -120,7 +117,7 @@ you have access to read and write to the respective blob store.`, httpClient := http.Client{Transport: transport} gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient) - query, err := sc.NewCertifier(gqlclient, opts.daysSinceLastScan, opts.batchSize, opts.addedLatency) + query, err := sc.NewCertifier(gqlclient, opts.batchSize, opts.addedLatency) if err != nil { logger.Errorf("unable to create source query: %v\n", err) os.Exit(1) @@ -138,7 +135,6 @@ func validateScorecardFlags( interval string, poll bool, pubToQueue bool, - daysSince int, certifierLatencyStr string, batchSize int) (scorecardOptions, error) { @@ -156,7 +152,6 @@ func validateScorecardFlags( return opts, fmt.Errorf("failed to parser duration with error: %w", err) } opts.interval = i - opts.daysSinceLastScan = daysSince if certifierLatencyStr != "" { addedLatency, err := time.ParseDuration(certifierLatencyStr) @@ -175,7 +170,7 @@ func validateScorecardFlags( func init() { set, err := cli.BuildFlags([]string{"interval", - "last-scan", "header-file", "certifier-latency", + "header-file", "certifier-latency", "certifier-batch-size"}) if err != nil { fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err) diff --git a/cmd/guacone/cmd/license.go b/cmd/guacone/cmd/license.go index 7da15dcf2a..a5261206c8 100644 --- a/cmd/guacone/cmd/license.go +++ b/cmd/guacone/cmd/license.go @@ -96,7 +96,7 @@ var cdCmd = &cobra.Command{ httpClient := http.Client{Transport: transport} gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient) - packageQuery := root_package.NewPackageQuery(gqlclient, 0, opts.batchSize, opts.addedLatency) + packageQuery := root_package.NewPackageQuery(gqlclient, opts.batchSize, opts.addedLatency) totalNum := 0 docChan := make(chan *processor.Document) diff --git a/cmd/guacone/cmd/osv.go b/cmd/guacone/cmd/osv.go index 022a1a86f1..419587286d 100644 --- a/cmd/guacone/cmd/osv.go +++ b/cmd/guacone/cmd/osv.go @@ -96,7 +96,7 @@ var osvCmd = &cobra.Command{ httpClient := http.Client{Transport: transport} gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient) - packageQuery := root_package.NewPackageQuery(gqlclient, 0, opts.batchSize, opts.addedLatency) + packageQuery := root_package.NewPackageQuery(gqlclient, opts.batchSize, opts.addedLatency) totalNum := 0 docChan := make(chan *processor.Document) diff --git a/cmd/guacone/cmd/scorecard.go b/cmd/guacone/cmd/scorecard.go index 5ccdd9ff5a..29b895d970 100644 --- a/cmd/guacone/cmd/scorecard.go +++ b/cmd/guacone/cmd/scorecard.go @@ -112,8 +112,7 @@ var scorecardCmd = &cobra.Command{ } // scorecard certifier is the certifier that gets the scorecard data graphQL - // setting "daysSinceLastScan" to 0 does not check the timestamp on the scorecard that exist - query, err := sc.NewCertifier(gqlclient, 0, opts.batchSize, opts.addedLatency) + query, err := sc.NewCertifier(gqlclient, opts.batchSize, opts.addedLatency) if err != nil { fmt.Printf("unable to create scorecard certifier: %v\n", err) diff --git a/container_files/guac/guac.yaml b/container_files/guac/guac.yaml index d9c26382be..a8ec87d482 100644 --- a/container_files/guac/guac.yaml +++ b/container_files/guac/guac.yaml @@ -22,8 +22,7 @@ use-csub: true poll: true # certifier interval interval: 20m -# days since the last vulnerability scan was run. 0 means only run once -last-scan: 0 + # set the batch size for the package pagination query certifier-batch-size: 60000 # add artificial latency to throttle the certifier diff --git a/guac.yaml b/guac.yaml index f36ae12810..049903877f 100644 --- a/guac.yaml +++ b/guac.yaml @@ -26,8 +26,7 @@ blob-addr: file:///tmp/blobstore?no_tmp_dir=true # certifier interval interval: 20m -# days since the last vulnerability scan was run. 0 means only run once -last-scan: 0 + # set the batch size for the package pagination query certifier-batch-size: 60000 # add artificial latency to throttle the certifier diff --git a/internal/testing/cmd/pubsub_test/cmd/osv.go b/internal/testing/cmd/pubsub_test/cmd/osv.go index 79c63e6e9e..bafca187b4 100644 --- a/internal/testing/cmd/pubsub_test/cmd/osv.go +++ b/internal/testing/cmd/pubsub_test/cmd/osv.go @@ -92,7 +92,7 @@ func getCertifierPublish(ctx context.Context, blobStore *blob.BlobStore, pubsub func getPackageQuery(client graphql.Client) (func() certifier.QueryComponents, error) { return func() certifier.QueryComponents { - packageQuery := root_package.NewPackageQuery(client, 0, 60000, nil) + packageQuery := root_package.NewPackageQuery(client, 60000, nil) return packageQuery }, nil } diff --git a/pkg/certifier/components/root_package/root_package.go b/pkg/certifier/components/root_package/root_package.go index f758586a57..7b5b2129e5 100644 --- a/pkg/certifier/components/root_package/root_package.go +++ b/pkg/certifier/components/root_package/root_package.go @@ -18,7 +18,6 @@ package root_package import ( "context" "fmt" - "math" "time" "github.com/Khan/genqlient/graphql" @@ -36,8 +35,6 @@ type PackageNode struct { type packageQuery struct { client graphql.Client - // daysSinceLastScan sets the days since the last vulnerability scan was run - daysSinceLastScan int // set the batch size for the package pagination query batchSize int // add artificial latency to throttle the pagination query @@ -45,21 +42,18 @@ type packageQuery struct { } var getPackages func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) -var getNeighbors func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) // NewPackageQuery initializes the packageQuery to query from the graph database -func NewPackageQuery(client graphql.Client, daysSinceLastScan, batchSize int, addedLatency *time.Duration) certifier.QueryComponents { +func NewPackageQuery(client graphql.Client, batchSize int, addedLatency *time.Duration) certifier.QueryComponents { getPackages = generated.PackagesList - getNeighbors = generated.Neighbors return &packageQuery{ - client: client, - daysSinceLastScan: daysSinceLastScan, - batchSize: batchSize, - addedLatency: addedLatency, + client: client, + batchSize: batchSize, + addedLatency: addedLatency, } } -// GetComponents get all the packages that do not have a certify vulnerability attached or last scanned is more than daysSinceLastScan +// GetComponents get all the packages func (p *packageQuery) GetComponents(ctx context.Context, compChan chan<- interface{}) error { if compChan == nil { return fmt.Errorf("compChan cannot be nil") @@ -131,7 +125,6 @@ func (p *packageQuery) getPackageNodes(ctx context.Context, nodeChan chan<- *Pac var afterCursor *string first := p.batchSize - //first := 60000 for { pkgConn, err := getPackages(ctx, p.client, generated.PkgSpec{}, afterCursor, &first) if err != nil { @@ -149,37 +142,10 @@ func (p *packageQuery) getPackageNodes(ctx context.Context, nodeChan chan<- *Pac for _, namespace := range pkgNode.Node.Namespaces { for _, name := range namespace.Names { for _, version := range name.Versions { - response, err := getNeighbors(ctx, p.client, version.Id, []generated.Edge{generated.EdgePackageCertifyVuln}) - if err != nil { - return fmt.Errorf("failed neighbors query: %w", err) - } - vulnList := []*generated.NeighborsNeighborsCertifyVuln{} - certifyVulnFound := false - for _, neighbor := range response.Neighbors { - if certifyVuln, ok := neighbor.(*generated.NeighborsNeighborsCertifyVuln); ok { - vulnList = append(vulnList, certifyVuln) - } - } - // collect all certifyVulnerability and then check timestamp else if not checking timestamp, - // if a certifyVulnerability is found break out - for _, vulns := range vulnList { - if p.daysSinceLastScan != 0 { - now := time.Now() - difference := vulns.Metadata.TimeScanned.Sub(now) - if math.Abs(difference.Hours()) < float64(p.daysSinceLastScan*24) { - certifyVulnFound = true - } - } else { - certifyVulnFound = true - break - } - } - if !certifyVulnFound { - packNode := PackageNode{ - Purl: version.Purl, - } - nodeChan <- &packNode + packNode := PackageNode{ + Purl: version.Purl, } + nodeChan <- &packNode } } } diff --git a/pkg/certifier/components/root_package/root_package_test.go b/pkg/certifier/components/root_package/root_package_test.go index 8de62a25c6..bd9d909257 100644 --- a/pkg/certifier/components/root_package/root_package_test.go +++ b/pkg/certifier/components/root_package/root_package_test.go @@ -32,10 +32,9 @@ func TestNewPackageQuery(t *testing.T) { gqlclient := graphql.NewClient("inmemeory", &httpClient) type args struct { - client graphql.Client - daysSinceLastScan int - batchSize int - addedLatency *time.Duration + client graphql.Client + batchSize int + addedLatency *time.Duration } tests := []struct { name string @@ -44,21 +43,19 @@ func TestNewPackageQuery(t *testing.T) { }{{ name: "newPackageQuery", args: args{ - client: gqlclient, - daysSinceLastScan: 0, - batchSize: 60000, - addedLatency: nil, + client: gqlclient, + batchSize: 60000, + addedLatency: nil, }, want: &packageQuery{ - client: gqlclient, - daysSinceLastScan: 0, - batchSize: 60000, - addedLatency: nil, + client: gqlclient, + batchSize: 60000, + addedLatency: nil, }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := NewPackageQuery(tt.args.client, tt.args.daysSinceLastScan, tt.args.batchSize, tt.args.addedLatency); !reflect.DeepEqual(got, tt.want) { + if got := NewPackageQuery(tt.args.client, tt.args.batchSize, tt.args.addedLatency); !reflect.DeepEqual(got, tt.want) { t.Errorf("NewPackageQuery() = %v, want %v", got, tt.want) } }) @@ -66,7 +63,6 @@ func TestNewPackageQuery(t *testing.T) { } func Test_packageQuery_GetComponents(t *testing.T) { - tm, _ := time.Parse(time.RFC3339, "2022-11-21T17:45:50.52Z") testPypiPackage := generated.PackagesListPackagesListPackageConnectionEdgesPackageEdgeNodePackage{} testPypiPackage.Type = "pypi" @@ -104,27 +100,14 @@ func Test_packageQuery_GetComponents(t *testing.T) { }, }) - neighborCertifyVulnTimeStamp := generated.NeighborsNeighborsCertifyVuln{} - neighborCertifyVulnTimeStamp.Metadata = generated.AllCertifyVulnMetadataScanMetadata{ - TimeScanned: tm.UTC(), - } - - neighborCertifyVulnTimeNow := generated.NeighborsNeighborsCertifyVuln{} - neighborCertifyVulnTimeNow.Metadata = generated.AllCertifyVulnMetadataScanMetadata{ - TimeScanned: time.Now().UTC(), - } - tests := []struct { - name string - daysSinceLastScan int - getPackages func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) - getNeighbors func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) - wantPackNode []*PackageNode - wantErr bool + name string + getPackages func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) + wantPackNode []*PackageNode + wantErr bool }{ { - name: "django: daysSinceLastScan=0", - daysSinceLastScan: 0, + name: "django:", getPackages: func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) { return &generated.PackagesListResponse{ PackagesList: &generated.PackagesListPackagesListPackageConnection{ @@ -141,11 +124,6 @@ func Test_packageQuery_GetComponents(t *testing.T) { }, }, nil }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{}, - }, nil - }, wantPackNode: []*PackageNode{ { Purl: "pkg:pypi/django@1.11.1", @@ -153,142 +131,7 @@ func Test_packageQuery_GetComponents(t *testing.T) { }, wantErr: false, }, { - name: "django with certifyVuln", - daysSinceLastScan: 0, - getPackages: func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) { - return &generated.PackagesListResponse{ - PackagesList: &generated.PackagesListPackagesListPackageConnection{ - TotalCount: 1, - Edges: []generated.PackagesListPackagesListPackageConnectionEdgesPackageEdge{ - { - Node: testPypiPackage, - Cursor: "", - }, - }, - PageInfo: generated.PackagesListPackagesListPackageConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{&neighborCertifyVulnTimeStamp}, - }, nil - }, - wantPackNode: []*PackageNode{}, - wantErr: false, - }, { - name: "django with certifyVuln, daysSinceLastScan=30", - daysSinceLastScan: 30, - getPackages: func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) { - return &generated.PackagesListResponse{ - PackagesList: &generated.PackagesListPackagesListPackageConnection{ - TotalCount: 1, - Edges: []generated.PackagesListPackagesListPackageConnectionEdgesPackageEdge{ - { - Node: testPypiPackage, - Cursor: "", - }, - }, - PageInfo: generated.PackagesListPackagesListPackageConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{&neighborCertifyVulnTimeStamp}, - }, nil - }, - wantPackNode: []*PackageNode{{ - Purl: "pkg:pypi/django@1.11.1", - }}, - wantErr: false, - }, { - name: "django with certifyVuln, timestamp: time now, daysSinceLastScan=30", - daysSinceLastScan: 30, - getPackages: func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) { - return &generated.PackagesListResponse{ - PackagesList: &generated.PackagesListPackagesListPackageConnection{ - TotalCount: 1, - Edges: []generated.PackagesListPackagesListPackageConnectionEdgesPackageEdge{ - { - Node: testPypiPackage, - Cursor: "", - }, - }, - PageInfo: generated.PackagesListPackagesListPackageConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{&neighborCertifyVulnTimeNow}, - }, nil - }, - wantPackNode: []*PackageNode{}, - wantErr: false, - }, { - name: "django with certifyVuln, daysSinceLastScan=0, IsOccurrence", - daysSinceLastScan: 0, - getPackages: func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) { - return &generated.PackagesListResponse{ - PackagesList: &generated.PackagesListPackagesListPackageConnection{ - TotalCount: 1, - Edges: []generated.PackagesListPackagesListPackageConnectionEdgesPackageEdge{ - { - Node: testPypiPackage, - Cursor: "", - }, - }, - PageInfo: generated.PackagesListPackagesListPackageConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{&neighborCertifyVulnTimeStamp}, - }, nil - }, - wantPackNode: []*PackageNode{}, - wantErr: false, - }, { - name: "django, daysSinceLastScan=0, IsOccurrence", - daysSinceLastScan: 0, - getPackages: func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) { - return &generated.PackagesListResponse{ - PackagesList: &generated.PackagesListPackagesListPackageConnection{ - TotalCount: 1, - Edges: []generated.PackagesListPackagesListPackageConnectionEdgesPackageEdge{ - { - Node: testPypiPackage, - Cursor: "", - }, - }, - PageInfo: generated.PackagesListPackagesListPackageConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{}, - }, nil - }, - wantPackNode: []*PackageNode{{ - Purl: "pkg:pypi/django@1.11.1", - }}, - wantErr: false, - }, { - name: "multiple packages", - daysSinceLastScan: 0, + name: "multiple packages", getPackages: func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error) { return &generated.PackagesListResponse{ PackagesList: &generated.PackagesListPackagesListPackageConnection{ @@ -309,11 +152,6 @@ func Test_packageQuery_GetComponents(t *testing.T) { }, }, nil }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{}, - }, nil - }, wantPackNode: []*PackageNode{{ Purl: "pkg:pypi/django@1.11.1", }, { @@ -329,13 +167,11 @@ func Test_packageQuery_GetComponents(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() p := &packageQuery{ - client: nil, - daysSinceLastScan: tt.daysSinceLastScan, - batchSize: 1, - addedLatency: &addedLatency, + client: nil, + batchSize: 1, + addedLatency: &addedLatency, } getPackages = tt.getPackages - getNeighbors = tt.getNeighbors // compChan to collect query components compChan := make(chan interface{}, 1) diff --git a/pkg/certifier/components/source/source.go b/pkg/certifier/components/source/source.go index 8917c71062..84b50a7c6f 100644 --- a/pkg/certifier/components/source/source.go +++ b/pkg/certifier/components/source/source.go @@ -18,7 +18,6 @@ package source import ( "context" "fmt" - "math" "path" "strings" "time" @@ -30,8 +29,6 @@ import ( type sourceQuery struct { client graphql.Client - // daysSinceLastScan sets the days since the last vulnerability scan was run - daysSinceLastScan int // set the batch size for the package pagination query batchSize int // add artificial latency to throttle the pagination query @@ -45,9 +42,8 @@ type SourceNode struct { } var getSources func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) -var getNeighbors func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) -// GetComponents get all the sources that do not have a certify scorecard attached or last scanned is more than daysSinceLastScan +// GetComponents get all the sources func (s sourceQuery) GetComponents(ctx context.Context, compChan chan<- interface{}) error { if compChan == nil { return fmt.Errorf("compChan cannot be nil") @@ -67,40 +63,12 @@ func (s sourceQuery) GetComponents(ctx context.Context, compChan chan<- interfac for _, srcNode := range srcEdges { for _, namespace := range srcNode.Node.Namespaces { for _, names := range namespace.Names { - response, err := getNeighbors(ctx, s.client, names.Id, []generated.Edge{generated.EdgeSourceCertifyScorecard}) - if err != nil { - return fmt.Errorf("failed neighbors query: %w", err) - } - scorecardList := []*generated.NeighborsNeighborsCertifyScorecard{} - scoreCardFound := false - for _, neighbor := range response.Neighbors { - scorecardNode, ok := neighbor.(*generated.NeighborsNeighborsCertifyScorecard) - if ok { - scorecardList = append(scorecardList, scorecardNode) - } - } - // collect all scorecardNodes and then check timestamp else if not checking timestamp, - // if a scorecard is found break out - for _, scorecardNode := range scorecardList { - if s.daysSinceLastScan != 0 { - now := time.Now() - difference := scorecardNode.Scorecard.TimeScanned.Sub(now) - if math.Abs(difference.Hours()) < float64(s.daysSinceLastScan*24) { - scoreCardFound = true - } - } else { - scoreCardFound = true - break - } - } - if !scoreCardFound { - sourceNode := SourceNode{ - Repo: path.Join(namespace.Namespace, names.Name), - Commit: trimAlgorithm(nilOrEmpty(names.Commit)), - Tag: nilOrEmpty(names.Tag), - } - compChan <- &sourceNode + sourceNode := SourceNode{ + Repo: path.Join(namespace.Namespace, names.Name), + Commit: trimAlgorithm(nilOrEmpty(names.Commit)), + Tag: nilOrEmpty(names.Tag), } + compChan <- &sourceNode } } } @@ -139,16 +107,14 @@ func trimAlgorithm(commit string) string { } // NewCertifier returns a new sourceArtifacts certifier -func NewCertifier(client graphql.Client, daysSinceLastScan, batchSize int, addedLatency *time.Duration) (certifier.QueryComponents, error) { +func NewCertifier(client graphql.Client, batchSize int, addedLatency *time.Duration) (certifier.QueryComponents, error) { if client == nil { return nil, fmt.Errorf("client cannot be nil") } getSources = generated.SourcesList - getNeighbors = generated.Neighbors return &sourceQuery{ - client: client, - daysSinceLastScan: daysSinceLastScan, - batchSize: batchSize, - addedLatency: addedLatency, + client: client, + batchSize: batchSize, + addedLatency: addedLatency, }, nil } diff --git a/pkg/certifier/components/source/source_test.go b/pkg/certifier/components/source/source_test.go index 6b5dccc40b..214a395820 100644 --- a/pkg/certifier/components/source/source_test.go +++ b/pkg/certifier/components/source/source_test.go @@ -33,10 +33,9 @@ func TestNewCertifier(t *testing.T) { gqlclient := graphql.NewClient("inmemeory", &httpClient) type args struct { - client graphql.Client - daysSinceLastScan int - batchSize int - addedLatency *time.Duration + client graphql.Client + batchSize int + addedLatency *time.Duration } tests := []struct { name string @@ -46,21 +45,19 @@ func TestNewCertifier(t *testing.T) { }{{ name: "newSourceQuery", args: args{ - client: gqlclient, - daysSinceLastScan: 0, - batchSize: 60000, - addedLatency: nil, + client: gqlclient, + batchSize: 60000, + addedLatency: nil, }, want: &sourceQuery{ - client: gqlclient, - daysSinceLastScan: 0, - batchSize: 60000, - addedLatency: nil, + client: gqlclient, + batchSize: 60000, + addedLatency: nil, }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewCertifier(tt.args.client, tt.args.daysSinceLastScan, tt.args.batchSize, tt.args.addedLatency) + got, err := NewCertifier(tt.args.client, tt.args.batchSize, tt.args.addedLatency) if (err != nil) != tt.wantErr { t.Errorf("NewCertifier() error = %v, wantErr %v", err, tt.wantErr) return @@ -73,8 +70,6 @@ func TestNewCertifier(t *testing.T) { } func Test_sourceArtifacts_GetComponents(t *testing.T) { - tm, _ := time.Parse(time.RFC3339, "2022-11-21T17:45:50.52Z") - testSourceDjangoTag := generated.SourcesListSourcesListSourceConnectionEdgesSourceEdgeNodeSource{} testSourceDjangoTag.Type = "git" testSourceDjangoTag.Namespaces = append(testSourceDjangoTag.Namespaces, generated.AllSourceTreeNamespacesSourceNamespace{ @@ -131,27 +126,14 @@ func Test_sourceArtifacts_GetComponents(t *testing.T) { }, }) - neighborCertifyScorecardTimeStamp := generated.NeighborsNeighborsCertifyScorecard{} - neighborCertifyScorecardTimeStamp.Scorecard = generated.AllCertifyScorecardScorecard{ - TimeScanned: tm.UTC(), - } - - neighborCertifyScorecardTimeNow := generated.NeighborsNeighborsCertifyScorecard{} - neighborCertifyScorecardTimeNow.Scorecard = generated.AllCertifyScorecardScorecard{ - TimeScanned: time.Now().UTC(), - } - tests := []struct { - name string - daysSinceLastScan int - getSources func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) - getNeighbors func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) - wantSourceNode []*SourceNode - wantErr bool + name string + getSources func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) + wantSourceNode []*SourceNode + wantErr bool }{ { - name: "django: daysSinceLastScan=0, tag specified", - daysSinceLastScan: 0, + name: "django", getSources: func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) { return &generated.SourcesListResponse{ SourcesList: &generated.SourcesListSourcesListSourceConnection{ @@ -168,11 +150,6 @@ func Test_sourceArtifacts_GetComponents(t *testing.T) { }, }, nil }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{}, - }, nil - }, wantSourceNode: []*SourceNode{ { Repo: "github.com/django/django", @@ -182,8 +159,7 @@ func Test_sourceArtifacts_GetComponents(t *testing.T) { }, wantErr: false, }, { - name: "django: daysSinceLastScan=0, commit specified", - daysSinceLastScan: 0, + name: "django: commit specified", getSources: func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) { return &generated.SourcesListResponse{ SourcesList: &generated.SourcesListSourcesListSourceConnection{ @@ -200,11 +176,6 @@ func Test_sourceArtifacts_GetComponents(t *testing.T) { }, }, nil }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{}, - }, nil - }, wantSourceNode: []*SourceNode{ { Repo: "github.com/django/django", @@ -214,124 +185,7 @@ func Test_sourceArtifacts_GetComponents(t *testing.T) { }, wantErr: false, }, { - name: "django: daysSinceLastScan=0, commit with algorithm specified", - daysSinceLastScan: 0, - getSources: func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) { - return &generated.SourcesListResponse{ - SourcesList: &generated.SourcesListSourcesListSourceConnection{ - TotalCount: 1, - Edges: []generated.SourcesListSourcesListSourceConnectionEdgesSourceEdge{ - { - Node: testSourceDjangoCommitWithAlgo, - Cursor: "", - }, - }, - PageInfo: generated.SourcesListSourcesListSourceConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{}, - }, nil - }, - wantSourceNode: []*SourceNode{ - { - Repo: "github.com/django/django", - Commit: "e829b0a239cffdeab5781df450a6b0e0026faa2d", - Tag: "", - }, - }, - wantErr: false, - }, { - name: "django with scorecard, daysSinceLastScan=0", - daysSinceLastScan: 0, - getSources: func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) { - return &generated.SourcesListResponse{ - SourcesList: &generated.SourcesListSourcesListSourceConnection{ - TotalCount: 1, - Edges: []generated.SourcesListSourcesListSourceConnectionEdgesSourceEdge{ - { - Node: testSourceDjangoTag, - Cursor: "", - }, - }, - PageInfo: generated.SourcesListSourcesListSourceConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{&neighborCertifyScorecardTimeStamp}, - }, nil - }, - wantSourceNode: []*SourceNode{}, - wantErr: false, - }, { - name: "django with scorecard, timestamp: time past, daysSinceLastScan=30", - daysSinceLastScan: 30, - getSources: func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) { - return &generated.SourcesListResponse{ - SourcesList: &generated.SourcesListSourcesListSourceConnection{ - TotalCount: 1, - Edges: []generated.SourcesListSourcesListSourceConnectionEdgesSourceEdge{ - { - Node: testSourceDjangoTag, - Cursor: "", - }, - }, - PageInfo: generated.SourcesListSourcesListSourceConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{&neighborCertifyScorecardTimeStamp}, - }, nil - }, - wantSourceNode: []*SourceNode{ - { - Repo: "github.com/django/django", - Commit: "", - Tag: "1.11.1", - }, - }, - wantErr: false, - }, { - name: "django with scorecard, timestamp: time now, daysSinceLastScan=30", - daysSinceLastScan: 30, - getSources: func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) { - return &generated.SourcesListResponse{ - SourcesList: &generated.SourcesListSourcesListSourceConnection{ - TotalCount: 1, - Edges: []generated.SourcesListSourcesListSourceConnectionEdgesSourceEdge{ - { - Node: testSourceDjangoTag, - Cursor: "", - }, - }, - PageInfo: generated.SourcesListSourcesListSourceConnectionPageInfo{ - HasNextPage: false, - }, - }, - }, nil - }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{&neighborCertifyScorecardTimeNow}, - }, nil - }, - wantSourceNode: []*SourceNode{}, - wantErr: false, - }, { - name: "multiple packages", - daysSinceLastScan: 0, + name: "multiple sources", getSources: func(ctx context.Context, client graphql.Client, filter generated.SourceSpec, after *string, first *int) (*generated.SourcesListResponse, error) { return &generated.SourcesListResponse{ SourcesList: &generated.SourcesListSourcesListSourceConnection{ @@ -356,11 +210,6 @@ func Test_sourceArtifacts_GetComponents(t *testing.T) { }, }, nil }, - getNeighbors: func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error) { - return &generated.NeighborsResponse{ - Neighbors: []generated.NeighborsNeighborsNode{}, - }, nil - }, wantSourceNode: []*SourceNode{ { Repo: "github.com/django/django", @@ -387,12 +236,10 @@ func Test_sourceArtifacts_GetComponents(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() p := &sourceQuery{ - client: nil, - daysSinceLastScan: tt.daysSinceLastScan, - addedLatency: &addedLatency, + client: nil, + addedLatency: &addedLatency, } getSources = tt.getSources - getNeighbors = tt.getNeighbors // compChan to collect query components compChan := make(chan interface{}, 1) diff --git a/pkg/cli/store.go b/pkg/cli/store.go index 5ef4f15b5e..8e4d7f7c4e 100644 --- a/pkg/cli/store.go +++ b/pkg/cli/store.go @@ -117,8 +117,6 @@ func init() { set.StringP("interval", "i", "5m", "if polling set interval, m, h, s, etc.") - set.IntP("last-scan", "l", 0, "days since the last vulnerability scan was run. Default 0 means only run once") - set.BoolP("cert-good", "g", false, "enable to certifyGood, otherwise defaults to certifyBad") set.BoolP("package-name", "n", false, "if type is package, enable if attestation is at package-name level (for all versions), defaults to specific version")