remove daysSinceLastScan as it is redundant with certifier interval

Signed-off-by: pxp928 <parth.psu@gmail.com>

cmd/guaccollect/cmd/license.go

@@ -45,9 +45,6 @@ type cdOptions struct {
 	interval time.Duration
 	// enable/disable message publish to queue
 	publishToQueue bool
-	// days since the last vulnerability scan was run.
-	// 0 means only run once
-	daysSinceLastScan int
 	// sets artificial latency on the certifier (default to nil)
 	addedLatency *time.Duration
 	// sets the batch size for pagination query for the certifier
@@ -81,7 +78,6 @@ you have access to read and write to the respective blob store.`,
 			viper.GetString("interval"),
 			viper.GetBool("service-poll"),
 			viper.GetBool("publish-to-queue"),
-			viper.GetInt("last-scan"),
 			viper.GetString("certifier-latency"),
 			viper.GetInt("certifier-batch-size"),
 		)
@@ -102,7 +98,7 @@ you have access to read and write to the respective blob store.`,
 		httpClient := http.Client{Transport: transport}
 		gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient)
 
-		packageQueryFunc, err := getPackageQuery(gqlclient, opts.daysSinceLastScan, opts.batchSize, opts.addedLatency)
+		packageQueryFunc, err := getPackageQuery(gqlclient, opts.batchSize, opts.addedLatency)
 		if err != nil {
 			logger.Errorf("error: %v", err)
 			os.Exit(1)
@@ -120,7 +116,6 @@ func validateCDFlags(
 	interval string,
 	poll bool,
 	pubToQueue bool,
-	daysSince int,
 	certifierLatencyStr string,
 	batchSize int) (cdOptions, error) {
 
@@ -138,7 +133,6 @@ func validateCDFlags(
 		return opts, fmt.Errorf("failed to parser duration with error: %w", err)
 	}
 	opts.interval = i
-	opts.daysSinceLastScan = daysSince
 
 	if certifierLatencyStr != "" {
 		addedLatency, err := time.ParseDuration(certifierLatencyStr)
@@ -157,7 +151,7 @@ func validateCDFlags(
 
 func init() {
 	set, err := cli.BuildFlags([]string{"interval",
-		"last-scan", "header-file", "certifier-latency",
+		"header-file", "certifier-latency",
 		"certifier-batch-size"})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)

cmd/guaccollect/cmd/osv.go

@@ -54,9 +54,6 @@ type osvOptions struct {
 	interval time.Duration
 	// enable/disable message publish to queue
 	publishToQueue bool
-	// days since the last vulnerability scan was run.
-	// 0 means only run once
-	daysSinceLastScan int
 	// sets artificial latency on the certifier (default to nil)
 	addedLatency *time.Duration
 	// sets the batch size for pagination query for the certifier
@@ -90,7 +87,6 @@ you have access to read and write to the respective blob store.`,
 			viper.GetString("interval"),
 			viper.GetBool("service-poll"),
 			viper.GetBool("publish-to-queue"),
-			viper.GetInt("last-scan"),
 			viper.GetString("certifier-latency"),
 			viper.GetInt("certifier-batch-size"),
 		)
@@ -111,7 +107,7 @@ you have access to read and write to the respective blob store.`,
 		httpClient := http.Client{Transport: transport}
 		gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient)
 
-		packageQueryFunc, err := getPackageQuery(gqlclient, opts.daysSinceLastScan, opts.batchSize, opts.addedLatency)
+		packageQueryFunc, err := getPackageQuery(gqlclient, opts.batchSize, opts.addedLatency)
 		if err != nil {
 			logger.Errorf("error: %v", err)
 			os.Exit(1)
@@ -129,7 +125,6 @@ func validateOSVFlags(
 	interval string,
 	poll bool,
 	pubToQueue bool,
-	daysSince int,
 	certifierLatencyStr string,
 	batchSize int) (osvOptions, error) {
 
@@ -147,7 +142,6 @@ func validateOSVFlags(
 		return opts, fmt.Errorf("failed to parser duration with error: %w", err)
 	}
 	opts.interval = i
-	opts.daysSinceLastScan = daysSince
 
 	if certifierLatencyStr != "" {
 		addedLatency, err := time.ParseDuration(certifierLatencyStr)
@@ -170,9 +164,9 @@ func getCertifierPublish(ctx context.Context, blobStore *blob.BlobStore, pubsub
 	}, nil
 }
 
-func getPackageQuery(client graphql.Client, daysSinceLastScan int, batchSize int, addedLatency *time.Duration) (func() certifier.QueryComponents, error) {
+func getPackageQuery(client graphql.Client, batchSize int, addedLatency *time.Duration) (func() certifier.QueryComponents, error) {
 	return func() certifier.QueryComponents {
-		packageQuery := root_package.NewPackageQuery(client, daysSinceLastScan, batchSize, addedLatency)
+		packageQuery := root_package.NewPackageQuery(client, batchSize, addedLatency)
 		return packageQuery
 	}, nil
 }
@@ -255,7 +249,7 @@ func initializeNATsandCertifier(ctx context.Context, blobAddr, pubsubAddr string
 
 func init() {
 	set, err := cli.BuildFlags([]string{"interval",
-		"last-scan", "header-file", "certifier-latency",
+		"header-file", "certifier-latency",
 		"certifier-batch-size"})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)

cmd/guaccollect/cmd/scorecard.go

@@ -46,8 +46,6 @@ type scorecardOptions struct {
 	interval time.Duration
 	// enable/disable message publish to queue
 	publishToQueue bool
-	// setting "daysSinceLastScan" to 0 does not check the timestamp on the scorecard that exist
-	daysSinceLastScan int
 	// sets artificial latency on the certifier (default to nil)
 	addedLatency *time.Duration
 	// sets the batch size for pagination query for the certifier
@@ -81,7 +79,6 @@ you have access to read and write to the respective blob store.`,
 			viper.GetString("interval"),
 			viper.GetBool("service-poll"),
 			viper.GetBool("publish-to-queue"),
-			viper.GetInt("last-scan"),
 			viper.GetString("certifier-latency"),
 			viper.GetInt("certifier-batch-size"),
 		)
@@ -120,7 +117,7 @@ you have access to read and write to the respective blob store.`,
 		httpClient := http.Client{Transport: transport}
 		gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient)
 
-		query, err := sc.NewCertifier(gqlclient, opts.daysSinceLastScan, opts.batchSize, opts.addedLatency)
+		query, err := sc.NewCertifier(gqlclient, opts.batchSize, opts.addedLatency)
 		if err != nil {
 			logger.Errorf("unable to create source query: %v\n", err)
 			os.Exit(1)
@@ -138,7 +135,6 @@ func validateScorecardFlags(
 	interval string,
 	poll bool,
 	pubToQueue bool,
-	daysSince int,
 	certifierLatencyStr string,
 	batchSize int) (scorecardOptions, error) {
 
@@ -156,7 +152,6 @@ func validateScorecardFlags(
 		return opts, fmt.Errorf("failed to parser duration with error: %w", err)
 	}
 	opts.interval = i
-	opts.daysSinceLastScan = daysSince
 
 	if certifierLatencyStr != "" {
 		addedLatency, err := time.ParseDuration(certifierLatencyStr)
@@ -175,7 +170,7 @@ func validateScorecardFlags(
 
 func init() {
 	set, err := cli.BuildFlags([]string{"interval",
-		"last-scan", "header-file", "certifier-latency",
+		"header-file", "certifier-latency",
 		"certifier-batch-size"})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)

cmd/guacone/cmd/license.go

@@ -96,7 +96,7 @@ var cdCmd = &cobra.Command{
 
 		httpClient := http.Client{Transport: transport}
 		gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient)
-		packageQuery := root_package.NewPackageQuery(gqlclient, 0, opts.batchSize, opts.addedLatency)
+		packageQuery := root_package.NewPackageQuery(gqlclient, opts.batchSize, opts.addedLatency)
 
 		totalNum := 0
 		docChan := make(chan *processor.Document)

cmd/guacone/cmd/osv.go

@@ -96,7 +96,7 @@ var osvCmd = &cobra.Command{
 
 		httpClient := http.Client{Transport: transport}
 		gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient)
-		packageQuery := root_package.NewPackageQuery(gqlclient, 0, opts.batchSize, opts.addedLatency)
+		packageQuery := root_package.NewPackageQuery(gqlclient, opts.batchSize, opts.addedLatency)
 
 		totalNum := 0
 		docChan := make(chan *processor.Document)

cmd/guacone/cmd/scorecard.go

@@ -112,8 +112,7 @@ var scorecardCmd = &cobra.Command{
 		}
 
 		// scorecard certifier is the certifier that gets the scorecard data graphQL
-		// setting "daysSinceLastScan" to 0 does not check the timestamp on the scorecard that exist
-		query, err := sc.NewCertifier(gqlclient, 0, opts.batchSize, opts.addedLatency)
+		query, err := sc.NewCertifier(gqlclient, opts.batchSize, opts.addedLatency)
 
 		if err != nil {
 			fmt.Printf("unable to create scorecard certifier: %v\n", err)

container_files/guac/guac.yaml

@@ -22,8 +22,7 @@ use-csub: true
 poll: true
 # certifier interval
 interval: 20m
-# days since the last vulnerability scan was run. 0 means only run once
-last-scan: 0
+
 # set the batch size for the package pagination query
 certifier-batch-size: 60000
 # add artificial latency to throttle the certifier

guac.yaml

@@ -26,8 +26,7 @@ blob-addr: file:///tmp/blobstore?no_tmp_dir=true
 
 # certifier interval
 interval: 20m
-# days since the last vulnerability scan was run. 0 means only run once
-last-scan: 0
+
 # set the batch size for the package pagination query
 certifier-batch-size: 60000
 # add artificial latency to throttle the certifier

internal/testing/cmd/pubsub_test/cmd/osv.go

@@ -92,7 +92,7 @@ func getCertifierPublish(ctx context.Context, blobStore *blob.BlobStore, pubsub
 
 func getPackageQuery(client graphql.Client) (func() certifier.QueryComponents, error) {
 	return func() certifier.QueryComponents {
-		packageQuery := root_package.NewPackageQuery(client, 0, 60000, nil)
+		packageQuery := root_package.NewPackageQuery(client, 60000, nil)
 		return packageQuery
 	}, nil
 }

pkg/certifier/components/root_package/root_package.go

@@ -18,7 +18,6 @@ package root_package
 import (
 	"context"
 	"fmt"
-	"math"
 	"time"
 
 	"github.com/Khan/genqlient/graphql"
@@ -36,30 +35,25 @@ type PackageNode struct {
 
 type packageQuery struct {
 	client graphql.Client
-	// daysSinceLastScan sets the days since the last vulnerability scan was run
-	daysSinceLastScan int
 	// set the batch size for the package pagination query
 	batchSize int
 	// add artificial latency to throttle the pagination query
 	addedLatency *time.Duration
 }
 
 var getPackages func(ctx context.Context, client graphql.Client, filter generated.PkgSpec, after *string, first *int) (*generated.PackagesListResponse, error)
-var getNeighbors func(ctx context.Context, client graphql.Client, node string, usingOnly []generated.Edge) (*generated.NeighborsResponse, error)
 
 // NewPackageQuery initializes the packageQuery to query from the graph database
-func NewPackageQuery(client graphql.Client, daysSinceLastScan, batchSize int, addedLatency *time.Duration) certifier.QueryComponents {
+func NewPackageQuery(client graphql.Client, batchSize int, addedLatency *time.Duration) certifier.QueryComponents {
 	getPackages = generated.PackagesList
-	getNeighbors = generated.Neighbors
 	return &packageQuery{
-		client:            client,
-		daysSinceLastScan: daysSinceLastScan,
-		batchSize:         batchSize,
-		addedLatency:      addedLatency,
+		client:       client,
+		batchSize:    batchSize,
+		addedLatency: addedLatency,
 	}
 }
 
-// GetComponents get all the packages that do not have a certify vulnerability attached or last scanned is more than daysSinceLastScan
+// GetComponents get all the packages
 func (p *packageQuery) GetComponents(ctx context.Context, compChan chan<- interface{}) error {
 	if compChan == nil {
 		return fmt.Errorf("compChan cannot be nil")
@@ -131,7 +125,6 @@ func (p *packageQuery) getPackageNodes(ctx context.Context, nodeChan chan<- *Pac
 	var afterCursor *string
 
 	first := p.batchSize
-	//first := 60000
 	for {
 		pkgConn, err := getPackages(ctx, p.client, generated.PkgSpec{}, afterCursor, &first)
 		if err != nil {
@@ -149,37 +142,10 @@ func (p *packageQuery) getPackageNodes(ctx context.Context, nodeChan chan<- *Pac
 			for _, namespace := range pkgNode.Node.Namespaces {
 				for _, name := range namespace.Names {
 					for _, version := range name.Versions {
-						response, err := getNeighbors(ctx, p.client, version.Id, []generated.Edge{generated.EdgePackageCertifyVuln})
-						if err != nil {
-							return fmt.Errorf("failed neighbors query: %w", err)
-						}
-						vulnList := []*generated.NeighborsNeighborsCertifyVuln{}
-						certifyVulnFound := false
-						for _, neighbor := range response.Neighbors {
-							if certifyVuln, ok := neighbor.(*generated.NeighborsNeighborsCertifyVuln); ok {
-								vulnList = append(vulnList, certifyVuln)
-							}
-						}
-						// collect all certifyVulnerability and then check timestamp else if not checking timestamp,
-						// if a certifyVulnerability is found break out
-						for _, vulns := range vulnList {
-							if p.daysSinceLastScan != 0 {
-								now := time.Now()
-								difference := vulns.Metadata.TimeScanned.Sub(now)
-								if math.Abs(difference.Hours()) < float64(p.daysSinceLastScan*24) {
-									certifyVulnFound = true
-								}
-							} else {
-								certifyVulnFound = true
-								break
-							}
-						}
-						if !certifyVulnFound {
-							packNode := PackageNode{
-								Purl: version.Purl,
-							}
-							nodeChan <- &packNode
+						packNode := PackageNode{
+							Purl: version.Purl,
 						}
+						nodeChan <- &packNode
 					}
 				}
 			}