From d3c1e068d2a51914b065d4b753a902890dcbf93d Mon Sep 17 00:00:00 2001 From: pxp928 Date: Tue, 16 Jul 2024 14:24:07 -0400 Subject: [PATCH 1/2] add indexes for common queries on ENT Signed-off-by: pxp928 --- .../migrations/20240716182144_ent_diff.sql | 12 ++++++ .../backends/ent/migrate/migrations/atlas.sum | 3 +- pkg/assembler/backends/ent/migrate/schema.go | 42 +++++++++++++++++++ .../backends/ent/schema/billofmaterials.go | 2 + .../backends/ent/schema/certifylegal.go | 1 + .../backends/ent/schema/dependency.go | 3 +- .../backends/ent/schema/occurrence.go | 2 + 7 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 pkg/assembler/backends/ent/migrate/migrations/20240716182144_ent_diff.sql diff --git a/pkg/assembler/backends/ent/migrate/migrations/20240716182144_ent_diff.sql b/pkg/assembler/backends/ent/migrate/migrations/20240716182144_ent_diff.sql new file mode 100644 index 0000000000..7abb3b9f46 --- /dev/null +++ b/pkg/assembler/backends/ent/migrate/migrations/20240716182144_ent_diff.sql @@ -0,0 +1,12 @@ +-- Create index "billofmaterials_artifact_id" to table: "bill_of_materials" +CREATE INDEX "billofmaterials_artifact_id" ON "bill_of_materials" ("artifact_id") WHERE ((package_id IS NULL) AND (artifact_id IS NOT NULL)); +-- Create index "billofmaterials_package_id" to table: "bill_of_materials" +CREATE INDEX "billofmaterials_package_id" ON "bill_of_materials" ("package_id") WHERE ((package_id IS NOT NULL) AND (artifact_id IS NULL)); +-- Create index "certifylegal_package_id" to table: "certify_legals" +CREATE INDEX "certifylegal_package_id" ON "certify_legals" ("package_id") WHERE ((package_id IS NOT NULL) AND (source_id IS NULL)); +-- Create index "dependency_dependent_package_version_id" to table: "dependencies" +CREATE INDEX "dependency_dependent_package_version_id" ON "dependencies" ("dependent_package_version_id"); +-- Create index "occurrence_artifact_id" to table: "occurrences" +CREATE INDEX "occurrence_artifact_id" ON "occurrences" ("artifact_id"); +-- Create index "query_occurrence_package_id" to table: "occurrences" +CREATE INDEX "query_occurrence_package_id" ON "occurrences" ("package_id") WHERE ((package_id IS NOT NULL) AND (source_id IS NULL)); diff --git a/pkg/assembler/backends/ent/migrate/migrations/atlas.sum b/pkg/assembler/backends/ent/migrate/migrations/atlas.sum index ffc532557b..c86285bb10 100644 --- a/pkg/assembler/backends/ent/migrate/migrations/atlas.sum +++ b/pkg/assembler/backends/ent/migrate/migrations/atlas.sum @@ -1,6 +1,7 @@ -h1:tylM27WD3aiFco8Hl4NNfSVCY9g42uwOh5O1GMmkWP8= +h1:2v3i5iU6MalDXCn9jNXp+75vnIcNccKwMbKG2r1uBkc= 20240503123155_baseline.sql h1:oZtbKI8sJj3xQq7ibfvfhFoVl+Oa67CWP7DFrsVLVds= 20240626153721_ent_diff.sql h1:FvV1xELikdPbtJk7kxIZn9MhvVVoFLF/2/iT/wM5RkA= 20240702195630_ent_diff.sql h1:y8TgeUg35krYVORmC7cN4O96HqOc3mVO9IQ2lYzIzwg= 20240712131658_ent_diff.sql h1:N54EB3Wh2NC2v9RToXo/BfpezLnSeHJBP1ha6VXfxD8= 20240712193834_ent_diff.sql h1:gHTfEzlqvgYi6NKwT/Mb+wooWsVjAkp98zfg1OvdoZw= +20240716182144_ent_diff.sql h1:Pm0/+7zkBUGOY/AiF3bCJzW8giL5+uSg/j/0SUDsuNg= diff --git a/pkg/assembler/backends/ent/migrate/schema.go b/pkg/assembler/backends/ent/migrate/schema.go index 06e56c643d..be032048a9 100644 --- a/pkg/assembler/backends/ent/migrate/schema.go +++ b/pkg/assembler/backends/ent/migrate/schema.go @@ -82,6 +82,22 @@ var ( Where: "package_id IS NULL AND artifact_id IS NOT NULL", }, }, + { + Name: "billofmaterials_package_id", + Unique: false, + Columns: []*schema.Column{BillOfMaterialsColumns[13]}, + Annotation: &entsql.IndexAnnotation{ + Where: "package_id IS NOT NULL AND artifact_id IS NULL", + }, + }, + { + Name: "billofmaterials_artifact_id", + Unique: false, + Columns: []*schema.Column{BillOfMaterialsColumns[14]}, + Annotation: &entsql.IndexAnnotation{ + Where: "package_id IS NULL AND artifact_id IS NOT NULL", + }, + }, }, } // BuildersColumns holds the columns for the "builders" table. @@ -234,6 +250,14 @@ var ( Where: "package_id IS NOT NULL AND source_id IS NULL", }, }, + { + Name: "certifylegal_package_id", + Unique: false, + Columns: []*schema.Column{CertifyLegalsColumns[11]}, + Annotation: &entsql.IndexAnnotation{ + Where: "package_id IS NOT NULL AND source_id IS NULL", + }, + }, }, } // CertifyScorecardsColumns holds the columns for the "certify_scorecards" table. @@ -417,6 +441,11 @@ var ( Unique: false, Columns: []*schema.Column{DependenciesColumns[6]}, }, + { + Name: "dependency_dependent_package_version_id", + Unique: false, + Columns: []*schema.Column{DependenciesColumns[7]}, + }, }, } // HasMetadataColumns holds the columns for the "has_metadata" table. @@ -667,6 +696,19 @@ var ( Where: "package_id IS NULL AND source_id IS NOT NULL", }, }, + { + Name: "query_occurrence_package_id", + Unique: false, + Columns: []*schema.Column{OccurrencesColumns[6]}, + Annotation: &entsql.IndexAnnotation{ + Where: "package_id IS NOT NULL AND source_id IS NULL", + }, + }, + { + Name: "occurrence_artifact_id", + Unique: false, + Columns: []*schema.Column{OccurrencesColumns[5]}, + }, }, } // PackageNamesColumns holds the columns for the "package_names" table. diff --git a/pkg/assembler/backends/ent/schema/billofmaterials.go b/pkg/assembler/backends/ent/schema/billofmaterials.go index f23141047a..9f34e22a37 100644 --- a/pkg/assembler/backends/ent/schema/billofmaterials.go +++ b/pkg/assembler/backends/ent/schema/billofmaterials.go @@ -74,5 +74,7 @@ func (BillOfMaterials) Indexes() []ent.Index { index.Fields("algorithm", "digest", "uri", "download_location", "known_since", "included_packages_hash", "included_artifacts_hash", "included_dependencies_hash", "included_occurrences_hash", "origin", "collector", "document_ref").Edges("artifact").Unique(). Annotations(entsql.IndexWhere("package_id IS NULL AND artifact_id IS NOT NULL")).StorageKey("sbom_artifact_id"), + index.Fields("package_id").Annotations(entsql.IndexWhere("package_id IS NOT NULL AND artifact_id IS NULL")), // query when subject is package ID + index.Fields("artifact_id").Annotations(entsql.IndexWhere("package_id IS NULL AND artifact_id IS NOT NULL")), // query when subject is artifact ID } } diff --git a/pkg/assembler/backends/ent/schema/certifylegal.go b/pkg/assembler/backends/ent/schema/certifylegal.go index e0249eb16d..782b6b191d 100644 --- a/pkg/assembler/backends/ent/schema/certifylegal.go +++ b/pkg/assembler/backends/ent/schema/certifylegal.go @@ -71,5 +71,6 @@ func (CertifyLegal) Indexes() []ent.Index { "origin", "collector", "document_ref", "declared_licenses_hash", "discovered_licenses_hash"). Unique(). Annotations(entsql.IndexWhere("package_id IS NOT NULL AND source_id IS NULL")), + index.Fields("package_id").Annotations(entsql.IndexWhere("package_id IS NOT NULL AND source_id IS NULL")), // query when subject is package ID } } diff --git a/pkg/assembler/backends/ent/schema/dependency.go b/pkg/assembler/backends/ent/schema/dependency.go index 95b968a913..8a22f31536 100644 --- a/pkg/assembler/backends/ent/schema/dependency.go +++ b/pkg/assembler/backends/ent/schema/dependency.go @@ -73,6 +73,7 @@ func (Dependency) Edges() []ent.Edge { func (Dependency) Indexes() []ent.Index { return []ent.Index{ index.Fields("dependency_type", "justification", "origin", "collector", "document_ref", "package_id", "dependent_package_version_id").Unique(), - index.Fields("package_id"), // speed up frequently run queries to check for deps with a certain package ID + index.Fields("package_id"), // speed up frequently run queries to check for deps with a certain package ID + index.Fields("dependent_package_version_id"), // query via the dependent package ID } } diff --git a/pkg/assembler/backends/ent/schema/occurrence.go b/pkg/assembler/backends/ent/schema/occurrence.go index b49fa727f3..ba0afd7ef7 100644 --- a/pkg/assembler/backends/ent/schema/occurrence.go +++ b/pkg/assembler/backends/ent/schema/occurrence.go @@ -69,5 +69,7 @@ func (Occurrence) Indexes() []ent.Index { Annotations(entsql.IndexWhere("package_id IS NOT NULL AND source_id IS NULL")).StorageKey("occurrence_package_id"), index.Fields("justification", "origin", "collector", "document_ref").Edges("artifact", "source").Unique(). Annotations(entsql.IndexWhere("package_id IS NULL AND source_id IS NOT NULL")).StorageKey("occurrence_source_id"), + index.Fields("package_id").Annotations(entsql.IndexWhere("package_id IS NOT NULL AND source_id IS NULL")).StorageKey("query_occurrence_package_id"), //querying subject - package ID + index.Fields("artifact_id"), //querying object - artifact ID } } From cddb9868aa10edbeae710be80831f7c21b987435 Mon Sep 17 00:00:00 2001 From: pxp928 Date: Tue, 16 Jul 2024 14:51:51 -0400 Subject: [PATCH 2/2] fix lint issues on main on testdata Signed-off-by: pxp928 --- .../exampledata/ingest_predicates.json | 6 ---- internal/testing/testdata/testdata.go | 36 +++++++------------ 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/internal/testing/testdata/exampledata/ingest_predicates.json b/internal/testing/testdata/exampledata/ingest_predicates.json index 26b0792393..7494b121d1 100644 --- a/internal/testing/testdata/exampledata/ingest_predicates.json +++ b/internal/testing/testdata/exampledata/ingest_predicates.json @@ -87,9 +87,6 @@ ], "subpath": "" }, - "depPkgMatchFlag" : { - "pkg": "SPECIFIC_VERSION" - }, "isDependency": { "dependencyType": "UNKNOWN", "justification": "top level package dependency", @@ -127,9 +124,6 @@ ], "subpath": "" }, - "depPkgMatchFlag" : { - "pkg": "SPECIFIC_VERSION" - }, "isDependency": { "dependencyType": "UNKNOWN", "justification": "top level package dependency", diff --git a/internal/testing/testdata/testdata.go b/internal/testing/testdata/testdata.go index 4a3dabdd73..f11045ad64 100644 --- a/internal/testing/testdata/testdata.go +++ b/internal/testing/testdata/testdata.go @@ -1126,62 +1126,50 @@ var ( cdxLegalDeps = []assembler.IsDependencyIngest{ { - Pkg: cdxTopQuarkusPack, - DepPkg: cdxResteasyPack, - DepPkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + Pkg: cdxTopQuarkusPack, + DepPkg: cdxResteasyPack, IsDependency: &model.IsDependencyInputSpec{ DependencyType: model.DependencyTypeDirect, - VersionRange: "2.13.4.Final", Justification: isCDXDepJustifyDependsJustification, }, }, { - Pkg: cdxTopQuarkusPack, - DepPkg: cdxReactiveCommonPack, - DepPkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + Pkg: cdxTopQuarkusPack, + DepPkg: cdxReactiveCommonPack, IsDependency: &model.IsDependencyInputSpec{ DependencyType: model.DependencyTypeIndirect, - VersionRange: "2.13.4.Final", Justification: isCDXDepJustifyDependsJustification, }, }, { - Pkg: cdxResteasyPack, - DepPkg: cdxReactiveCommonPack, - DepPkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + Pkg: cdxResteasyPack, + DepPkg: cdxReactiveCommonPack, IsDependency: &model.IsDependencyInputSpec{ DependencyType: model.DependencyTypeDirect, - VersionRange: "2.13.4.Final", Justification: isCDXDepJustifyDependsJustification, }, }, { - Pkg: cdxTopQuarkusPack, - DepPkg: cdxSmallRye, - DepPkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + Pkg: cdxTopQuarkusPack, + DepPkg: cdxSmallRye, IsDependency: &model.IsDependencyInputSpec{ DependencyType: model.DependencyTypeUnknown, - VersionRange: "2.27.0", Justification: isDepJustifyTopPkgJustification, }, }, { - Pkg: cdxTopQuarkusPack, - DepPkg: cdxNetbasePack, - DepPkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + Pkg: cdxTopQuarkusPack, + DepPkg: cdxNetbasePack, IsDependency: &model.IsDependencyInputSpec{ DependencyType: model.DependencyTypeUnknown, - VersionRange: "6.3", Justification: isDepJustifyTopPkgJustification, }, }, { - Pkg: cdxTopQuarkusPack, - DepPkg: cdxMicroprofilePack, - DepPkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + Pkg: cdxTopQuarkusPack, + DepPkg: cdxMicroprofilePack, IsDependency: &model.IsDependencyInputSpec{ DependencyType: model.DependencyTypeUnknown, - VersionRange: "1.2", Justification: isDepJustifyTopPkgJustification, }, },