Skip to content

Commit

Permalink
Support offline validation (no network) using only built-in, local sc…
Browse files Browse the repository at this point in the history 
…hema (#114)

* Enable loading of remote JSON schemas on --force flag

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Enable loading of remote JSON schemas on --force flag

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Change use of accent char. to single quote char. in log msgs.

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Support offline validation (no network) using only built-in, local schema

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Load/compile dependency schemas for any supported schema

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Assure v1.2 and v1.3 schemas include their dep. schemas

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Fix README duplication

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Fix README duplication

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Add off-line validation desc. for supported schemas to README

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Add off-line validation desc. for supported schemas to README

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Rename local function to better describe what it does

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Provide more INFO messages around dep. schema loading and compile

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Remove temporary files generated during manual testing

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

---------

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>
  • Loading branch information
@mrutkows
mrutkows authored Nov 18, 2024
1 parent 39bfb28 commit 7a27192
Show file tree
Hide file tree
Showing 9 changed files with 1,240 additions and 40 deletions.
21 changes: 21 additions & 0 deletions .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,27 @@
"version": "0.2.0",
"configurations": [

{
"showGlobalVariables": true,
"name": "Debug: validate",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "main.go", // "program": "${file}",
"args": ["validate", "-i", "examples/cyclonedx/SBOM/protonmail-webclient-v4-0912dff/bom.json"],
"dlvFlags": ["--check-go-version=false"]
},
{
"showGlobalVariables": true,
"name": "Debug: validate (offline)",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "main.go", // "program": "${file}",
"args": ["validate", "-i", "test/cyclonedx/cdx-1-5-mature-example-1.json"],
"dlvFlags": ["--check-go-version=false"]
},

{
"showGlobalVariables": true,
"name": "Debug: validate",
Expand Down
10 changes: 8 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,11 +318,17 @@ See each command's section for contextual examples of the `--where` flag filter

### Validate

This command will parse standardized SBOMs and validate it against its declared format and version (e.g., SPDX 2.3, CycloneDX 1.6). Custom variants of standard JSON schemas can be used for validation by supplying the `--variant` name as a flag. Explicit JSON schemas can be specified using the `--force` flag.
This command will parse standardized SBOMs and validate it against its declared format and version (e.g., SPDX 2.3, CycloneDX 1.6).

- Custom variants of standard JSON schemas can be used for validation by supplying the `--variant` name as a flag.
- Explicit JSON schemas can be specified using the `--force` flag.

#### Validating using supported schemas

Use the [schema](#schema) command to list supported schemas formats, versions and variants.
Use the [schema](#schema) command to list supported schemas formats, versions and variants.

- A "supported" schema is already **"built-in"** to the utility resources along with any dependent schemas it imports.
- This means that BOM files **can be validated when there is no network connection** to load the schemas from remote locations (a.k.a., *"off-line"* mode).

#### Validating using "custom" schemas

Expand Down
110 changes: 93 additions & 17 deletions cmd/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,71 @@ func validationError(document *schema.BOM, valid bool, err error) {
getLogger().Info(message)
}

func LoadSchemaDependencies(depSchemaLoader *gojsonschema.SchemaLoader, schemas []schema.FormatSchemaInstance, schemaNames []string) (err error) {
for _, schemaName := range schemaNames {
formatSchemaInstance, errMatch := schema.FindMatchingFormatSchemaInstance(
schemas, schemaName)
if errMatch != nil {
return fmt.Errorf("schema '%s' match not found in resources: '%s'", schemaName, schema.SCHEMA_FORMAT_COMMON)
}
getLogger().Debugf("Found: '%s': %v", schemaName, formatSchemaInstance)

getLogger().Infof("Added schema '%s' to loader:...", formatSchemaInstance.File)
err = AddDependencySchemaToLoader(depSchemaLoader, formatSchemaInstance)
if err != nil {
return
}
}
return
}

func AddDependencySchemaToLoader(depSchemaLoader *gojsonschema.SchemaLoader, formatSchemaInstance schema.FormatSchemaInstance) (err error) {
getLogger().Debugf("Reading schema: '%s'...", formatSchemaInstance.File)
bSchema, errRead := resources.BOMSchemaFiles.ReadFile(formatSchemaInstance.File)

if errRead != nil {
return errRead
}
getLogger().Tracef("schema: %s", bSchema)
sharedSchemaLoader := gojsonschema.NewBytesLoader(bSchema)
depSchemaLoader.AddSchema(formatSchemaInstance.Url, sharedSchemaLoader)
return
}

func LoadCompileSchemaDependencies(
bomSchemaLoader gojsonschema.JSONLoader,
bomSchemaInstance schema.FormatSchemaInstance,
bomSchemaDependencies []string,
) (jsonBOMSchema *gojsonschema.Schema, err error) {

if len(bomSchemaDependencies) > 0 {
getLogger().Infof("Found schema dependencies: %v", bomSchemaDependencies)
// Create a schema loader we will add all dep. schemas into
depSchemaLoader := gojsonschema.NewSchemaLoader()

// Load common schema from application resources
var commonSchemas schema.FormatSchema
commonSchemas, err = SupportedFormatConfig.FindMatchingFormatSchema(schema.SCHEMA_FORMAT_COMMON)
if err != nil {
return
}
getLogger().Tracef("Found '%s' schemas: %v", schema.SCHEMA_FORMAT_COMMON, commonSchemas)

err = LoadSchemaDependencies(depSchemaLoader, commonSchemas.Schemas, bomSchemaDependencies)
if err != nil {
return
}

// Compile BOM schema (JSON) with the dependency schemas and return it
getLogger().Infof("Compiling schema: '%s'...", bomSchemaInstance.File)
jsonBOMSchema, err = depSchemaLoader.Compile(bomSchemaLoader)
if err != nil {
return
}
}
return
}

func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, validateFlags utils.ValidateCommandFlags) (valid bool, bom *schema.BOM, schemaErrors []gojsonschema.ResultError, err error) {
getLogger().Enter()
defer getLogger().Exit()
Expand Down Expand Up @@ -213,8 +278,8 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va

// Create a loader for the BOM (JSON) document
var documentLoader gojsonschema.JSONLoader
var schemaLoader gojsonschema.JSONLoader
var errRead error
var jsonBOMSchemaLoader gojsonschema.JSONLoader
var errRead, errLoadCompile error
var bSchema, bDocument []byte

if bDocument = bom.GetRawBytes(); len(bDocument) > 0 {
Expand All @@ -233,14 +298,15 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
return INVALID, bom, schemaErrors, fmt.Errorf("unable to load document: '%s'", bom.GetFilename())
}

// Regardless of how or where we load JSON schemas from the final
// we define the overall Schema object used to validate the BOM document
var jsonBOMSchema *gojsonschema.Schema
schemaName := bom.SchemaInfo.File

// If caller "forced" a specific schema file (version), load it instead of
// any SchemaInfo found in config.json
// TODO: support remote schema load (via URL) with a flag (default should always be local file for security)
forcedSchemaFile := validateFlags.ForcedJsonSchemaFile
if forcedSchemaFile != "" {

if !isValidURIPrefix(forcedSchemaFile) {
// attempt to load as a local file
forcedSchemaFile = "file://" + forcedSchemaFile
Expand All @@ -254,7 +320,7 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
}

getLogger().Infof("Loading schema from '--force' flag: '%s'...", forcedSchemaFile)
schemaLoader = gojsonschema.NewReferenceLoader(forcedSchemaFile)
jsonBOMSchemaLoader = gojsonschema.NewReferenceLoader(forcedSchemaFile)
getLogger().Infof("Validating document using forced schema (i.e., '--force %s')", forcedSchemaFile)
} else {
// Load the matching JSON schema (format, version and variant) from embedded resources
Expand All @@ -268,10 +334,19 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
return INVALID, bom, schemaErrors, errRead
}

schemaLoader = gojsonschema.NewBytesLoader(bSchema)
// Create a schema loader for the primary BOM schema file
jsonBOMSchemaLoader = gojsonschema.NewBytesLoader(bSchema)

// If the BOM schema has $refs to other schemas, attempt to load and compile
// them from those included as built-in resources
jsonBOMSchema, errLoadCompile = LoadCompileSchemaDependencies(jsonBOMSchemaLoader, bom.SchemaInfo, bom.SchemaInfo.Dependencies)
if err != nil {
return INVALID, bom, schemaErrors, errLoadCompile
}
}

if schemaLoader == nil {
// At this point we should have a BOM schema loader
if jsonBOMSchemaLoader == nil {
// we force result to INVALID as any errors from the library means
// we could NOT actually confirm the input documents validity
return INVALID, bom, schemaErrors, fmt.Errorf("unable to read schema: '%s'", schemaName)
Expand All @@ -280,27 +355,28 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
// create a reusable schema object (TODO: validate multiple documents)
var errLoad error = nil
const RETRY int = 3
var jsonBOMSchema *gojsonschema.Schema

// we force result to INVALID as any errors from the library means
// we could NOT actually confirm the input documents validity
// WARNING: if schemas reference "remote" schemas which are loaded
// over http... then there is a chance of 503 errors (as the pkg. loads
// externally referenced schemas over network)... attempt fixed retry...
for i := 0; i < RETRY; i++ {
jsonBOMSchema, errLoad = gojsonschema.NewSchema(schemaLoader)
if jsonBOMSchema == nil {
for i := 0; i < RETRY; i++ {
jsonBOMSchema, errLoad = gojsonschema.NewSchema(jsonBOMSchemaLoader)

if errLoad == nil {
break
if errLoad == nil {
break
}
getLogger().Warningf("unable to load referenced schema over HTTP: \"%v\"\n retrying...", errLoad)
}
getLogger().Warningf("unable to load referenced schema over HTTP: \"%v\"\n retrying...", errLoad)
}

if errLoad != nil {
return INVALID, bom, schemaErrors, fmt.Errorf("unable to load schema: '%s'", schemaName)
if errLoad != nil {
return INVALID, bom, schemaErrors, fmt.Errorf("unable to load schema: `%s`", schemaName)
}
}

getLogger().Infof("Schema '%s' loaded.", schemaName)
getLogger().Infof("Schema '%s' loaded", schemaName)

// Validate against the schema and save result determination
getLogger().Infof("Validating '%s'...", bom.GetFilenameInterpolated())
Expand Down
53 changes: 44 additions & 9 deletions resources/config/config.json
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,31 @@
{
"formats": [
{
"canonicalName": "common",
"propertyKeyFormat": "",
"propertyKeyVersion": "version",
"propertyValueFormat": "latest",
"schemas": [
{
"version": "",
"variant": "",
"name": "jsf-0.82.schema.json",
"file": "schema/cyclonedx/common/jsf-0.82.schema.json",
"development": "",
"url": "http://cyclonedx.org/schema/jsf-0.82.schema.json",
"default": false
},
{
"version": "",
"variant": "",
"name": "spdx.schema.json",
"file": "schema/cyclonedx/common/spdx.schema.json",
"development": "",
"url": "http://cyclonedx.org/schema/spdx.schema.json",
"default": false
}
]
},
{
"canonicalName": "SPDX",
"propertyKeyFormat": "SPDXID",
Expand Down Expand Up @@ -57,7 +83,8 @@
"file": "schema/cyclonedx/1.2/bom-1.2.schema.json",
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.2.schema.json",
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.2.schema.json",
"default": false
"default": false,
"dependencies": ["spdx.schema.json"]
},
{
"version": "1.2",
Expand All @@ -66,7 +93,8 @@
"file": "schema/cyclonedx/1.2/bom-1.2-strict.schema.json",
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.2-strict.schema.json",
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.2-strict.schema.json",
"default": false
"default": false,
"dependencies": ["spdx.schema.json"]
},
{
"version": "1.3",
Expand All @@ -75,7 +103,8 @@
"file": "schema/cyclonedx/1.3/bom-1.3.schema.json",
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.3.schema.json",
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.3.schema.json",
"default": false
"default": false,
"dependencies": ["spdx.schema.json"]
},
{
"version": "1.3",
Expand All @@ -84,7 +113,8 @@
"file": "schema/cyclonedx/1.3/bom-1.3-strict.schema.json",
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.3-strict.schema.json",
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.3-strict.schema.json",
"default": false
"default": false,
"dependencies": ["spdx.schema.json"]
},
{
"version": "1.4",
Expand All @@ -93,7 +123,8 @@
"file": "schema/cyclonedx/1.4/bom-1.4.schema.json",
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.4.schema.json",
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.4.schema.json",
"default": false
"default": false,
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
},
{
"version": "1.5",
Expand All @@ -102,7 +133,8 @@
"file": "schema/cyclonedx/1.5/bom-1.5.schema.json",
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.5.schema.json",
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.5.schema.json",
"default": false
"default": false,
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
},
{
"version": "1.6",
Expand All @@ -111,7 +143,8 @@
"file": "schema/cyclonedx/1.6/bom-1.6.schema.json",
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.6.schema.json",
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.6.schema.json",
"default": true
"default": true,
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
},
{
"version": "1.3",
Expand All @@ -120,7 +153,8 @@
"file": "schema/test/bom-1.3-custom.schema.json",
"development":"https://github.com/CycloneDX/sbom-utility/blob/main/resources/schema/test/bom-1.3-custom.schema.json",
"url": "",
"default": false
"default": false,
"dependencies": ["spdx.schema.json"]
},
{
"version": "1.4",
Expand All @@ -129,7 +163,8 @@
"file": "schema/test/bom-1.4-custom.schema.json",
"development":"https://github.com/CycloneDX/sbom-utility/blob/main/resources/schema/test/bom-1.4-custom.schema.json",
"url": "",
"default": false
"default": false,
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
}
]
}
Expand Down
8 changes: 4 additions & 4 deletions resources/resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func LoadConfigFile(baseFilename string) (bData []byte, err error) {
return
}

func LoadSchemaFile(baseFilename string) (bData []byte, err error) {
bData, err = ConfigFiles.ReadFile(RESOURCES_SCHEMA_DIR + baseFilename)
return
}
// func LoadSchemaFile(baseFilename string) (bData []byte, err error) {
// bData, err = BOMSchemaFiles.ReadFile(RESOURCES_SCHEMA_DIR + baseFilename)
// return
// }
Loading

0 comments on commit 7a27192

Please sign in to comment.