Validate in exhaustive mode by default and add --fast to validate (…

…#238)

Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>

docs/validate.markdown

@@ -7,7 +7,8 @@ Validating
 ```sh
 jsonschema validate <schema.json|.yaml> <instance.json|.jsonl|.yaml...> [--http/-h]
   [--verbose/-v] [--resolve/-r <schemas-or-directories> ...] [--benchmark/-b]
-  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>] [--trace/-t]
+  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>] [--trace/-t] 
+  [--fast/-f]
 ```
 
 The most popular use case of JSON Schema is to validate JSON documents. The
@@ -21,6 +22,10 @@ information on unsuccessful validation.
 To help scripts distinguish validation errors, these are reported using exit
 code 2.
 
+By default, schemas are validated in exhaustive mode, which results in better
+error messages, at the expense of speed. The `--fast`/`-f` option makes the
+schema compiler optimise for speed, at the expense of error messages.
+
 Examples
 --------
 
@@ -57,6 +62,12 @@ error: The target document is expected to be of the given type
 jsonschema validate path/to/my/schema.json path/to/my/instance.json
 ```
 
+### Validate a JSON instance against a schema in fast mode
+
+```sh
+jsonschema validate path/to/my/schema.json path/to/my/instance.json --fast
+```
+
 ### Validate a multiple JSON instances against a schema
 
 ```sh

src/command_metaschema.cc

@@ -42,7 +42,8 @@ auto sourcemeta::jsonschema::cli::metaschema(
     if (!cache.contains(dialect.value())) {
       const auto metaschema_template{sourcemeta::blaze::compile(
           metaschema, sourcemeta::core::schema_official_walker, custom_resolver,
-          sourcemeta::blaze::default_schema_compiler)};
+          sourcemeta::blaze::default_schema_compiler,
+          sourcemeta::blaze::Mode::Exhaustive)};
       cache.insert({dialect.value(), metaschema_template});
     }
 

src/command_validate.cc

@@ -19,8 +19,8 @@
 // TODO: Add a flag to take a pre-compiled schema as input
 auto sourcemeta::jsonschema::cli::validate(
     const std::span<const std::string> &arguments) -> int {
-  const auto options{
-      parse_options(arguments, {"h", "http", "b", "benchmark", "t", "trace"})};
+  const auto options{parse_options(
+      arguments, {"h", "http", "b", "benchmark", "t", "trace", "f", "fast"})};
 
   if (options.at("").size() < 1) {
     std::cerr
@@ -51,11 +51,15 @@ auto sourcemeta::jsonschema::cli::validate(
     return EXIT_FAILURE;
   }
 
+  const auto fast_mode{options.contains("f") || options.contains("fast")};
+
   const auto benchmark{options.contains("b") || options.contains("benchmark")};
   const auto trace{options.contains("t") || options.contains("trace")};
   const auto schema_template{sourcemeta::blaze::compile(
       schema, sourcemeta::core::schema_official_walker, custom_resolver,
-      sourcemeta::blaze::default_schema_compiler)};
+      sourcemeta::blaze::default_schema_compiler,
+      fast_mode ? sourcemeta::blaze::Mode::FastValidation
+                : sourcemeta::blaze::Mode::Exhaustive)};
   sourcemeta::blaze::Evaluator evaluator;
 
   bool result{true};
@@ -93,6 +97,8 @@ auto sourcemeta::jsonschema::cli::validate(
           } else if (trace) {
             subresult = evaluator.validate(schema_template, instance,
                                            std::ref(trace_output));
+          } else if (fast_mode) {
+            subresult = evaluator.validate(schema_template, instance);
           } else {
             subresult =
                 evaluator.validate(schema_template, instance, std::ref(output));
@@ -153,6 +159,8 @@ auto sourcemeta::jsonschema::cli::validate(
       } else if (trace) {
         subresult = evaluator.validate(schema_template, instance,
                                        std::ref(trace_output));
+      } else if (fast_mode) {
+        subresult = evaluator.validate(schema_template, instance);
       } else {
         subresult =
             evaluator.validate(schema_template, instance, std::ref(output));

src/main.cc

@@ -23,10 +23,15 @@ Global Options:
 
    validate <schema.json|.yaml> <instance.json|.jsonl|.yaml...> [--http/-h]
             [--benchmark/-b] [--extension/-e <extension>]
-            [--ignore/-i <schemas-or-directories>] [--trace/-t]
+            [--ignore/-i <schemas-or-directories>] [--trace/-t] [--fast/-f]
 
        Validate one or more instances against the given schema.
 
+       By default, schemas are validated in exhaustive mode, which results in
+       better error messages, at the expense of speed. The --fast/-f option
+       makes the schema compiler optimise for speed, at the expense of error
+       messages.
+
    metaschema [schemas-or-directories...] [--http/-h]
               [--extension/-e <extension>]
               [--ignore/-i <schemas-or-directories>] [--trace/-t]

test/CMakeLists.txt

@@ -45,9 +45,11 @@ add_jsonschema_test_unix(validate/fail_schema_enoent)
 add_jsonschema_test_unix(validate/fail_schema_invalid_json)
 add_jsonschema_test_unix(validate/fail_schema_non_schema)
 add_jsonschema_test_unix(validate/fail_schema_unknown_dialect)
-add_jsonschema_test_unix(validate/fail_trace)
 add_jsonschema_test_unix(validate/fail_jsonl_bigint)
+add_jsonschema_test_unix(validate/fail_trace)
+add_jsonschema_test_unix(validate/fail_trace_fast)
 add_jsonschema_test_unix(validate/pass_trace)
+add_jsonschema_test_unix(validate/pass_trace_fast)
 add_jsonschema_test_unix(validate/pass_resolve)
 add_jsonschema_test_unix(validate/pass_resolve_custom_extension)
 add_jsonschema_test_unix(validate/pass_resolve_verbose)
@@ -57,11 +59,13 @@ add_jsonschema_test_unix(validate/pass_draft6)
 add_jsonschema_test_unix(validate/pass_draft7)
 add_jsonschema_test_unix(validate/pass_2019_09)
 add_jsonschema_test_unix(validate/pass_2020_12)
+add_jsonschema_test_unix(validate/pass_2020_12_fast)
 add_jsonschema_test_unix(validate/fail_draft4)
 add_jsonschema_test_unix(validate/fail_draft6)
 add_jsonschema_test_unix(validate/fail_draft7)
 add_jsonschema_test_unix(validate/fail_2019_09)
 add_jsonschema_test_unix(validate/fail_2020_12)
+add_jsonschema_test_unix(validate/fail_2020_12_fast)
 add_jsonschema_test_unix(validate/pass_jsonl)
 add_jsonschema_test_unix(validate/pass_jsonl_empty)
 add_jsonschema_test_unix(validate/pass_jsonl_empty_verbose)

test/metaschema/fail_directory.sh

@@ -29,10 +29,13 @@ test "$CODE" = "2" || exit 1
 cat << EOF > "$TMP/expected.txt"
 fail: $(realpath "$TMP")/schemas/schema_2.json
 error: Schema validation failure
-  The integer value 1 was expected to equal one of the given declared values
+  The integer value 1 was expected to equal one of the following values: "array", "boolean", "integer", "null", "number", "object", and "string"
     at instance location "/type"
     at evaluate path "/properties/type/anyOf/0/\$ref/enum"
-  The value was expected to consist of an array of at least 1 item
+  The integer value was expected to validate against the statically referenced schema
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/0/\$ref"
+  The value was expected to be of type array but it was of type integer
     at instance location "/type"
     at evaluate path "/properties/type/anyOf/1/type"
   The integer value was expected to validate against at least one of the 2 given subschemas

test/metaschema/fail_single.sh

@@ -20,10 +20,13 @@ test "$CODE" = "2" || exit 1
 cat << EOF > "$TMP/expected.txt"
 fail: $(realpath "$TMP")/schema.json
 error: Schema validation failure
-  The integer value 1 was expected to equal one of the given declared values
+  The integer value 1 was expected to equal one of the following values: "array", "boolean", "integer", "null", "number", "object", and "string"
     at instance location "/type"
     at evaluate path "/properties/type/anyOf/0/\$ref/enum"
-  The value was expected to consist of an array of at least 1 item
+  The integer value was expected to validate against the statically referenced schema
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/0/\$ref"
+  The value was expected to be of type array but it was of type integer
     at instance location "/type"
     at evaluate path "/properties/type/anyOf/1/type"
   The integer value was expected to validate against at least one of the 2 given subschemas

test/metaschema/fail_yaml.sh

@@ -18,10 +18,13 @@ test "$CODE" = "2" || exit 1
 cat << EOF > "$TMP/expected.txt"
 fail: $(realpath "$TMP")/schema.yaml
 error: Schema validation failure
-  The integer value 1 was expected to equal one of the given declared values
+  The integer value 1 was expected to equal one of the following values: "array", "boolean", "integer", "null", "number", "object", and "string"
     at instance location "/type"
     at evaluate path "/properties/type/anyOf/0/\$ref/enum"
-  The value was expected to consist of an array of at least 1 item
+  The integer value was expected to validate against the statically referenced schema
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/0/\$ref"
+  The value was expected to be of type array but it was of type integer
     at instance location "/type"
     at evaluate path "/properties/type/anyOf/1/type"
   The integer value was expected to validate against at least one of the 2 given subschemas

test/validate/fail_2019_09.sh

@@ -33,6 +33,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/fail_2020_12.sh

@@ -33,6 +33,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/fail_2020_12_fast.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "foo": 1 }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --fast 2> "$TMP/stderr.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "2" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+fail: $(realpath "$TMP")/instance.json
+error: Schema validation failure
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/fail_draft4.sh

@@ -33,6 +33,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/fail_draft6.sh

@@ -33,6 +33,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/fail_draft7.sh

@@ -33,6 +33,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/fail_many.sh

@@ -45,6 +45,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/fail_many_verbose.sh

@@ -47,6 +47,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 ok: $(realpath "$TMP")/instance_3.json
   matches $(realpath "$TMP")/schema.json
 EOF

test/validate/fail_trace.sh

@@ -27,13 +27,21 @@ EOF
 test "$CODE" = "2" || exit 1
 
 cat << EOF > "$TMP/expected.txt"
+-> (push) "/properties"
+   at ""
+   at keyword location "#/properties"
+
 -> (push) "/properties/foo/type"
    at "/foo"
    at keyword location "#/properties/foo/type"
 
 <- (fail) "/properties/foo/type"
    at "/foo"
    at keyword location "#/properties/foo/type"
+
+<- (fail) "/properties"
+   at ""
+   at keyword location "#/properties"
 EOF
 
 diff "$TMP/output.txt" "$TMP/expected.txt"

test/validate/fail_trace_fast.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "foo": 1 }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --trace --fast > "$TMP/output.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "2" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+-> (push) "/properties/foo/type"
+   at "/foo"
+   at keyword location "#/properties/foo/type"
+
+<- (fail) "/properties/foo/type"
+   at "/foo"
+   at keyword location "#/properties/foo/type"
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"

test/validate/fail_yaml.sh

@@ -29,6 +29,9 @@ error: Schema validation failure
   The value was expected to be of type string but it was of type integer
     at instance location "/foo"
     at evaluate path "/properties/foo/type"
+  The object value was expected to validate against the single defined property subschema
+    at instance location ""
+    at evaluate path "/properties"
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/validate/pass_2020_12_fast.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "foo": "bar" }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --fast