Merge pull request #169 from OkieOth/add_json_sanity_check

Add json sanity check

bin/runGithubActions.sh

@@ -77,7 +77,7 @@ fi
 
 if ! python \
     validate.py --schema resources/models/json/yacg_config_schema.json \
-    --inputFile resources/configurations/conf_with_vars.json --draft07hack &> /dev/null; then
+    --inputFile resources/configurations/conf_with_vars.json --draft07hack; then
     echo "can't validate schema"
     popd > /dev/null
     exit 1
@@ -127,6 +127,31 @@ if python yacg.py \
     exit 1
 fi
 
+# Tests for the schema validation
+
+if python validateSchemas.py --schema tests/resources/evil/evil1.json --noEmptySchemas > /dev/null; then
+    echo "Didn't identify violated schema: evil1.json"
+    popd &> /dev/null
+    exit 1
+fi
+
+if python validateSchemas.py --schema tests/resources/evil/evil2.json  --noEmptySchemas > /dev/null; then
+    echo "Didn't identify violated schema: evil2.json"
+    popd &> /dev/null
+    exit 1
+fi
+
+if ! python validateSchemas.py --inputDir tests/resources/models  --noEmptySchemas > /dev/null; then
+    echo "didn't validate inputDir: tests/resources/models"
+    popd &> /dev/null
+    exit 1
+fi
+
+if python validateSchemas.py --inputDir tests/resources/evil  --noEmptySchemas > /dev/null; then
+    echo "didn't fail on inputDir: tests/resources/evil"
+    popd &> /dev/null
+    exit 1
+fi
 
 echo "all good :)"
 popd > /dev/null

requirements.txt

@@ -3,26 +3,27 @@ charset-normalizer==3.3.0
 colorama==0.4.6
 debugpy==1.8.1
 Faker==18.4.0
-flake8==4.0.1
+flake8==7.1.1
 idna==3.7
 importlib-metadata==5.2.0
 jedi==0.19.1
 json-spec==0.11.0
 Mako==1.2.4
 MarkupSafe==2.1.3
-mccabe==0.6.1
+mccabe==0.7.0
 packaging==23.2
 parso==0.8.3
 ptvsd==4.3.2
 pudb==2022.1.3
-pycodestyle==2.8.0
-pyflakes==2.4.0
+pycodestyle==2.12.1
+pyflakes==3.2.0
 Pygments==2.16.1
 pyparsing==3.0.9
 python-dateutil==2.8.2
-PyYAML==6.0
-requests==2.32.0
+PyYAML==6.0.2
+requests==2.32.3
 semver==3.0.0
+setuptools==75.3.0
 shell==1.0.1
 six==1.16.0
 urllib3==2.2.2

tests/incrVersion/test_incrementVersion.py

@@ -17,7 +17,7 @@ def testGetJsonSchemaFileNames(self):
         for file in ret:
             self.assertTrue(doesFileExist(file))
         ret2 = incrementVersion._getJsonSchemaFileNames('./tests')
-        self.assertEqual(54, len(ret2))
+        self.assertEqual(57, len(ret2))
         for file in ret2:
             self.assertTrue(doesFileExist(file))
         # negative test

tests/resources/evil/evil1.json

@@ -0,0 +1,43 @@
+{
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"title": "Demo Array",
+	"xproperties": {
+		"prop1": {
+			"type": "array",
+			"items": {
+				"type": "object",
+				"properties": {
+					"int_value": {
+						"type": "integer"
+					}
+				}
+			}
+		},
+		"prop2": {
+			"type": "array",
+			"items": {
+				"type": "array",
+				"items": {
+					"type": "object",
+					"properties": {
+						"int_value": {
+							"type": "integer"
+						}
+					}
+				}
+			}
+		},
+		"prop3": {
+			"type": "array",
+			"items": {
+				"type": "array",
+				"items": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			}
+		}
+	}
+}

tests/resources/evil/evil2.json

@@ -0,0 +1,79 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "version": "1.1.0",
+	"$comment": "working source: tests/resources/models/json/examples/generic-t.json",
+    "ddefinitions": {
+      "TTT": {
+        "type": "object",
+        "properties": {
+          "guid": {
+            "description": "primary key",
+            "x-comment": "use uuid v4",
+            "type": "string",
+            "format": "uuid"
+          },
+          "name": {
+            "type": "string",
+            "$comment": "normal name thing",
+            "x-tags": ["filter"]
+          },
+          "nodes": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/TTTNode"
+            },
+            "minItems": 1
+          }
+        }
+      },
+      "TTTNode": {
+        "type": "object",
+        "allOf": [
+          {
+            "$ref": "./id_name_pair.json"
+          },
+          {
+            "properties": {
+              "yyy": {
+                "type": "string",
+                "format": "uuid"
+              },
+              "subNodes": {
+                "type": "array",
+                "items": {
+                  "$ref": "#/definitions/TTTSubNode"
+                },
+                "examples": [
+                  {
+                    "xxx": 1,
+                    "y": true
+                  },
+                  123,
+                  "bla"
+                ],
+                "minItems": 0
+              },
+              "detectors": {
+                "description": "loops, cameras, buttons, ...",
+                "type": "array",
+                "items": {
+                  "$ref": "#/definitions/DDD"
+                },
+                "minItems": 0
+              }
+            }
+          }
+        ]
+      },
+      "TTTSubNode": {
+        "description": "representation of an intersection, all things are running in the same program",
+        "$ref": "./id_name_pair.json"
+      },
+      "DDD": {
+        "description": "representation of an intersection, all things are running in the same program",
+        "$ref": "./id_name_pair.json"
+      }
+    },
+    "type": "object"
+  }
+

tests/resources/evil/wrong_comma.json

@@ -0,0 +1,43 @@
+{
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"title": "Demo Array",
+	"properties": {
+		"prop1": {
+			"type": "array",
+			"items": {
+				"type": "object",
+				"properties": {
+					"int_value": {
+						"type": "integer"
+					}
+				}
+			},
+		},
+		"prop2": {
+			"type": "array",
+			"items": {
+				"type": "array",
+				"items": {
+					"type": "object",
+					"properties": {
+						"int_value": {
+							"type": "integer"
+						}
+					}
+				}
+			}
+		},
+		"prop3": {
+			"type": "array",
+			"items": {
+				"type": "array",
+				"items": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			}
+		}
+	}
+}

validateSchemas.py

@@ -0,0 +1,67 @@
+import argparse
+import sys
+import os
+import json
+import yacg.builder.impl.dictionaryBuilder as builder
+from yacg.util.outputUtils import printError
+
+description = """Takes a schema or a base directory and tries to load the found schemas for 
+validation purposes. The validation isn't done against standards, but the loading capabilities
+are used for it.
+"""
+
+
+parser = argparse.ArgumentParser(prog='validate', description=description)
+parser.add_argument('--schema', help='path to model schema')
+parser.add_argument('--inputDir', help='path to start to search for JSON schemas')
+parser.add_argument('--noEmptySchemas', help='Checks that at least one type is found in the schema', action='store_true')  # noqa: E501
+parser.add_argument('--blackListed', nargs='+', help='schemas that should not be handled in the template')
+
+
+def main():
+    args = parser.parse_args()
+    if (args.schema is None) and (args.inputDir is None):
+        printError("\nNeither a schema file nor a inputDir is given, so don't know what to validate")
+        sys.exit(1)
+    if args.schema is not None:
+        try:
+            schemaAsDict = builder.getParsedSchemaFromJson(args.schema)
+            if args.noEmptySchemas:
+                extractedTypes = builder.extractTypes(schemaAsDict, args.schema, [], False)
+                if len(extractedTypes) == 0:
+                    printError(f"Schema file doesn't contain a type: ${args.schema}")
+                    sys.exit(1)            
+        except Exception as e:
+            printError(f"Error while validating schema: ${args.schema}")
+            sys.exit(1)            
+    if args.inputDir is not None:
+        violatedFiles = []
+        for root, _, files in os.walk(args.inputDir):
+            for file in files:
+                if file.endswith(".json"):
+                    if (args.blackListed is not None) and (file in args.blackListed):
+                        continue
+                    file_path = os.path.join(root, file)
+                    try:
+                        with open(file_path, "r") as f:
+                            json_data = json.load(f)
+                            if not isinstance(json_data, dict):
+                                continue
+                            if json_data.get("$schema", None) is not None:
+                                schemaAsDict = builder.getParsedSchemaFromJson(file_path)
+                                if args.noEmptySchemas:
+                                    extractedTypes = builder.extractTypes(schemaAsDict, file_path, [], False)
+                                    if len(extractedTypes) == 0:
+                                        violatedFiles.append(file_path)            
+                    except (json.JSONDecodeError, FileNotFoundError, IndexError) as e:
+                        # Handle errors: skip files with bad JSON or that can't be read
+                        violatedFiles.append(file_path)
+        if len(violatedFiles) > 0:
+            print(f"Error while validating schemas in dir: ${args.inputDir}")
+            print(f"corrupted files are: ${violatedFiles}")
+            sys.exit(1)            
+
+
+if __name__ == '__main__':
+    main()
+

version.txt

@@ -1 +1 @@
-6.7.0
+6.8.0

yacg/util/stringUtils.py

@@ -21,6 +21,9 @@ def toUpperCamelCase(text, separator=None):
     """converts a given Text to a upper camel case text
     this is an example -> ThisIsAnExample"""
 
+    if text is None:
+        return ""
+
     splittedText = text.split() if separator is None else text.split(separator)
     upperCamelCase = ''
     for t in splittedText: