Feat(kclvm-test): add konfig test for kclvm on unix. (#415)

Feat(kclvm-test): add konfig test for kclvm on unix. issue #379.

.github/workflows/macos_test.yaml

@@ -54,3 +54,7 @@ jobs:
         working-directory: ./kclvm
         run: export PATH=$PATH:$PWD/../_build/dist/Darwin/kclvm/bin:/usr/local/opt/llvm@12/bin && make install-rustc-wasm && make && make codecov-lcov
         shell: bash
+      - name: Rust konfig test
+        working-directory: ./kclvm
+        run: export PATH=$PATH:$PWD/../_build/dist/Darwin/kclvm/bin:/usr/local/opt/llvm@12/bin && make install-rustc-wasm && make && make test-konfig
+        shell: bash

.github/workflows/ubuntu_test.yaml

@@ -45,6 +45,10 @@ jobs:
         working-directory: ./kclvm
         run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make install-rustc-wasm && make && make codecov-lcov
         shell: bash
+      - name: Rust konfig test
+        working-directory: ./kclvm
+        run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make install-rustc-wasm && make && make test-konfig
+        shell: bash
       - name: Coveralls upload
         uses: coverallsapp/github-action@master
         with:

internal/scripts/test.sh

@@ -23,6 +23,8 @@ help_message=$(cat <<-END
             trigger unit test
         test_grammar
             trigger grammar test
+        test_konfig
+            trigger konfig test
         all
             trigger all tests
 END
@@ -44,14 +46,18 @@ done
 
 if [ "$action" == "" ]; then
     PS3='Please select the test scope: '
-    options=("test_grammar")
+    options=("test_grammar", "test_konfig")
     select action in "${options[@]}"
     do
         case $action in
             "test_grammar")
                 $topdir/internal/scripts/test_grammar.sh
                 break
                 ;;
+            "test_konfig")
+                $topdir/internal/scripts/test_konfig.sh
+                break
+                ;;
             *) echo "Invalid action $REPLY:$action"
                 exit 1
                 break

internal/scripts/test_konfig.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Environment
+if [ -f "/etc/os-release" ]; then
+    source /etc/os-release
+    os=$ID
+else
+    os=$(uname)
+fi
+topdir=$(realpath $(dirname $0)/../../)
+kclvm_source_dir="$topdir"
+
+echo PATH=$PATH:$kclvm_source_dir/_build/dist/$os/kclvm/bin >> ~/.bash_profile
+source ~/.bash_profile
+
+export PATH=$PATH:$topdir/_build/dist/$os/kclvm/bin
+
+# Grammar test
+cd $kclvm_source_dir/test/integration
+python3 -m pip install --upgrade pip
+python3 -m pip install -U kclvm
+python3 -m pip install pytest pytest-xdist
+python3 -m pytest -v -n 10

kclvm/makefile

@@ -70,6 +70,10 @@ test-runtime:
 test-grammar:
 	cd tests/integration/grammar && kclvm -m pytest -v -n 5
 
+# E2E konfig tests.
+test-konfig:
+	cd tests/integration/konfig && kclvm -m pytest -v -n 5
+
 # Parser fuzz.
 fuzz-parser:
 	cd tests && cargo fuzz run fuzz_parser

kclvm/tests/integration/konfig/test_konfig_kcl.py

@@ -0,0 +1,110 @@
+"""
+this testing framework is developed based on pytest.
+see quick start of pytest: https://docs.pytest.org/en/latest/example/simple.html
+
+"""
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+from ruamel.yaml import YAML
+from collections.abc import Mapping, Sequence
+
+TEST_FILE = "kcl.yaml"
+CI_TEST_DIR = "ci-test"
+STDOUT_GOLDEN = "stdout.golden.yaml"
+SETTINGS_FILE = "settings.yaml"
+
+ROOT_STR = "test/integration/konfig"
+ROOT = str(Path(__file__).parent.parent.parent.parent.parent.joinpath(ROOT_STR))
+
+yaml = YAML(typ="unsafe", pure=True)
+
+
+def find_test_dirs():
+    result = []
+    root_dirs = [ROOT]
+    for root_dir in root_dirs:
+        for root, _, files in os.walk(root_dir):
+            for name in files:
+                if name == TEST_FILE:
+                    result.append(root)
+    return result
+
+
+def compare_results(result, golden_result):
+    # Convert result and golden_result string to string lines with line ending stripped, then compare.
+
+    assert compare_unordered_yaml_objects(list(yaml.load_all(result)), list(yaml.load_all(golden_result)))
+
+# Comparing the contents of two YAML objects for equality in an unordered manner
+def compare_unordered_yaml_objects(result, golden_result):
+    if isinstance(result, Mapping) and isinstance(golden_result, Mapping):
+        if result.keys() != golden_result.keys():
+            return False
+        for key in result.keys():
+            if not compare_unordered_yaml_objects(result[key], golden_result[key]):
+                return False
+
+        return True
+    elif isinstance(result, Sequence) and isinstance(golden_result, Sequence):
+        if len(result) != len(golden_result):
+            return False
+        for item in result:
+            if item not in golden_result:
+                return False
+        for item in golden_result:
+            if item not in result:
+                return False
+        return True
+    else:
+        return result == golden_result
+
+def has_settings_file(directory):
+    settings_file = directory / SETTINGS_FILE
+    return settings_file.is_file()
+
+
+print("##### K Language Grammar Test Suite #####")
+test_dirs = find_test_dirs()
+pwd = str(Path(__file__).parent.parent.parent.parent)
+os.environ["PYTHONPATH"] = pwd
+
+
+@pytest.mark.parametrize("test_dir", test_dirs)
+def test_konfigs(test_dir):
+    print(f"Testing {test_dir}")
+    test_dir = Path(test_dir)
+    kcl_file_name = test_dir / TEST_FILE
+    ci_test_dir = test_dir / CI_TEST_DIR
+    if not ci_test_dir.is_dir():
+        # Skip invalid test cases
+        return
+    golden_file = ci_test_dir / STDOUT_GOLDEN
+    if not golden_file.is_file():
+        # Skip invalid test cases
+        return
+    kcl_command = ["kcl"]
+    if has_settings_file(ci_test_dir):
+        kcl_command.append("-Y")
+        kcl_command.append(f"{CI_TEST_DIR}/{SETTINGS_FILE}")
+        kcl_command.append(f"kcl.yaml")
+    else:
+        kcl_command.append(f"{TEST_FILE}")
+    kcl_command.append("--target")
+    kcl_command.append("native")
+    process = subprocess.run(
+        kcl_command, capture_output=True, cwd=test_dir, env=dict(os.environ)
+    )
+    stdout, stderr = process.stdout, process.stderr
+    print(f"STDOUT:\n{stdout.decode()}")
+    assert (
+        process.returncode == 0 and len(stderr) == 0
+    ), f"Error executing file {kcl_file_name}.\nexit code = {process.returncode}\nstderr = {stderr}"
+    if process.returncode == 0 and len(stderr) == 0:
+        try:
+            with open(golden_file, "r") as golden:
+                compare_results(stdout.decode(), golden)
+        except FileNotFoundError:
+            raise Exception(f"Error reading expected result from file {golden_file}")