Introduce post-upgrade data comparison test

Test is intended to run only in upgrade pipeline and expects reference
(pre-upgrade) and actual (post-upgrade) data snapshot directories to
exist. `scripts/snapshot-data.py` can be used to create either. In
Discovery project, discovery-ci is responsible for running the script
and ensuring both directories exist.

camayoc/config.py

@@ -20,6 +20,8 @@
     Validator("camayoc.run_scans", default=False),
     Validator("camayoc.scan_timeout", default=600),
     Validator("camayoc.db_cleanup", default=True),
+    Validator("camayoc.snapshot_test_reference_path", default=None),
+    Validator("camayoc.snapshot_test_actual_path", default=None),
     Validator("quipucords_server.hostname", default=""),
     Validator("quipucords_server.https", default=False),
     Validator("quipucords_server.port", default=8000),

camayoc/db_snapshot.py

@@ -0,0 +1,129 @@
+import dataclasses
+import json
+import logging
+from operator import itemgetter
+from pathlib import Path
+from typing import Any
+
+from camayoc.constants import DBSERIALIZER_CONNECTIONJOBS_DIR_PATH
+from camayoc.constants import DBSERIALIZER_CREDENTIALS_FILE_PATH
+from camayoc.constants import DBSERIALIZER_REPORTS_DIR_PATH
+from camayoc.constants import DBSERIALIZER_SCANJOBS_DIR_PATH
+from camayoc.constants import DBSERIALIZER_SCANS_FILE_PATH
+from camayoc.constants import DBSERIALIZER_SOURCES_FILE_PATH
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass(frozen=True)
+class DBSnapshot:
+    """DB Snapshot data representation suitable for test usage.
+
+    The intended usage is through `from_dir()` constructor, which takes a path
+    to snapshot data directory.
+
+    DBSnapshot is responsible for transforming a data to a format suitable for
+    native equality comparison. It means that lists may be reordered, fields
+    with dynamic data can be removed etc.
+    """
+
+    credentials: list[dict[str, Any]]
+    sources: list[dict[str, Any]]
+    scans: list[dict[str, Any]]
+    connection_jobs: dict[str, Any]
+    scan_jobs: dict[str, Any]
+    reports: dict[str, Any]
+
+    @classmethod
+    def from_dir(cls, source: Path):
+        credentials = read_credentials(source / DBSERIALIZER_CREDENTIALS_FILE_PATH)
+        sources = read_sources(source / DBSERIALIZER_SOURCES_FILE_PATH)
+        scans = read_scans(source / DBSERIALIZER_SCANS_FILE_PATH)
+        connection_jobs = read_connection_jobs(source / DBSERIALIZER_CONNECTIONJOBS_DIR_PATH)
+        scan_jobs = read_scan_jobs(source / DBSERIALIZER_SCANJOBS_DIR_PATH)
+        reports = read_reports(source / DBSERIALIZER_REPORTS_DIR_PATH)
+        return cls(credentials, sources, scans, connection_jobs, scan_jobs, reports)
+
+
+def read_credentials(source: Path) -> list[dict[str, Any]]:
+    with source.open() as fh:
+        all_credentials = json.load(fh)
+    return sorted(all_credentials, key=itemgetter("name"))
+
+
+def read_sources(source: Path) -> list[dict[str, Any]]:
+    with source.open() as fh:
+        all_sources = json.load(fh)
+    return sorted(all_sources, key=itemgetter("name"))
+
+
+def read_scans(source: Path) -> list[dict[str, Any]]:
+    with source.open() as fh:
+        all_scans = json.load(fh)
+    return sorted(all_scans, key=itemgetter("name"))
+
+
+def read_connection_jobs(source: Path) -> dict[str, Any]:
+    connection_jobs = {}
+
+    for file in source.rglob("*"):
+        if file.is_dir():
+            continue
+        key = file.stem
+        with file.open() as fh:
+            jobs = json.load(fh)
+        connection_jobs[key] = sorted(jobs, key=_connection_job_itemgetter)
+
+    return connection_jobs
+
+
+def read_scan_jobs(source: Path) -> dict[str, Any]:
+    scan_jobs = {}
+
+    for file in source.rglob("*"):
+        if file.is_dir():
+            continue
+        key = file.stem
+        with file.open() as fh:
+            jobs = json.load(fh)
+        scan_jobs[key] = sorted(jobs, key=itemgetter("id"))
+
+    return scan_jobs
+
+
+def read_reports(source: Path) -> dict[str, Any]:
+    all_reports = {}
+    for directory in source.glob("*"):
+        if directory.is_file():
+            continue
+        key = directory.name
+        all_reports[key] = _read_report_directory(directory)
+    return all_reports
+
+
+def _read_report_directory(directory: Path):
+    details_data = _read_report_details(directory / "details.json")
+    aggregate_data = _read_report_aggregate(directory / "aggregate.json")
+    return {
+        "details": details_data,
+        "aggregate": aggregate_data,
+    }
+
+
+def _read_report_details(file: Path):
+    with file.open() as fh:
+        data = json.load(fh)
+    return data
+
+
+def _read_report_aggregate(file: Path):
+    with file.open() as fh:
+        data = json.load(fh)
+    return data
+
+
+def _connection_job_itemgetter(item):
+    source_id = item.get("source", {}).get("id", 0)
+    credential_id = item.get("credential", {}).get("id", 0)
+    name = item.get("name", "")
+    return f"{name}-{credential_id}-{source_id}"

camayoc/tests/qpc/snapshots/test_compare.py

@@ -0,0 +1,48 @@
+import os
+from pathlib import Path
+from pprint import pformat
+
+import pytest
+from deepdiff.diff import DeepDiff
+
+from camayoc.config import settings
+from camayoc.db_snapshot import DBSnapshot
+
+
+def compare_snapshots_handler(differences: DeepDiff):
+    node_name = os.getenv("NODE_NAME", "local")
+    results_dir = Path(".").cwd() / "test-results" / f"compare-snapshots-{node_name}"
+    results_dir.mkdir(parents=True, exist_ok=True)
+    for key, value in differences.items():
+        filepath = results_dir / f"{key}.txt"
+        filepath.write_text(pformat(value))
+    return f"Details of differences are saved in {results_dir.as_posix()}/"
+
+
+@pytest.mark.upgrade_only
+@pytest.mark.skipif(
+    not all(
+        (settings.camayoc.snapshot_test_reference_path, settings.camayoc.snapshot_test_actual_path)
+    ),
+    reason=(
+        "Both camayoc.snapshot_test_reference_path and "
+        "camayoc.snapshot_test_actual_path must be set"
+    ),
+)
+def test_compare_snapshots():
+    """Verify that user-facing data survives the upgrade.
+
+    :id: f85334a1-85db-4338-ba41-982fed14d978
+    :description: Verify that user-facing data survives the upgrade.
+    :steps:
+        1) Read reference snapshot data (created before upgrade)
+        2) Read actual snapshot data (created after upgrade)
+        3) Compare them
+    :expectedresults: Post-upgrade data matches pre-upgrade data.
+    """
+    reference = DBSnapshot.from_dir(settings.camayoc.snapshot_test_reference_path)
+    actual = DBSnapshot.from_dir(settings.camayoc.snapshot_test_actual_path)
+
+    differences = DeepDiff(reference, actual)
+
+    assert not differences, compare_snapshots_handler(differences)

camayoc/types/settings.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Any
 from typing import Literal
 from typing import Optional
@@ -13,6 +14,8 @@ class CamayocOptions(BaseModel):
     run_scans: Optional[bool] = False
     scan_timeout: Optional[int] = 600
     db_cleanup: Optional[bool] = True
+    snapshot_test_reference_path: Optional[Path] = None
+    snapshot_test_actual_path: Optional[Path] = None
 
 
 class QuipucordsServerOptions(BaseModel):

pyproject.toml

@@ -29,6 +29,7 @@ requires = ["poetry-core"]
 python = ">= 3.11, < 3.14"
 attrs = "^24.2.0"
 dynaconf = "^3.2.4"
+deepdiff = "^8.2.0"
 factory_boy = "^3.2.1"
 littletable = "^3.0.1"
 pexpect = "^4.8.0"
@@ -65,6 +66,7 @@ markers = [
     "slow: tests that take a long time to run (on average, more than 30 seconds)",
     "nightly_only: tests to execute only during nightly (or full) run, i.e. not during PR check; note that actual selection is implemented in discovery-ci",
     "pr_only: tests to execute only during PR check run, i.e. not during nightly run; note that actual selection is implemented in discovery-ci",
+    "upgrade_only: tests to execute only during upgrade testing; note that actual selection is implemented in discovery-ci",
 ]
 
 [tool.ruff]