#75/#79: Implemented ci for gh workflows check if build needed and fixed print docker images

doc/changes/unreleased.md

@@ -2,4 +2,12 @@
 
 ## Refactorings
 
- - #76: Updated pyproject.toml and PTB
+ - #76: Updated pyproject.toml and PTB
+
+## Features
+
+ - #75: Implemented CI for Github workflows - check if build needed
+
+## Bugs
+
+ - #79: Fixed print docker images

exasol/slc_ci/cli/cli.py

@@ -0,0 +1,21 @@
+import click
+
+
+@click.group()
+def cli():
+    """
+    EXASLC_CI - Exasol Script Languages Continuous Integration
+
+    Provides a CLI to build/test/deploy and release Exasol's Script-Languages-Containters in a Github CI environment.
+
+    Examples:
+
+        Print this help message:
+
+            $ exaslci --help
+
+        Get the list of available flavors and write to $GITHUB_OUT:
+
+            $ exaslct get-flavors --github-var flavors
+    """
+    pass

exasol/slc_ci/cli/commands/__init__.py

@@ -0,0 +1,3 @@
+from .check_if_build_needed import check_if_build_needed
+from .get_build_runner import get_build_runner
+from .get_flavors import get_flavors

exasol/slc_ci/cli/commands/check_if_build_needed.py

@@ -0,0 +1,30 @@
+import click
+from exasol_integration_test_docker_environment.lib.utils.cli_function_decorators import (
+    add_options,
+)
+
+import exasol.slc_ci.lib.check_if_build_needed as lib_check_if_build_needed
+from exasol.slc_ci.cli.cli import cli
+from exasol.slc_ci.cli.options.branch_options import branch_options
+from exasol.slc_ci.cli.options.flavor_options import flavor_options
+from exasol.slc_ci.cli.options.github_options import github_options
+from exasol.slc_ci.lib.git_access import GitAccess
+from exasol.slc_ci.lib.github_access import GithubAccess
+
+
+@cli.command()
+@add_options(flavor_options)
+@add_options(branch_options)
+@add_options(github_options)
+def check_if_build_needed(
+    flavor: str, branch_name: str, base_branch_name: str, github_var: str
+) -> None:
+    git_access: GitAccess = GitAccess()
+    github_access: GithubAccess = GithubAccess(github_var)
+    lib_check_if_build_needed.check_if_need_to_build(
+        branch_name=branch_name,
+        base_branch_name=base_branch_name,
+        flavor=flavor,
+        github_access=github_access,
+        git_access=git_access,
+    )

exasol/slc_ci/cli/commands/get_build_runner.py

@@ -0,0 +1,17 @@
+from exasol_integration_test_docker_environment.lib.utils.cli_function_decorators import (
+    add_options,
+)
+
+import exasol.slc_ci.lib.get_build_runner as lib_get_build_runner
+from exasol.slc_ci.cli.cli import cli
+from exasol.slc_ci.cli.options.flavor_options import flavor_options
+from exasol.slc_ci.cli.options.github_options import github_options
+from exasol.slc_ci.lib.github_access import GithubAccess
+
+
+@cli.command()
+@add_options(flavor_options)
+@add_options(github_options)
+def get_build_runner(flavor: str, github_var: str):
+    github_access = GithubAccess(github_var)
+    lib_get_build_runner.get_build_runner(flavor=flavor, github_access=github_access)

exasol/slc_ci/cli/commands/get_flavors.py

@@ -0,0 +1,20 @@
+from exasol_integration_test_docker_environment.lib.utils.cli_function_decorators import (
+    add_options,
+)
+
+import exasol.slc_ci.lib.get_flavors as lib_get_flavors
+from exasol.slc_ci.cli.cli import cli
+from exasol.slc_ci.cli.options.github_options import github_options
+from exasol.slc_ci.lib.github_access import GithubAccess
+
+
+@cli.command()
+@add_options(github_options)
+def get_flavors(
+    github_var: str,
+):
+    """
+    Searches for all available flavors and writes result as JSON array to Github variable <github-var>.
+    """
+    github_access: GithubAccess = GithubAccess(github_var=github_var)
+    lib_get_flavors.get_flavors(github_access=github_access)

exasol/slc_ci/cli/main.py

@@ -0,0 +1,12 @@
+#! /usr/bin/env python3
+
+import exasol.slc_ci.cli.commands
+from exasol.slc_ci.cli.cli import cli
+
+
+def main():
+    cli()
+
+
+if __name__ == "__main__":
+    main()

exasol/slc_ci/cli/options/branch_options.py

@@ -0,0 +1,17 @@
+import click
+
+branch_option = click.option(
+    "--branch-name",
+    type=str,
+    required=True,
+    help="In case of a PR, the source branch of the PR.",
+)
+
+base_branch_option = click.option(
+    "--base-branch-name",
+    type=str,
+    required=True,
+    help="In case of a PR, the target branch of the PR.",
+)
+
+branch_options = [branch_option, base_branch_option]

exasol/slc_ci/cli/options/flavor_options.py

@@ -0,0 +1,5 @@
+import click
+
+flavor_options = [
+    click.option("--flavor", type=str, required=True, help="Selects the flavor. ")
+]

exasol/slc_ci/cli/options/github_options.py

@@ -0,0 +1,10 @@
+import click
+
+github_options = [
+    click.option(
+        "--github-var",
+        type=str,
+        required=True,
+        help="Sets the github variable where the result of the operation will be stored.",
+    )
+]

exasol/slc_ci/lib/branch_config.py

@@ -0,0 +1,52 @@
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class BuildActions:
+    build_always: bool
+    rebuild: bool
+    push_to_docker_release_repo: bool
+
+
+@dataclass(frozen=True)
+class BranchConfig:
+    develop = BuildActions(
+        build_always=True, rebuild=True, push_to_docker_release_repo=False
+    )
+    main = BuildActions(
+        build_always=True, rebuild=True, push_to_docker_release_repo=True
+    )
+    rebuild = BuildActions(
+        build_always=True, rebuild=True, push_to_docker_release_repo=False
+    )
+    other = BuildActions(
+        build_always=False, rebuild=False, push_to_docker_release_repo=False
+    )
+
+
+def _get_branch_config(branch_name: str) -> BuildActions:
+    matches = (
+        (re.compile(r"refs/heads/(master|main)"), BranchConfig.main),
+        (re.compile(r"refs/heads/develop"), BranchConfig.develop),
+        (re.compile(r"refs/heads/rebuild/.*"), BranchConfig.rebuild),
+    )
+
+    branch_cfg = BranchConfig.other
+    for branch_regex, branch_config in matches:
+        if branch_regex.match(branch_name):
+            branch_cfg = branch_config
+            break
+    return branch_cfg
+
+
+def build_always(branch_name: str) -> bool:
+    return _get_branch_config(branch_name).build_always
+
+
+def rebuild(branch_name) -> bool:
+    return _get_branch_config(branch_name).rebuild
+
+
+def push_to_docker_release_repo(branch_name: str) -> bool:
+    return _get_branch_config(branch_name).push_to_docker_release_repo

exasol/slc_ci/lib/check_if_build_needed.py

@@ -0,0 +1,67 @@
+import logging
+from pathlib import Path
+from typing import List
+
+from exasol.slc_ci.lib.branch_config import build_always
+from exasol.slc_ci.lib.get_build_config_model import get_build_config_model
+from exasol.slc_ci.lib.git_access import GitAccess
+from exasol.slc_ci.lib.github_access import GithubAccess
+
+
+def get_all_affected_files(git_access: GitAccess, base_branch: str) -> List[Path]:
+    base_last_commit_sha = git_access.get_head_commit_sha_of_branch(base_branch)
+    changed_files = set()  # type: ignore
+    for commit in git_access.get_last_commits():
+        if commit == base_last_commit_sha:
+            break
+        changed_files.update(git_access.get_files_of_commit(commit))
+    return [Path(changed_file) for changed_file in changed_files]
+
+
+def _run_check_if_need_to_build(
+    branch_name: str, base_branch_name: str, flavor: str, git_access: GitAccess
+) -> bool:
+    build_config = get_build_config_model()
+    if build_always(branch_name):
+        return True
+    if "[rebuild]" in git_access.get_last_commit_message():
+        return True
+    affected_files = list(get_all_affected_files(git_access, base_branch_name))
+    logging.debug(
+        f"check_if_need_to_build: Found files of last commits: {affected_files}"
+    )
+    affected_files = [
+        file
+        for file in affected_files
+        if not any(
+            [
+                file.is_relative_to(ignore_path)
+                for ignore_path in build_config.ignore_paths
+            ]
+        )
+    ]
+
+    if len(affected_files) > 0:
+        # Now filter out also other flavor folders
+        this_flavor_path = build_config.flavors_path / flavor
+        affected_files = [
+            file
+            for file in affected_files
+            if (
+                file.is_relative_to(this_flavor_path)
+                or not file.is_relative_to(build_config.flavors_path)
+            )
+        ]
+    logging.debug(f"check_if_need_to_build: filtered files: {affected_files}")
+    return len(affected_files) > 0
+
+
+def check_if_need_to_build(
+    branch_name: str,
+    base_branch_name: str,
+    flavor: str,
+    github_access: GithubAccess,
+    git_access: GitAccess,
+) -> None:
+    res = _run_check_if_need_to_build(branch_name, base_branch_name, flavor, git_access)
+    github_access.write_result("True" if res else "False")

exasol/slc_ci/lib/get_build_config_model.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+from exasol.slc_ci.model.build_config_model import BuildConfig
+
+
+def get_build_config_model() -> BuildConfig:
+    build_config_path = Path.cwd() / "build_config.json"
+    return BuildConfig.model_validate_json(build_config_path.read_text(), strict=True)

exasol/slc_ci/lib/get_build_runner.py

@@ -0,0 +1,9 @@
+from exasol.slc_ci.lib.get_build_config_model import get_build_config_model
+from exasol.slc_ci.lib.get_flavor_ci_model import get_flavor_ci_model
+from exasol.slc_ci.lib.github_access import GithubAccess
+
+
+def get_build_runner(flavor: str, github_access: GithubAccess):
+    build_config = get_build_config_model()
+    flavor_config = get_flavor_ci_model(build_config, flavor)
+    github_access.write_result(flavor_config.build_runner)

exasol/slc_ci/lib/get_flavor_ci_model.py

@@ -0,0 +1,9 @@
+from exasol.slc_ci.model.build_config_model import BuildConfig
+from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig
+
+
+def get_flavor_ci_model(build_config: BuildConfig, flavor: str) -> FlavorCiConfig:
+    flavor_ci_config_path = build_config.flavors_path / flavor / "ci.json"
+    return FlavorCiConfig.model_validate_json(
+        flavor_ci_config_path.read_text(), strict=True
+    )

exasol/slc_ci/lib/get_flavors.py

@@ -0,0 +1,17 @@
+import json
+from typing import List
+
+from exasol.slc_ci.lib.get_build_config_model import get_build_config_model
+from exasol.slc_ci.lib.github_access import GithubAccess
+
+
+def get_flavors(github_access: GithubAccess) -> None:
+    build_config = get_build_config_model()
+    if not build_config.flavors_path.exists():
+        raise ValueError(f"Flavor path '{build_config.flavors_path}' does not exist")
+    flavors: List[str] = list()
+    for p in build_config.flavors_path.iterdir():
+        if p.is_dir():
+            flavors.append(p.name)
+
+    github_access.write_result(json.dumps(flavors))

exasol/slc_ci/lib/git_access.py

@@ -0,0 +1,43 @@
+from typing import Iterable
+
+from git import Repo
+
+
+class GitAccess:
+
+    def get_last_commit_message(self):
+        """
+        Assumes that PWD belongs to a GIT repository. Get's the last commit message of this repo and returns it as string
+        :return: Last commit message of current working directory GIT repository.
+        """
+        return Repo().head.commit.message
+
+    def get_head_commit_sha_of_branch(self, branch_name) -> str:
+        """
+        Returns the last commit sha of given branch.
+        :raise: ValueError: if the refs with label 'branch_name' does not exists or is not unique.
+        """
+        repo = Repo()
+        branch = [b for b in repo.refs if b.name == branch_name]  # type: ignore
+        if len(branch) == 0:
+            ex_msg = f"Branch '{branch_name}' does not exist."
+            raise ValueError(ex_msg)
+        elif len(branch) > 1:
+            ex_msg = f"Branch '{branch_name}' found more than once."
+            raise ValueError(ex_msg)
+        return str(branch[0].commit)
+
+    def get_last_commits(self) -> Iterable[str]:
+        """
+        Returns all commit-sha's of the current branch of the repo in the cwd.
+        """
+        repo = Repo()
+        for c in repo.iter_commits(repo.head):
+            yield str(c)
+
+    def get_files_of_commit(self, commit_sha) -> Iterable[str]:
+        """
+        Returns the files of the specific commits of the repo in the cwd.
+        """
+        repo = Repo()
+        return repo.commit(commit_sha).stats.files.keys()  # type: ignore

exasol/slc_ci/lib/github_access.py

@@ -0,0 +1,16 @@
+import os
+from pathlib import Path
+
+
+class GithubAccess:
+    def __init__(self, github_var: str):
+        self.github_var = github_var
+
+    @property
+    def _get_github_output_file(self):
+        get_github_output_file = Path(os.getenv("GITHUB_OUTPUT"))
+        return get_github_output_file
+
+    def write_result(self, value: str):
+        with open(self._get_github_output_file, "a") as f:
+            f.write(f"{self.github_var}={value}\n")