From a6e06f59f0a2cae0f4ddc631caad3c5685a59439 Mon Sep 17 00:00:00 2001
From: Filippo Cremonese <filippocremonese@rev.ng>
Date: Fri, 26 Feb 2021 16:46:51 +0100
Subject: [PATCH] Checkout and promote orchestra next-* branches
Checkout the appropriate orchestra configuration branch
before invoking the CI script.
If the build succeeds and the branch was named next-*,
it gets promoted as *.
---
.orchestra/ci/ci-run.sh | 265 ++++++++++++++++++++++++++++++++++++++++
.orchestra/ci/ci.sh | 253 +++++++-------------------------------
2 files changed, 306 insertions(+), 212 deletions(-)
create mode 100755 .orchestra/ci/ci-run.sh
diff --git a/.orchestra/ci/ci-run.sh b/.orchestra/ci/ci-run.sh
new file mode 100755
index 00000000..619001d3
--- /dev/null
+++ b/.orchestra/ci/ci-run.sh
@@ -0,0 +1,265 @@
+#!/bin/bash
+
+# rev.ng CI script
+# This script runs orchestra to build the required components.
+# If the build is successful the script can push the newly produced binary
+# archives and promote next-<name> branches to <name>.
+#
+# Parameters are supplied as environment variables.
+#
+# Target component is mandatory and done with these parameters:
+#
+# TARGET_COMPONENTS: list of components to build
+# TARGET_COMPONENTS_URL:
+# list of glob patterns used to select additional target components
+# by matching their remote URL
+#
+# Optional parameters:
+#
+# BASE_USER_OPTIONS_YML:
+# user_options.yml is initialized to this value.
+# %GITLAB_ROOT% is replaced with the base URL of the Gitlab instance.
+# COMPONENT_TARGET_BRANCH:
+# branch name to try first when checking out component sources
+# PUSH_CHANGES:
+# if != 1 do not push binary archives and do not promote next-* branches
+# PUSH_BINARY_ARCHIVE_EMAIL: used as author's email in binary archive commit
+# PUSH_BINARY_ARCHIVE_NAME: used as author's name in binary archive commit
+# SSH_PRIVATE_KEY: private key used to push binary archives
+# REVNG_ORCHESTRA_URL:
+# alternative URL from where orchestra is installed (passed to pip install)
+
+set -e
+set -x
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ORCHESTRA_ROOT="$(realpath "$DIR/../..")"
+ORCHESTRA_DOTDIR="$ORCHESTRA_ROOT/.orchestra"
+USER_OPTIONS="$ORCHESTRA_DOTDIR/config/user_options.yml"
+
+function log() {
+ echo "$1" > /dev/stderr
+}
+
+PUSH_BINARY_ARCHIVE_EMAIL="${PUSH_BINARY_ARCHIVE_EMAIL:-sysadmin@rev.ng}"
+PUSH_BINARY_ARCHIVE_NAME="${PUSH_BINARY_ARCHIVE_NAME:-rev.ng CI}"
+
+cd "$DIR"
+
+# Install dependencies
+"$DIR/install-dependencies.sh"
+
+#
+# Register deploy key, if any
+#
+set +x
+if test -n "$SSH_PRIVATE_KEY"; then
+ eval "$(ssh-agent -s)"
+ echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+ unset SSH_PRIVATE_KEY
+ mkdir -p ~/.ssh
+ chmod 700 ~/.ssh
+
+ # Disable checking the host key
+ if ! test -e ~/.ssh/config; then
+ cat > ~/.ssh/config <<EOF
+Host *
+ StrictHostKeyChecking no
+ UserKnownHostsFile=/dev/null
+EOF
+ fi
+fi
+set -x
+
+#
+# Install orchestra
+#
+if test -n "$REVNG_ORCHESTRA_URL"; then
+ pip3 install --user "$REVNG_ORCHESTRA_URL"
+else
+ pip3 install --user revng-orchestra
+fi
+
+# Make sure we can run orchestra
+export PATH="$HOME/.local/bin:$PATH"
+which orc
+
+#
+# Prepare the user_options.yml file
+#
+if test -e "$USER_OPTIONS"; then
+ log "$USER_OPTIONS already exists!"
+ exit 1
+fi
+
+REMOTE="$(git remote get-url origin | sed 's|^\([^:]*:\)\([^/]\)|\1/\2|')"
+GITLAB_ROOT="$(dirname "$(dirname "$REMOTE")")"
+echo "${BASE_USER_OPTIONS_YML//\%GITLAB_ROOT\%/$GITLAB_ROOT}" > "$USER_OPTIONS"
+
+# Register target components
+if test -n "$TARGET_COMPONENTS_URL"; then
+ # Add components by repository URL
+ for TARGET_COMPONENT_URL in $TARGET_COMPONENTS_URL; do
+ NEW_COMPONENT="$(orc components --repository-url "$TARGET_COMPONENT_URL" \
+ | grep '^Component' \
+ | cut -d' ' -f2)"
+ if test -z "$NEW_COMPONENT"; then
+ log "Warning: ignoring URL $TARGET_COMPONENT_URL since it doesn't "\
+ "match any component"
+ else
+ TARGET_COMPONENTS="$NEW_COMPONENT $TARGET_COMPONENTS"
+ fi
+ done
+fi
+
+if test -z "$TARGET_COMPONENTS"; then
+ log "Nothing to do!"
+ exit 1
+fi
+
+# Register components to build from source
+cat >> "$USER_OPTIONS" <<EOF
+#@overlay/replace
+build_from_source:
+EOF
+for TARGET_COMPONENT in $TARGET_COMPONENTS; do
+ echo " - $TARGET_COMPONENT" >> "$USER_OPTIONS"
+done
+
+# Build branches list
+cat >> "$USER_OPTIONS" <<EOF
+#@overlay/replace
+branches:
+EOF
+
+if [[ -n "$COMPONENT_TARGET_BRANCH" ]] \
+ && ! [[ "$COMPONENT_TARGET_BRANCH" =~ ^(next-develop|develop|next-master|master)$ ]]; then
+
+ echo " - $COMPONENT_TARGET_BRANCH" >> "$USER_OPTIONS"
+ if test "${COMPONENT_TARGET_BRANCH:0:5}" == "next-"; then
+ echo " - ${COMPONENT_TARGET_BRANCH:5}" >> "$USER_OPTIONS"
+ fi
+fi
+
+cat >> "$USER_OPTIONS" <<EOF
+ - next-develop
+ - develop
+ - next-master
+ - master
+EOF
+
+# Print debug information
+cat "$USER_OPTIONS"
+find ..
+
+orc update --no-config
+
+# Print debugging information
+# Full dependency graph
+orc graph -b
+# Solved dependency graph for the target component
+orc graph --solved -b "$TARGET_COMPONENT"
+# Information about the components
+orc components --hashes --deps
+# Binary archives commit
+for BINARY_ARCHIVE_PATH in $(orc ls --binary-archives); do
+ echo "Commit for $BINARY_ARCHIVE_PATH: "\
+ "$(git -C "$BINARY_ARCHIVE_PATH" rev-parse HEAD)"
+done
+
+#
+# Actually run the build
+#
+RESULT=0
+for TARGET_COMPONENT in $TARGET_COMPONENTS; do
+ if ! orc install -b --test --create-binary-archives "$TARGET_COMPONENT"; then
+ RESULT=1
+ break
+ fi
+done
+
+if test "$PUSH_CHANGES" = 1; then
+ #
+ # Promote `next-*` branches to `*`
+ #
+ if test "$RESULT" -eq 0; then
+
+ # Clone all the components having branch next-*
+ for COMPONENT in $(orc components --branch 'next-*' \
+ | grep '^Component' \
+ | awk '{ print $2 }'); do
+ # TODO: find a more robust way to clone if not already cloned
+ if ! test -d "$ORCHESTRA_ROOT/sources/$COMPONENT"; then
+ orc clone "$COMPONENT"
+ fi
+ done
+
+ # Promote next-* to *.
+ # We also promote orchestra config because fix-binary-archive-symlinks
+ # uses the current branch name
+ for SOURCE_PATH in $(orc ls --git-sources) "$ORCHESTRA_ROOT"; do
+ if test -e "$SOURCE_PATH/.git"; then
+ cd "$SOURCE_PATH"
+ BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+ if test "${BRANCH:0:5}" == "next-"; then
+ PUSH_TO="${BRANCH:5}"
+ git branch -d "$PUSH_TO" || true
+ git checkout -b "$PUSH_TO" "$BRANCH"
+ git push origin "$PUSH_TO"
+ fi
+ cd -
+ fi
+ done
+
+ orc fix-binary-archives-symlinks
+ fi
+
+ #
+ # Push to binary archives
+ #
+ for BINARY_ARCHIVE_PATH in $(orc ls --binary-archives); do
+
+ cd "$BINARY_ARCHIVE_PATH"
+
+ # Ensure we have git lfs
+ git lfs >& /dev/null
+
+ git config user.email "$PUSH_BINARY_ARCHIVE_EMAIL"
+ git config user.name "$PUSH_BINARY_ARCHIVE_NAME"
+
+ if ! test -e .gitattributes; then
+ git lfs track "*.tar.gz"
+ git add .gitattributes
+ git commit -m'Initialize .gitattributes'
+ fi
+
+ ls -lh
+ git add .
+
+ # TODO: cleanup-binary-archives.sh
+
+ if ! git diff --cached --quiet; then
+ git commit -m'Automatic binary archives'
+ git status
+ git stash
+ GIT_LFS_SKIP_SMUDGE=1 git fetch
+ GIT_LFS_SKIP_SMUDGE=1 git rebase -Xtheirs origin/master
+
+ git config --add lfs.dialtimeout 300
+ git config --add lfs.tlstimeout 300
+ git config --add lfs.activitytimeout 300
+ git config --add lfs.keepalive 300
+ git push
+ git lfs push origin master
+ else
+ log "Nothing new to push"
+ fi
+
+ done
+
+ exit "$RESULT"
+
+else
+ echo "PUSH_CHANGES != 1, exiting without pushing changes"
+ exit $RESULT
+fi
diff --git a/.orchestra/ci/ci.sh b/.orchestra/ci/ci.sh
index 66a2c9df..3d604cae 100755
--- a/.orchestra/ci/ci.sh
+++ b/.orchestra/ci/ci.sh
@@ -1,240 +1,69 @@
#!/bin/bash
-set -e
-set -x
+# rev.ng CI entrypoint script
+# This script checks out the correct configuration branch and initializes
+# variables for the actual CI script (ci-run.sh)
+#
+# Parameters are supplied as environment variables.
+#
+# Optional parameters:
+#
+# PUSHED_REF:
+# orchestra config commit/branch to use.
+# Normally set by Gitlab or whoever triggers the CI.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-ORCHESTRA_ROOT="$DIR/../.."
+ORCHESTRA_DIR="$DIR/../.."
+
+# Runs git in the orchestra directory
+function ogit () {
+ git -C "$ORCHESTRA_DIR" "$@"
+}
function log() {
echo "$1" > /dev/stderr
}
-PUSH_BINARY_ARCHIVE_EMAIL="${PUSH_BINARY_ARCHIVE_EMAIL:-sysadmin@rev.ng}"
-PUSH_BINARY_ARCHIVE_NAME="${PUSH_BINARY_ARCHIVE_NAME:-rev.ng CI}"
-
-cd "$DIR"
+set -e
+set -x
+# Determine target branch
#
-# Register deploy key, if any
+# PUSHED_REF contains the git ref that was pushed and triggered the CI.
#
-set +x
-if test -n "$SSH_PRIVATE_KEY"; then
- eval "$(ssh-agent -s)"
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- unset SSH_PRIVATE_KEY
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
+# If this ref is a branch, it will be used as the first default branch to try
+# for all components and for orchestra configuration
- # Disable checking the host key
- if ! test -e ~/.ssh/config; then
- cat > ~/.ssh/config <<EOF
-Host *
- StrictHostKeyChecking no
- UserKnownHostsFile=/dev/null
-EOF
- fi
-fi
-set -x
+COMPONENT_TARGET_BRANCH=""
-#
-# Detect target branch
-#
-BRANCH=""
-if test -n "$PUSHED_REF"; then
+if [[ -n "$PUSHED_REF" ]]; then
if [[ "$PUSHED_REF" = refs/heads/* ]]; then
- BRANCH="${PUSHED_REF#refs/heads/}"
-
- # Switch orchestra to that branch too, if it exists
- git fetch
- git checkout "$BRANCH" || true
+ COMPONENT_TARGET_BRANCH="${PUSHED_REF#refs/heads/}"
else
log "PUSHED_REF ($PUSHED_REF) is not a branch, bailing out"
exit 0
fi
fi
-#
-# Install orchestra
-#
-if test -n "$REVNG_ORCHESTRA_URL"; then
- pip3 install --user "$REVNG_ORCHESTRA_URL"
-else
- pip3 install --user revng-orchestra
-fi
-
-# Make sure we can run orchestra
-export PATH="$HOME/.local/bin:$PATH"
-which orc
-
-#
-# Prepare the user_options.yml file
-#
-if test -e ../config/user_options.yml; then
- log "user_options.yml already exists!"
- exit 1
-fi
-
-REMOTE="$(git remote get-url origin | sed 's|^\([^:]*:\)\([^/]\)|\1/\2|')"
-GITLAB_ROOT="$(dirname "$(dirname "$REMOTE")")"
-echo "$BASE_USER_OPTIONS_YML" | sed "s|%GITLAB_ROOT%|$GITLAB_ROOT|g" > ../config/user_options.yml
-
-# Register target components
-if test -n "$TARGET_COMPONENTS_URL"; then
- # Add components by repository URL
- for TARGET_COMPONENT_URL in $TARGET_COMPONENTS_URL; do
- NEW_COMPONENT="$(orc components --repository-url "$TARGET_COMPONENT_URL" \
- | grep '^Component' \
- | cut -d' ' -f2)"
- if test -z "$NEW_COMPONENT"; then
- log "Warning: ignoring URL $TARGET_COMPONENT_URL since it doesn't match any component"
- else
- TARGET_COMPONENTS="$NEW_COMPONENT $TARGET_COMPONENTS"
- fi
- done
-fi
-
-if test -z "$TARGET_COMPONENTS"; then
- log "Nothing to do!"
- exit 1
-fi
-
-# Register components to build from source
-cat >> ../config/user_options.yml <<EOF
-#@overlay/replace
-build_from_source:
-EOF
-for TARGET_COMPONENT in $TARGET_COMPONENTS; do
- echo " - $TARGET_COMPONENT" >> ../config/user_options.yml
-done
-
-# Build branches list
-cat >> ../config/user_options.yml <<EOF
-#@overlay/replace
-branches:
-EOF
-
-if test -n "$BRANCH" && ! [[ "$BRANCH" =~ ^(next-develop|develop|next-master|master)$ ]]; then
- echo " - $BRANCH" >> ../config/user_options.yml
- if test "${BRANCH:0:5}" == "next-"; then
- echo " - ${BRANCH:5}" >> ../config/user_options.yml
- fi
-fi
-
-cat >> ../config/user_options.yml <<EOF
- - next-develop
- - develop
- - next-master
- - master
-EOF
-
-# Print debug information
-cat ../config/user_options.yml
-find ..
-
-orc update --no-config
-
-# Print debugging information
-# Full dependency graph
-orc graph -b
-# Solved dependency graph for the target component
-orc graph --solved -b "$TARGET_COMPONENT"
-# Information about the components
-orc components --hashes --deps
-# Binary archives commit
-for BINARY_ARCHIVE_PATH in $(orc ls --binary-archives); do
- echo "Commit for $BINARY_ARCHIVE_PATH:" "$(git -C "$BINARY_ARCHIVE_PATH" rev-parse HEAD)"
-done
-
-#
-# Actually run the build
-#
-RESULT=0
-for TARGET_COMPONENT in $TARGET_COMPONENTS; do
- if ! orc install -b --test --create-binary-archives "$TARGET_COMPONENT"; then
- RESULT=1
+# Switch orchestra to the target branch or try the default list
+ogit fetch
+for B in "$COMPONENT_TARGET_BRANCH" next-develop develop next-master master; do
+ if ogit checkout "$B"; then
+ ORCHESTRA_TARGET_BRANCH="$B"
break
fi
done
-if test "$PUSH_CHANGES" = 1; then
- #
- # Promote `next-*` branches to `*`
- #
- if test "$RESULT" -eq 0; then
-
- # Clone all the components having branch next-*
- for COMPONENT in $(orc components --branch 'next-*' | grep '^Component' | awk '{ print $2 }'); do
- # TODO: find a more robust way to clone if not already cloned
- if ! test -d "$ORCHESTRA_ROOT/sources/$COMPONENT"; then
- orc clone "$COMPONENT"
- fi
- done
-
- # Promote next-* to *
- for SOURCE_PATH in $(orc ls --git-sources); do
- if test -e "$SOURCE_PATH/.git"; then
- cd "$SOURCE_PATH"
- BRANCH="$(git rev-parse --abbrev-ref HEAD)"
- if test "${BRANCH:0:5}" == "next-"; then
- PUSH_TO="${BRANCH:5}"
- git branch -d "$PUSH_TO" || true
- git checkout -b "$PUSH_TO" "$BRANCH"
- git push origin "$PUSH_TO"
- fi
- cd -
- fi
- done
-
- orc fix-binary-archives-symlinks
- fi
-
- #
- # Push to binary archives
- #
- for BINARY_ARCHIVE_PATH in $(orc ls --binary-archives); do
-
- cd "$BINARY_ARCHIVE_PATH"
-
- # Ensure we have git lfs
- git lfs >& /dev/null
-
- git config user.email "$PUSH_BINARY_ARCHIVE_EMAIL"
- git config user.name "$PUSH_BINARY_ARCHIVE_NAME"
-
- if ! test -e .gitattributes; then
- git lfs track "*.tar.gz"
- git add .gitattributes
- git commit -m'Initialize .gitattributes'
- fi
-
- ls -lh
- git add .
-
- # TODO: cleanup-binary-archives.sh
-
- if ! git diff --cached --quiet; then
- git commit -m'Automatic binary archives'
- git status
- git stash
- GIT_LFS_SKIP_SMUDGE=1 git fetch
- GIT_LFS_SKIP_SMUDGE=1 git rebase -Xtheirs origin/master
-
- git config --add lfs.dialtimeout 300
- git config --add lfs.tlstimeout 300
- git config --add lfs.activitytimeout 300
- git config --add lfs.keepalive 300
- git push
- git lfs push origin master
- else
- log "Nothing new to push"
- fi
+if [[ -z "$ORCHESTRA_TARGET_BRANCH" ]]; then
+ echo "[!] All checkout attempts failed, aborting"
+ exit 1
+fi
- done
+ORCHESTRA_CONFIG_COMMIT="$(ogit rev-parse --short "$ORCHESTRA_TARGET_BRANCH")"
+echo "[+] Using configuration branch $ORCHESTRA_TARGET_BRANCH "\
+ "(commit $ORCHESTRA_CONFIG_COMMIT)"
- exit "$RESULT"
+export COMPONENT_TARGET_BRANCH
-else
- echo "PUSH_CHANGES != 1, exiting without pushing changes"
- exit $RESULT
-fi
+# Run "true" CI script
+"$DIR/ci-run.sh"