From 7556bde10f49d0c87d56b4b647c165a236b5fc7b Mon Sep 17 00:00:00 2001 From: Iteron-dev Date: Tue, 28 Jan 2025 15:11:50 +0100 Subject: [PATCH] Refactor&Build: Update Dockerfile and implement sandbox caching --- Dockerfile | 37 ++-- docker-compose-dev.yml | 12 ++ docker-compose.yml | 11 ++ download_sandboxes.sh | 166 ++++++++++++++++++ .../management/commands/download_sandboxes.py | 101 ++++------- worker_init.sh | 2 +- 6 files changed, 250 insertions(+), 79 deletions(-) create mode 100644 download_sandboxes.sh diff --git a/Dockerfile b/Dockerfile index 347029593..3bc6e9e0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.10 AS base ENV PYTHONUNBUFFERED 1 @@ -66,23 +66,34 @@ RUN pip3 install -r requirements_static.txt --user COPY --chown=oioioi:oioioi . /sio2/oioioi - -ENV OIOIOI_DB_ENGINE 'django.db.backends.postgresql' -ENV RABBITMQ_HOST 'broker' -ENV RABBITMQ_PORT '5672' -ENV RABBITMQ_USER 'oioioi' -ENV RABBITMQ_PASSWORD 'oioioi' -ENV FILETRACKER_LISTEN_ADDR '0.0.0.0' -ENV FILETRACKER_LISTEN_PORT '9999' -ENV FILETRACKER_URL 'http://web:9999' - RUN oioioi-create-config /sio2/deployment WORKDIR /sio2/deployment RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver} -# Download sandboxes +FROM python:3.10 AS development-sandboxes + +ENV DOWNLOAD_DIR=/sio2/sandboxes +ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest + +RUN apt-get update && \ + apt-get install -y curl wget bash && \ + apt-get clean + +ADD $MANIFEST_URL /sio2/Manifest + +COPY download_sandboxes.sh /download_sandboxes.sh +RUN chmod +x /download_sandboxes.sh + +RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL + +FROM base AS development + +COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes +RUN chmod +x /sio2/oioioi/download_sandboxes.sh + RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \ - ./manage.py download_sandboxes -q -y -c /sio2/sandboxes && \ + /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \ + ./manage.py download_sandboxes -q -y -c /sio2/sandboxes -p /sio2/oioioi/download_sandboxes.sh && \ ./manage.py supervisor stop all diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 37a93b3c8..9c2b8f4fa 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -14,10 +14,22 @@ services: build: context: . dockerfile: Dockerfile + target: development args: - "oioioi_uid=${OIOIOI_UID}" extra_hosts: - "web:127.0.0.1" + environment: + OIOIOI_DB_ENGINE: 'django.db.backends.postgresql' + RABBITMQ_HOST: 'broker' + RABBITMQ_PORT: '5672' + RABBITMQ_USER: 'oioioi' + RABBITMQ_PASSWORD: 'oioioi' + FILETRACKER_LISTEN_ADDR: '0.0.0.0' + FILETRACKER_LISTEN_PORT: '9999' + FILETRACKER_URL: 'http://web:9999' + DATABASE_HOST: 'db' + DATABASE_PORT: '5432' ports: # web server - "8000:8000" diff --git a/docker-compose.yml b/docker-compose.yml index 4eb9cc100..197d23505 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,17 @@ services: web: image: sio2project/oioioi:$OIOIOI_VERSION command: ["/sio2/oioioi/oioioi_init.sh"] + environment: + OIOIOI_DB_ENGINE: 'django.db.backends.postgresql' + RABBITMQ_HOST: 'broker' + RABBITMQ_PORT: '5672' + RABBITMQ_USER: 'oioioi' + RABBITMQ_PASSWORD: 'oioioi' + FILETRACKER_LISTEN_ADDR: '0.0.0.0' + FILETRACKER_LISTEN_PORT: '9999' + FILETRACKER_URL: 'http://web:9999' + DATABASE_HOST: 'db' + DATABASE_PORT: '5432' ports: - "8000:8000" stop_grace_period: 3m diff --git a/download_sandboxes.sh b/download_sandboxes.sh new file mode 100644 index 000000000..ce4ee4dc9 --- /dev/null +++ b/download_sandboxes.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +DEFAULT_MANIFEST_URL="https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest" +DEFAULT_DOWNLOAD_DIR="sandboxes-download" +DEFAULT_WGET="wget" +QUIET=false +AGREE_LICENSE=false + +echoerr() { echo "$@" 1>&2; } + +usage() { + echo "Usage: $0 [options] [sandbox1 sandbox2 ...]" + echo "" + echo "Options:" + echo " -m, --manifest URL Specifies URL with the Manifest file listing available sandboxes (default: $DEFAULT_MANIFEST_URL)" + echo " -d, --download-dir DIR Specify the download directory (default: $DEFAULT_DOWNLOAD_DIR)" + echo " -c, --cache-dir DIR Load cached sandboxes from a local directory (default: None)" + echo " --wget PATH Specify the wget binary to use (default: $DEFAULT_WGET)" + echo " -y, --yes Enabling this options means that you agree to the license terms and conditions, so no license prompt will be displayed" + echo " -q, --quiet Disables wget interactive progress bars" + echo " -h, --help Display this help message" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -m|--manifest) + MANIFEST_URL="$2" + shift 2 + ;; + -d|--download-dir) + DOWNLOAD_DIR="$2" + shift 2 + ;; + -c|--cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --wget) + WGET_CMD="$2" + shift 2 + ;; + -y|--yes) + AGREE_LICENSE=true + shift + ;; + -q|--quiet) + QUIET=true + shift + ;; + -h|--help) + usage + ;; + --) + shift + break + ;; + -*) + echoerr "Unknown argument: $1" + usage + ;; + *) + break + ;; + esac +done + +MANIFEST_URL="${MANIFEST_URL:-$DEFAULT_MANIFEST_URL}" +DOWNLOAD_DIR="${DOWNLOAD_DIR:-$DEFAULT_DOWNLOAD_DIR}" +WGET_CMD="${WGET_CMD:-$DEFAULT_WGET}" + +SANDBOXES=("$@") + + +if ! MANIFEST_CONTENT=$(curl -fsSL "$MANIFEST_URL"); then + echoerr "Error: Unable to download manifest from $MANIFEST_URL" + exit 1 +fi + +IFS=$'\n' read -d '' -r -a MANIFEST <<< "$MANIFEST_CONTENT" + + +BASE_URL=$(dirname "$MANIFEST_URL")/ +LICENSE_URL="${BASE_URL}LICENSE" + +LICENSE_CONTENT=$(curl -fsSL "$LICENSE_URL") +LICENSE_STATUS=$? + +if [[ $LICENSE_STATUS -eq 0 ]]; then + if ! $AGREE_LICENSE; then + echoerr "" + echoerr "The sandboxes are accompanied with a license:" + echoerr "$LICENSE_CONTENT" + while true; do + read -rp "Do you accept the license? (yes/no): " yn + case "$yn" in + yes ) break;; + no ) echoerr "License not accepted. Exiting..."; exit 1;; + * ) echoerr 'Please enter either "yes" or "no".';; + esac + done + fi +elif [[ $LICENSE_STATUS -ne 22 ]]; then + echoerr "Error: Unable to download LICENSE from $LICENSE_URL" + exit 1 +fi + +if [[ ${#SANDBOXES[@]} -eq 0 ]]; then + SANDBOXES=("${MANIFEST[@]}") +fi + + +URLS=() +for SANDBOX in "${SANDBOXES[@]}"; do + found=false + for item in "${MANIFEST[@]}"; do + if [[ "$item" == "$SANDBOX" ]]; then + found=true + break + fi + done + + if [[ $found == false ]]; then + echoerr "Error: Sandbox '$SANDBOX' not available (not in Manifest)" + exit 1 + fi + + echo "$SANDBOX"; + + BASENAME="${SANDBOX}.tar.gz" + + if [[ -n "$CACHE_DIR" && -f "$CACHE_DIR/$BASENAME" ]]; then + continue + fi + + URL="${BASE_URL}${BASENAME}" + URLS+=("$URL") +done + +if [[ ! -d "$DOWNLOAD_DIR" ]]; then + if ! mkdir -p "$DOWNLOAD_DIR"; then + echoerr "Error: Unable to create download directory '$DOWNLOAD_DIR'" + exit 1 + fi +fi + +if ! command -v "$WGET_CMD" &> /dev/null; then + echoerr "Error: '$WGET_CMD' is not installed or not in PATH." + exit 1 +fi + +WGET_OPTIONS=("--no-check-certificate") +if $QUIET; then + WGET_OPTIONS+=("-nv") +fi + +for URL in "${URLS[@]}"; do + BASENAME=$(basename "$URL") + OUTPUT_PATH="$DOWNLOAD_DIR/$BASENAME" + if ! "$WGET_CMD" "${WGET_OPTIONS[@]}" -O "$OUTPUT_PATH" "$URL"; then + echoerr "Error: Failed to download $BASENAME" + exit 1 + fi +done + +exit 0 diff --git a/oioioi/sioworkers/management/commands/download_sandboxes.py b/oioioi/sioworkers/management/commands/download_sandboxes.py index d061d1f8b..bcff2a710 100644 --- a/oioioi/sioworkers/management/commands/download_sandboxes.py +++ b/oioioi/sioworkers/management/commands/download_sandboxes.py @@ -2,6 +2,7 @@ import os import os.path +from subprocess import check_output import urllib.error import urllib.parse @@ -59,8 +60,8 @@ def add_arguments(self, parser): default=False, action='store_true', help="Enabling this options means that you agree to the license " - "terms and conditions, so no license prompt will be " - "displayed", + "terms and conditions, so no license prompt will be " + "displayed", ) parser.add_argument( '-q', @@ -70,6 +71,14 @@ def add_arguments(self, parser): action='store_true', help="Disables wget interactive progress bars", ) + parser.add_argument( + '-p', + '--script-path', + metavar='FILEPATH', + dest='script_path', + default=None, + help="Path to script that downloads the sandboxes", + ) parser.add_argument( 'sandboxes', type=str, nargs='*', help='List of sandboxes to be downloaded' ) @@ -92,87 +101,49 @@ def display_license(self, license): break def handle(self, *args, **options): - print("--- Downloading Manifest ...", file=self.stdout) - try: - manifest_url = options['manifest_url'] - manifest = ( - urllib.request.urlopen(manifest_url).read().decode('utf-8') - ) - manifest = manifest.strip().splitlines() - except Exception as e: - raise CommandError("Error downloading manifest: %s" % (e,)) - - print("--- Looking for license ...", file=self.stdout) - try: - license_url = urllib.parse.urljoin(manifest_url, 'LICENSE') - license = ( - urllib.request.urlopen(license_url).read().decode('utf-8') - ) - if not options['license_agreement']: - self.display_license(license) - except urllib.error.HTTPError as e: - if e.code != 404: - raise - + if not options.get('script_path'): + raise CommandError("You must specify a script path") + + license_agreement = "" + if options['license_agreement']: + license_agreement = "-y" + cache_dir = "-c " + if options['cache_dir']: + cache_dir += options['cache_dir'] + + args_str = " ".join(args) + manifest_output = check_output( + f"{options['script_path']} -m {options['manifest_url']} -d {options['download_dir']} --wget {options['wget']} -q {license_agreement} {cache_dir} {args_str}", + shell=True, text=True) + if manifest_output == "": + raise CommandError(f"Manifest output cannot be empty") + + manifest = manifest_output.strip().splitlines() args = options['sandboxes'] if not args: args = manifest - print("--- Preparing ...", file=self.stdout) - urls = [] - cached_args = [] - for arg in args: - basename = arg + '.tar.gz' - if options['cache_dir']: - path = os.path.join(options['cache_dir'], basename) - if os.path.isfile(path): - cached_args.append(arg) - continue - if arg not in manifest: - raise CommandError( - "Sandbox '%s' not available (not in Manifest)" % (arg,) - ) - urls.append(urllib.parse.urljoin(manifest_url, basename)) + print("--- Preparing to save sandboxes to the Filetracker ...", file=self.stdout) + cached_args = [ + arg for arg in args + if options['cache_dir'] and os.path.isfile(os.path.join(options['cache_dir'], arg + '.tar.gz')) + ] filetracker = get_client() - download_dir = options['download_dir'] - if not os.path.exists(download_dir): - os.makedirs(download_dir) - - try: - execute([options['wget'], '--version']) - except ExecuteError: - raise CommandError( - "Wget not working. Please specify a working " - "Wget binary using --wget option." - ) - - if len(urls) > 0: - print("--- Downloading sandboxes ...", file=self.stdout) - - quiet_flag = ['-nv'] if options['quiet'] else [] - execute( - [options['wget'], '-N', '--no-check-certificate', '-i', '-'] + quiet_flag, - stdin='\n'.join(urls).encode('utf-8'), - capture_output=False, - cwd=download_dir, - ) - print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) for arg in args: basename = arg + '.tar.gz' if arg in cached_args: local_file = os.path.join(options['cache_dir'], basename) else: - local_file = os.path.join(download_dir, basename) - print(" ", basename, file=self.stdout) + local_file = os.path.join(options['download_dir'], basename) filetracker.put_file('/sandboxes/' + basename, local_file) if arg not in cached_args: os.unlink(local_file) try: - os.rmdir(download_dir) + os.rmdir(options['download_dir']) except OSError: print( "--- Done, but couldn't remove the downloads directory.", diff --git a/worker_init.sh b/worker_init.sh index 32911c01a..0234ac19c 100755 --- a/worker_init.sh +++ b/worker_init.sh @@ -4,7 +4,7 @@ set -x sudo apt install -y proot -/sio2/oioioi/wait-for-it.sh -t 60 "db:5432" +/sio2/oioioi/wait-for-it.sh -t 60 "${DATABASE_HOST}:${DATABASE_PORT}" /sio2/oioioi/wait-for-it.sh -t 0 "web:8000" mkdir -pv /sio2/deployment/logs/database