03-running-sklearn-with-docker-in-condor-executor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a8571181-862a-42ee-8167-5accdac50670",
   "metadata": {},
   "source": [
    "## Executing a simple sklearn machine learning in HTCondor Executor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60d75551-5695-48ec-8c8a-ad321f079aac",
   "metadata": {},
   "source": [
    "### Pre-requisites\n",
    "- Make sure all those Condor Executors have `scikit-learn` Python library installed via `sudo pip3 installl scikit-learn`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "813238fe-a0fa-424b-95ff-20bdd50058d7",
   "metadata": {},
   "source": [
    "### Running `scikit-learn` in HTCondor Executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "86672b00-bf1b-4e34-b4b7-3a797122f6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Installing necessary libraries\n",
    "import os\n",
    "import htcondor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3b527a49-ad76-4c2b-8eed-d9bf4d9580b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting /mnt/nfs/scripts/train_iris_classifier.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile /mnt/nfs/scripts/train_iris_classifier.py\n",
    "#!/usr/bin/env python3\n",
    "# Import necessary libraries\n",
    "print(\"------------\")\n",
    "print(\"03-running-sklearn-with-docker-in-condor-executor.ipynb\")\n",
    "print(\"------------\")\n",
    "\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Load the Iris dataset\n",
    "data = load_iris()\n",
    "X = data.data  # Features\n",
    "y = data.target  # Target labels\n",
    "\n",
    "# Split the data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "# Standardize the features (important for algorithms like Logistic Regression)\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "# Initialize the Logistic Regression model\n",
    "model = LogisticRegression(max_iter=200)\n",
    "\n",
    "# Train the model\n",
    "model.fit(X_train_scaled, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = model.predict(X_test_scaled)\n",
    "\n",
    "# Evaluate the model's accuracy\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(f\"Accuracy: {accuracy:.2f}\")\n",
    "print(\"Task in Docker runtime is completed!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ab909ab2-c924-4f15-baca-391914b3d0fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure the python script is executable\n",
    "!chmod 764 /mnt/nfs/scripts/train_iris_classifier.py"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90cf5607-9d71-42dc-8be0-2fd607585bd3",
   "metadata": {},
   "source": [
    "- Setting up Configuration file for HTCondor Submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8916fd57-446b-4f87-8970-3707fe5e1125",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "universe = docker\n",
      "docker_image = pyenv310:v1\n",
      "executable = /mnt/nfs/scripts/train_iris_classifier.py\n",
      "transfer_input_files = /mnt/nfs/scripts/train_iris_classifier.py\n",
      "should_transfer_files = YES\n",
      "request_cpus = 1\n",
      "request_memory = 128MB\n",
      "request_disk = 128MB\n",
      "output = /mnt/nfs/output/train_iris_classifier.out\n",
      "error = /mnt/nfs/error/train_iris_classifier.err\n",
      "log = /mnt/nfs/log/train_iris_classifier.log\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_ml_job = htcondor.Submit({\n",
    "    \"universe\": \"docker\",\n",
    "    \"docker_image\": \"pyenv310:v1\",\n",
    "    \"executable\": \"/mnt/nfs/scripts/train_iris_classifier.py\",  # Python script with shebang\n",
    "    \"transfer_input_files\": \"/mnt/nfs/scripts/train_iris_classifier.py\",\n",
    "    \"should_transfer_files\": \"YES\",\n",
    "    # \"when_to_transfer_output\": \"ON_EXIT\",\n",
    "    \"request_cpus\": \"1\",            # Number of CPU cores required\n",
    "    \"request_memory\": \"128MB\",      # Memory required\n",
    "    \"request_disk\": \"128MB\",        # Disk space required\n",
    "    \"output\": \"/mnt/nfs/output/train_iris_classifier.out\",  # Standard output file\n",
    "    \"error\": \"/mnt/nfs/error/train_iris_classifier.err\",    # Standard error file\n",
    "    \"log\": \"/mnt/nfs/log/train_iris_classifier.log\",        # Log file\n",
    "})\n",
    "\n",
    "print(train_ml_job)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7481db84-0fd5-406f-b8ca-af43339e45e9",
   "metadata": {},
   "source": [
    "- Submit the task to HTCondor task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "69994b02-24fd-4b44-9450-a66aa9ef7ed8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "253\n"
     ]
    }
   ],
   "source": [
    "schedd = htcondor.Schedd()                   # get the Python representation of the scheduler\n",
    "submit_result = schedd.submit(train_ml_job)  # submit the job\n",
    "print(submit_result.cluster())               # print the job's ClusterId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "64e1a650-2273-4c07-b5aa-fd48ca7f342a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[ ProcId = 0; ClusterId = 253; JobStatus = 1; ServerTime = 1737732442; EnteredCurrentStatus = 1737732442 ]]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "schedd.query(\n",
    "    constraint='ClusterId =?= {}'.format(submit_result.cluster()),\n",
    "    projection=[\"ClusterId\", \"ProcId\", \"JobStatus\", \"EnteredCurrentStatus\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c534b5f2-bd88-43d1-8d1f-9f11fd50e23f",
   "metadata": {},
   "source": [
    "## Monitoring condor_status in Jupyter notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38c47044-e1f4-463f-a8d4-f00d4dd9725e",
   "metadata": {},
   "source": [
    "- Basically, it's same as running `wait -n 1 condor_status` in terminal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d551f955-19c1-4fcc-97e5-a73329ac1921",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style='white-space: pre-wrap; word-wrap: break-word; max-height: 600px; overflow: auto;'>Name                                                                   OpSys      Arch   State     Activity LoadAv Mem   ActvtyTime\n",
       "\n",
       "slot1@condor-executor-01.us-central1-b.c.wqd7008-assignment.internal   LINUX      X86_64 Unclaimed Idle      0.000 3913  0+00:24:31\n",
       "slot1@condor-executor-02.us-central1-b.c.wqd7008-assignment.internal   LINUX      X86_64 Unclaimed Idle      0.000 1837  0+00:10:57\n",
       "slot1_1@condor-executor-02.us-central1-b.c.wqd7008-assignment.internal LINUX      X86_64 Claimed   Busy      0.000  128  0+00:00:00\n",
       "\n",
       "               Total Owner Claimed Unclaimed Matched Preempting  Drain Backfill BkIdle\n",
       "\n",
       "  X86_64/LINUX     3     0       1         2       0          0      0        0      0\n",
       "\n",
       "         Total     3     0       1         2       0          0      0        0      0\n",
       "</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 23\u001b[0m\n\u001b[1;32m     20\u001b[0m         time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     22\u001b[0m \u001b[38;5;66;03m# Run the display function\u001b[39;00m\n\u001b[0;32m---> 23\u001b[0m \u001b[43mdisplay_condor_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[7], line 20\u001b[0m, in \u001b[0;36mdisplay_condor_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m display(HTML(html_output))\n\u001b[1;32m     19\u001b[0m \u001b[38;5;66;03m# Wait for 1 second before updating\u001b[39;00m\n\u001b[0;32m---> 20\u001b[0m \u001b[43mtime\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msleep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import subprocess\n",
    "import time\n",
    "from IPython.display import clear_output, display, HTML\n",
    "\n",
    "# Function to run the condor_status command and display the output\n",
    "def display_condor_status():\n",
    "    while True:\n",
    "        result = subprocess.run(['condor_status'], capture_output=True, text=True)\n",
    "        \n",
    "        # Clear the previous output before updating\n",
    "        clear_output(wait=True)\n",
    "\n",
    "        # Format the output with proper wrapping and limit cell size\n",
    "        html_output = f\"<pre style='white-space: pre-wrap; word-wrap: break-word; max-height: 600px; overflow: auto;'>{result.stdout}</pre>\"\n",
    "        \n",
    "        # Display the formatted output\n",
    "        display(HTML(html_output))\n",
    "        \n",
    "        # Wait for 1 second before updating\n",
    "        time.sleep(1)\n",
    "\n",
    "# Run the display function\n",
    "display_condor_status()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}