05-loading-csv-from-nfs-in-htcondor-executor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a8571181-862a-42ee-8167-5accdac50670",
   "metadata": {},
   "source": [
    "## Executing a simple sklearn machine learning in HTCondor Executor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60d75551-5695-48ec-8c8a-ad321f079aac",
   "metadata": {},
   "source": [
    "### Pre-requisites\n",
    "- Make sure all those Condor Executors have `pandas` and `scikit-learn` Python library installed via `sudo pip3 installl pandas scikit-learn`.\n",
    "- Install NFS server on Condor Submit, and mount it to Condor Executors\n",
    "- For your Submit Condor and Condor Executors, you need to change the ownership and permission for your home directory and NFS directory"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "813238fe-a0fa-424b-95ff-20bdd50058d7",
   "metadata": {},
   "source": [
    "### Running `scikit-learn` in HTCondor Executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "86672b00-bf1b-4e34-b4b7-3a797122f6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Installing necessary libraries\n",
    "import os\n",
    "import htcondor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3b527a49-ad76-4c2b-8eed-d9bf4d9580b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting /mnt/nfs/scripts/train_loan_prediction.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile /mnt/nfs/scripts/train_loan_prediction.py\n",
    "#!/usr/bin/env python3\n",
    "# Import necessary libraries\n",
    "print(\"------------\")\n",
    "print(\"05-loading-csv-from-nfs-in-htcondor-executor.ipynb\")\n",
    "print(\"------------\")\n",
    "\n",
    "# Import necessary libraries\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "\n",
    "# Load the loan dataset from NFS filesystem\n",
    "base_dir = \"/mnt/nfs\"\n",
    "\n",
    "CSV_file = os.path.join(base_dir, \"loan_data.csv\")\n",
    "data = pd.read_csv(CSV_file)\n",
    "\n",
    "# Inspect the dataset\n",
    "print(data.head())\n",
    "\n",
    "# Preprocess the data: Handle missing values, encoding categorical features, etc.\n",
    "# Example of filling missing values with the mean or mode\n",
    "data.fillna(data.mean(), inplace=True)\n",
    "\n",
    "# Encoding categorical features (example: 'Gender', 'Married' columns)\n",
    "data = pd.get_dummies(data, drop_first=True)\n",
    "\n",
    "# Define the target and features\n",
    "X = data.drop('Loan_Status', axis=1)  # Assuming 'Loan_Status' is the target column\n",
    "y = data['Loan_Status']\n",
    "\n",
    "# Split the data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Standardize the features (optional but recommended)\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "\n",
    "# Initialize and train the logistic regression model\n",
    "model = LogisticRegression()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Evaluate the model\n",
    "print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
    "print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, y_pred))\n",
    "print(\"Classification Report:\\n\", classification_report(y_test, y_pred))\n",
    "\n",
    "print(f\"Accuracy: {accuracy:.2f}\")\n",
    "print(\"Task completed!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ab909ab2-c924-4f15-baca-391914b3d0fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure the python script is executable\n",
    "!chmod 764 /mnt/nfs/scripts/train_loan_prediction.py"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90cf5607-9d71-42dc-8be0-2fd607585bd3",
   "metadata": {},
   "source": [
    "- Setting up Configuration file for HTCondor Submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8916fd57-446b-4f87-8970-3707fe5e1125",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "executable = /scripts/train_loan_prediction.py\n",
      "request_cpus = 1\n",
      "request_memory = 128MB\n",
      "request_disk = 128MB\n",
      "output = /mnt/nfs/output/train_loan_prediction.out\n",
      "error = /mnt/nfs/.error/train_loan_prediction.err\n",
      "log = /mnt/nfs/log/train_loan_prediction.log\n",
      "\n"
     ]
    }
   ],
   "source": [
    "base_dir = \"/mnt/nfs\"\n",
    "\n",
    "train_ml_job = htcondor.Submit({\n",
    "    \"executable\": os.path.join(base_dir, \"/scripts/train_loan_prediction.py\"),  # Use bash to execute shell commands\n",
    "    \"request_cpus\": \"1\",            # Number of CPU cores required\n",
    "    \"request_memory\": \"128MB\",      # Memory required\n",
    "    \"request_disk\": \"128MB\",        # Disk space required\n",
    "    \"output\": os.path.join(base_dir, \"output/train_loan_prediction.out\"),  # Standard output file\n",
    "    \"error\": os.path.join(base_dir, \".error/train_loan_prediction.err\"),    # Standard error file\n",
    "    \"log\": os.path.join(base_dir, \"log/train_loan_prediction.log\"),        # Log file\n",
    "})\n",
    "\n",
    "print(train_ml_job)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7481db84-0fd5-406f-b8ca-af43339e45e9",
   "metadata": {},
   "source": [
    "- Submit the task to HTCondor task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "69994b02-24fd-4b44-9450-a66aa9ef7ed8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "254\n"
     ]
    }
   ],
   "source": [
    "schedd = htcondor.Schedd()                   # get the Python representation of the scheduler\n",
    "submit_result = schedd.submit(train_ml_job)  # submit the job\n",
    "print(submit_result.cluster())               # print the job's ClusterId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "64e1a650-2273-4c07-b5aa-fd48ca7f342a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[ ProcId = 0; ClusterId = 254; JobStatus = 1; ServerTime = 1737732689; EnteredCurrentStatus = 1737732689 ]]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "schedd.query(\n",
    "    constraint='ClusterId =?= {}'.format(submit_result.cluster()),\n",
    "    projection=[\"ClusterId\", \"ProcId\", \"JobStatus\", \"EnteredCurrentStatus\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "813244e8-2bd3-4bca-834c-dd15a102ff23",
   "metadata": {},
   "source": [
    "## Monitoring condor_status in Jupyter notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82ccdd0e-66e3-4178-8559-6a0d0b3c6c4d",
   "metadata": {},
   "source": [
    "- Basically, it's same as running `wait -n 1 condor_status` in terminal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d551f955-19c1-4fcc-97e5-a73329ac1921",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style='white-space: pre-wrap; word-wrap: break-word; max-height: 600px; overflow: auto;'>Name                                                                 OpSys      Arch   State     Activity LoadAv Mem   ActvtyTime\n",
       "\n",
       "slot1@condor-executor-01.us-central1-b.c.wqd7008-assignment.internal LINUX      X86_64 Unclaimed Idle      0.000 3913  0+00:29:31\n",
       "slot1@condor-executor-02.us-central1-b.c.wqd7008-assignment.internal LINUX      X86_64 Unclaimed Idle      0.000 1837  0+00:15:10\n",
       "\n",
       "               Total Owner Claimed Unclaimed Matched Preempting  Drain Backfill BkIdle\n",
       "\n",
       "  X86_64/LINUX     2     0       0         2       0          0      0        0      0\n",
       "\n",
       "         Total     2     0       0         2       0          0      0        0      0\n",
       "</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 23\u001b[0m\n\u001b[1;32m     20\u001b[0m         time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     22\u001b[0m \u001b[38;5;66;03m# Run the display function\u001b[39;00m\n\u001b[0;32m---> 23\u001b[0m \u001b[43mdisplay_condor_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[7], line 20\u001b[0m, in \u001b[0;36mdisplay_condor_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m display(HTML(html_output))\n\u001b[1;32m     19\u001b[0m \u001b[38;5;66;03m# Wait for 1 second before updating\u001b[39;00m\n\u001b[0;32m---> 20\u001b[0m \u001b[43mtime\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msleep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import subprocess\n",
    "import time\n",
    "from IPython.display import clear_output, display, HTML\n",
    "\n",
    "# Function to run the condor_status command and display the output\n",
    "def display_condor_status():\n",
    "    while True:\n",
    "        result = subprocess.run(['condor_status'], capture_output=True, text=True)\n",
    "        \n",
    "        # Clear the previous output before updating\n",
    "        clear_output(wait=True)\n",
    "\n",
    "        # Format the output with proper wrapping and limit cell size\n",
    "        html_output = f\"<pre style='white-space: pre-wrap; word-wrap: break-word; max-height: 600px; overflow: auto;'>{result.stdout}</pre>\"\n",
    "        \n",
    "        # Display the formatted output\n",
    "        display(HTML(html_output))\n",
    "        \n",
    "        # Wait for 1 second before updating\n",
    "        time.sleep(1)\n",
    "\n",
    "# Run the display function\n",
    "display_condor_status()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}