Explainable Air Quality Management- Adaptive Federated XGBoost Enhanced by Quantum Optimization and SHAP Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on node: District 1\n",
      "Training on node: District 2\n",
      "Training on node: District 3\n",
      "Training on node: District 4\n",
      "Training on node: District 5\n",
      "Training on node: District 6\n",
      "Training on node: District 7\n",
      "Training on node: District 8\n",
      "Training on node: District 9\n",
      "Training on node: District 10\n",
      "Training on node: District 11\n",
      "Training on node: District 12\n",
      "Training on node: District 13\n",
      "Training on node: District 14\n",
      "Training on node: District 15\n",
      "Training on node: District 16\n",
      "Training on node: District 17\n",
      "Training on node: District 18\n",
      "Training on node: District 19\n",
      "Training on node: District 20\n",
      "Training on node: District 21\n",
      "Training on node: District 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|===================| 8019/8052 [02:09<00:00]        "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All results, SHAP analysis, and visualizations have been saved successfully.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from prefixspan import PrefixSpan\n",
    "import pennylane as qml\n",
    "import shap\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import seaborn as sns\n",
    "\n",
    "# Dataset Path\n",
    "dataset_path = \"your_path\"\n",
    "result_path = \"your_path\"\n",
    "\n",
    "# Load Dataset\n",
    "data = pd.read_excel(dataset_path)\n",
    "\n",
    "# Preprocessing: Convert date to datetime and sort by node/date\n",
    "data['date'] = pd.to_datetime(data['date'])\n",
    "data.sort_values(['node', 'date'], inplace=True)\n",
    "\n",
    "# Feature and Target Columns\n",
    "features = ['CO', 'O3', 'NO2', 'SO2', 'PM10', 'PM2.5', 'Highest tempreture: 12pm', 'Wind:km/h']\n",
    "target = 'AQI'\n",
    "\n",
    "# Split Data by Nodes (Federated Nodes)\n",
    "nodes = sorted(data['node'].unique(), key=lambda x: int(x.split(' ')[-1]))  # Sort nodes numerically\n",
    "federated_data = {node: data[data['node'] == node] for node in nodes}\n",
    "\n",
    "# Quantum Optimization for Hyperparameters\n",
    "def quantum_hyperparameter_tuning():\n",
    "    dev = qml.device(\"default.qubit\", wires=2)\n",
    "    \n",
    "    @qml.qnode(dev)\n",
    "    def quantum_circuit(params):\n",
    "        qml.RX(params[0], wires=0)\n",
    "        qml.RY(params[1], wires=1)\n",
    "        qml.CNOT(wires=[0, 1])\n",
    "        return qml.expval(qml.PauliZ(0))\n",
    "    \n",
    "    def cost(params):\n",
    "        return quantum_circuit(params)\n",
    "    \n",
    "    from scipy.optimize import minimize\n",
    "    init_params = np.random.uniform(0, np.pi, 2)\n",
    "    res = minimize(cost, init_params, method=\"BFGS\")\n",
    "    return res.x  # Optimal parameters\n",
    "\n",
    "# Adaptive Federated Averaging\n",
    "def adaptive_federated_averaging(global_model, local_model, X_global, y_global, local_weight):\n",
    "    \"\"\"\n",
    "    Adaptive aggregation: Assign higher weights to nodes with better performance.\n",
    "    \"\"\"\n",
    "    # Generate predictions for the global dataset\n",
    "    global_preds = global_model.predict(X_global)\n",
    "    local_preds = local_model.predict(X_global)\n",
    "    \n",
    "    # Weighted average predictions\n",
    "    aggregated_preds = ((1 - local_weight) * global_preds + local_weight * local_preds)\n",
    "    \n",
    "    # Retrain the global model on the aggregated predictions\n",
    "    global_model.fit(X_global, aggregated_preds)\n",
    "    return global_model\n",
    "\n",
    "# Initialize Global Model and Results\n",
    "global_model = None\n",
    "evaluation_results = []\n",
    "\n",
    "# Accuracy Calculation\n",
    "def calculate_accuracy(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Calculates accuracy as the percentage of predictions within a certain threshold of the true value.\n",
    "    \"\"\"\n",
    "    threshold = 0.1 * np.mean(y_true)  # 10% of the mean value as a threshold\n",
    "    within_threshold = np.abs(y_true - y_pred) <= threshold\n",
    "    return np.sum(within_threshold) / len(y_true)\n",
    "\n",
    "# Split data into a global validation set for federated averaging\n",
    "X_global = data[features]\n",
    "y_global = data[target]\n",
    "\n",
    "for node, node_data in federated_data.items():\n",
    "    print(f\"Training on node: {node}\")\n",
    "    \n",
    "    # Train-Test Split\n",
    "    X = node_data[features]\n",
    "    y = node_data[target]\n",
    "    \n",
    "    # Check and handle missing values in the dataset\n",
    "    if X.isnull().values.any() or y.isnull().values.any():\n",
    "        print(f\"Missing values detected in node {node}. Filling missing values with the mean.\")\n",
    "        X = X.fillna(X.mean())\n",
    "        y = y.fillna(y.mean())\n",
    "    \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    \n",
    "    # Hyperparameter Tuning via Quantum Optimization\n",
    "    quantum_params = quantum_hyperparameter_tuning()\n",
    "    learning_rate = quantum_params[0] * 0.1\n",
    "    max_depth = int(quantum_params[1] * 10)\n",
    "    \n",
    "    # Train Local XGBoost Model\n",
    "    model = XGBRegressor(learning_rate=learning_rate, max_depth=max_depth, random_state=42)\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    # Evaluate\n",
    "    y_pred = model.predict(X_test)\n",
    "    \n",
    "    # Check if y_pred contains NaN values\n",
    "    if np.isnan(y_pred).any():\n",
    "        print(f\"NaN values detected in predictions for node {node}. Skipping evaluation for this node.\")\n",
    "        continue  # Skip this node if predictions are invalid\n",
    "    \n",
    "    # Calculate metrics\n",
    "    mse = mean_squared_error(y_test, y_pred)\n",
    "    mae = mean_absolute_error(y_test, y_pred)\n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "    accuracy = calculate_accuracy(y_test, y_pred)\n",
    "    evaluation_results.append([node, mse, mae, r2, accuracy])\n",
    "    \n",
    "    # Adaptive Federated Averaging\n",
    "    local_weight = 1 / (1 + mse)  # Nodes with lower MSE get higher weights\n",
    "    if global_model is None:\n",
    "        global_model = model\n",
    "    else:\n",
    "        global_model = adaptive_federated_averaging(global_model, model, X_global, y_global, local_weight)\n",
    "\n",
    "# Anomaly Detection using PrefixSpan\n",
    "def detect_anomalies(data):\n",
    "    sequences = [list(row[features]) for _, row in data.iterrows()]\n",
    "    ps = PrefixSpan(sequences)\n",
    "    ps.minlen = 2\n",
    "    ps.maxlen = 5\n",
    "    return ps.topk(10, closed=True)\n",
    "\n",
    "# Collect anomaly results\n",
    "anomaly_results = []\n",
    "for node, node_data in federated_data.items():\n",
    "    patterns = detect_anomalies(node_data)\n",
    "    for pattern in patterns:\n",
    "        anomaly_results.append({'Node': node, 'Pattern': pattern[1], 'Frequency': pattern[0]})\n",
    "\n",
    "# Save Anomaly Detection Results\n",
    "anomaly_df = pd.DataFrame(anomaly_results)\n",
    "anomaly_df.to_excel(os.path.join(result_path, \"anomaly_results.xlsx\"), index=False)\n",
    "\n",
    "# Save Evaluation Results\n",
    "result_df = pd.DataFrame(evaluation_results, columns=['Node', 'MSE', 'MAE', 'R2', 'Accuracy'])\n",
    "result_df.to_excel(os.path.join(result_path, \"evaluation_results.xlsx\"), index=False)\n",
    "\n",
    "# SHAP Explainability\n",
    "explainer = shap.Explainer(global_model, X_global)\n",
    "shap_values = explainer(X_global)\n",
    "\n",
    "# Save SHAP Summary Plot\n",
    "plt.figure(figsize=(10, 6))\n",
    "shap.summary_plot(shap_values, X_global, show=False)\n",
    "plt.savefig(os.path.join(result_path, \"SHAP_Summary_Plot.png\"))\n",
    "plt.close()\n",
    "\n",
    "# Visualizations\n",
    "# Top 5 Nodes with Highest AQI\n",
    "top_nodes = data.groupby('node')['AQI'].mean().sort_values(ascending=False).head(5).index\n",
    "plt.figure(figsize=(8, 4))\n",
    "for node in top_nodes:\n",
    "    node_data = federated_data[node]\n",
    "    plt.plot(node_data['date'], node_data['AQI'], label=f\"Node {node}\")\n",
    "plt.title(\"Top 5 Nodes with Highest AQI\")\n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"AQI\")\n",
    "plt.legend()\n",
    "plt.savefig(os.path.join(result_path, \"Top_5_Nodes_AQI.png\"))\n",
    "plt.close()\n",
    "\n",
    "print(\"All results, SHAP analysis, and visualizations have been saved successfully.\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}