add: n_clients benchmarks

Medfl/LearningManager/flpipeline.py

@@ -7,6 +7,7 @@
 # File: create_query.py
 from sqlalchemy import text
 from torch.utils.data import DataLoader, TensorDataset
+import torch 
 
 from Medfl.LearningManager.server import FlowerServer
 from Medfl.LearningManager.utils import params, test
@@ -99,7 +100,8 @@ def delete(self) -> None:
         # Placeholder code for deleting the FLpipeline entry from the database based on the name.
         # You need to implement the actual deletion based on your database setup.
         my_eng.execute(DELETE_FLPIPELINE_QUERY.format(self.name))
-
+
+
     def test_by_node(self, node_name: str, test_frac=1) -> dict:
         """
         Test the FLpipeline by node with the specified test_frac.
@@ -117,22 +119,28 @@ def test_by_node(self, node_name: str, test_frac=1) -> dict:
             self.server.global_model,
             self.server.fed_dataset.testloaders[idx],
         )
+
+        # Move model to GPU if available
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        global_model.model.to(device)
+
+        # Prepare test data
         test_data = test_loader.dataset
-        test_data = TensorDataset(
-            test_data[: int(test_frac * len(test_data))][0],
-            test_data[: int(test_frac * len(test_data))][1],
-        )
-        test_loader = DataLoader(
-            test_data, batch_size=params["test_batch_size"]
-        )
-        classification_report = test(
-            model=global_model.model, test_loader=test_loader
-        )
+        num_samples = int(test_frac * len(test_data))
+        test_data = TensorDataset(test_data[:num_samples][0].to(device), test_data[:num_samples][1].to(device))
+
+        # Create DataLoader for test data
+        test_loader = DataLoader(test_data, batch_size=params["test_batch_size"])
+
+        # Perform testing
+        classification_report = test(model=global_model.model, test_loader=test_loader, device=device)
+
         return {
             "node_name": node_name,
             "classification_report": str(classification_report),
         }
 
+
     def auto_test(self, test_frac=1) -> List[dict]:
         """
         Automatically test the FLpipeline on all nodes with the specified test_frac.

Medfl/LearningManager/model.py

@@ -159,9 +159,7 @@ def evaluate(self, val_loader, device=torch.device("cpu")) -> Tuple[float, float
                 X_test, y_test = X_test.to(device), y_test.to(device)  # Move data to device
 
                 y_hat = torch.squeeze(self.model(X_test), 1)
-                print('/////////////////////////////////////////////////////////////////////')
-                print(y_hat.device)
-                print(y_test.device) 
+
 
                 criterion = self.criterion.to(y_hat.device)
                 loss += criterion(y_hat, y_test).item()

Medfl/LearningManager/params.yaml

@@ -1,22 +1,14 @@
-
----
-  task : BinaryClassification
-  optimizer : SGD
-  train_batch_size : 32
-  test_batch_size : 1
-  train_epochs : 116
-  lr : 0.01
-  diff_privacy: True
-  MAX_GRAD_NORM: 1.0
-  EPSILON: 5.0
-  DELTA: 1e-5
-  num_rounds: 12
-  min_evalclient: 2
-  # path_to_master_csv : 'D:\ESI\3CS\PFE\last_year\Code\MEDfl\notebooks\sapsii_score_knnimputed_eicu.csv'
-  path_to_master_csv : '/home/local/USHERBROOKE/saho6810/MEDfl/code/MEDfl/notebooks/data/masterDataSet/Mimic_ouael.csv'
-
-  path_to_test_csv : '/home/local/USHERBROOKE/saho6810/MEDfl/code/MEDfl/notebooks/data/masterDataSet/Mimic_train.csv'
-
-...
-
-
+DELTA: 1e-5
+EPSILON: 5.2
+MAX_GRAD_NORM: 1.0
+diff_privacy: true
+lr: 0.01
+min_evalclient: 2
+num_rounds: 12
+optimizer: SGD
+path_to_master_csv: /home/local/USHERBROOKE/saho6810/MEDfl/code/MEDfl/notebooks/data/masterDataSet/Mimic_ouael.csv
+path_to_test_csv: /home/local/USHERBROOKE/saho6810/MEDfl/code/MEDfl/notebooks/data/masterDataSet/Mimic_train.csv
+task: BinaryClassification
+test_batch_size: 1
+train_batch_size: 32
+train_epochs: 116

Medfl/LearningManager/server.py

@@ -110,18 +110,11 @@ def client_fn(self, cid) -> FlowerClient:
         Returns:
             FlowerClient: A FlowerClient object representing the individual client.
         """
-        print('////////////////////////////////////////////////////////////////////////////////////////////////////////')
-        print("i m here")
+
         device = torch.device(
             f"cuda:{int(cid) % 4}" if torch.cuda.is_available() else "cpu"
         )
         client_model = copy.deepcopy(self.global_model)
-
-
-        if device.type == 'cuda':
-            print("the model is runing on CUDA")
-        else:
-            print("the model is runing on CPU")
 
         trainloader = self.fed_dataset.trainloaders[int(cid)]
         valloader = self.fed_dataset.valloaders[int(cid)]
@@ -158,9 +151,7 @@ def evaluate(
         self.auc.append(auc)
         self.losses.append(loss)
         self.accuracies.append(accuracy)
-        # if(server_round > 1 ):
-        #     self.strategy.study.tell(server_round-1 , accuracy)
-        # print(f"Server-side evaluation loss {loss} / accuracy {accuracy}")
+
         return loss, {"accuracy": accuracy}
 
     def run(self) -> None:

Medfl/LearningManager/utils.py

@@ -21,7 +21,7 @@
     global_params = yaml.load(g, Loader=SafeLoader)
 
 
-def custom_classification_report(y_true, y_pred):
+def custom_classification_report(y_true, y_pred_prob):
     """
     Compute custom classification report metrics including accuracy, sensitivity, specificity, precision, NPV,
     F1-score, false positive rate, and true positive rate.
@@ -33,6 +33,9 @@ def custom_classification_report(y_true, y_pred):
     Returns:
         dict: A dictionary containing custom classification report metrics.
     """
+    y_pred = y_pred_prob.round()
+
+    auc = roc_auc_score(y_true, y_pred_prob)  # Calculate AUC
 
     tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
 
@@ -78,6 +81,7 @@ def custom_classification_report(y_true, y_pred):
         "F1-score": round(f1, 3),
         "False positive rate": round(fpr, 3),
         "True positive rate": round(tpr, 3),
+        "auc": auc
     }
 
 
@@ -96,10 +100,10 @@ def test(model, test_loader, device=torch.device("cpu")):
 
     model.eval()
     with torch.no_grad():
-        X_test, y_test = test_loader.dataset[:][0], test_loader.dataset[:][1]
-        y_hat = torch.squeeze(model(X_test), 1).round()
+        X_test, y_test = test_loader.dataset[:][0].to(device), test_loader.dataset[:][1].to(device)
+        y_hat_prob = torch.squeeze(model(X_test), 1).cpu()
 
-    return custom_classification_report(y_test, y_hat)
+    return custom_classification_report(y_test.cpu().numpy(), y_hat_prob.cpu().numpy())
 
 
 column_map = {"object": "VARCHAR(255)", "int64": "INT", "float64": "FLOAT"}