Testing LFS and adding some testing for anomaly detection

.devcontainer/raspberry/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:22.04
 RUN apt-get update && apt-get install -y \
     sudo \
     git \
+    git-lfs \
     tig \
     bash-completion \
     python3 \
@@ -16,6 +17,8 @@ RUN apt-get update && apt-get install -y \
     wget \
     && rm -rf /var/lib/apt/lists/*
 
+RUN git lfs install
+
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
     && locale-gen
 ENV LANG en_US.UTF-8

.devcontainer/stm32/Dockerfile

@@ -12,6 +12,7 @@ RUN ln -s /usr/bin/gdb-multiarch /usr/bin/arm-none-eabi-gdb
 RUN apt-get update && apt-get install -y \
     sudo \
     git \
+    git-lfs \
     tig \
     bash-completion \
     python3 \
@@ -22,6 +23,8 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 RUN python3 -m pip install pymodbus flask
 
+RUN git lfs install
+
 RUN addgroup --gid 1000 docker \
     && adduser --uid 1000 --ingroup docker --home /home/docker --disabled-password --gecos "" docker \
     && echo 'docker:docker' | chpasswd \

.devcontainer/virtual/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:22.04
 RUN apt-get update && apt-get install -y \
     sudo \
     git \
+    git-lfs \
     tig \
     bash-completion \
     python3 \
@@ -14,6 +15,8 @@ RUN apt-get update && apt-get install -y \
     iputils-ping \
     wget
 
+RUN git lfs install
+
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y tshark
 
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \

.gitattributes

@@ -0,0 +1 @@
+*.pcap filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -98,7 +98,7 @@ coverage.xml
 cover/
 
 # do not push network captures / pcaps
-*.pcap
+#*.pcap
 
 # do not push trained models
 *.keras

training/anomaly_detection/validation.py

@@ -0,0 +1,90 @@
+import pyshark
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+from sklearn.preprocessing import MinMaxScaler
+
+def parse_modbus_layer_fields(pcap_file):
+    """
+    Parse a PCAP file and extract all fields dynamically from the Modbus layer.
+    """
+    print(f"Parsing Modbus TCP traffic from PCAP file: {pcap_file}")
+    capture = pyshark.FileCapture(pcap_file, display_filter="modbus")
+    traffic_data = []
+
+    for packet in capture:
+        if 'MODBUS' in packet:
+            modbus_layer = packet['MODBUS']
+            packet_data = {}
+            for field in modbus_layer.field_names:
+                try:
+                    packet_data[field] = getattr(modbus_layer, field)
+                except AttributeError:
+                    packet_data[field] = None  # If field is not available
+            traffic_data.append(packet_data)
+
+    capture.close()
+    return pd.DataFrame(traffic_data)
+
+def preprocess_data(data, scaler=None):
+    """
+    Preprocess Modbus TCP data for TensorFlow.
+    """
+    # Fill missing values and convert non-numeric fields to numeric
+    data = data.fillna(0)
+    for col in data.columns:
+        try:
+            data[col] = pd.to_numeric(data[col], errors='coerce').fillna(0)
+        except ValueError:
+            pass
+
+    # Normalize the data
+    if scaler is None:
+        scaler = MinMaxScaler()
+        normalized_data = scaler.fit_transform(data)
+    else:
+        normalized_data = scaler.transform(data)
+
+    return normalized_data, scaler
+
+def detect_anomalies(model, data, threshold=0.5):
+    """
+    Use a trained model to detect anomalies in the data.
+    """
+    predictions = model.predict(data)
+    anomalies = predictions > threshold
+    return anomalies, predictions
+
+if __name__ == "__main__":
+    # Replace with your new PCAP file path
+    new_pcap_file = "modbus_traffic_flood.pcap"
+
+    # Step 1: Extract Modbus TCP traffic
+    new_modbus_data = parse_modbus_layer_fields(new_pcap_file)
+
+    if not new_modbus_data.empty:
+        print("Extracted Modbus Data:")
+        print(new_modbus_data.head())
+
+        # Step 2: Preprocess the data
+        # Load the same scaler used during training
+        preprocessed_data, _ = preprocess_data(new_modbus_data)
+
+        # Step 3: Load the trained TensorFlow model
+        model = tf.keras.models.load_model("modbus_tf_model.keras")
+        print("Model loaded successfully.")
+
+        # Step 4: Detect anomalies
+        anomalies, predictions = detect_anomalies(model, preprocessed_data)
+
+        # Display results
+        new_modbus_data['Prediction'] = predictions
+        new_modbus_data['Anomaly'] = anomalies
+        print("Detection Results:")
+        print(new_modbus_data[['Prediction', 'Anomaly']])
+
+        # Save results to a CSV
+        new_modbus_data.to_csv("anomaly_detection_results.csv", index=False)
+        print("Results saved to anomaly_detection_results.csv")
+    else:
+        print("No Modbus TCP traffic found in the PCAP file.")