-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcomputervision_courseproject.py
627 lines (426 loc) · 28.4 KB
/
computervision_courseproject.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
# -*- coding: utf-8 -*-
"""ComputerVision_CourseProject.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1c1G4RfSC7zId7tU10KgWo7_LAMkd1_WT
"""
# Commented out IPython magic to ensure Python compatibility.
# Import the required libraries.
import os
import cv2
import pafy
import math
import random
import numpy as np
import datetime as dt
import tensorflow as tf
from collections import deque
import matplotlib.pyplot as plt
from moviepy.editor import *
# %matplotlib inline
from sklearn.model_selection import train_test_split
from tensorflow.keras.layers import *
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.utils import plot_model
"""And will set `Numpy`, `Python`, and `Tensorflow` seeds to get consistent results on every execution."""
seed_constant = 27
np.random.seed(seed_constant)
random.seed(seed_constant)
tf.random.set_seed(seed_constant)
"""## **<font style="color:rgb(134,19,348)">Step 2: Preprocess the Dataset</font>**
Next, we will perform some preprocessing on the dataset. First, we will read the video files from the dataset and resize the frames of the videos to a fixed width and height, to reduce the computations and normalized the data to range `[0-1]` by dividing the pixel values with `255`, which makes convergence faster while training the network.
*But first, let's initialize some constants.*
"""
# Specify the height and width to which each video frame will be resized in our dataset.
IMAGE_HEIGHT , IMAGE_WIDTH = 64, 64
# Specify the number of frames of a video that will be fed to the model as one sequence.
SEQUENCE_LENGTH = 10
# Specify the directory containing the UCF50 dataset.
DATASET_DIR = ""
# Specify the list containing the names of the classes used for training. Feel free to choose any set of classes.
CLASSES_LIST = all_classes_names
print(CLASSES_LIST)
"""**Note:** *The **`IMAGE_HEIGHT`**, **`IMAGE_WIDTH`** and **`SEQUENCE_LENGTH`** constants can be increased for better results, although increasing the sequence length is only effective to a certain point, and increasing the values will result in the process being more computationally expensive.*
### **<font style="color:rgb(134,19,348)">Create a Function to Extract, Resize & Normalize Frames</font>**
We will create a function **`frames_extraction()`** that will create a list containing the resized and normalized frames of a video whose path is passed to it as an argument. The function will read the video file frame by frame, although not all frames are added to the list as we will only need an evenly distributed sequence length of frames.
"""
def frames_extraction(video_path):
'''
This function will extract the required frames from a video after resizing and normalizing them.
Args:
video_path: The path of the video in the disk, whose frames are to be extracted.
Returns:
frames_list: A list containing the resized and normalized frames of the video.
'''
# Declare a list to store video frames.
frames_list = []
# Read the Video File using the VideoCapture object.
video_reader = cv2.VideoCapture(video_path)
# Get the total number of frames in the video.
video_frames_count = int(video_reader.get(cv2.CAP_PROP_FRAME_COUNT))
# Calculate the the interval after which frames will be added to the list.
skip_frames_window = max(int(video_frames_count/SEQUENCE_LENGTH), 1)
# Iterate through the Video Frames.
for frame_counter in range(SEQUENCE_LENGTH):
# Set the current frame position of the video.
video_reader.set(cv2.CAP_PROP_POS_FRAMES, frame_counter * skip_frames_window)
# Reading the frame from the video.
success, frame = video_reader.read()
# Check if Video frame is not successfully read then break the loop
if not success:
break
# Resize the Frame to fixed height and width.
resized_frame = cv2.resize(frame, (IMAGE_HEIGHT, IMAGE_WIDTH))
# Normalize the resized frame by dividing it with 255 so that each pixel value then lies between 0 and 1
normalized_frame = resized_frame / 255
# Append the normalized frame into the frames list
frames_list.append(normalized_frame)
# Release the VideoCapture object.
video_reader.release()
# Return the frames list.
#print(frames_list)
return frames_list
"""### **<font style="color:rgb(134,19,348)">Create a Function for Dataset Creation</font>**
Now we will create a function **`create_dataset()`** that will iterate through all the classes specified in the **`CLASSES_LIST`** constant and will call the function **`frame_extraction()`** on every video file of the selected classes and return the frames (**`features`**), class index ( **`labels`**), and video file path (**`video_files_paths`**).
"""
def create_dataset():
'''
This function will extract the data of the selected classes and create the required dataset.
Returns:
features: A list containing the extracted frames of the videos.
labels: A list containing the indexes of the classes associated with the videos.
video_files_paths: A list containing the paths of the videos in the disk.
'''
# Declared Empty Lists to store the features, labels and video file path values.
features = []
labels = []
video_files_paths = []
# Iterating through all the classes mentioned in the classes list
for class_index, class_name in enumerate(CLASSES_LIST):
# Display the name of the class whose data is being extracted.
print(f'Extracting Data of Class: {class_name}')
# Get the list of video files present in the specific class name directory.
files_list = os.listdir(os.path.join(DATASET_DIR, class_name))
print(files_list)
# Iterate through all the files present in the files list.
for file_name in files_list:
#print(len(file_name))
#print(vn)
# Get the complete video path.
#print(sn)
video_file_path = os.path.join(DATASET_DIR, class_name, file_name)
# Extract the frames of the video file.
frames = frames_extraction(video_file_path)
#print(video_file_path)
# Check if the extracted frames are equal to the SEQUENCE_LENGTH specified above.
# So ignore the vides having frames less than the SEQUENCE_LENGTH.
if len(frames) == SEQUENCE_LENGTH:
# Append the data to their repective lists.
features.append(frames)
labels.append(class_index)
video_files_paths.append(video_file_path)
# Converting the list to numpy arrays
features = np.asarray(features)
labels = np.array(labels)
# Return the frames, class index, and video file path.
#print(video_files_paths)
return features, labels, video_files_paths
"""Now we will utilize the function **`create_dataset()`** created above to extract the data of the selected classes and create the required dataset."""
# Create the dataset.
features, labels, video_files_paths = create_dataset()
"""Now we will convert `labels` (class indexes) into one-hot encoded vectors."""
# Using Keras's to_categorical method to convert labels into one-hot-encoded vectors
one_hot_encoded_labels = to_categorical(labels)
"""## **<font style="color:rgb(134,19,348)">Step 3: Split the Data into Train and Test Set</font>**
As of now, we have the required **`features`** (a NumPy array containing all the extracted frames of the videos) and **`one_hot_encoded_labels`** (also a Numpy array containing all class labels in one hot encoded format). So now, we will split our data to create training and testing sets. We will also shuffle the dataset before the split to avoid any bias and get splits representing the overall distribution of the data.
"""
# Split the Data into Train ( 75% ) and Test Set ( 25% ).
features_train, features_test, labels_train, labels_test = train_test_split(features, one_hot_encoded_labels,
test_size = 0.25, shuffle = True,
random_state = seed_constant)
"""## **<font style="color:rgb(134,19,348)">Step 4: Implement the ConvLSTM Approach</font>**
In this step, we will implement the first approach by using a combination of ConvLSTM cells. A ConvLSTM cell is a variant of an LSTM network that contains convolutions operations in the network. it is an LSTM with convolution embedded in the architecture, which makes it capable of identifying spatial features of the data while keeping into account the temporal relation.
<center>
<img src="https://drive.google.com/uc?export=view&id=1KHN_JFWJoJi1xQj_bRdxy2QgevGOH1qP" width= 500px>
</center>
For video classification, this approach effectively captures the spatial relation in the individual frames and the temporal relation across the different frames. As a result of this convolution structure, the ConvLSTM is capable of taking in 3-dimensional input `(width, height, num_of_channels)` whereas a simple LSTM only takes in 1-dimensional input hence an LSTM is incompatible for modeling Spatio-temporal data on its own.
You can read the paper [**Convolutional LSTM Network: A Machine Learning Approach for Precipitation Nowcasting**](https://arxiv.org/abs/1506.04214v1) by **Xingjian Shi** (NIPS 2015), to learn more about this architecture.
### **<font style="color:rgb(134,19,348)">Step 4.1: Construct the Model</font>**
To construct the model, we will use Keras [**`ConvLSTM2D`**](https://keras.io/api/layers/recurrent_layers/conv_lstm2d) recurrent layers. The **`ConvLSTM2D`** layer also takes in the number of filters and kernel size required for applying the convolutional operations. The output of the layers is flattened in the end and is fed to the **`Dense`** layer with softmax activation which outputs the probability of each action category.
We will also use **`MaxPooling3D`** layers to reduce the dimensions of the frames and avoid unnecessary computations and **`Dropout`** layers to prevent [overfitting](https://en.wikipedia.org/wiki/Overfitting) the model on the data. The architecture is a simple one and has a small number of trainable parameters. This is because we are only dealing with a small subset of the dataset which does not require a large-scale model.
"""
def create_convlstm_model():
'''
This function will construct the required convlstm model.
Returns:
model: It is the required constructed convlstm model.
'''
# We will use a Sequential model for model construction
model = Sequential()
# Define the Model Architecture.
########################################################################################################################
model.add(ConvLSTM2D(filters = 4, kernel_size = (3, 3), activation = 'tanh',data_format = "channels_last",
recurrent_dropout=0.2, return_sequences=True, input_shape = (SEQUENCE_LENGTH,
IMAGE_HEIGHT, IMAGE_WIDTH, 3)))
model.add(MaxPooling3D(pool_size=(1, 2, 2), padding='same', data_format='channels_last'))
model.add(TimeDistributed(Dropout(0.2)))
model.add(ConvLSTM2D(filters = 8, kernel_size = (3, 3), activation = 'tanh', data_format = "channels_last",
recurrent_dropout=0.2, return_sequences=True))
model.add(MaxPooling3D(pool_size=(1, 2, 2), padding='same', data_format='channels_last'))
model.add(TimeDistributed(Dropout(0.2)))
model.add(ConvLSTM2D(filters = 14, kernel_size = (3, 3), activation = 'tanh', data_format = "channels_last",
recurrent_dropout=0.2, return_sequences=True))
model.add(MaxPooling3D(pool_size=(1, 2, 2), padding='same', data_format='channels_last'))
model.add(TimeDistributed(Dropout(0.2)))
model.add(ConvLSTM2D(filters = 16, kernel_size = (3, 3), activation = 'tanh', data_format = "channels_last",
recurrent_dropout=0.2, return_sequences=True))
model.add(MaxPooling3D(pool_size=(1, 2, 2), padding='same', data_format='channels_last'))
#model.add(TimeDistributed(Dropout(0.2)))
model.add(Flatten())
model.add(Dense(len(CLASSES_LIST), activation = "softmax"))
########################################################################################################################
# Display the models summary.
model.summary()
# Return the constructed convlstm model.
return model
"""Now we will utilize the function **`create_convlstm_model()`** created above, to construct the required `convlstm` model."""
# Construct the required convlstm model.
convlstm_model = create_convlstm_model()
# Display the success message.
print("Model Created Successfully!")
"""#### **<font style="color:rgb(134,19,348)">Check Model’s Structure:</font>**
Now we will use the **`plot_model()`** function, to check the structure of the constructed model, this is helpful while constructing a complex network and making that the network is created correctly.
"""
# Plot the structure of the contructed model.
#plot_model(convlstm_model, to_file = 'convlstm_model_structure_plot.png', show_shapes = True, show_layer_names = True)
"""### **<font style="color:rgb(134,19,348)">Step 4.2: Compile & Train the Model</font>**
Next, we will add an early stopping callback to prevent [overfitting](https://en.wikipedia.org/wiki/Overfitting) and start the training after compiling the model.
"""
# Create an Instance of Early Stopping Callback
early_stopping_callback = EarlyStopping(monitor = 'val_loss', patience = 10, mode = 'min', restore_best_weights = True)
# Compile the model and specify loss function, optimizer and metrics values to the model
convlstm_model.compile(loss = 'categorical_crossentropy', optimizer = 'Adam', metrics = ["accuracy"])
# Start training the model.
convlstm_model_training_history = convlstm_model.fit(x = features_train, y = labels_train, epochs = 100, batch_size = 64,
shuffle = True, validation_split = 0.2,
callbacks = [early_stopping_callback])
"""#### **<font style="color:rgb(134,19,348)">Evaluate the Trained Model</font>**
After training, we will evaluate the model on the test set.
"""
# Evaluate the trained model.
model_evaluation_history = convlstm_model.evaluate(features_test, labels_test)
from sklearn.metrics import classification_report
# ...
# Evaluate the trained model.
model_evaluation = convlstm_model.evaluate(features_test, labels_test)
# Get the predicted labels for the test set.
predicted_labels = convlstm_model.predict(features_test)
predicted_labels = np.argmax(predicted_labels, axis=1)
# Convert one-hot encoded labels back to original labels.
true_labels = np.argmax(labels_test, axis=1)
# Generate the classification report.
classification_rep = classification_report(true_labels, predicted_labels, target_names=CLASSES_LIST)
# Print the classification report.
print(classification_rep)
"""#### **<font style="color:rgb(134,19,348)">Save the Model</font>**
Now we will save the model to avoid training it from scratch every time we need the model.
"""
# Get the loss and accuracy from model_evaluation_history.
model_evaluation_loss, model_evaluation_accuracy = model_evaluation_history
# Define the string date format.
# Get the current Date and Time in a DateTime Object.
# Convert the DateTime object to string according to the style mentioned in date_time_format string.
date_time_format = '%Y_%m_%d__%H_%M_%S'
current_date_time_dt = dt.datetime.now()
current_date_time_string = dt.datetime.strftime(current_date_time_dt, date_time_format)
# Define a useful name for our model to make it easy for us while navigating through multiple saved models.
model_file_name = f'convlstm_model___Date_Time_{current_date_time_string}___Loss_{model_evaluation_loss}___Accuracy_{model_evaluation_accuracy}.h5'
# Save your Model.
convlstm_model.save(model_file_name)
"""### **<font style="color:rgb(134,19,348)">Step 4.3: Plot Model’s Loss & Accuracy Curves</font>**
Now we will create a function **`plot_metric()`** to visualize the training and validation metrics. We already have separate metrics from our training and validation steps so now we just have to visualize them.
"""
def plot_metric(model_training_history, metric_name_1, metric_name_2, plot_name):
'''
This function will plot the metrics passed to it in a graph.
Args:
model_training_history: A history object containing a record of training and validation
loss values and metrics values at successive epochs
metric_name_1: The name of the first metric that needs to be plotted in the graph.
metric_name_2: The name of the second metric that needs to be plotted in the graph.
plot_name: The title of the graph.
'''
# Get metric values using metric names as identifiers.
metric_value_1 = model_training_history.history[metric_name_1]
metric_value_2 = model_training_history.history[metric_name_2]
# Construct a range object which will be used as x-axis (horizontal plane) of the graph.
epochs = range(len(metric_value_1))
# Plot the Graph.
plt.plot(epochs, metric_value_1, 'blue', label = metric_name_1)
plt.plot(epochs, metric_value_2, 'red', label = metric_name_2)
# Add title to the plot.
plt.title(str(plot_name))
# Add legend to the plot.
plt.legend()
"""Now we will utilize the function **`plot_metric()`** created above, to visualize and understand the metrics."""
# Visualize the training and validation loss metrices.
#plot_metric(convlstm_model_training_history, 'loss', 'val_loss', 'Explosion Class Total Loss vs Total Validation Loss')
# Visualize the training and validation accuracy metrices.
#plot_metric(convlstm_model_training_history, 'accuracy', 'val_accuracy', 'Explosion Class Total Accuracy vs Total Validation Accuracy')
"""## **<font style="color:rgb(134,19,348)">Step 5: Implement the LRCN Approach</font>**
In this step, we will implement the LRCN Approach by combining Convolution and LSTM layers in a single model. Another similar approach can be to use a CNN model and LSTM model trained separately. The CNN model can be used to extract spatial features from the frames in the video, and for this purpose, a pre-trained model can be used, that can be fine-tuned for the problem. And the LSTM model can then use the features extracted by CNN, to predict the action being performed in the video.
But here, we will implement another approach known as the Long-term Recurrent Convolutional Network (LRCN), which combines CNN and LSTM layers in a single model. The Convolutional layers are used for spatial feature extraction from the frames, and the extracted spatial features are fed to LSTM layer(s) at each time-steps for temporal sequence modeling. This way the network learns spatiotemporal features directly in an end-to-end training, resulting in a robust model.
<center>
<img src='https://drive.google.com/uc?export=download&id=1I-q5yLsIoNh2chfzT7JYvra17FsXvdme'>
</center>
You can read the paper [Long-term Recurrent Convolutional Networks for Visual Recognition and Description](https://arxiv.org/abs/1411.4389?source=post_page---------------------------) by Jeff Donahue (CVPR 2015), to learn more about this architecture.
We will also use [**`TimeDistributed`**](https://keras.io/api/layers/recurrent_layers/time_distributed/) wrapper layer, which allows applying the same layer to every frame of the video independently. So it makes a layer (around which it is wrapped) capable of taking input of shape `(no_of_frames, width, height, num_of_channels)` if originally the layer's input shape was `(width, height, num_of_channels)` which is very beneficial as it allows to input the whole video into the model in a single shot.
<center>
<img src='https://drive.google.com/uc?export=download&id=1CbauSm5XTY7ypHYBHH7rDSnJ5LO9CUWX' width=400>
</center>
### **<font style="color:rgb(134,19,348)">Step 5.1: Construct the Model</font>**
To implement our LRCN architecture, we will use time-distributed **`Conv2D`** layers which will be followed by **`MaxPooling2D`** and **`Dropout`** layers. The feature extracted from the **`Conv2D`** layers will be then flattened using the **`Flatten`** layer and will be fed to a **`LSTM`** layer. The **`Dense`** layer with softmax activation will then use the output from the **`LSTM`** layer to predict the action being performed.
"""
def create_LRCN_model():
'''
This function will construct the required LRCN model.
Returns:
model: It is the required constructed LRCN model.
'''
# We will use a Sequential model for model construction.
model = Sequential()
# Define the Model Architecture.
########################################################################################################################
model.add(TimeDistributed(Conv2D(16, (3, 3), padding='same',activation = 'relu'),
input_shape = (SEQUENCE_LENGTH, IMAGE_HEIGHT, IMAGE_WIDTH, 3)))
model.add(TimeDistributed(MaxPooling2D((4, 4))))
model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Conv2D(32, (3, 3), padding='same',activation = 'relu')))
model.add(TimeDistributed(MaxPooling2D((4, 4))))
model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Conv2D(64, (3, 3), padding='same',activation = 'relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2))))
model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Conv2D(64, (3, 3), padding='same',activation = 'relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2))))
#model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(32))
model.add(Dense(len(CLASSES_LIST), activation = 'softmax'))
########################################################################################################################
# Display the models summary.
model.summary()
# Return the constructed LRCN model.
return model
"""Now we will utilize the function **`create_LRCN_model()`** created above to construct the required `LRCN` model."""
# Construct the required LRCN model.
LRCN_model = create_LRCN_model()
# Display the success message.
print("Model Created Successfully!")
"""#### **<font style="color:rgb(134,19,348)">Check Model’s Structure:</font>**
Now we will use the **`plot_model()`** function to check the structure of the constructed `LRCN` model. As we had checked for the previous model.
"""
# Plot the structure of the contructed LRCN model.
#plot_model(LRCN_model, to_file = 'LRCN_model_structure_plot.png', show_shapes = True, show_layer_names = True)
"""### **<font style="color:rgb(134,19,348)">Step 5.2: Compile & Train the Model</font>**
After checking the structure, we will compile and start training the model.
"""
# Create an Instance of Early Stopping Callback.
early_stopping_callback = EarlyStopping(monitor = 'val_loss', patience = 15, mode = 'min', restore_best_weights = True)
# Compile the model and specify loss function, optimizer and metrics to the model.
LRCN_model.compile(loss = 'categorical_crossentropy', optimizer = 'Adam', metrics = ["accuracy"])
# Start training the model.
LRCN_model_training_history = LRCN_model.fit(x = features_train, y = labels_train, epochs = 100, batch_size = 64 ,
shuffle = True, validation_split = 0.2, callbacks = [early_stopping_callback])
"""#### **<font style="color:rgb(134,19,348)">Evaluating the trained Model</font>**
As done for the previous one, we will evaluate the `LRCN` model on the test set.
"""
# Evaluate the trained model.
model_evaluation_history = LRCN_model.evaluate(features_test, labels_test)
from sklearn.metrics import classification_report
# ...
# Evaluate the trained model.
model_evaluation = LRCN_model.evaluate(features_test, labels_test)
# Get the predicted labels for the test set.
predicted_labels = LRCN_model.predict(features_test)
predicted_labels = np.argmax(predicted_labels, axis=1)
# Convert one-hot encoded labels back to original labels.
true_labels = np.argmax(labels_test, axis=1)
# Generate the classification report.
classification_rep = classification_report(true_labels, predicted_labels, target_names=CLASSES_LIST)
# Print the classification report.
print(classification_rep)
"""#### **<font style="color:rgb(134,19,348)">Save the Model</font>**
After that, we will save the model for future uses using the same technique we had used for the previous model.
"""
# Get the loss and accuracy from model_evaluation_history.
model_evaluation_loss, model_evaluation_accuracy = model_evaluation_history
# Define the string date format.
# Get the current Date and Time in a DateTime Object.
# Convert the DateTime object to string according to the style mentioned in date_time_format string.
date_time_format = '%Y_%m_%d__%H_%M_%S'
current_date_time_dt = dt.datetime.now()
current_date_time_string = dt.datetime.strftime(current_date_time_dt, date_time_format)
# Define a useful name for our model to make it easy for us while navigating through multiple saved models.
model_file_name = f'LRCN_model___Date_Time_{current_date_time_string}___Loss_{model_evaluation_loss}___Accuracy_{model_evaluation_accuracy}.h5'
# Save the Model.
LRCN_model.save(model_file_name)
"""#Ensemble the LRCN and CONVLSTM"""
import math
from sklearn.metrics import classification_report
# Get predictions from LRCN model
LRCN_predictions = LRCN_model.predict(features_test)
LRCN_predictions = np.argmax(LRCN_predictions, axis=1)
# Get predictions from ConvLSTM model
convlstm_predictions = convlstm_model.predict(features_test)
convlstm_predictions = np.argmax(convlstm_predictions, axis=1)
# Average the predictions of both models
ensemble_predictions = (LRCN_predictions + convlstm_predictions)
#print(ensemble_predictions)
#print(ensemble_predictions/2)
ensemble_predictions =(np.ceil(ensemble_predictions/2)).astype(int)
# Convert one-hot encoded labels back to original labels
true_labels = np.argmax(labels_test, axis=1)
# Generate the classification report for the ensemble model
classification_rep = classification_report(true_labels, ensemble_predictions, target_names=CLASSES_LIST)
# Print the classification report
print(classification_rep)
"""### **<font style="color:rgb(134,19,348)">Step 5.3: Plot Model’s Loss & Accuracy Curves</font>**
Now we will utilize the function **`plot_metric()`** we had created above to visualize the training and validation metrics of this model.
"""
# Visualize the training and validation loss metrices.
#plot_metric(LRCN_model_training_history, 'loss', 'val_loss', 'Total Loss vs Total Validation Loss')
# Visualize the training and validation accuracy metrices.
#plot_metric(LRCN_model_training_history, 'accuracy', 'val_accuracy', 'Total Accuracy vs Total Validation Accuracy')
import matplotlib.pyplot as plt
def plot_metric(metric_value_1, metric_value_2, label_1, label_2, plot_name):
'''
This function will plot the accuracy of two models in a single graph.
Args:
metric_value_1: The accuracy values for model 1.
metric_value_2: The accuracy values for model 2.
label_1: The label for model 1.
label_2: The label for model 2.
plot_name: The title of the graph.
'''
# Construct a range object which will be used as x-axis (horizontal plane) of the graph.
epochs = range(len(metric_value_1))
# Plot the accuracy values for both models.
plt.plot(epochs, metric_value_1, 'blue', label=label_1)
plt.plot(epochs, metric_value_2, 'red', label=label_2)
# Add title to the plot.
plt.title(str(plot_name))
# Add legend to the plot.
plt.legend()
# Show the plot
plt.show()
#plot_metric(convlstm_model_training_history, 'accuracy', 'val_accuracy', 'Explosion Class Total Accuracy vs Total Validation Accuracy')
# Get metric values using metric names as identifiers.
m1 = LRCN_model_training_history.history['accuracy']
m2 = convlstm_model_training_history.history['accuracy']
plot_metric(m2, m1, "ConvLSTM", "LRCN", "Training Accuracy Graph")
m1 = LRCN_model_training_history.history['val_accuracy']
m2 = convlstm_model_training_history.history['val_accuracy']
plot_metric(m2, m1, "ConvLSTM", "LRCN", "Validation Accuracy Graph")