mnist_webcam_draw.py

import cv2
import numpy as np
import time
import keras 
import os
os.environ["CUDA_VISIBLE_DEVICES"]="1"

model = keras.models.load_model('model.h5')


load_from_disk = True
if load_from_disk:
    penval = np.load('F:\Fazil Photography\Photoshop Rough\Demos\\virtual-pen\penval.npy')

cap = cv2.VideoCapture(0)

cap.set(3, 1280)
cap.set(4, 720)

kernel = np.ones((5, 5), np.uint8)

# Initializing the canvas on which we will draw upon
canvas = None

# Initilize x1,y1 points
x1, y1 = 0, 0

# Threshold for noise
noiseth = 800

while (1):
    _, frame = cap.read()
    frame = cv2.flip(frame, 1)

    # Initialize the canvas as a black image of the same size as the frame.
    if canvas is None:
        canvas = np.zeros_like(frame)

    # Convert BGR to HSV
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

    # If you're reading from memory then load the upper and lower ranges
    # from there
    if load_from_disk:
        lower_range = penval[0]
        upper_range = penval[1]

    # Otherwise define your own custom values for upper and lower range.
    else:
        lower_range = np.array([26, 80, 147])
        upper_range = np.array([81, 255, 255])

    mask = cv2.inRange(hsv, lower_range, upper_range)

    # Perform morphological operations to get rid of the noise
    mask = cv2.erode(mask, kernel, iterations=1)
    mask = cv2.dilate(mask, kernel, iterations=2)
    res = cv2.bitwise_and(frame, frame, mask=mask)

    mask_3 = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)

    # Find Contours
    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL,
                                           cv2.CHAIN_APPROX_SIMPLE)

    # Make sure there is a contour present and also its size is bigger than
    # the noise threshold.
    if contours and cv2.contourArea(max(contours,
                                        key=cv2.contourArea)) > noiseth:

        c = max(contours, key=cv2.contourArea)
        x2, y2, w, h = cv2.boundingRect(c)

        # If there were no previous points then save the detected x2,y2
        # coordinates as x1,y1.
        # This is true when we writing for the first time or when writing
        # again when the pen had disappeared from view.
        if x1 == 0 and y1 == 0:
            x1, y1 = x2, y2

        else:
            # Draw the line on the canvas
            canvas = cv2.line(canvas, (x1, y1), (x2, y2), [255, 255, 255], 15)

        # After the line is drawn the new points become the previous points.
        x1, y1 = x2, y2

    else:
        # If there were no contours detected then make x1,y1 = 0
        x1, y1 = 0, 0

    # Merge the canvas and the frame.
    frame = cv2.add(frame, canvas)

    # The MNIST Deep Learning Part
    cv2.rectangle(canvas,(400,100),(800,500),(255,255,255),2)
    roi = np.copy(canvas[100:500,400:800])
    digit = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
    img_size = 28
    new_array = cv2.resize(digit, (img_size,img_size))
    p = np.reshape(new_array,(1,784))
    predicted = model.predict(p,batch_size=1)
    print(np.argmax(predicted[0]),np.max(predicted[0]))
    x = np.argmax(predicted)
    xp = int(predicted[0][x]*100)
    
    # cv2.imshow('roi',roi) 
    # cv2.imshow('roi_digit',new_array) 
    stack1 = np.hstack((canvas, frame))


    # stack2 = np.hstack((mask_3,res))
    # stacked = np.vstack((stack1,stack2))
    cv2.putText(stack1, str(int(x)), (80,150), cv2.FONT_HERSHEY_DUPLEX, 4, (255, 255, 255), 3)
    # cv2.putText(stack1, str(xp)+'%', (80,210), cv2.FONT_HERSHEY_DUPLEX, 2, (255, 255, 255), 2)
    cv2.imshow('Magic Wand', cv2.resize(stack1, None, fx=0.6, fy=0.6))

    k = cv2.waitKey(1) & 0xFF
    if k == 27:
        break

    # When c is pressed clear the canvas
    if k == ord('c'):
        canvas = None

cv2.destroyAllWindows()
cap.release()