Skip to content

Commit

Permalink
updating models
Browse files Browse the repository at this point in the history
  • Loading branch information
@thiagocosenza
thiagocosenza authored Jul 24, 2022
1 parent 06712a9 commit 0b763b7
Show file tree
Hide file tree
Showing 3 changed files with 1,120 additions and 0 deletions.
326 changes: 326 additions & 0 deletions extraction_model.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,326 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#%pip install tensorflow tensorflow-gpu opencv-python mediapipe sklearn matplotlib"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import cv2\n",
"import numpy as np\n",
"import os\n",
"from matplotlib import pyplot as plt\n",
"import time\n",
"import datetime\n",
"import mediapipe as mp"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Diretório para os pontos coletados\n",
"DATA_DIR = os.path.join('Dados_extraidos') \n",
"\n",
"# Sinais\n",
"sinais = np.array(['Ola','Tudo bem','Obrigado','Entendo','Calma'])\n",
"\n",
"# Quantidade de vídeos por sinal\n",
"num_videos = 10\n",
"\n",
"# Quantidade de frames por vídeo\n",
"num_frames = 30"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"mp_holistic = mp.solutions.holistic \n",
"mp_drawing = mp.solutions.drawing_utils\n",
"mp_drawing_styles = mp.solutions.drawing_styles\n",
"cam = cv2.VideoCapture(0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def mediapipe_extrai(imagem, modelo):\n",
" imagem = cv2.cvtColor(imagem, cv2.COLOR_BGR2RGB) \n",
" imagem.flags.writeable = False \n",
" resultados = modelo.process(imagem) \n",
" imagem.flags.writeable = True \n",
" imagem = cv2.cvtColor(imagem, cv2.COLOR_RGB2BGR) \n",
" return imagem, resultados"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def mediapipe_desenha(imagem, resultados):\n",
" # Pontos do rosto\n",
" mp_drawing.draw_landmarks(imagem, resultados.face_landmarks, mp_holistic.FACEMESH_CONTOURS, \n",
" mp_drawing.DrawingSpec(color=(153,0,153), thickness=1, circle_radius=1), \n",
" mp_drawing_styles.get_default_face_mesh_contours_style()\n",
" ) \n",
" # Pontos do corpo\n",
" mp_drawing.draw_landmarks(imagem, resultados.pose_landmarks, mp_holistic.POSE_CONNECTIONS,\n",
" mp_drawing.DrawingSpec(color=(153,0,153), thickness=2, circle_radius=3), \n",
" mp_drawing.DrawingSpec(color=(255,40,130), thickness=2, circle_radius=1)\n",
" ) \n",
" # Pontos da mão esquerda\n",
" mp_drawing.draw_landmarks(imagem, resultados.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
" mp_drawing.DrawingSpec(color=(153,0,153), thickness=1, circle_radius=3), \n",
" mp_drawing.DrawingSpec(color=(35,100,255), thickness=1, circle_radius=1)\n",
" ) \n",
" # Pontos da mão direita \n",
" mp_drawing.draw_landmarks(imagem, resultados.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
" mp_drawing.DrawingSpec(color=(153,0,153), thickness=1, circle_radius=3), \n",
" mp_drawing.DrawingSpec(color=(247,0,247), thickness=1, circle_radius=1)\n",
" ) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def mediapipe_desenha_extrai(cam, modelo):\n",
" ret, frame = cam.read() # Ler o frame\n",
" imagem, resultados = mediapipe_extrai(frame, modelo)# Fazer detecções\n",
" mediapipe_desenha(imagem, resultados) # Desenhar pontos\n",
" return imagem, resultados"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def trata_pontos(resultados):\n",
"\n",
" rosto = []\n",
" corpo = []\n",
" mao_esquerda = []\n",
" mao_direita = []\n",
"\n",
" if resultados.face_landmarks:\n",
" for r in resultados.face_landmarks.landmark:\n",
" ponto = np.array([r.x, r.y, r.z])\n",
" rosto.append(ponto)\n",
" else:\n",
" rosto = np.zeros(468*3) \n",
"\n",
" if resultados.pose_landmarks:\n",
" for r in resultados.pose_landmarks.landmark:\n",
" ponto = np.array([r.x, r.y, r.z, r.visibility])\n",
" corpo.append(ponto)\n",
" else:\n",
" corpo = np.zeros(33*4) \n",
" \n",
" if resultados.left_hand_landmarks:\n",
" for r in resultados.left_hand_landmarks.landmark:\n",
" ponto = np.array([r.x, r.y, r.z])\n",
" mao_esquerda.append(ponto)\n",
" else:\n",
" mao_esquerda = np.zeros(21*3) \n",
"\n",
" if resultados.right_hand_landmarks:\n",
" for r in resultados.right_hand_landmarks.landmark:\n",
" ponto = np.array([r.x, r.y, r.z])\n",
" mao_direita.append(ponto)\n",
" else:\n",
" mao_direita = np.zeros(21*3)\n",
"\n",
" rosto = np.array(rosto).flatten() \n",
" corpo = np.array(corpo).flatten()\n",
" mao_esquerda = np.array(mao_esquerda).flatten()\n",
" mao_direita = np.array(mao_direita).flatten()\n",
"\n",
" return np.concatenate([corpo, rosto, mao_esquerda, mao_direita]) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Monta toda estrutura de diretórios\n",
"for sinal in sinais: \n",
" for video in range(num_videos):\n",
" try: \n",
" os.makedirs(os.path.join(DATA_DIR, sinal, str(video)))\n",
" except:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"with mp_holistic.Holistic() as holistic:\n",
" \n",
" while cam.isOpened():\n",
"\n",
" # Desenha pontos na imagem e extrai características em resultados\n",
" imagem, resultados = mediapipe_desenha_extrai(cam, holistic)\n",
"\n",
" # Enunciado na tela\n",
" cv2.putText(imagem,'Por favor, posicione-se da cintura para cima',(20, 40),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=0.8,color=(0, 255, 255), thickness = 2,lineType=cv2.LINE_AA) \n",
" cv2.putText(imagem,'Tecle barra para continuar',(20, 80),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2,lineType=cv2.LINE_AA) \n",
" \n",
" # Mostra a tela\n",
" cv2.imshow('LIBRAS', imagem)\n",
"\n",
" # Espera pela tecla\n",
" if cv2.waitKey(10) & 0xFF == ord(' '):\n",
" cv2.destroyAllWindows()\n",
" break\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"x_polegar_direito = resultados.pose_landmarks.landmark[22].x\n",
"pontos_tratados = trata_pontos(resultados)\n",
"indice = np.where(pontos_tratados == x_polegar_direito)[0]\n",
"print(indice)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"with mp_holistic.Holistic() as holistic:\n",
"\n",
" for sinal in sinais:\n",
"\n",
" #video inicial\n",
" video = 0\n",
"\n",
" while video < num_videos:\n",
" \n",
" # Enunciado\n",
" while cam.isOpened():\n",
" imagem, resultados = mediapipe_desenha_extrai(cam, holistic)\n",
" cv2.putText(imagem,'Sinal {}, video numero {}'.format(sinal, video),(20, 40),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2) \n",
" cv2.putText(imagem,'Tecle barra para gravar',(20, 80),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2) \n",
" cv2.imshow('LIBRAS', imagem)\n",
" if cv2.waitKey(10) & 0xFF == ord(' '):\n",
" break\n",
"\n",
" # Contagem regressiva\n",
" contador = 3\n",
" inicio = time.time()\n",
" while contador > 0:\n",
" imagem, resultados = mediapipe_desenha_extrai(cam, holistic)\n",
" cv2.putText(imagem,'Preparando pra gravar: {}'.format(contador),(20, 40),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2)\n",
" cv2.imshow('LIBRAS', imagem)\n",
" cv2.waitKey(10)\n",
" final = time.time()\n",
" if final - inicio >= 1:\n",
" contador-=1\n",
" inicio=final\n",
"\n",
" # Gravando\n",
" for frame_num in range(num_frames):\n",
"\n",
" # Desenha pontos na imagem e extrai características em resultados\n",
" imagem, resultados = mediapipe_desenha_extrai(cam, holistic)\n",
"\n",
" # Mostra na tela\n",
" cv2.putText(imagem,'Sinal {}, video numero {}'.format(sinal, video),(20, 40),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2) \n",
" cv2.putText(imagem,'Gravando...',(20, 80),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 0, 255), thickness = 2) \n",
" cv2.imshow('LIBRAS', imagem)\n",
" cv2.waitKey(10)\n",
"\n",
" # Trata os pontos e arquiva\n",
" pontos_tratados = trata_pontos(resultados)\n",
" nome_arquivo = os.path.join(DATA_DIR, sinal, str(video), str(frame_num))\n",
" np.save(nome_arquivo, pontos_tratados)\n",
"\n",
" # Gravar novamente\n",
" while cam.isOpened():\n",
" # Desenha pontos na imagem e extrai características em resultados\n",
" imagem, resultados = mediapipe_desenha_extrai(cam, holistic)\n",
" cv2.putText(imagem,'Sinal {}, video numero {}'.format(sinal, video),(20, 40),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2) \n",
" cv2.putText(imagem,'Para repetir, tecle \"r\"',(20, 80),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2)\n",
" cv2.putText(imagem,'Para prosseguir, tecle barra',(20, 120),fontFace=cv2.FONT_HERSHEY_SIMPLEX,fontScale=1,color=(0, 255, 255), thickness = 2) \n",
" cv2.imshow('LIBRAS', imagem)\n",
" k = cv2.waitKey(10)\n",
" if k & 0xFF == ord('r'):\n",
" video-=1\n",
" break\n",
" if k & 0xFF == ord(' '):\n",
" break\n",
" video +=1\n",
"\n",
"cam.release()\n",
"cv2.destroyAllWindows()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Zipando a pasta dos dados coletados\n",
"import shutil\n",
"shutil.make_archive('Dados_extraidos', 'zip', 'Dados_extraidos')"
]
}
],
"metadata": {
"interpreter": {
"hash": "362f9b98393e3f0a487724f0683d667170a7d1c551755b1686e5e3f1c636f922"
},
"kernelspec": {
"display_name": "Python 3.9.5 64-bit",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Loading

0 comments on commit 0b763b7

Please sign in to comment.