Merge pull request #13 from Capstone-Projects-2022-Spring/ASLIntegration

model integrated into web app

Author: tun60968

.idea/vcs.xml

@@ -1,26 +1,16 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-# In[ ]:
-
-
 import tensorflow as tf
 from object_detection.utils import config_util
-from object_detection.protos import pipeline_pb2
-from google.protobuf import text_format
 import os
 from object_detection.utils import label_map_util
 from object_detection.utils import visualization_utils as viz_utils
 from object_detection.builders import model_builder
 import cv2 
 import numpy as np
-from gtts import gTTS
 from playsound import playsound
-from matplotlib import pyplot as plt
 
-WORKSPACE_PATH = 'Tensorflow/workspace'
-SCRIPTS_PATH = 'Tensorflow/scripts'
-APIMODEL_PATH = 'Tensorflow/models'
+WORKSPACE_PATH = 'RealTimeObjectDetection/Tensorflow/workspace'
+SCRIPTS_PATH = 'RealTimeObjectDetection/Tensorflow/scripts'
+APIMODEL_PATH = 'RealTimeObjectDetection/Tensorflow/models'
 ANNOTATION_PATH = WORKSPACE_PATH+'/annotations'
 IMAGE_PATH = WORKSPACE_PATH+'/images'
 MODEL_PATH = WORKSPACE_PATH+'/models'
@@ -33,16 +23,20 @@
 config = config_util.get_configs_from_pipeline_file(CONFIG_PATH)
 configs = config_util.get_configs_from_pipeline_file(CONFIG_PATH)
 detection_model = model_builder.build(model_config=configs['model'], is_training=False)
-
-phrase_buffer = []
+print("model built")
 
 cap = cv2.VideoCapture(0)
 width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
 height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-
+print("camera created")
 
 ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)
 ckpt.restore(os.path.join(CHECKPOINT_PATH, 'ckpt-6')).expect_partial()
+category_index = label_map_util.create_category_index_from_labelmap(ANNOTATION_PATH+'/label_map.pbtxt')
+print("checkpoints reached")
+
+def getCamera():
+    return cap
 
 @tf.function
 def detect_fn(image):
@@ -51,71 +45,63 @@ def detect_fn(image):
     detections = detection_model.postprocess(prediction_dict, shapes)
     return detections
 
-category_index = label_map_util.create_category_index_from_labelmap(ANNOTATION_PATH+'/label_map.pbtxt')
-
-while True:
-    ret, frame = cap.read()
-    image_np = np.array(frame)
-    input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)
-    detections = detect_fn(input_tensor)
-    num_detections = int(detections.pop('num_detections'))
-    detections = {key: value[0, :num_detections].numpy()
-                  for key, value in detections.items()}
-    detections['num_detections'] = num_detections
-    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
-    label_id_offset = 1
-    image_np_with_detections = image_np.copy()
-    res = [(i, j) for i, j in zip(detections['detection_classes'], detections['detection_scores'])]
-    for i in res:
-            if i[1] >= .50:
-                gesture_id = int(i[0])
-                if gesture_id == 0:
-                    playsound("hello_en.mp3")
-                if gesture_id == 1:
-                    playsound("help_en.mp3")
-                if gesture_id == 2:
-                    playsound("yes_en.mp3")
-                if gesture_id == 3:
-                    playsound("no_en.mp3")
-                if gesture_id == 4:
-                    playsound("i_en.mp3")
-                if gesture_id == 5:
-                    playsound("i_love_you_en.mp3")
-                if gesture_id == 6:
-                    playsound("stand_en.mp3")
-                if gesture_id == 7:
-                    playsound("telephone_en.mp3")
-                if gesture_id == 8:
-                    playsound("mom_en.mp3")
-                if gesture_id == 9:
-                    playsound("thank_you_en.mp3")
-                    
-    viz_utils.visualize_boxes_and_labels_on_image_array(
-                image_np_with_detections,
-                detections['detection_boxes'],
-                detections['detection_classes']+label_id_offset,
-                detections['detection_scores'],
-                category_index,
-                use_normalized_coordinates=True,
-                max_boxes_to_draw=5,
-                min_score_thresh=.5,
-                agnostic_mode=False)
-
-    cv2.imshow('object detection',  cv2.resize(image_np_with_detections, (800, 600)))
-    
-    if cv2.waitKey(1) & 0xFF == ord('q'):
-        cap.release()
-        break
-
-
-# In[ ]:
-
-
-
-
-
-# In[ ]:
-
-
-
-
+def activateModel():
+    while True:
+        ret, frame = cap.read()
+        frame = cv2.flip(frame, 1)
+
+        image_np = np.array(frame)
+        input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)
+        detections = detect_fn(input_tensor)
+        num_detections = int(detections.pop('num_detections'))
+        detections = {key: value[0, :num_detections].numpy()
+                      for key, value in detections.items()}
+        detections['num_detections'] = num_detections
+        detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
+        label_id_offset = 1
+        image_np_with_detections = image_np.copy()
+
+        res = [(i, j) for i, j in zip(detections['detection_classes'], detections['detection_scores'])]
+        for i in res:
+                if i[1] >= .50:
+                    gesture_id = int(i[0])
+                    if gesture_id == 0:
+                        playsound("RealTimeObjectDetection/hello_en.mp3")
+                    if gesture_id == 1:
+                        playsound("RealTimeObjectDetection/help_en.mp3")
+                    if gesture_id == 2:
+                        playsound("RealTimeObjectDetection/yes_en.mp3")
+                    if gesture_id == 3:
+                        playsound("RealTimeObjectDetection/no_en.mp3")
+                    if gesture_id == 4:
+                        playsound("RealTimeObjectDetection/i_en.mp3")
+                    if gesture_id == 5:
+                        playsound("RealTimeObjectDetection/i_love_you_en.mp3")
+                    if gesture_id == 6:
+                        playsound("RealTimeObjectDetection/stand_en.mp3")
+                    if gesture_id == 7:
+                        playsound("RealTimeObjectDetection/telephone_en.mp3")
+                    if gesture_id == 8:
+                        playsound("RealTimeObjectDetection/mom_en.mp3")
+                    if gesture_id == 9:
+                        playsound("RealTimeObjectDetection/thank_you_en.mp3")
+
+        viz_utils.visualize_boxes_and_labels_on_image_array(
+                    image_np_with_detections,
+                    detections['detection_boxes'],
+                    detections['detection_classes']+label_id_offset,
+                    detections['detection_scores'],
+                    category_index,
+                    use_normalized_coordinates=True,
+                    max_boxes_to_draw=5,
+                    min_score_thresh=.5,
+                    agnostic_mode=False)
+
+        # cv2.imshow('object detection',  cv2.resize(image_np_with_detections, (800, 600)))
+        ret, buffer = cv2.imencode('.jpg', image_np_with_detections)
+        frame = buffer.tobytes()
+        yield (b'--frame\r\n'b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
+
+        if cv2.waitKey(1) & 0xFF == ord('q'):
+            cap.release()
+            break

RealTimeObjectDetection

@@ -0,0 +1,195 @@
+absl-py==1.0.0
+apache-beam==2.37.0
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+asttokens==2.0.5
+astunparse==1.6.3
+attrs==21.4.0
+avro-python3==1.10.2
+backcall==0.2.0
+beautifulsoup4==4.10.0
+bleach==4.1.0
+cachetools==5.0.0
+certifi==2021.10.8
+cffi==1.15.0
+charset-normalizer==2.0.12
+click==7.1.2
+cloudpickle==2.0.0
+colorama==0.4.4
+contextlib2==21.6.0
+crcmod==1.7
+cryptography==36.0.0
+cycler==0.11.0
+Cython==0.29.28
+debugpy==1.6.0
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.1.1
+dm-tree==0.1.6
+dnspython==2.2.1
+docopt==0.6.2
+docutils==0.18.1
+email-validator==1.1.3
+entrypoints==0.4
+executing==0.8.3
+fastavro==1.4.10
+Flask==1.1.2
+Flask-SQLAlchemy==2.5.1
+Flask-WTF==0.14.3
+flatbuffers==2.0
+fonttools==4.31.2
+gast==0.5.3
+gin-config==0.5.0
+google-api-core==2.7.1
+google-api-python-client==2.42.0
+google-auth==2.6.2
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.6
+google-pasta==0.2.0
+googleapis-common-protos==1.56.0
+greenlet==1.1.2
+grpcio==1.44.0
+gTTS==2.2.4
+h5py==3.6.0
+hdfs==2.7.0
+httplib2==0.19.1
+idna==3.3
+importlib-metadata==4.11.3
+ipykernel==6.10.0
+ipython==8.2.0
+ipython-genutils==0.2.0
+ipywidgets==7.7.0
+itsdangerous==1.1.0
+jedi==0.18.1
+Jinja2==2.11.2
+joblib==1.1.0
+jsonschema==4.4.0
+jupyter==1.0.0
+jupyter-client==7.1.2
+jupyter-console==6.4.3
+jupyter-core==4.9.2
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.1.0
+kaggle==1.5.12
+keras==2.8.0
+Keras-Preprocessing==1.1.2
+keyring==23.5.0
+kiwisolver==1.4.1
+libclang==13.0.0
+lvis==0.5.3
+lxml==4.8.0
+Markdown==3.3.6
+MarkupSafe==2.0.1
+matplotlib==3.5.1
+matplotlib-inline==0.1.3
+mistune==0.8.4
+nbclient==0.5.13
+nbconvert==6.4.5
+nbformat==5.2.0
+nest-asyncio==1.5.4
+nltk==3.5
+notebook==6.4.10
+numpy==1.22.3
+oauth2client==4.1.3
+oauthlib==3.2.0
+object-detection==0.1
+opencv-python==4.5.5
+opencv-python-headless==4.5.5.64
+opt-einsum==3.3.0
+orjson==3.6.7
+packaging==21.3
+pandas==1.4.1
+pandocfilters==1.5.0
+parso==0.8.3
+pickleshare==0.7.5
+Pillow==9.0.1
+pip==21.2.4
+pkginfo==1.8.2
+playsound==1.3.0
+portalocker==2.4.0
+prometheus-client==0.13.1
+promise==2.3
+prompt-toolkit==3.0.28
+proto-plus==1.20.3
+protobuf==3.19.4
+psutil==5.9.0
+pure-eval==0.2.2
+py-cpuinfo==8.0.0
+pyarrow==6.0.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycocotools==2.0
+pycparser==2.21
+pydot==1.4.2
+Pygments==2.11.2
+pymongo==3.12.3
+PyMySQL==1.0.2
+pyparsing==2.4.7
+pyrsistent==0.18.1
+python-dateutil==2.8.2
+python-slugify==6.1.1
+pytz==2022.1
+pywin32==303
+pywin32-ctypes==0.2.0
+pywinpty==2.0.5
+PyYAML==5.4.1
+pyzmq==22.3.0
+qtconsole==5.3.0
+QtPy==2.0.1
+readme-renderer==34.0
+regex==2022.3.15
+requests==2.27.1
+requests-oauthlib==1.3.1
+requests-toolbelt==0.9.1
+rfc3986==2.0.0
+rsa==4.8
+sacrebleu==2.0.0
+scikit-learn==1.0.2
+scipy==1.8.0
+Send2Trash==1.8.0
+sentencepiece==0.1.96
+seqeval==1.2.2
+setuptools==61.2.0
+six==1.16.0
+soupsieve==2.3.1
+SQLAlchemy==1.4.32
+stack-data==0.2.0
+tabulate==0.8.9
+tensorboard==2.8.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorflow==2.8.0
+tensorflow-addons==0.16.1
+tensorflow-datasets==4.5.2
+tensorflow-hub==0.12.0
+tensorflow-io==0.24.0
+tensorflow-io-gcs-filesystem==0.24.0
+tensorflow-metadata==1.7.0
+tensorflow-model-optimization==0.7.2
+tensorflow-object-detection-api==0.1.1
+tensorflow-text==2.8.1
+termcolor==1.1.0
+terminado==0.13.3
+testpath==0.6.0
+text-unidecode==1.3
+tf-estimator-nightly==2.8.0.dev2021122109
+tf-models-official==2.8.0
+tf-slim==1.1.0
+threadpoolctl==3.1.0
+tornado==6.1
+tqdm==4.63.1
+traitlets==5.1.1
+twine==3.8.0
+typeguard==2.13.3
+typing_extensions==4.1.1
+uritemplate==4.1.1
+urllib3==1.26.9
+wcwidth==0.2.5
+webencodings==0.5.1
+Werkzeug==2.0.3
+wheel==0.37.1
+widgetsnbextension==3.6.0
+wincertstore==0.2
+wrapt==1.14.0
+WTForms==2.3.3
+zipp==3.7.0