Merge pull request #14 from Capstone-Projects-2022-Spring/ASLIntegration-DynamicPageUpdates

updated model integration

Author: tun60968

__pycache__/detection.cpython-39.pyc

@@ -1,11 +1,12 @@
+import os
 import tensorflow as tf
 from object_detection.utils import config_util
-import os
 from object_detection.utils import label_map_util
 from object_detection.utils import visualization_utils as viz_utils
 from object_detection.builders import model_builder
-import cv2 
+import cv2
 import numpy as np
+import keyboard
 from playsound import playsound
 
 WORKSPACE_PATH = 'RealTimeObjectDetection/Tensorflow/workspace'
@@ -17,7 +18,7 @@
 PRETRAINED_MODEL_PATH = WORKSPACE_PATH+'/pre-trained-models'
 CONFIG_PATH = MODEL_PATH+'/my_ssd_mobnet/pipeline.config'
 CHECKPOINT_PATH = MODEL_PATH+'/my_ssd_mobnet/'
-CUSTOM_MODEL_NAME = 'my_ssd_mobnet' 
+CUSTOM_MODEL_NAME = 'my_ssd_mobnet'
 CONFIG_PATH = MODEL_PATH+'/'+CUSTOM_MODEL_NAME+'/pipeline.config'
 
 config = config_util.get_configs_from_pipeline_file(CONFIG_PATH)
@@ -30,10 +31,43 @@
 height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
 print("camera created")
 
+def read_label_map(label_map_path):
+    item_id = None
+    item_name = None
+    items = {}
+
+    with open(label_map_path, "r") as file:
+        for line in file:
+            line.replace(" ", "")
+            if line == "item{":
+                pass
+            elif line == "}":
+                pass
+            elif "id" in line:
+                item_id = int(line.split(":", 1)[1].strip())-1
+            elif "name" in line:
+                item_name = line.split(":", 1)[1].replace("'", "").strip()
+
+            if item_id is not None and item_name is not None:
+                items[item_id] = item_name
+                item_id = None
+                item_name = None
+
+    return items
+
 ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)
 ckpt.restore(os.path.join(CHECKPOINT_PATH, 'ckpt-6')).expect_partial()
 category_index = label_map_util.create_category_index_from_labelmap(ANNOTATION_PATH+'/label_map.pbtxt')
-print("checkpoints reached")
+label_map = read_label_map(ANNOTATION_PATH+'/label_map.pbtxt')
+
+class myWords:
+    wordList = []
+    modelRunning = 0
+
+    def addWord(self, word):
+        self.wordList.append(word)
+    def resetList(self):
+        self.wordList = []
 
 def getCamera():
     return cap
@@ -45,12 +79,14 @@ def detect_fn(image):
     detections = detection_model.postprocess(prediction_dict, shapes)
     return detections
 
-def activateModel():
+def activateModel(m: myWords):
+    oldWord = ''
+
     while True:
         ret, frame = cap.read()
-        frame = cv2.flip(frame, 1)
+        frame2 = cv2.flip(frame, 1)
 
-        image_np = np.array(frame)
+        image_np = np.array(frame2)
         input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)
         detections = detect_fn(input_tensor)
         num_detections = int(detections.pop('num_detections'))
@@ -60,48 +96,59 @@ def activateModel():
         detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
         label_id_offset = 1
         image_np_with_detections = image_np.copy()
-
         res = [(i, j) for i, j in zip(detections['detection_classes'], detections['detection_scores'])]
+
         for i in res:
                 if i[1] >= .50:
                     gesture_id = int(i[0])
-                    if gesture_id == 0:
-                        playsound("RealTimeObjectDetection/hello_en.mp3")
-                    if gesture_id == 1:
-                        playsound("RealTimeObjectDetection/help_en.mp3")
-                    if gesture_id == 2:
-                        playsound("RealTimeObjectDetection/yes_en.mp3")
-                    if gesture_id == 3:
-                        playsound("RealTimeObjectDetection/no_en.mp3")
-                    if gesture_id == 4:
-                        playsound("RealTimeObjectDetection/i_en.mp3")
-                    if gesture_id == 5:
-                        playsound("RealTimeObjectDetection/i_love_you_en.mp3")
-                    if gesture_id == 6:
-                        playsound("RealTimeObjectDetection/stand_en.mp3")
-                    if gesture_id == 7:
-                        playsound("RealTimeObjectDetection/telephone_en.mp3")
-                    if gesture_id == 8:
-                        playsound("RealTimeObjectDetection/mom_en.mp3")
-                    if gesture_id == 9:
-                        playsound("RealTimeObjectDetection/thank_you_en.mp3")
+
+                    newWord = label_map.get(gesture_id)
+                    if oldWord != newWord:
+                        m.addWord(newWord)
+                        oldWord = newWord
+
+
+                    # if gesture_id == 0:
+                    #     playsound("hello_en.mp3")
+                    # if gesture_id == 1:
+                    #     playsound("help_en.mp3")
+                    # if gesture_id == 2:
+                    #     playsound("yes_en.mp3")
+                    # if gesture_id == 3:
+                    #     playsound("no_en.mp3")
+                    # if gesture_id == 4:
+                    #     playsound("i_en.mp3")
+                    # if gesture_id == 5:
+                    #     playsound("i_love_you_en.mp3")
+                    # if gesture_id == 6:
+                    #     playsound("stand_en.mp3")
+                    # if gesture_id == 7:
+                    #     playsound("telephone_en.mp3")
+                    # if gesture_id == 8:
+                    #     playsound("mom_en.mp3")
+                    # if gesture_id == 9:
+                    #     playsound("thank_you_en.mp3")
 
         viz_utils.visualize_boxes_and_labels_on_image_array(
-                    image_np_with_detections,
-                    detections['detection_boxes'],
-                    detections['detection_classes']+label_id_offset,
-                    detections['detection_scores'],
-                    category_index,
-                    use_normalized_coordinates=True,
-                    max_boxes_to_draw=5,
-                    min_score_thresh=.5,
-                    agnostic_mode=False)
+            image_np_with_detections,
+            detections['detection_boxes'],
+            detections['detection_classes'] + label_id_offset,
+            detections['detection_scores'],
+            category_index,
+            use_normalized_coordinates=True,
+            max_boxes_to_draw=5,
+            min_score_thresh=.5,
+            agnostic_mode=False)
 
         # cv2.imshow('object detection',  cv2.resize(image_np_with_detections, (800, 600)))
         ret, buffer = cv2.imencode('.jpg', image_np_with_detections)
-        frame = buffer.tobytes()
-        yield (b'--frame\r\n'b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
+        frame3 = buffer.tobytes()
+        yield (b'--frame\r\n'b'Content-Type: image/jpeg\r\n\r\n' + frame3 + b'\r\n')
 
-        if cv2.waitKey(1) & 0xFF == ord('q'):
-            cap.release()
-            break
+        try:
+            if keyboard.is_pressed('q'):
+                print("exiting model")
+                cap.release()
+                return m
+        except:
+            pass

detection.py

@@ -4,11 +4,10 @@
 
 db = SQLAlchemy()
 
+
 def create_app():
-    
     # Create the Object of Flask
     app = Flask(__name__)
-    #app.config['SECRET_KEY'] = 'as;dlfkja;sdlkjf'
 
     # Load the Configuration from the config.py file
     app.config.from_object(Config)
@@ -17,7 +16,6 @@ def create_app():
     db.init_app(app)
 
     with app.app_context():
-
         # Import Views and Auth (Routes)
         from .views import views
         from .auth import auth
@@ -29,4 +27,4 @@ def create_app():
         # Create SQL Tables for our Data Models
         db.create_all()
 
-        return app
+        return app

website/__init__.py

@@ -15,8 +15,6 @@
 
         <!-- video.js base CSS -->
         <link href="https://vjs.zencdn.net/7.18.1/video-js.css" rel="stylesheet" />
-
-
         <style>
             ::-moz-selection { /* Code for Firefox */
                 color: white;
@@ -94,11 +92,7 @@
 
 
         {% endblock %}
-         <script
-            src="https://code.jquery.com/jquery-3.5.1.slim.min.js"
-            integrity="sha384-DfXdz2htPH0lsSSs5nCTpuj/zy4C+OGpamoFVy38MVBnE+IbbVYUew+OrCXaRkfj"
-            crossorigin="anonymous"
-        ></script>
+         <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
 
 
         <script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.9.2/dist/umd/popper.min.js" integrity="sha384-IQsoLXl5PILFhosVNubq5LC7Qb9DXgDA9i+tQ8Zj3iwWAwPtgFTxbJ8NT4GN1R8p" crossorigin="anonymous"></script>

website/__pycache__/__init__.cpython-39.pyc

@@ -7,32 +7,97 @@
 {% endblock %}
 
 {% block content%}
-<div class="row">
-    <div class="col align-items-start">
-        <form method="post">
-            <input type="submit" class="btn btn-success btn-lg" value="Stop Translation" name="stop"/>
-        </form>
 
-        <div class="form-check form-switch">
-          <input class="form-check-input" type="checkbox" role="switch" id="textSwitch" checked>
-          <label class="form-check-label" for="textSwitch">Text output</label>
-        </div>
-
-        <div class="form-check form-switch">
-          <input class="form-check-input" type="checkbox" role="switch" id="audioSwitch">
-          <label class="form-check-label" for="audioSwitch">Audio output</label>
-        </div>
-    </div>
+<div class="row">
+    <div class="col align-items-start"></div>
     <div class="col align-items-center">
+        {% if translating and wordsModel %}
+<!--        <h2>Press 'q' to end the translation</h2>-->
         <img src="{{ url_for('views.video_feed') }}">
+        {% elif translating %}
+        this is the view for the alphabet translator
+        {% endif %}
+
     </div>
+
     <div class="col align-items-end">
-        <textarea class="form-control" id="exampleFormControlTextarea1" rows="3" disabled readonly>This is where the output text generated by the ASL interpretation will be printed.
-        </textarea>
+        {% if translating %}
+        <h3>Translation output</h3>
+        <textarea class="form-control" id="words" rows="5" disabled readonly></textarea>
+        <button type="button" id="audio" class="btn btn-primary" onclick="getAudio()">Play Audio</button>
+        {% endif %}
     </div>
 </div>
 
-
+<div class="container">
+  <div class="row">
+    <div class="col text-center">
+        <form method="POST">
+        {% if not translating %}
+        <input type="submit" class="btn btn-lg btn-success" name="translateButton" value="Words">
+        <input type="submit" class="btn btn-lg btn-success" name="translateButton" value="Alphabet">
+        {% elif translating %}
+        <input type="submit" class="btn btn-lg btn-danger" name="translateButton" value="Restart Translation">
+        {% endif %}
+        </form>
+    </div>
+  </div>
+</div>
 
 {% endblock %}
 
+
+{% block script %}
+<script>
+   $('#launchModal').on('click', function (e) {
+
+        $("#stopTranslate").modal('show');
+
+    });
+
+    var myTimer = setInterval(updateText, 3000);
+
+    function updateText(){
+        var updatedText = $.get("{{ url_for('views.get_words') }}");
+
+        updatedText.done(function(results) {
+            var result = "";
+            results.results.forEach(item => {
+                item = item.replaceAll("_", " ");
+                result = result.concat(" ", item);
+            });
+            $('#words').text(result);
+        });
+
+    }
+
+    function getAudio(){
+        var updatedText = $.get("{{ url_for('views.get_words') }}");
+
+        updatedText.done(function(results) {
+            var result = "";
+
+            results.results.forEach(item => {
+                item = item.replaceAll("_", " ");
+                result = result.concat(" ", item);
+            });
+
+            console.log(result)
+            var msg = new SpeechSynthesisUtterance('Translated output:'+result);
+            window.speechSynthesis.speak(msg);
+
+        })
+    }
+
+    document.addEventListener('keypress', (event) => {
+      var name = event.key;
+      var code = event.code;
+      // Alert the key name and key code on keydown
+      console.log(`Key pressed ${name} \r\n Key code value: ${code}`);
+      if (name === 'q') {
+        clearInterval(myTimer);
+      }
+    }, false);
+
+</script>
+{% endblock %}

website/__pycache__/views.cpython-39.pyc

@@ -23,20 +23,10 @@
 
 {% block content %}
 <div class="d-grid gap-4 col-5 mx-auto d-md-block">
-  <button class="btn btn-primary btn-lg" type="submit" onclick="location.href = '/translate/fromASL';">Translate from ASL</button>
-  <button class="btn btn-primary btn-lg" type="submit" onclick="location.href = '/translate/toASL';">Translate to ASL</button>
-  <button class="btn btn-primary btn-lg" type="submit" onclick="location.href = '/translate/audio';">Audio to Text</button>
+  <button class="btn btn-primary btn-lg" type="submit" onclick="location.href = '/translate/fromASL';">ASL to Text</button>
+  <button class="btn btn-primary btn-lg" type="submit" onclick="location.href = '/translate/toASL';">Text to ASL</button>
+  <button class="btn btn-primary btn-lg" type="submit" onclick="location.href = '/translate/audio';">Audio to ASL</button>
   <button class="btn btn-primary btn-lg" type="submit" onclick="location.href = '/translate/connect';">Connect Online</button>
 </div>
 
-<!--<div class="container">
-    <div class="row">
-        <div class="col-lg-8  offset-lg-2">
-            <form method="post">
-                <input type="submit" value="Start Translation" />
-            </form>
-            <img src="{{ url_for('views.video_feed') }}">
-        </div>
-    </div>
-</div>-->
 {% endblock %}