Merge pull request #21 from Capstone-Projects-2022-Spring/ConnectOnline-ChatMessage

Connect online chat message

Author: ArchaePi

__pycache__/camera.cpython-39.pyc

@@ -8,6 +8,7 @@
 import tensorflow as tf
 from object_detection.utils import visualization_utils as viz_utils
 from alphabet import letterModel
+import nltk
 
 def base64ToOpenCV(base64_img):
     im_bytes = base64.b64decode(base64_img)
@@ -24,16 +25,17 @@ def openCVTobase64(opencv_img):
     return im_b64
 
 class Camera(object):
-    def __init__(self):
+    def __init__(self, w: wordsModel, l: letterModel):
         self.to_process = []
         self.to_output = []
         self.wordList = []
         self.modelType = 0
+        self.lastTen = []
 
         # self.cam = cv2.VideoCapture(0)
 
-        self.words = wordsModel()
-        self.letters = letterModel()
+        self.words = w
+        self.letters = l
 
         thread = threading.Thread(target=self.keep_processing, args=())
         thread.daemon = True
@@ -44,6 +46,7 @@ def restartModel(self):
         self.to_process.clear()
 
     def updateModel(self, num):
+        self.lastTen.clear()
         if num < 1:
             self.modelType = 0
         else:
@@ -190,30 +193,37 @@ def processLettersModel(self, input_img):
 
         newLetter = self.letters.getLetter(lmList, handsType)
 
-        if newLetter == '':
-            pass
-        elif len(self.wordList) == 0:
-            # there is nothing in the word list currently - this is the first entry
-            if (newLetter != ' ') and (newLetter != '*'):
-                self.wordList.append(newLetter)
-        else:
-            oldWord = next(reversed(self.wordList))
-            if len(oldWord) > 0:
-                oldLetter = oldWord[-1]
+        if len(self.lastTen) < 10:
+            self.lastTen.append(newLetter)
+        else: # this is the case that it is at 10 frames
+            frequency_distribution = nltk.FreqDist(self.lastTen)
+            newLetter = frequency_distribution.max()
+            self.lastTen.clear()
+
+            if newLetter == '':
+                pass
+            elif len(self.wordList) == 0:
+                # there is nothing in the word list currently - this is the first entry
+                if (newLetter != ' ') and (newLetter != '*'):
+                    self.wordList.append(newLetter)
             else:
-                oldLetter = ''
-
-            if oldLetter != newLetter:
-                string = oldWord+newLetter
-                if newLetter == '*':
-                    string = string[:-2]
-                self.wordList[len(self.wordList)-1] = string
-
-        if newLetter == ' ':
-            newLetter = 'SPACE'
-        elif newLetter == '*':
-            newLetter = 'DELETE'
-
-        cv2.putText(frame, newLetter, (0, 70), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 0), 3)
+                oldWord = next(reversed(self.wordList))
+                if len(oldWord) > 0:
+                    oldLetter = oldWord[-1]
+                else:
+                    oldLetter = ''
+
+                if oldLetter != newLetter:
+                    string = oldWord+newLetter
+                    if newLetter == '*':
+                        string = string[:-2]
+                    self.wordList[len(self.wordList)-1] = string
+
+            if newLetter == ' ':
+                newLetter = 'SPACE'
+            elif newLetter == '*':
+                newLetter = 'DELETE'
+
+            cv2.putText(frame, newLetter, (0, 70), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 0), 3)
 
         return frame

__pycache__/detection.cpython-39.pyc

@@ -11,5 +11,5 @@
     #app.run(debug=True)
 
     # Run the Flask Application using SocketIO
-    socketio.run(app, debug=True)
+    socketio.run(app, debug=True, port=5000)
 

__pycache__/text_to_asl.cpython-39.pyc

@@ -1,7 +1,16 @@
 import json, os, requests, string
+from nltk.corpus import wordnet as wn
+from nltk.stem.wordnet import WordNetLemmatizer
 from nltk.tokenize import word_tokenize
+from nltk import pos_tag
+from collections import defaultdict
 from bs4 import BeautifulSoup
 
+tag_map = defaultdict(lambda: wn.NOUN)
+tag_map['J'] = wn.ADJ
+tag_map['V'] = wn.VERB
+tag_map['R'] = wn.ADV
+
 def getVideoPath(s):
 
     # split input string into tokens
@@ -20,8 +29,15 @@ def getTokensFromString(s):
     s = s.translate(translator)
     tokenized = word_tokenize(s)
 
+    lemmatized = []
+
+    lemma_function = WordNetLemmatizer()
+    for token, tag in pos_tag(tokenized):
+        lemma = lemma_function.lemmatize(token, tag_map[tag[0]])
+        lemmatized.append(lemma)
+
     # should remove articles / useless words from list first
-    return tokenized
+    return lemmatized
 
 def getVideosFromTokens(tokens):
     URL = "https://www.signasl.org/sign/"

camera.py

@@ -81,6 +81,9 @@ def check_password(self, password):
         # Check Hashed Password
         return check_password_hash(self.user_password, password)
 
+    def update_username(self, username):
+        self.username = username
+
     def __repr__(self):
 
         return 'User {}'.format(self.username)

main.py

@@ -0,0 +1,102 @@
+var myVideo, myCanvas;
+var namespace = "/chat";
+var localMediaStream = null;
+var mySID;
+$(document).ready(function(){
+
+    var socket = io(location.protocol + '//' + document.domain + ':' + location.port + namespace, {autoConnect: false});
+
+    myVideo = document.getElementById("videoStream");
+    myCanvas = document.getElementById("canvasElement");
+    var chatLog = document.getElementById('scrollable');
+
+    function sendSnapshot() {
+        if (!localMediaStream) {
+          return;
+        }
+
+        myCanvas.getContext('2d').drawImage(myVideo, 0, 0, myVideo.videoWidth, myVideo.videoHeight, 0, 0, 300, 150);
+
+        let dataURL = myCanvas.toDataURL('image/jpeg');
+        socket.emit('input image', dataURL);
+    }
+
+    socket.on('connect', function() {
+        socket.emit('join-room', {});
+        console.log('Client connected!');
+    });
+
+    socket.on('get-sid', function(data) {
+        mySID = data.sid;
+        console.log(mySID);
+    });
+
+    socket.on('status', function(data) {
+        var newText = document.createTextNode(data.msg);
+        chatLog.appendChild(newText);
+
+        $('#scrollable').animate({ scrollTop: 		$('#scrollable').prop('scrollHeight')}, 1000);
+    });
+
+    socket.on('output-message', function(data) {
+        var newText = document.createTextNode(data.msg);
+        chatLog.appendChild(newText);
+
+        $('#scrollable').animate({ scrollTop: 		$('#scrollable').prop('scrollHeight')}, 1000);
+
+        let sid = data.sid;
+        if (sid !== mySID){
+            getVideoPlaylist(data.msg);
+        }
+    });
+
+    var constraints = {
+        video: {
+          width: { min: 640 },
+          height: { min: 480 }
+        }
+      };
+
+    navigator.mediaDevices.getUserMedia(constraints).then(function(stream) {
+        myVideo.srcObject = stream;
+        localMediaStream = stream;
+        socket.connect()
+
+        setInterval(function () {
+          sendSnapshot();
+        }, 100);
+      }).catch(function(error) {
+        console.log(error);
+      });
+
+    $('#send').click(function (){
+        message = document.getElementById("words").innerText;
+        clearList();
+        socket.emit('input-message', {'msg': message});
+    });
+
+    $('#audio').click(function (){
+        getAudio();
+    });
+
+    $('#clear').click(function (){
+        clearList();
+    });
+
+    $('#restart').click(function (){
+        restart();
+    });
+
+    $('#leave').click(function (){
+        socket.emit('leave-room', {});
+        setTimeout(() => { location.href = '/'; }, 1);
+
+    });
+
+    var myTimer = setInterval(updateText, 2000);
+
+
+
+});
+
+