Additional debugging commands

Author: breandan

META-INF/plugin.xml

@@ -4,15 +4,10 @@
     <version>1.0</version>
     <vendor email="breandan.considine@jetbrains.com" url="http://www.jetbrains.com">JetBrains</vendor>
 
-    <description><![CDATA[
-      Enter short description for your plugin here.<br>
-      <em>most HTML tags may be used</em>
-    ]]></description>
-
-    <change-notes><![CDATA[
-      Add change notes here.<br>
-      <em>most HTML tags may be used</em>
-    ]]>
+    <description>Speech-based user interface for your favorite IDE. Idear listens to voice commands and performs basic actions like navigating the IDE.</description>
+
+    <change-notes>
+
     </change-notes>
 
     <!-- please see https://confluence.jetbrains.com/display/IDEADEV/Build+Number+Ranges for description -->
@@ -29,19 +24,19 @@
 
     <extensions defaultExtensionNs="com.intellij">
 
-        <applicationService serviceInterface="com.jetbrains.idear.asr.ASRService" serviceImplementation="com.jetbrains.idear.asr.ASRServiceImpl">
+        <applicationService serviceInterface="com.jetbrains.idear.asr.ASRService" serviceImplementation="com.jetbrains.idear.asr.ASRService">
         </applicationService>
 
-        <applicationService serviceInterface="com.jetbrains.idear.tts.TTSService" serviceImplementation="com.jetbrains.idear.tts.TTSServiceImpl">
+        <applicationService serviceInterface="com.jetbrains.idear.tts.TTSService" serviceImplementation="com.jetbrains.idear.tts.TTSService">
         </applicationService>
 
-        <applicationService serviceInterface="com.jetbrains.idear.asr.GrammarService" serviceImplementation="com.jetbrains.idear.asr.GrammarServiceImpl">
+        <applicationService serviceInterface="com.jetbrains.idear.asr.GrammarService" serviceImplementation="com.jetbrains.idear.asr.GrammarService">
         </applicationService>
 
         <applicationService serviceInterface="com.jetbrains.idear.nlp.ParserService" serviceImplementation="com.jetbrains.idear.nlp.NlpParserService">
         </applicationService>
 
-        <applicationService serviceInterface="com.jetbrains.idear.ide.IDEService" serviceImplementation="com.jetbrains.idear.ide.IDEServiceImpl">
+        <applicationService serviceInterface="com.jetbrains.idear.ide.IDEService" serviceImplementation="com.jetbrains.idear.ide.IDEService">
         </applicationService>
 
     </extensions>

src/com/jetbrains/idear/actions/recognition/FindUsagesActionRecognizer.java

@@ -1,17 +1,13 @@
 package com.jetbrains.idear.actions.recognition;
 
-import com.intellij.codeInsight.TargetElementUtil;
-import com.intellij.find.FindManager;
-import com.intellij.find.findUsages.FindUsagesOptions;
 import com.intellij.find.findUsages.PsiElement2UsageTargetAdapter;
 import com.intellij.openapi.actionSystem.*;
 import com.intellij.openapi.actionSystem.impl.SimpleDataContext;
 import com.intellij.openapi.components.ServiceManager;
-import com.intellij.openapi.editor.Document;
 import com.intellij.openapi.editor.Editor;
 import com.intellij.openapi.project.Project;
-import com.intellij.psi.*;
-import com.intellij.psi.util.PsiTreeUtil;
+import com.intellij.psi.PsiClass;
+import com.intellij.psi.PsiElement;
 import com.intellij.usages.UsageTarget;
 import com.intellij.usages.UsageView;
 import com.jetbrains.idear.psi.PsiUtil;

src/com/jetbrains/idear/asr/ASRControlLoop.java

@@ -21,11 +21,13 @@
 import javax.sound.sampled.AudioSystem;
 import javax.sound.sampled.Clip;
 import java.awt.*;
-import java.awt.event.KeyEvent;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import static java.awt.event.KeyEvent.*;
+
 /**
  * Created by breandan on 10/23/2015.
  */
@@ -54,6 +56,7 @@ public ASRControlLoop(CustomLiveSpeechRecognizer recognizer) {
     public static final String SHRINK = "shrink";
     public static final String PRESS = "press";
     public static final String DELETE = "delete";
+    public static final String DEBUG = "debug";
     public static final String ENTER = "enter";
     public static final String ESCAPE = "escape";
     public static final String TAB = "tab";
@@ -105,6 +108,7 @@ private String getResultFromRecognizer() {
         logger.info("\tTop H:       " + result.getResult() + " / " + result.getResult().getBestToken() + " / " + result.getResult().getBestPronunciationResult());
         logger.info("\tTop 3H:      " + result.getNbest(3));
 
+        Collection<String> c = result.getNbest(3);
         return result.getHypothesis();
     }
 
@@ -119,7 +123,7 @@ private void applyAction(String c) {
             } else if (c.endsWith(RECENT)) {
                 ideService.invokeAction(IdeActions.ACTION_RECENT_FILES);
             } else if (c.endsWith(TERMINAL)) {
-                pressKeystroke(KeyEvent.VK_ALT, KeyEvent.VK_F12);
+                pressKeystroke(VK_ALT, VK_F12);
             }
         } else if (c.startsWith(NAVIGATE)) {
             ideService.invokeAction("GotoDeclaration");
@@ -130,53 +134,67 @@ private void applyAction(String c) {
             ideService.invokeAction("Idear.WhereAmI");
         } else if (c.startsWith("focus")) {
             if (c.endsWith(EDITOR)) {
-                pressKeystroke(KeyEvent.VK_ESCAPE);
+                pressKeystroke(VK_ESCAPE);
             } else if (c.endsWith(PROJECT)) {
-                pressKeystroke(KeyEvent.VK_ALT, KeyEvent.VK_1);
+                pressKeystroke(VK_ALT, VK_1);
             } else if(c.endsWith("symbols")) {
                 ideService.invokeAction("AceJumpAction");
+                ideService.type(VK_SPACE);
                 ideService.type(("" + recognizeNumber()).toCharArray());
             }
+        } else if (c.startsWith(GOTO)) {
+
         } else if (c.startsWith(EXPAND)) {
-            pressKeystroke(KeyEvent.VK_CONTROL, KeyEvent.VK_W);
+//            ActionManager instance = ActionManager.getInstance();
+//            AnAction a = instance.getAction("EditorSelectWord");
+//            AnActionEvent event = new AnActionEvent(null, DataManager.getInstance().getDataContext(),
+//                    ActionPlaces.UNKNOWN, a.getTemplatePresentation(), instance, 0);
+//            a.actionPerformed(event);
+            ideService.invokeAction("EditorSelectWord");
         } else if (c.startsWith(SHRINK)) {
-            pressKeystroke(KeyEvent.VK_CONTROL, KeyEvent.VK_SHIFT, KeyEvent.VK_W);
+            pressKeystroke(VK_CONTROL, VK_SHIFT, VK_W);
         } else if (c.startsWith("press")) {
             if (c.contains("delete")) {
-                pressKeystroke(KeyEvent.VK_DELETE);
-            } else if (c.contains("return")) {
-                pressKeystroke(KeyEvent.VK_ENTER);
+                pressKeystroke(VK_DELETE);
+            } else if (c.contains("return") || c.contains("enter")) {
+                pressKeystroke(VK_ENTER);
             } else if (c.contains(ESCAPE)) {
-                pressKeystroke(KeyEvent.VK_ESCAPE);
+                pressKeystroke(VK_ESCAPE);
             } else if (c.contains(TAB)) {
-                pressKeystroke(KeyEvent.VK_TAB);
+                pressKeystroke(VK_TAB);
             } else if (c.contains(UNDO)) {
-                pressKeystroke(KeyEvent.VK_CONTROL, KeyEvent.VK_Z);
+                pressKeystroke(VK_CONTROL, VK_Z);
             }
         } else if (c.startsWith("following")) {
             if (c.endsWith("line")) {
-                pressKeystroke(KeyEvent.VK_DOWN);
+                pressKeystroke(VK_DOWN);
             } else if (c.endsWith("page")) {
-                pressKeystroke(KeyEvent.VK_PAGE_DOWN);
+                pressKeystroke(VK_PAGE_DOWN);
             } else if (c.endsWith("method")) {
-                pressKeystroke(KeyEvent.VK_ALT, KeyEvent.VK_DOWN);
+                pressKeystroke(VK_ALT, VK_DOWN);
+            } else if (c.endsWith("tab")) {
+                pressKeystroke(VK_CONTROL, VK_TAB);
+            } else if (c.endsWith("page")) {
+                pressKeystroke(VK_PAGE_DOWN);
             }
         } else if (c.startsWith("previous")) {
             if (c.endsWith("line")) {
-                pressKeystroke(KeyEvent.VK_UP);
+                pressKeystroke(VK_UP);
             } else if (c.endsWith("page")) {
-                pressKeystroke(KeyEvent.VK_PAGE_UP);
+                pressKeystroke(VK_PAGE_UP);
             } else if (c.endsWith("method")) {
-                pressKeystroke(KeyEvent.VK_ALT, KeyEvent.VK_UP);
+                pressKeystroke(VK_ALT, VK_UP);
+            } else if (c.endsWith("page")) {
+                pressKeystroke(VK_PAGE_UP);
             }
         } else if (c.startsWith("extract this")) {
             if (c.endsWith("method")) {
-                pressKeystroke(KeyEvent.VK_CONTROL, KeyEvent.VK_ALT, KeyEvent.VK_M);
+                pressKeystroke(VK_CONTROL, VK_ALT, VK_M);
             } else if (c.endsWith("parameter")) {
-                pressKeystroke(KeyEvent.VK_CONTROL, KeyEvent.VK_ALT, KeyEvent.VK_P);
+                pressKeystroke(VK_CONTROL, VK_ALT, VK_P);
             }
         } else if (c.startsWith("inspect code")) {
-            pressKeystroke(KeyEvent.VK_ALT, KeyEvent.VK_SHIFT, KeyEvent.VK_I);
+            pressKeystroke(VK_ALT, VK_SHIFT, VK_I);
         } else if (c.startsWith("speech pause")) {
             pauseSpeech();
         } else if (c.startsWith(OK_IDEA) || c.startsWith(OKAY_IDEA)) {
@@ -187,16 +205,22 @@ private void applyAction(String c) {
             fireGoogleSearch();
         } else if (c.contains("break point")) {
             if (c.startsWith("toggle")) {
-                pressKeystroke(KeyEvent.VK_CONTROL, KeyEvent.VK_F8);
+                pressKeystroke(VK_CONTROL, VK_F8);
             } else if (c.startsWith("view")) {
-                pressKeystroke(KeyEvent.VK_CONTROL, KeyEvent.VK_SHIFT, KeyEvent.VK_F8);
+                pressKeystroke(VK_CONTROL, VK_SHIFT, VK_F8);
             }
+        } else if(c.startsWith("debug")){
+            pressKeystroke(VK_SHIFT, VK_F9);
         } else if (c.startsWith("step")) {
             if (c.endsWith("over")) {
-                pressKeystroke(KeyEvent.VK_F8);
+                pressKeystroke(VK_F8);
             } else if (c.endsWith("into")) {
-                pressKeystroke(KeyEvent.VK_F7);
+                pressKeystroke(VK_F7);
+            } else if (c.endsWith("return")) {
+                pressKeystroke(VK_SHIFT, VK_F8);
             }
+        } else if (c.startsWith("resume")) {
+            pressKeystroke(VK_F9);
         } else if (c.startsWith("tell me a joke")) {
             tellJoke();
         } else if (c.contains("check")) {
@@ -338,7 +362,7 @@ public static synchronized void beep() {
             try {
                 Clip clip = AudioSystem.getClip();
                 AudioInputStream inputStream = AudioSystem.getAudioInputStream(
-                        ASRServiceImpl.class.getResourceAsStream("/com.jetbrains.idear/sounds/beep.wav"));
+                        ASRService.class.getResourceAsStream("/com.jetbrains.idear/sounds/beep.wav"));
                 clip.open(inputStream);
                 clip.start();
             } catch (Exception e) {

src/com/jetbrains/idear/asr/ASRService.java

@@ -1,8 +1,76 @@
 package com.jetbrains.idear.asr;
 
-public interface ASRService {
-    void init();
-    void dispose();
-    boolean activate();
-    boolean deactivate();
+import com.jetbrains.idear.recognizer.CustomLiveSpeechRecognizer;
+import edu.cmu.sphinx.api.Configuration;
+
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class ASRService {
+
+    public static final double MASTER_GAIN = 0.85;
+    public static final double CONFIDENCE_LEVEL_THRESHOLD = 0.5;
+
+    private Thread speechThread;
+
+    private static final String ACOUSTIC_MODEL = "resource:/edu.cmu.sphinx.models.en-us/en-us";
+    private static final String DICTIONARY_PATH = "resource:/edu.cmu.sphinx.models.en-us/cmudict-en-us.dict";
+    private static final String GRAMMAR_PATH = "resource:/com.jetbrains.idear/grammars";
+
+    private static final Logger logger = Logger.getLogger(ASRService.class.getSimpleName());
+
+    private CustomLiveSpeechRecognizer recognizer;
+
+    public void init() {
+        Configuration configuration = new Configuration();
+        configuration.setAcousticModelPath(ACOUSTIC_MODEL);
+        configuration.setDictionaryPath(DICTIONARY_PATH);
+        configuration.setGrammarPath(GRAMMAR_PATH);
+        configuration.setUseGrammar(true);
+
+        configuration.setGrammarName("command");
+
+        try {
+            recognizer = new CustomLiveSpeechRecognizer(configuration);
+//            recognizer.setMasterGain(MASTER_GAIN);
+            speechThread = new Thread(new ASRControlLoop(recognizer), "ASR Thread");
+            recognizer.startRecognition(true);
+            // Fire up control-loop
+            speechThread.start();
+        } catch (IOException e) {
+            logger.log(Level.SEVERE, "Couldn't initialize speech recognizer:", e);
+        }
+    }
+
+    public boolean activate() {
+//        if (getStatus() == Status.INIT) {
+//            // Cold start prune cache
+//            recognizer.startRecognition(true);
+//        }
+
+        return ListeningState.activate();
+    }
+
+    public boolean deactivate() {
+        return ListeningState.standBy();
+    }
+
+    public void dispose() {
+        // Deactivate in the first place, therefore actually
+        // prevent activation upon the user-input
+        deactivate();
+        terminate();
+    }
+
+    private void terminate() {
+        recognizer.stopRecognition();
+    }
+
+    // This is for testing purposes solely
+    public static void main(String[] args) {
+        ASRService asrService = new ASRService();
+        asrService.init();
+        ListeningState.activate();
+    }
 }