AI function calling, state summary, and spatial references

tools/.identity/spatialSensor/settings.json

@@ -0,0 +1,3 @@
+{
+	"enabled": true
+}

tools/spatialDraw/DrawingManager.js

@@ -336,6 +336,160 @@ class DrawingManager {
             this.setCursor(this.cursorMap['PROJECTION']);
         }
     }
+
+    apiDrawLine(startPoint, endPoint, color) {
+        // TODO: convert from world coordinates to the tool coordinates
+        let startPointVec3 = this.convertToVec3(startPoint);
+        let endPointVec3 = this.convertToVec3(endPoint);
+        // const worldPosition_end = endPointVec3.clone().applyMatrix4(this.groundPlaneContainer.matrixWorld);
+        // const inverseTargetMatrixWorld_end = new THREE.Matrix4().copy(this.scene.matrixWorld).invert();
+        // const targetLocalPosition_end = worldPosition_end.applyMatrix4(inverseTargetMatrixWorld_end);
+        // console.log(targetLocalPosition_end);
+        //
+        // const worldPosition_start = startPointVec3.clone().applyMatrix4(this.groundPlaneContainer.matrixWorld);
+        // const inverseTargetMatrixWorld_start = new THREE.Matrix4().copy(this.scene.matrixWorld).invert();
+        // const targetLocalPosition_start = worldPosition_end.applyMatrix4(inverseTargetMatrixWorld_start);
+        // console.log(targetLocalPosition_start);
+
+        const targetLocalPosition_start = this.convertWorldToTool(startPointVec3);
+        const targetLocalPosition_end = this.convertWorldToTool(endPointVec3);
+
+        try {
+            this.setColor(color);
+            this.setTool(drawingManager.toolMap['LINE']);
+
+            console.log('startDraw', targetLocalPosition_start);
+            this.tool.startDraw(this.drawingGroup, targetLocalPosition_start, this.cursor.getNormal());
+            this.tool.moveDraw(this.drawingGroup, targetLocalPosition_end, this.cursor.getNormal());
+            console.log('endDraw', targetLocalPosition_end);
+
+            // this.tool.startDraw(this.drawingGroup, this.convertToVec3(startPoint), this.cursor.getNormal());
+            // this.tool.moveDraw(this.drawingGroup, this.convertToVec3(endPoint), this.cursor.getNormal());
+            this.tool.endDraw();
+        } catch (e) {
+            console.warn(e);
+        }
+    }
+
+    apiDrawPath(points, color) {
+        if (!points || !points.length || points.length < 2) return;
+
+        this.setColor(color);
+        this.setTool(drawingManager.toolMap['LINE']);
+
+        for (let i = 0; i < points.length; i++) {
+            let startPoint = points[i];
+            // let endPoint = points[i+1];
+            let startPointVec3 = this.convertToVec3(startPoint);
+            // let endPointVec3 = this.convertToVec3(endPoint);
+
+            const targetLocalPosition_start = this.convertWorldToTool(startPointVec3);
+            // const targetLocalPosition_end = this.convertWorldToTool(endPointVec3);
+
+            try {
+
+                if (i === 0) {
+                    this.tool.startDraw(this.drawingGroup, targetLocalPosition_start, this.cursor.getNormal());
+                } else {
+                    let prevPoint = points[i - 1];
+                    let prevPointVec3 = this.convertToVec3(prevPoint);
+                    const targetLocalPosition_prev = this.convertWorldToTool(prevPointVec3);
+                    this.tool.startDraw(this.drawingGroup, targetLocalPosition_prev, this.cursor.getNormal());
+                    this.tool.moveDraw(this.drawingGroup, targetLocalPosition_start, this.cursor.getNormal());
+                    this.tool.endDraw();
+                }
+
+                // this.tool.startDraw(this.drawingGroup, this.convertToVec3(startPoint), this.cursor.getNormal());
+                // this.tool.moveDraw(this.drawingGroup, this.convertToVec3(endPoint), this.cursor.getNormal());
+            } catch (e) {
+                console.warn(e);
+            }
+        }
+
+        // this.tool.endDraw();
+    }
+
+    updateCoordinateSystems(toolOrigin, worldOrigin) {
+        let toolMatrixThree = new THREE.Matrix4();
+        this.setMatrixFromArray(toolMatrixThree, toolOrigin);
+        let worldMatrixThree = new THREE.Matrix4();
+        this.setMatrixFromArray(worldMatrixThree, worldOrigin);
+
+//         // Create an inverse of the world matrix
+//         let inverseWorldMatrix = new THREE.Matrix4().copy(worldMatrixThree).invert();
+//
+// // Create the worldToToolMatrix by multiplying toolMatrixThree by the inverse of worldMatrixThree
+//         this.worldToToolMatrix = new THREE.Matrix4().multiplyMatrices(toolMatrixThree, inverseWorldMatrix);
+
+        // Create an inverse of the world matrix
+        let inverseToolMatrix = new THREE.Matrix4().copy(toolMatrixThree).invert();
+
+// Create the worldToToolMatrix by multiplying toolMatrixThree by the inverse of worldMatrixThree
+        this.worldToToolMatrix = new THREE.Matrix4().multiplyMatrices(worldMatrixThree, inverseToolMatrix);
+    }
+
+    convertWorldToTool(point) {
+        if (!this.worldToToolMatrix) return point;
+        let worldPoint = new THREE.Vector3(point.x, point.y, point.z);
+        let toolPoint = worldPoint.applyMatrix4(this.worldToToolMatrix);
+        return toolPoint;
+    }
+
+    /**
+     * This is just a helper function to set a three.js matrix using an array
+     * @param {*} matrix
+     * @param {*} array
+     */
+    setMatrixFromArray(matrix, array) {
+        matrix.set(array[0], array[4], array[8], array[12],
+            array[1], array[5], array[9], array[13],
+            array[2], array[6], array[10], array[14],
+            array[3], array[7], array[11], array[15]);
+    }
+
+    convertToVec3(input) {
+        let x, y, z;
+
+        if (Array.isArray(input)) {
+            // Input format is an array [x, y, z]
+            [x, y, z] = input;
+        } else if (typeof input === 'string') {
+            // Input format is a string "(x, y, z)"
+            const match = input.match(/\(([^)]+)\)/);
+            if (match) {
+                [x, y, z] = match[1].split(',').map(Number);
+            } else {
+                throw new Error('Invalid string format');
+            }
+        } else if (typeof input === 'object' && input !== null) {
+            // Input format is an object {x, y, z}
+            if ('x' in input && 'y' in input && 'z' in input) {
+                x = input.x;
+                y = input.y;
+                z = input.z;
+            } else {
+                throw new Error('Object does not have x, y, and z properties');
+            }
+        } else {
+            throw new Error('Unsupported input type');
+        }
+
+        return new THREE.Vector3(x, y, z);
+    }
+
+    apiCountLines() {
+        return this.eventStack.length; // todo; this only counts lines from active session
+    }
+
+    apiClearCanvas() {
+        // let numUndoEvents = this.eventStack.filter(item => item.type === 'erase').length;
+        // for (let i = 0; i < numUndoEvents; i++) {
+        //    
+        // }
+        while (this.eventStack.length > 0) {
+            drawingManager.popUndoEvent(); // todo; this only clears lines from active session
+        }
+    }
 }
 
 /* ========== Classes ========== */

tools/spatialDraw/index.html

@@ -7,6 +7,8 @@
     <script src="objectDefaultFiles/envelope.js"></script>
     <script src="objectDefaultFiles/pep.min.js"></script>
     <script src="objectDefaultFiles/gl-worker.js"></script>
+    <script src="objectDefaultFiles/SpatialApplicationAPI.js"></script>
+    <script src="objectDefaultFiles/LanguageInterface.js"></script>
 
     <style>
         @font-face {

tools/spatialDraw/index.js

@@ -41,6 +41,129 @@ if (!spatialInterface) {
     spatialInterface = new SpatialInterface();
 }
 
+let languageInterface = null;
+
+function setupAPI() {
+    const api = new SpatialApplicationAPI('spatialDraw', spatialObject.object, spatialObject.frame);
+
+    api.defineAPI('addLine', [
+        {name: 'startPoint', type: 'Point', description: 'Start point of the line'},
+        {name: 'endPoint', type: 'Point', description: 'End point of the line'},
+        {name: 'color', type: 'String', description: 'Color of the line'}
+    ], {
+        type: 'boolean',
+        description: 'Draws a line on the canvas; returns success or error'
+    }, (startPoint, endPoint, color) => {
+        console.log('>> spatialDraw addLine', startPoint, endPoint, color);
+        drawingManager.apiDrawLine(startPoint, endPoint, color);
+        return true;
+    });
+
+    api.defineAPI('drawMultipointPath', [
+        {name: 'points', type: 'Point[]', description: 'Array of points'},
+        {name: 'color', type: 'String', description: 'Color of the path'}
+    ], {
+        type: 'boolean',
+        description: 'Draws a colored path on the canvas, starting at the first point in the array and connected piecewise until reaching the last point in the array; returns success or error'
+    }, (points, color) => {
+        console.log('>> spatialDraw drawMultipointPath', color);
+        drawingManager.apiDrawPath(points, color);
+        return true;
+    });
+
+    api.defineAPI('clearCanvas', [
+        // no parameters
+    ], {
+        type: 'boolean',
+        description: 'Erases all drawings, resetting the canvas to an empty state; returns success or error'
+    }, () => {
+        console.log('>> spatialDraw clearCanvas');
+        drawingManager.apiClearCanvas();
+        return true;
+    });
+
+    api.defineAPI('countLines', [
+        // no parameters
+    ], {
+        type: 'number',
+        description: 'Returns the number of visible strokes that users have drawn onto the canvas'
+    }, () => {
+        console.log('>> spatialDraw countLines');
+        return drawingManager.apiCountLines();
+    });
+
+    api.sendAPIDefinitionsToParent();
+    api.listenForCalls();
+
+    languageInterface = new LanguageInterface('spatialDraw', spatialObject.object, spatialObject.frame);
+
+    // TODO: populate these with correct data as the user draws into the scene
+    // languageInterface.updateSummarizedState('numStrokes', 0);
+    // languageInterface.updateSummarizedState('centroid', {x: 0, y: 0, z: 0});
+    // languageInterface.updateSummarizedState('bbox', { min: {x: 0, y: 0, z: 0}, max: {x: 0, y: 0, z: 0} });
+    // languageInterface.updateSummarizedState('colorsList', []);
+    // languageInterface.sendSummarizedStateToParent();
+
+    if (loadedDrawing) {
+        let drawingCopy = copyDrawingForAiSummary(loadedDrawing);
+        setTimeout(() => {
+            // TODO: round the coordinates
+            // TODO: try to normalize the coordinates
+            languageInterface.updateStateToBeProcessedByAiPrompts('drawing', drawingCopy); // TODO: maybe we can dropout some of the intermediate points for high-density user-drawn paths
+            languageInterface.sendAiProcessingStateToParent();
+        }, 1000);
+    }
+
+    languageInterface.setStateToStringReducer((summarizedState) => {
+        let contentsString = ' It is currently an empty canvas with no strokes drawn.';
+        let distributionString = '';
+        if (summarizedState.numStrokes > 0) {
+            contentsString = ` It contains some content: currently, ${summarizedState.numStrokes} strokes have been drawn.`;
+
+            let centroid = summarizedState.centroid;
+            let bbox = summarizedState.bbox;
+            let colorsList = summarizedState.colorsList;
+            distributionString = ` The lines/strokes of the drawing seem to be clustered into ${summarizedState.numClusters} groups.
+            The centroid of the drawings are at position (${centroid.x}, ${centroid.y}, ${centroid.z}).
+            The bounding box of the drawings goes from (${bbox.min.x}, ${bbox.min.y}, ${bbox.min.z}) to (${bbox.max.x}, ${bbox.max.y}, ${bbox.max.z}).
+            The colors used, from most to least frequent are: ${colorsList}.`;
+        }
+        return `This spatialDraw tool is a 3D drawing tool that allows users to annotate the 3D scene with sketches.${contentsString}${distributionString}`;
+
+        // return `This is the data of the spatialDraw tool, a 3D drawing tool that allows users to annotate the 3D scene with sketches:\n${JSON.stringify(summarizedState.drawing)}`;
+    });
+
+    languageInterface.addAiStateProcessingPrompt(`Please analyze the following dataset, which is generated by a 3D drawing application, and try to determine some plain-English description of the shapes/geometry/content of the 3D drawing, as a human might perceive it (for example, if you were to narrate the contents of the drawing to a blind person for accessibility reasons).
+
+Here is how to interpret the structure of the data. The top level "drawing" contains multiple strokes, each considered one of the child "drawings". A stroke begins with "putting the pen down" in the 3D space at the first of the "points" within the drawing. The points is a list of all of the subsequent positions that the pen moves to (while being pressed down / drawing a straight line from point Xn to point X(n+1)). At the final point in the list of points for a particular drawing (stroke), the pen is lifted from the 3D canvas and the completed stroke is displayed on the screen in the given color. The next "drawing" is unrelated to the previous drawing – they are considered separate strokes. It is possible that the start point of a subsequent drawing may align with the end point of a previous drawing, which will visually compose those drawings into a more complex path or shape, but it is also possible that the subsequent drawings are in a completely disconnected position and are unrelated to the previous drawings.
+
+Some of the most commonly-drawn shapes that you might try to categorize these strokes as include: arrows (lines/curves with arrow-heads on one end), circles, squares, and X shapes (a cross of two short lines).
+
+Please analyze this dataset and describe its contents.`);
+    languageInterface.addAiStateProcessingPrompt(`
+    I've asked another GPT agent to look at a dataset produced by a 3D drawing application and attempt to describe its contents to a blind audience. I will provide to you its output (delimited by -- characters):
+    Can you "summarize" that description into an even shorter but higher-level description of the contents of the 3D line/path drawing application? For example, something akin to "it contains a red circle beside a green X, and far away there is a yellow square" but with more detail.
+    --
+    `);
+    languageInterface.sendAiProcessingPromptsToParent();
+}
+
+function copyDrawingForAiSummary(drawing) {
+    let newDrawing = JSON.parse(JSON.stringify(drawing));
+    if (newDrawing.drawings) {
+        newDrawing.drawings = newDrawing.drawings.map(drawing => {
+            let newThisDrawing = JSON.parse(JSON.stringify(drawing));
+            newThisDrawing.points.forEach(point => {
+                point.x = Math.round(point.x) / 1000;
+                point.y = Math.round(point.y) / 1000;
+                point.z = Math.round(point.z) / 1000;
+            });
+            return newThisDrawing;
+        });
+    }
+    return newDrawing;
+}
+
 spatialInterface.setMoveDelay(500);
 spatialInterface.useWebGlWorker();
 spatialInterface.setAlwaysFaceCamera(true);
@@ -59,6 +182,13 @@ spatialInterface.addReadPublicDataListener('storage', 'drawing', function (drawi
         drawingManager.deserializeDrawing(drawing);
     } else {
         loadedDrawing = drawing;
+        if (languageInterface) {
+            let drawingCopy = copyDrawingForAiSummary(loadedDrawing);
+            setTimeout(() => {
+                languageInterface.updateStateToBeProcessedByAiPrompts('drawing', drawingCopy);
+                languageInterface.sendAiProcessingStateToParent();
+            }, 1000);
+        }
     }
 });
 
@@ -211,9 +341,13 @@ function initDrawingApp() {
         nextCircle.classList.add('active'); // Activates
     });
     drawingManager.addCallback('color', (color) => {
-        sizeCircles.forEach(sizeCircle => sizeCircle.setColor(color));
-        colorCircles.forEach(colorCircle => colorCircle.classList.remove('active'));
-        colorCircles.find(circle => circle.dataset.color === color).classList.add('active');
+        try {
+            sizeCircles.forEach(sizeCircle => sizeCircle.setColor(color));
+            colorCircles.forEach(colorCircle => colorCircle.classList.remove('active'));
+            colorCircles.find(circle => circle.dataset.color === color).classList.add('active');
+        } catch (e) {
+            console.warn('cant update 2D UI to match the colors value');
+        }
     });
     drawingManager.addCallback('eraseMode', (eraseMode) => {
         if (eraseMode) {
@@ -335,9 +469,26 @@ function initRenderer() {
             groundPlaneContainerObj.add(directionalLight2);
 
             spatialInterface.onSpatialInterfaceLoaded(function() {
+                setupAPI();
                 spatialInterface.subscribeToMatrix();
                 spatialInterface.addGroundPlaneMatrixListener(groundPlaneCallback);
                 spatialInterface.addMatrixListener(updateMatrices); // whenever we receive new matrices from the editor, update the 3d scene
+
+                spatialInterface.subscribeToCoordinateSystems([
+                    // spatialObject.COORDINATE_SYSTEMS.CAMERA,
+                    // spatialObject.COORDINATE_SYSTEMS.PROJECTION_MATRIX,
+                    spatialObject.COORDINATE_SYSTEMS.WORLD_ORIGIN,
+                    spatialObject.COORDINATE_SYSTEMS.TOOL_ORIGIN
+                ], (coordinateSystems) => {
+                    // let cameraMatrix = coordinateSystems[spatialObject.COORDINATE_SYSTEMS.CAMERA];
+                    // let projectionMatrix = coordinateSystems[spatialObject.COORDINATE_SYSTEMS.PROJECTION_MATRIX];
+                    let toolMatrix = coordinateSystems[spatialObject.COORDINATE_SYSTEMS.TOOL_ORIGIN];
+                    let worldMatrix = coordinateSystems[spatialObject.COORDINATE_SYSTEMS.WORLD_ORIGIN];
+                    if (toolMatrix && worldMatrix) {
+                        drawingManager.updateCoordinateSystems(toolMatrix, worldMatrix);
+                    }
+                });
+
                 spatialInterface.registerTouchDecider(touchDecider);
                 spatialInterface.getScreenDimensions((width, height) => {
                     adjustSidebarForWindowSize(height);