Setup Viz App Mock (#305)

Co-authored-by: Ajay Thorve <athorve@nvidia.com>

Author: matekdev

lerna.json

@@ -15,6 +15,7 @@
     "modules/demo/ssr/*",
     "modules/demo/tfjs/*",
     "modules/demo/client-server",
+    "modules/demo/viz-app",
     "modules/demo/sql/*"
   ]
 }

modules/deck.gl/src/layers/edges.ts

@@ -138,7 +138,7 @@ export class EdgeLayer extends (Layer as typeof DeckLayer) {
       this.internalState.selectedSourceNodeId = this.internalState.highlightedSourceNodeId;
       this.internalState.selectedTargetNodeId = this.internalState.highlightedTargetNodeId;
     }
-
+    info.object = info.index;  // deck.gl uses info.object to check if item has already been added
     return info;
   }
   _getModel({gl, shaderCache}: DeckContext) {

modules/deck.gl/src/layers/nodes.ts

@@ -133,6 +133,7 @@ export class NodeLayer extends (Layer as typeof DeckLayer) {
       this.internalState.selectedNodeId    = this.internalState.highlightedNodeId;
       this.internalState.selectedNodeIndex = this.internalState.highlightedNodeIndex;
     }
+    info.object = info.index;  // deck.gl uses info.object to check if item has already been added
     return info;
   }
   _getModel({gl, shaderCache}: DeckContext) {

modules/demo/.vscode/launch.json

@@ -112,7 +112,7 @@
             "command": "shellCommand.execute",
             "args": {
                 "description": "Select a demo to debug",
-                "command": "echo client-server luma graph spatial xterm $(find modules/demo/deck modules/demo/tfjs modules/demo/ipc modules/demo/ssr modules/demo/sql -maxdepth 2 -type f -name 'package.json' -print0 | grep -z -v node_modules | tr -d '\\0' | sed -r 's@modules/demo/@@g' | sed -r 's@/package.json@ @g') | sort -Vr | sed -r 's@\\s@\\n@g'",
+                "command": "echo client-server viz-app luma graph spatial xterm $(find modules/demo/deck modules/demo/tfjs modules/demo/ipc modules/demo/ssr modules/demo/sql -maxdepth 2 -type f -name 'package.json' -print0 | grep -z -v node_modules | tr -d '\\0' | sed -r 's@modules/demo/@@g' | sed -r 's@/package.json@ @g') | sort -Vr | sed -r 's@\\s@\\n@g'",
             }
         },
     ]

modules/demo/ssr/graph/plugins/graph/index.js

@@ -16,6 +16,7 @@ const wrtc            = require('wrtc');
 const {MemoryView}    = require('@nvidia/cuda');
 const {Float32Buffer} = require('@nvidia/cuda');
 const {GraphCOO}      = require('@rapidsai/cugraph');
+const {Series, Int32} = require('@rapidsai/cudf');
 
 const {loadNodes, loadEdges} = require('./loader');
 const {RenderCluster}        = require('../../render/cluster');
@@ -54,12 +55,32 @@ function graphSSRClients(fastify) {
 
     clients[stream.id] = {
       video: source,
-      state: {},
+      state: {
+        pickingMode: 'click',  // 'click', 'boxSelect'
+        selectedInfo: {},
+        boxSelectCoordinates: {rectdata: [{polygon: [[]], show: false}], startPos: null},
+        clearSelections: false
+      },
       event: {},
       props: {width, height, layout},
       graph: await loadGraph(graphId),
       frame: shmCreate(width * height * 3 / 2),
+      peer: peer,
     };
+    if (clients[stream.id].graph.dataframes[0]) {
+      const res = getPaginatedRows(clients[stream.id].graph.dataframes[0]);
+      peer.send(JSON.stringify({
+        type: 'data',
+        data: {nodes: {data: res, length: clients[stream.id].graph.dataframes[0].numRows}}
+      }));
+    }
+    if (clients[stream.id].graph.dataframes[1]) {
+      const res = getPaginatedRows(clients[stream.id].graph.dataframes[1]);
+      peer.send(JSON.stringify({
+        type: 'data',
+        data: {edges: {data: res, length: clients[stream.id].graph.dataframes[1].numRows}}
+      }));
+    }
 
     stream.addTrack(source.createTrack());
     peer.streams.push(stream);
@@ -79,6 +100,18 @@ function graphSSRClients(fastify) {
           clients[stream.id].event[data.type] = data;
           break;
         }
+        case 'pickingMode': {
+          clients[stream.id].state.pickingMode = data;
+          break;
+        }
+        case 'clearSelections': {
+          clients[stream.id].state.clearSelections = JSON.parse(data);
+          break;
+        }
+        case 'layout': {
+          clients[stream.id].props.layout = JSON.parse(data);
+          break;
+        }
       }
     }
   }
@@ -95,22 +128,24 @@ function graphSSRClients(fastify) {
   }
 
   async function loadGraph(id) {
+    let dataframes = [];
+
     if (!(id in graphs)) {
       const asDeviceMemory = (buf) => new (buf[Symbol.species])(buf);
-      const [nodes, edges]         = await Promise.all([loadNodes(id), loadEdges(id)]);
-      const src                    = edges.get('src');
-      const dst                    = edges.get('dst');
+      dataframes                   = await Promise.all([loadNodes(id), loadEdges(id)]);
+      const src                    = dataframes[1].get('src');
+      const dst                    = dataframes[1].get('dst');
       graphs[id]                   = {
         refCount: 0,
         nodes: {
-          nodeRadius: asDeviceMemory(nodes.get('size').data),
-          nodeFillColors: asDeviceMemory(nodes.get('color').data),
-          nodeElementIndices: asDeviceMemory(nodes.get('id').data),
+          nodeRadius: asDeviceMemory(dataframes[0].get('size').data),
+          nodeFillColors: asDeviceMemory(dataframes[0].get('color').data),
+          nodeElementIndices: asDeviceMemory(dataframes[0].get('id').data),
         },
         edges: {
-          edgeList: asDeviceMemory(edges.get('edge').data),
-          edgeColors: asDeviceMemory(edges.get('color').data),
-          edgeBundles: asDeviceMemory(edges.get('bundle').data),
+          edgeList: asDeviceMemory(dataframes[1].get('edge').data),
+          edgeColors: asDeviceMemory(dataframes[1].get('color').data),
+          edgeBundles: asDeviceMemory(dataframes[1].get('bundle').data),
         },
         graph: new GraphCOO(src._col, dst._col, {directedEdges: true}),
       };
@@ -124,24 +159,25 @@ function graphSSRClients(fastify) {
       ));
 
     return {
-      gravity: 1.0,
+      gravity: 0.0,
       linLogMode: false,
       scalingRatio: 5.0,
       barnesHutTheta: 0.0,
       jitterTolerance: 0.05,
       strongGravityMode: false,
       outboundAttraction: false,
       graph: graphs[id].graph,
-      edges: {
-        ...graphs[id].edges,
-        length: graphs[id].graph.numEdges(),
-      },
       nodes: {
         ...graphs[id].nodes,
         length: graphs[id].graph.numNodes(),
         nodeXPositions: pos.subarray(0, pos.length / 2),
         nodeYPositions: pos.subarray(pos.length / 2),
       },
+      edges: {
+        ...graphs[id].edges,
+        length: graphs[id].graph.numEdges(),
+      },
+      dataframes: dataframes
     };
   }
 }
@@ -152,8 +188,17 @@ function layoutAndRenderGraphs(clients) {
   return () => {
     for (const id in clients) {
       const client = clients[id];
+      const sendToClient =
+        ([nodes, edges]) => {
+          client.peer.send(JSON.stringify(
+            {type: 'data', data: {nodes: {data: getPaginatedRows(nodes), length: nodes.numRows}}}));
+          client.peer.send(JSON.stringify(
+            {type: 'data', data: {edges: {data: getPaginatedRows(edges), length: edges.numRows}}}));
+        }
 
-      if (client.isRendering) { continue; }
+      if (client.isRendering) {
+        continue;
+      }
 
       const state = {...client.state};
       const props = {...client.props};
@@ -170,55 +215,89 @@ function layoutAndRenderGraphs(clients) {
           'mouseup',
           'mouseleave',
           'wheel',
-          'beforeunload'
+          'beforeunload',
+          'shiftKey',
+          'dragStart',
+          'dragOver'
         ].map((x) => client.event[x])
           .filter(Boolean);
 
       if (event.length === 0 && !props.layout) { continue; }
       if (event.length !== 0) { client.event = Object.create(null); }
-      if (props.layout) { client.graph = forceAtlas2(client.graph); }
+      if (props.layout == true) { client.graph = forceAtlas2(client.graph); }
 
       const {
         width  = client.props.width ?? 800,
         height = client.props.height ?? 600,
       } = client.state;
 
+      state.window = {width: width, height: height, ...client.state.window};
+
       if (client.frame?.byteLength !== (width * height * 3 / 2)) {
         shmDetach(client.frame.key, true);
         client.frame = shmCreate(width * height * 3 / 2);
       }
-
       client.isRendering = true;
 
-      renderer.render(id,
-                      {
-                        state,
-                        props,
-                        event,
-                        frame: client.frame.key,
-                        graph: {
-                          ...client.graph,
-                          graph: undefined,
-                          edges: getIpcHandles(client.graph.edges),
-                          nodes: getIpcHandles(client.graph.nodes),
-                        },
-                      },
-                      (error, result) => {
-                        client.isRendering = false;
-                        if (id in clients) {
-                          if (error) { throw error; }
-                          result?.state && Object.assign(client.state, result.state);
-                          // console.log(state?.deck?.props?.initialViewState);
-                          // if (state?.deck?.props) {
-                          //   client.props.initialViewState = state.deck.props.initialViewState;
-                          // }
-                          client.video.onFrame({...result.frame, data: client.frame.buffer});
-                        }
-                      });
+      renderer.render(
+        id,
+        {
+          state,
+          props,
+          event,
+          frame: client.frame.key,
+          graph: {
+            ...client.graph,
+            graph: undefined,
+            edges: getIpcHandles(client.graph.edges),
+            nodes: getIpcHandles(client.graph.nodes),
+          },
+        },
+        (error, result) => {
+          client.isRendering = false;
+          if (id in clients) {
+            if (error) { throw error; }
+            if (client.state.clearSelections == true) {
+              // clear selection is called once
+              result.state.clearSelections = false;
+
+              // reset selected state
+              result.state.selectedInfo.selectedNodes       = [];
+              result.state.selectedInfo.selectedEdges       = [];
+              result.state.selectedInfo.selectedCoordinates = {};
+              result.state.boxSelectCoordinates.rectdata    = [{polygon: [[]], show: false}];
+
+              // send to client
+              if (client.graph.dataframes) { sendToClient(client.graph.dataframes); }
+            } else if (JSON.stringify(client.state.selectedInfo.selectedCoordinates) !==
+                       JSON.stringify(result.state.selectedInfo.selectedCoordinates)) {
+              // selections updated
+              const nodes =
+                Series.new({type: new Int32, data: result.state.selectedInfo.selectedNodes});
+              const edges =
+                Series.new({type: new Int32, data: result.state.selectedInfo.selectedEdges});
+              if (client.graph.dataframes) {
+                sendToClient([
+                  client.graph.dataframes[0].gather(nodes),
+                  client.graph.dataframes[1].gather(edges)
+                ]);
+              }
+            }
+            // copy result state to client's current state
+            result?.state && Object.assign(client.state, result.state);
+
+            client.video.onFrame({...result.frame, data: client.frame.buffer});
+          }
+        });
     }
   }
 }
 
+function getPaginatedRows(df, page = 1, rowsPerPage = 400) {
+  if (!df) { return {}; }
+  return df.head(page * rowsPerPage).tail(rowsPerPage).toArrow().toArray();
+}
+
 function forceAtlas2({graph, nodes, edges, ...params}) {
   graph.forceAtlas2({...params, positions: nodes.nodeXPositions.buffer});
   return {

modules/demo/ssr/graph/plugins/graph/loader.js

@@ -58,13 +58,7 @@ function getDefaultNodes() {
       'ustr::c',
       'ustr::d'
     ])),
-    id: Series.new({
-      type: new Uint32,
-      data: [
-        0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
-        20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39
-      ]
-    }),
+    id: Series.sequence({type: new Uint32, init: 0, step: 1, size: 40}),
     color: Series.new({
       type: new Uint32,
       data: [

modules/demo/ssr/graph/public/video.html

@@ -9,11 +9,21 @@
 </head>
 
 <body style="background:#2e2e2e; margin:0;">
+
     <video autoplay muted width="800" height="600"></video>
+    <div>
+        <input type="checkbox" id="box-select" onchange="updatePan()">
+        <label for="box-select">box-select</label>
+    </div>
+
+    <div>
+        <button onclick="updateClearSelections()">clear selections</button>
+    </div>
+
     <script src="https://cdn.jsdelivr.net/npm/simple-peer@9.11.0/simplepeer.min.js"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.1.3/socket.io.js"></script>
     <script>
-        const sock = io({ transports: ['websocket'], reconnection: false });
+        const sock = io({ transports: ['websocket'], reconnection: true });
         const video = document.querySelector('video');
         const peer = new SimplePeer({
             trickle: true,
@@ -31,6 +41,11 @@
         // Negotiate handshake
         sock.on('signal', (data) => peer.signal(data));
         peer.on('signal', (data) => sock.emit('signal', data));
+        peer.on('data', (data) => {
+            var decoded = new TextDecoder().decode(data);
+            var decodedjson = JSON.parse(decoded);
+            console.log("got data from peer: ", decodedjson.data);
+        });
 
         // Server video stream
         peer.on('stream', (stream) => {
@@ -64,6 +79,14 @@
             });
         }
 
+        function updatePan(){
+            let value = document.getElementById("box-select").checked;
+            peer.send(JSON.stringify({type: 'pickingMode', data: value? "boxSelect" : "click"}));
+        }
+        function updateClearSelections(){
+            peer.send(JSON.stringify({ type: 'clearSelections', data: true}));
+        }
+
         function serializeEvent(original) {
             return Object
                 .getOwnPropertyNames(Object.getPrototypeOf(original))