Merge pull request #3672 from BSd3v/fix/3661-compute-graphs2

adjustments for computeGraphs

Author: T4rk1n

dash/dash-renderer/src/APIController.react.js

@@ -143,7 +143,8 @@ function storeEffect(props, events, setErrorLoading) {
         graphs,
         hooks,
         layout,
-        layoutRequest
+        layoutRequest,
+        config
     } = props;
 
     batch(() => {
@@ -187,7 +188,8 @@ function storeEffect(props, events, setErrorLoading) {
                 setGraphs(
                     computeGraphs(
                         dependenciesRequest.content,
-                        dispatchError(dispatch)
+                        dispatchError(dispatch),
+                        config
                     )
                 )
             );

dash/dash-renderer/src/actions/dependencies.js

@@ -163,26 +163,52 @@ function addMap(depMap, id, prop, dependency) {
     callbacks.push(dependency);
 }
 
-function addPattern(depMap, idSpec, prop, dependency) {
+// Patterns are stored in a nested Map structure to avoid the overhead of
+// stringifying ids for every callback.
+function addPattern(patterns, idSpec, prop, dependency) {
     const keys = Object.keys(idSpec).sort();
     const keyStr = keys.join(',');
     const values = props(keys, idSpec);
-    const keyCallbacks = (depMap[keyStr] = depMap[keyStr] || {});
-    const propCallbacks = (keyCallbacks[prop] = keyCallbacks[prop] || []);
-    let valMatch = false;
-    for (let i = 0; i < propCallbacks.length; i++) {
-        if (equals(values, propCallbacks[i].values)) {
-            valMatch = propCallbacks[i];
-            break;
-        }
+    const valuesKey = values
+        .map(v =>
+            typeof v === 'object' && v !== null
+                ? v.wild
+                    ? v.wild
+                    : JSON.stringify(v)
+                : String(v)
+        )
+        .join('|');
+
+    if (!patterns.has(keyStr)) {
+        patterns.set(keyStr, new Map());
+    }
+    const propMap = patterns.get(keyStr);
+    if (!propMap.has(prop)) {
+        propMap.set(prop, new Map());
     }
+    const valueMap = propMap.get(prop);
+
+    let valMatch = valueMap.get(valuesKey);
     if (!valMatch) {
         valMatch = {keys, values, callbacks: []};
-        propCallbacks.push(valMatch);
+        valueMap.set(valuesKey, valMatch);
     }
     valMatch.callbacks.push(dependency);
 }
 
+// Convert the nested Map structure of patterns into the plain nested object structure
+// expected by the rest of the code, with stringified id keys.
+// This is only done once per pattern, at the end of graph construction,
+// to minimize the overhead of stringifying ids.
+function offloadPatterns(patternsMap, targetMap) {
+    for (const [keyStr, propMap] of patternsMap.entries()) {
+        targetMap[keyStr] = {};
+        for (const [prop, valueMap] of propMap.entries()) {
+            targetMap[keyStr][prop] = Array.from(valueMap.values());
+        }
+    }
+}
+
 function validateDependencies(parsedDependencies, dispatchError) {
     const outStrs = {};
     const outObjs = [];
@@ -626,9 +652,10 @@ export function validateCallbacksToLayout(state_, dispatchError) {
     validatePatterns(inputPatterns, 'Input');
 }
 
-export function computeGraphs(dependencies, dispatchError) {
+export function computeGraphs(dependencies, dispatchError, config) {
     // multiGraph is just for finding circular deps
     const multiGraph = new DepGraph();
+    const start = performance.now();
 
     const wildcardPlaceholders = {};
 
@@ -657,7 +684,9 @@ export function computeGraphs(dependencies, dispatchError) {
         hasError = true;
         dispatchError(message, lines);
     };
-    validateDependencies(parsedDependencies, wrappedDE);
+    if (config.validate_callbacks) {
+        validateDependencies(parsedDependencies, wrappedDE);
+    }
 
     /*
      * For regular ids, outputMap and inputMap are:
@@ -683,8 +712,10 @@ export function computeGraphs(dependencies, dispatchError) {
      */
     const outputMap = {};
     const inputMap = {};
-    const outputPatterns = {};
-    const inputPatterns = {};
+    const outputPatternMap = new Map();
+    const inputPatternMap = new Map();
+    let outputPatterns = {};
+    let inputPatterns = {};
 
     const finalGraphs = {
         MultiGraph: multiGraph,
@@ -701,12 +732,14 @@ export function computeGraphs(dependencies, dispatchError) {
         return finalGraphs;
     }
 
+    // builds up wildcardPlaceholders with all the wildcard keys and values used in the callbacks, so we can generate the full list of ids that each callback depends on.
     parsedDependencies.forEach(dependency => {
         const {outputs, inputs} = dependency;
 
-        outputs.concat(inputs).forEach(item => {
-            const {id} = item;
-            if (typeof id === 'object') {
+        outputs
+            .concat(inputs)
+            .filter(item => typeof item.id === 'object')
+            .forEach(item => {
                 forEachObjIndexed((val, key) => {
                     if (!wildcardPlaceholders[key]) {
                         wildcardPlaceholders[key] = {
@@ -722,11 +755,11 @@ export function computeGraphs(dependencies, dispatchError) {
                     } else if (keyPlaceholders.exact.indexOf(val) === -1) {
                         keyPlaceholders.exact.push(val);
                     }
-                }, id);
-            }
-        });
+                }, item.id);
+            });
     });
 
+    // Efficiently build wildcardPlaceholders.vals arrays
     forEachObjIndexed(keyPlaceholders => {
         const {exact, expand} = keyPlaceholders;
         const vals = exact.slice().sort(idValSort);
@@ -808,6 +841,7 @@ export function computeGraphs(dependencies, dispatchError) {
     const cbOut = [];
 
     function addInputToMulti(inIdProp, outIdProp, firstPass = true) {
+        if (!config.validate_callbacks) return;
         multiGraph.addNode(inIdProp);
         multiGraph.addDependency(inIdProp, outIdProp);
         // only store callback inputs and outputs during the first pass
@@ -825,6 +859,7 @@ export function computeGraphs(dependencies, dispatchError) {
         cbOut.push([]);
 
         function addOutputToMulti(outIdFinal, outIdProp) {
+            if (!config.validate_callbacks) return;
             multiGraph.addNode(outIdProp);
             inputs.forEach(inObj => {
                 const {id: inId, property} = inObj;
@@ -859,41 +894,50 @@ export function computeGraphs(dependencies, dispatchError) {
         outputs.forEach(outIdProp => {
             const {id: outId, property} = outIdProp;
             // check if this output is also an input to the same callback
-            const alsoInput = checkInOutOverlap(outIdProp, inputs);
+            let alsoInput;
+            if (config.validate_callbacks) {
+                alsoInput = checkInOutOverlap(outIdProp, inputs);
+            }
             if (typeof outId === 'object') {
-                const outIdList = makeAllIds(outId, {});
-                outIdList.forEach(id => {
-                    const tempOutIdProp = {id, property};
-                    let outIdName = combineIdAndProp(tempOutIdProp);
+                if (config.validate_callbacks) {
+                    const outIdList = makeAllIds(outId, {});
+                    outIdList.forEach(id => {
+                        const tempOutIdProp = {id, property};
+                        let outIdName = combineIdAndProp(tempOutIdProp);
+                        // if this output is also an input, add `outputTag` to the name
+                        if (alsoInput) {
+                            duplicateOutputs.push(tempOutIdProp);
+                            outIdName += outputTag;
+                        }
+                        addOutputToMulti(id, outIdName);
+                    });
+                }
+                addPattern(outputPatternMap, outId, property, finalDependency);
+            } else {
+                if (config.validate_callbacks) {
+                    let outIdName = combineIdAndProp(outIdProp);
                     // if this output is also an input, add `outputTag` to the name
                     if (alsoInput) {
-                        duplicateOutputs.push(tempOutIdProp);
+                        duplicateOutputs.push(outIdProp);
                         outIdName += outputTag;
                     }
-                    addOutputToMulti(id, outIdName);
-                });
-                addPattern(outputPatterns, outId, property, finalDependency);
-            } else {
-                let outIdName = combineIdAndProp(outIdProp);
-                // if this output is also an input, add `outputTag` to the name
-                if (alsoInput) {
-                    duplicateOutputs.push(outIdProp);
-                    outIdName += outputTag;
+                    addOutputToMulti({}, outIdName);
                 }
-                addOutputToMulti({}, outIdName);
                 addMap(outputMap, outId, property, finalDependency);
             }
         });
 
         inputs.forEach(inputObject => {
             const {id: inId, property: inProp} = inputObject;
             if (typeof inId === 'object') {
-                addPattern(inputPatterns, inId, inProp, finalDependency);
+                addPattern(inputPatternMap, inId, inProp, finalDependency);
             } else {
                 addMap(inputMap, inId, inProp, finalDependency);
             }
         });
     });
+    outputPatterns = offloadPatterns(outputPatternMap, outputPatterns);
+    inputPatterns = offloadPatterns(inputPatternMap, inputPatterns);
 
     // second pass for adding new output nodes as dependencies where needed
     duplicateOutputs.forEach(dupeOutIdProp => {
@@ -913,6 +957,11 @@ export function computeGraphs(dependencies, dispatchError) {
             }
         }
     });
+    const end = performance.now();
+    if (!window.dash_component_api) {
+        window.dash_component_api = {};
+    }
+    window.dash_component_api.callbackGraphTime = (end - start).toFixed(2);
 
     return finalGraphs;
 }

dash/dash-renderer/src/config.ts

@@ -21,6 +21,7 @@ export type DashConfig = {
     };
     serve_locally?: boolean;
     plotlyjs_url?: string;
+    validate_callbacks: boolean;
 };
 
 export default function getConfigFromDOM(): DashConfig {

dash/dash-renderer/src/dashApi.ts

@@ -35,7 +35,7 @@ function getLayout(componentPathOrId: DashLayoutPath | string): any {
     }
 }
 
-window.dash_component_api = {
+window.dash_component_api = Object.assign(window.dash_component_api || {}, {
     ExternalWrapper,
     DashContext,
     useDashContext,
@@ -46,4 +46,4 @@ window.dash_component_api = {
         useDevtool,
         useDevtoolMenuButtonClassName
     }
-};
+});

dash/dash.py

@@ -937,6 +937,7 @@ def _config(self):
             "dash_version_url": DASH_VERSION_URL,
             "ddk_version": ddk_version,
             "plotly_version": plotly_version,
+            "validate_callbacks": self._dev_tools.validate_callbacks,
         }
         if self._plotly_cloud is None:
             if os.getenv("DASH_ENTERPRISE_ENV") == "WORKSPACE":
@@ -1968,6 +1969,7 @@ def _setup_dev_tools(self, **kwargs):
             "hot_reload",
             "silence_routes_logging",
             "prune_errors",
+            "validate_callbacks",
         ):
             dev_tools[attr] = get_combined_config(
                 attr, kwargs.get(attr, None), default=debug
@@ -2003,6 +2005,7 @@ def enable_dev_tools(  # pylint: disable=too-many-branches
         dev_tools_silence_routes_logging: Optional[bool] = None,
         dev_tools_disable_version_check: Optional[bool] = None,
         dev_tools_prune_errors: Optional[bool] = None,
+        dev_tools_validate_callbacks: Optional[bool] = None,
     ) -> bool:
         """Activate the dev tools, called by `run`. If your application
         is served by wsgi and you want to activate the dev tools, you can call
@@ -2024,6 +2027,7 @@ def enable_dev_tools(  # pylint: disable=too-many-branches
             - DASH_SILENCE_ROUTES_LOGGING
             - DASH_DISABLE_VERSION_CHECK
             - DASH_PRUNE_ERRORS
+            - DASH_VALIDATE_CALLBACKS
 
         :param debug: Enable/disable all the dev tools unless overridden by the
             arguments or environment variables. Default is ``True`` when
@@ -2079,6 +2083,10 @@ def enable_dev_tools(  # pylint: disable=too-many-branches
             env: ``DASH_PRUNE_ERRORS``
         :type dev_tools_prune_errors: bool
 
+        :param dev_tools_validate_callbacks: Check for circular callback
+            dependencies and raise an error if any are found. env: ``DASH_VALIDATE_CALLBACKS``
+        :type dev_tools_validate_callbacks: bool
+
         :return: debug
         """
         if debug is None:
@@ -2096,6 +2104,7 @@ def enable_dev_tools(  # pylint: disable=too-many-branches
             silence_routes_logging=dev_tools_silence_routes_logging,
             disable_version_check=dev_tools_disable_version_check,
             prune_errors=dev_tools_prune_errors,
+            validate_callbacks=dev_tools_validate_callbacks,
         )
 
         if dev_tools.silence_routes_logging:
@@ -2319,6 +2328,7 @@ def run(
         dev_tools_silence_routes_logging: Optional[bool] = None,
         dev_tools_disable_version_check: Optional[bool] = None,
         dev_tools_prune_errors: Optional[bool] = None,
+        dev_tools_validate_callbacks: Optional[bool] = None,
         **flask_run_options,
     ):
         """Start the flask server in local mode, you should not run this on a
@@ -2409,6 +2419,10 @@ def run(
             env: ``DASH_PRUNE_ERRORS``
         :type dev_tools_prune_errors: bool
 
+        :param dev_tools_validate_callbacks: Check for circular callback
+            dependencies and raise an error if any are found. env: ``DASH_VALIDATE_CALLBACKS``
+        :type dev_tools_validate_callbacks: bool
+
         :param jupyter_mode: How to display the application when running
             inside a jupyter notebook.
 
@@ -2446,6 +2460,7 @@ def run(
             dev_tools_silence_routes_logging,
             dev_tools_disable_version_check,
             dev_tools_prune_errors,
+            dev_tools_validate_callbacks,
         )
 
         # Evaluate the env variables at runtime