SYNC: Merge pull request #100 from dbgate/feature/stale-loader-fix

Author: janproch

packages/api/src/proc/databaseConnectionProcess.js

@@ -69,6 +69,16 @@ async function checkedAsyncCall(promise) {
 }
 
 let loadingModel = false;
+let queuedSyncModelFullRefresh = null;
+
+function finishLoadingModel() {
+  loadingModel = false;
+  if (queuedSyncModelFullRefresh != null) {
+    const isFullRefresh = queuedSyncModelFullRefresh;
+    queuedSyncModelFullRefresh = null;
+    handleSyncModel({ isFullRefresh });
+  }
+}
 
 async function handleFullRefresh() {
   if (storedConnection.useSeparateSchemas && !isCompositeDbName(dbhan?.database)) {
@@ -86,7 +96,7 @@ async function handleFullRefresh() {
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
 
-  loadingModel = false;
+  finishLoadingModel();
   resolveAnalysedPromises();
 }
 
@@ -111,12 +121,15 @@ async function handleIncrementalRefresh(forceSend) {
 
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
-  loadingModel = false;
+  finishLoadingModel();
   resolveAnalysedPromises();
 }
 
 function handleSyncModel({ isFullRefresh }) {
-  if (loadingModel) return;
+  if (loadingModel) {
+    queuedSyncModelFullRefresh = queuedSyncModelFullRefresh || !!isFullRefresh;
+    return;
+  }
   if (isFullRefresh) handleFullRefresh();
   else handleIncrementalRefresh();
 }

packages/web/src/appobj/AppObjectList.svelte

@@ -7,8 +7,8 @@
   import { writable } from 'svelte/store';
   import Link from '../elements/Link.svelte';
   import { focusedConnectionOrDatabase } from '../stores';
-  import { tick } from 'svelte';
-  import { _tval } from '../translations';
+  import { tick, onDestroy, afterUpdate } from 'svelte';
+  import { _t, _tval } from '../translations';
 
   export let list;
   export let module;
@@ -34,8 +34,13 @@
 
   export let collapsedGroupNames = writable([]);
   export let onChangeFilteredList = undefined;
+  export let initialRenderCount = null;
+  export let renderBatchSize = 200;
 
   let expandLimited = false;
+  let renderedCount = null;
+  let renderTimer = null;
+  let lastLimitedList = null;
 
   $: matcher = module.createMatcher && module.createMatcher(filter, passProps?.searchSettings);
 
@@ -82,6 +87,9 @@
   $: filtered = dataLabeled.filter(x => x.isMatched).map(x => x.data);
   $: childrenMatched = dataLabeled.filter(x => x.isChildMatched).map(x => x.data);
   $: mainMatched = dataLabeled.filter(x => x.isMainMatched).map(x => x.data);
+  $: filteredSet = new Set(filtered);
+  $: childrenMatchedSet = new Set(childrenMatched);
+  $: mainMatchedSet = new Set(mainMatched);
 
   // let filtered = [];
 
@@ -112,12 +120,21 @@
     expandLimited = true;
   }
 
-  $: groups = groupFunc ? extendGroups(_.groupBy(dataLabeled, 'group'), emptyGroupNames) : null;
-
   $: listLimited =
     isExpandedBySearch && !expandLimited ? filtered.slice(0, filter.trim().length < 3 ? 1 : 3) : listTranslated;
   $: isListLimited = isExpandedBySearch && listLimited.length < filtered.length;
   $: listMissingItems = isListLimited ? filtered.slice(listLimited.length) : [];
+  $: if (initialRenderCount && listLimited !== lastLimitedList) {
+    lastLimitedList = listLimited;
+    renderedCount = Math.min(initialRenderCount, listLimited.length);
+  }
+  $: listRendered =
+    initialRenderCount && renderedCount != null ? listLimited.slice(0, renderedCount) : listLimited;
+  $: hasMoreToRender = initialRenderCount && renderedCount != null && renderedCount < listLimited.length;
+  $: loadingMoreObjectsMessage = _t('objectsList.loadingMoreObjects', { defaultMessage: 'Loading more objects...' });
+  $: dataLabeledRendered =
+    initialRenderCount && renderedCount != null ? dataLabeled.slice(0, renderedCount) : dataLabeled;
+  $: groups = groupFunc ? extendGroups(_.groupBy(dataLabeledRendered, 'group'), emptyGroupNames) : null;
 
   $: if (
     $focusedConnectionOrDatabase &&
@@ -127,6 +144,21 @@
   ) {
     tick().then(setExpandLimited);
   }
+
+  afterUpdate(() => {
+    if (hasMoreToRender && !renderTimer) {
+      renderTimer = setTimeout(() => {
+        renderTimer = null;
+        renderedCount = Math.min(listLimited.length, renderedCount + renderBatchSize);
+      }, 0);
+    }
+  });
+
+  onDestroy(() => {
+    if (renderTimer) {
+      clearTimeout(renderTimer);
+    }
+  });
 </script>
 
 {#if groupFunc}
@@ -152,10 +184,13 @@
       {collapsedGroupNames}
     />
   {/each}
+  {#if hasMoreToRender}
+    <div class="ml-2 grayed">{loadingMoreObjectsMessage}</div>
+  {/if}
 {:else}
-  {#each listLimited as data}
+  {#each listRendered as data}
     <AppObjectListItem
-      isHidden={!filtered.includes(data)}
+      isHidden={!filteredSet.has(data)}
       {module}
       {subItemsComponent}
       {expandOnClick}
@@ -166,8 +201,8 @@
       {checkedObjectsStore}
       {disableContextMenu}
       {filter}
-      isExpandedBySearch={filter && childrenMatched.includes(data)}
-      isMainMatched={filter && mainMatched.includes(data)}
+      isExpandedBySearch={filter && childrenMatchedSet.has(data)}
+      isMainMatched={filter && mainMatchedSet.has(data)}
       {passProps}
       {getIsExpanded}
       {setIsExpanded}
@@ -182,4 +217,7 @@
       >
     </div>
   {/if}
+  {#if hasMoreToRender}
+    <div class="ml-2 grayed">{loadingMoreObjectsMessage}</div>
+  {/if}
 {/if}

packages/web/src/utility/common.ts

@@ -126,13 +126,17 @@ export async function loadSchemaList(conid, database) {
   }
 }
 
+let switchCurrentDatabaseRequest = 0;
+
 export async function switchCurrentDatabase(data) {
+  const request = ++switchCurrentDatabaseRequest;
   if (data?.connection?.useSeparateSchemas && !isCompositeDbName(data.name)) {
     const conid = data.connection._id;
     const database = data.name;
     const storageKey = `selected-schema-${conid}-${database}`;
     const schemaInStorage = localStorage.getItem(storageKey);
     const schemas = await loadSchemaList(conid, database);
+    if (request != switchCurrentDatabaseRequest) return;
     if (!schemas) return;
     const driver = findEngineDriver(data.connection, getExtensions());
     const defaultSchema = findDefaultSchema(schemas, driver?.dialect, schemaInStorage);

packages/web/src/utility/metadataLoaders.ts

@@ -265,6 +265,7 @@ function useCore(loader, args) {
         }
       }
       openedCount += 1;
+      onChange(getCachedValue(cacheKey));
       handleReload();
 
       if (reloadTrigger) {

packages/web/src/widgets/SqlObjectList.svelte

@@ -371,6 +371,8 @@
           list={objectList
             .filter(x => x.schemaName == null || ($appliedCurrentSchema ? x.schemaName == $appliedCurrentSchema : true))
             .map(x => ({ ...x, conid, database }))}
+          initialRenderCount={200}
+          renderBatchSize={200}
           module={databaseObjectAppObject}
           groupFunc={getAppObjectGroup}
           subItemsComponent={(data, { isExpandedBySearch }) =>