optimize packagesList filtering: (#551)

* optimize packagesList filtering:
- implement initial sorting by last_modified_at date
- queue retrieval of bottles data

* cleanup

---------

Co-authored-by: neil molina <neil@neils-MacBook-Pro.local>

Author: getneil

modules/desktop/src/components/packages/packages.svelte

@@ -15,9 +15,6 @@
   const { packageList: allPackages } = packagesStore;
   export let packageFilter: SideMenuOptions = SideMenuOptions.all;
 
-  export let sortBy: "popularity" | "most recent" = "most recent";
-  export let sortDirection: "asc" | "desc" = "desc";
-
   export let scrollY = 0;
 
   let loadMore = 9;
@@ -43,14 +40,6 @@
     [SideMenuOptions.new_packages]: (pkg: GUIPackage) => {
       return moment(pkg.created).isAfter(moment().subtract(30, "days"));
     },
-    [SideMenuOptions.popular]: (pkg: GUIPackage) =>
-      pkg.categories.includes(SideMenuOptions.popular),
-    [SideMenuOptions.featured]: (pkg: GUIPackage) =>
-      pkg.categories.includes(SideMenuOptions.featured),
-    [SideMenuOptions.essentials]: (pkg: GUIPackage) =>
-      pkg.categories.includes(SideMenuOptions.essentials),
-    [SideMenuOptions.starstruck]: (pkg: GUIPackage) =>
-      pkg.categories.includes(SideMenuOptions.starstruck),
     [SideMenuOptions.made_by_tea]: (pkg: GUIPackage) => pkg.full_name.includes("tea.xyz")
   };
 
@@ -59,18 +48,7 @@
     scrollY = target.scrollTop || 0;
   };
 
-  $: packages = $allPackages.filter(pkgFilters[packageFilter] || pkgFilters.all).sort((a, b) => {
-    if (sortBy === "popularity") {
-      const aPop = +a.dl_count + a.installs;
-      const bPop = +b.dl_count + b.installs;
-      return sortDirection === "asc" ? aPop - bPop : bPop - aPop;
-    } else {
-      // most recent
-      const aDate = new Date(a.last_modified);
-      const bDate = new Date(b.last_modified);
-      return sortDirection === "asc" ? +aDate - +bDate : +bDate - +aDate;
-    }
-  });
+  $: packages = $allPackages.filter(pkgFilters[packageFilter] || pkgFilters.all);
 
   const onResize = (node: HTMLElement) => {
     const assumedCardHeight = 250;

modules/desktop/src/libs/stores/pkgs.ts

@@ -37,10 +37,28 @@ export default function initPackagesStore() {
   let refreshTimeoutId: ReturnType<typeof setTimeout> | null = null;
 
   const packageMap = writable<Packages>({ version: "0", packages: {} });
-  const packageList = derived(packageMap, ($packages) => Object.values($packages.packages));
+  const packageList = derived(packageMap, ($packages) =>
+    Object.values($packages.packages).sort((a, b) => {
+      // implement default sort by last_modified > descending
+      const aDate = new Date(a.last_modified);
+      const bDate = new Date(b.last_modified);
+      return +bDate - +aDate;
+    })
+  );
 
   let packagesIndex: Fuse<GUIPackage>;
 
+  // TODO: derive this concurrency relative to user's internet and computer performance?
+  const concurrency = 3;
+  const bottlesQueue = new Queue(concurrency, []);
+  bottlesQueue.setProcessor(async (pkgName: string) => {
+    // TODO: this api should take an architecture argument or else an architecture filter should be applied downstreawm
+    const bottles = await getPackageBottles(pkgName);
+    if (bottles?.length) {
+      updatePackage(pkgName, { bottles });
+    }
+  });
+
   const updateAllPackages = (guiPkgs: GUIPackage[]) => {
     packageMap.update((pkgs) => {
       guiPkgs.forEach((pkg) => {
@@ -266,11 +284,7 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
   };
 
   const fetchPackageBottles = async (pkgName: string) => {
-    // TODO: this api should take an architecture argument or else an architecture filter should be applied downstreawm
-    const bottles = await getPackageBottles(pkgName);
-    if (bottles?.length) {
-      updatePackage(pkgName, { bottles });
-    }
+    bottlesQueue.enqueue(pkgName);
   };
 
   const deletePkg = async (pkg: GUIPackage, version: string) => {
@@ -366,3 +380,55 @@ const setBadgeCountFromPkgs = (pkgs: Packages) => {
     log.error(error);
   }
 };
+
+type Processor = (input: string) => void;
+
+// TODO: move this to a generic design pattern then to another module
+class Queue {
+  private items: string[] = [];
+  private processor: Processor | null = null;
+  private processingCount = 0;
+  private concurrency: number;
+
+  constructor(concurrency = 3, initialItems: string[] = []) {
+    this.concurrency = concurrency;
+    this.items = initialItems;
+  }
+
+  setProcessor(processor: Processor): void {
+    this.processor = processor;
+  }
+
+  private async processQueue(): Promise<void> {
+    if (this.processingCount >= this.concurrency || this.items.length === 0 || !this.processor) {
+      return;
+    }
+
+    const item = this.dequeue();
+    if (item !== undefined) {
+      this.processingCount++;
+      Promise.resolve(this.processor(item))
+        .then(() => {
+          this.processingCount--;
+          this.processQueue();
+        })
+        .catch((error) => {
+          console.error(`Error processing item: ${error}`);
+          this.processingCount--;
+          this.processQueue();
+        });
+
+      // Start processing the next item(s) if concurrency allows
+      this.processQueue();
+    }
+  }
+
+  enqueue(item: string): void {
+    this.items.push(item);
+    this.processQueue();
+  }
+
+  dequeue(): string | undefined {
+    return this.items.shift();
+  }
+}

modules/desktop/src/routes/+page.svelte

@@ -21,9 +21,6 @@
 
   let sideMenuOption = (url.searchParams.get("tab") as SideMenuOptions) || SideMenuOptions.discover;
 
-  let sortBy: "popularity" | "most recent" = "most recent";
-  let sortDirection: "asc" | "desc" = "desc";
-
   let updating = false;
 
   let packagesScrollY = 0;
@@ -64,12 +61,7 @@
       {#if sideMenuOption == SideMenuOptions.discover}
         <DiscoverPackages bind:scrollY={packagesScrollY} />
       {:else}
-        <Packages
-          packageFilter={sideMenuOption}
-          {sortBy}
-          {sortDirection}
-          bind:scrollY={packagesScrollY}
-        />
+        <Packages packageFilter={sideMenuOption} bind:scrollY={packagesScrollY} />
       {/if}
     </ul>
     <header class="z-30 flex items-center justify-between" class:scrolling={packagesScrollY > 150}>