performance improvements & scrapeMemo index explanation

Author: Judahmeek

lib/extractor.rb

@@ -4,6 +4,7 @@
 require_relative "extractor/carousel"
 require_relative "extractor/item"
 
+# scrapeMemo psuedocode: enabling the Item class to detect whether a relevant scrapeMemo record has been found by the Carousel class would probably require changing the Extractor from a module into a class
 module Extractor
   # Public facade. Returns an Array<Hash> of carousel items.
   #

lib/extractor/carousel.rb

@@ -21,11 +21,19 @@ def self.tiles(document)
     # it without threading it through every method call.
     def initialize(document)
       @document = document
-      @root_selector = nil
     end
 
     def tiles
-      groups = candidate_groups
+      # scrapeMemo psuedocode: create empty scrapeMemo hash, which will serve as an index for future parsing of the same search result structure (data-attrid, tile grid container class, tile root class, tile count, name_attribute, image_script_variable_names)
+      target_section = @document.at_css('#search') || @document
+      # scrapeMemo psuedocode: if '#search' can't be found, add that to scrapeMemo hash
+      target_section = target_section.css('div').find { |d| d['data-attrid'] } || target_section
+      # scrapeMemo psuedocode: if div['data-attrid'] can't be found, add that to scrapeMemo hash
+      # scrapeMemo psuedocode: check database for any records containing the same ['data-attrid'] value
+      # scrapeMemo psuedocode: if one or more record(s) exist, scan for the tile grid container class, prioritizing the record most recently created
+      # scrapeMemo psuedocode: if the tile grid container exists & has the expected number of children with the expected tile root class, set them as the tile roots & skip the rest of this function
+
+      groups = candidate_groups(target_section)
       return [] if groups.empty?
 
       # Multiple stick-link groups can exist on one page. We prefer the group
@@ -36,38 +44,23 @@ def tiles
       best_score = scored.map(&:first).max
       best_groups = scored.select { |score, _| score == best_score }.map(&:last)
       best_groups.min_by { |g| document_position(g.first) } || []
+      # scrapeMemo psuedocode: just like with the div['data-attrid'] value before, we can now check the tile grid container class, tile root class, tile count, as well as the div['data-attrid'] value against recorded indexes to check for search result structure drift
     end
 
     private
 
-    # Build candidate groups by:
-    #   1. Finding every `/search?…&stick=…` anchor.
-    #   2. Walking each anchor up to its *tile root* — the highest ancestor
-    #      that still contains exactly one stick anchor.
-    #   3. Grouping tile roots by their common parent. A group with
-    #      MIN_TILES+ siblings is a carousel candidate.
-    def candidate_groups
-      # Structural fingerprint that avoids volatile CSS class names.
-      target_section = @document.at_css('#search') || @document
-      target_section = target_section.css('div').find { |d| d['data-attrid'] } || target_section
-      best_root_candidate = [
-        { elements: target_section.css('img[alt]'), priority: 0, selector: 'img[alt]' },
-        { elements: target_section.css('[title]'), priority: 1, selector: '[title]' },
-        { elements: target_section.css('[aria-label]'), priority: 2, selector: '[aria-label]' },
-        { elements: target_section.css('a[href*="stick="]'), priority: 3, selector: 'a[href*="stick="]' },
-      ].max_by { |entry| [ entry[:elements].size, -entry[:priority] ] }
-      @root_selector = best_root_candidate[:selector]
-
-      # Convert each anchor to the smallest "tile root" node that represents
-      # one tile (not a nested sub-node, not the whole carousel container).
-      tile_roots = best_root_candidate[:elements].map { |a| tile_root_for(a) }.compact.uniq
-
-      # Sibling tile roots under the same parent form one carousel candidate.
-      grouped = tile_roots.group_by { |root| root.parent.to_s.hash + root.parent.element_children.size }
-      grouped.delete(nil)
+    # before = Time.now
+    # for i in 1..1000
+    #   candidate_groups(target_section)
+    # end
+    # after = Time.now
+    # puts "new version benchmarked at #{after - before}"
 
-      # Drop weak candidates early.
-      grouped.values.select { |g| g.size >= MIN_TILES }
+    # Build candidate groups by finding the three biggest sibling groups within the target section
+    # I tested this versus the previous implementation with the quick & dirty benchmark commented out above
+    # against the van-gogh-paintings.html results and found this to be about x10 faster.
+    def candidate_groups(target_section)
+      target_section.css('div', 'section', 'main').max_by(3) { |element| element.element_children.count }.map { |e| e.element_children }
     end
 
     # Score shape:
@@ -84,13 +77,15 @@ def group_score(group)
       tile_anchor_weight    = ENV.fetch('TILE_ANCHOR_WEIGHT', default_weight).to_f
       tile_name_weight    = ENV.fetch('TILE_NAME_WEIGHT', default_weight).to_f
       # Prefer groups that look like media cards.
-      almost_all_tiles_have_images = @root_selector == 'img[alt]' || group.count { |tile| tile.at_css("img") } >= group.size - acceptable_number_of_misformed_tiles
+      almost_all_tiles_have_images = group.count { |tile| tile.at_css("img") } >= group.size - acceptable_number_of_misformed_tiles
       img_score = almost_all_tiles_have_images ? tile_img_weight : 1
       # properly formatted anchor links provide a second quality axis.
-      almost_all_tiles_have_anchors = @root_selector == 'a[href*="stick="]' || group.count { |tile| tile.at_css('a[href*="stick="]') } >= group.size - acceptable_number_of_misformed_tiles
+      almost_all_tiles_have_anchors = group.count { |tile| tile.at_css('a[href*="stick="]') } >= group.size - acceptable_number_of_misformed_tiles
       anchor_score = almost_all_tiles_have_anchors ? tile_anchor_weight : 1
       # Name-like signals provide a third quality axis.
-      almost_all_tiles_have_names = ['[title]', '[aria-label]', 'img[alt]'].include?(@root_selector) || group.count { |tile| tile.at_css('[title], [aria-label], img[alt]') } >= group.size - acceptable_number_of_misformed_tiles
+      # I didn't think that searching for each name candidate individually to enforce uniformity
+      # was worth the performance cost, but it is a tradeoff worth discussing in a code review
+      almost_all_tiles_have_names = group.count { |tile| tile.at_css('[title], [aria-label], img[alt]') } >= group.size - acceptable_number_of_misformed_tiles
       name_score = almost_all_tiles_have_names ? tile_name_weight : 1
       group.size * img_score * anchor_score * name_score
     end
@@ -100,22 +95,5 @@ def document_position(node)
       # DOM-order fallback is stable and explainable.
       @document.css("*").index(node) || Float::INFINITY
     end
-
-    # Walk up from the current root while the current node's parent still contains
-    # only one specific element. The last such node is the tile root — adding
-    # one more level would absorb sibling tiles.
-    def tile_root_for(current_root)
-      node = current_root
-      loop do
-        parent = node.parent
-        return node unless parent
-
-        # As soon as parent contains multiple stick anchors, walking higher
-        # would merge sibling tiles. Current node is the tile root boundary.
-        stick_count = parent.css(@root_selector).size
-        return node if stick_count != 1
-        node = parent
-      end
-    end
   end
 end

lib/extractor/item.rb

@@ -37,6 +37,7 @@ def to_h
     private
 
     def name
+      # scrapeMemo psuedocode: if the Carousel class detected a relevant scrapeMemo index, then the recorded name candidate can be checked for first
       @name ||= begin
         # Ordered fallback chain:
         # 1) <img alt> is usually canonical title on Google tiles.

lib/extractor/thumbnail_index.rb

@@ -8,12 +8,6 @@ module Extractor
   # thumbnail is injected at runtime against the image id(s) listed in `ii`.
   # We replicate that mapping at parse time so no JS execution is required.
   class ThumbnailIndex
-    # Greedy on the data URI body, anchored on the trailing `_setImagesSrc(ii,s,r)`
-    # to avoid mis-pairing s/ii from adjacent script blocks.
-      
-      
-    IMAGE_SETTER_REGEX = /_setImagesSrc\((?<id>[a-z]*),\s*(?<source>[a-z]*)/.freeze
-
     ID_REGEX = /'([^']+)'/.freeze
 
     # Public convenience API.
@@ -26,15 +20,22 @@ def initialize(document)
     end
 
     def build
+      # scrapeMemo psuedocode: if the Carousel class detected a relevant scrapeMemo index, then the recorded function name, variable names, and variable order (maybe?) could be used to construct a single regex instead of the three different regexes I'm using
+      image_setter_regex = /_setImagesSrc\((?<id>[a-z]*),\s*(?<source>[a-z]*)/.freeze
       mapping = {}
       @document.css("script").each do |script|
         body = script.content
-        desired_variables = body.match(IMAGE_SETTER_REGEX) 
+        desired_variables = body.match(image_setter_regex)
+        # instead of skipping if the image_setter_regex fails to match because the image_setter function has been renamed, we could modify the source & ids regexes to look for properly formatted variable assignments & then look for a function taking those variables as parameters near the very end of the script
+        # Something like...
+        # source_regex = /var\s+(?<source_variable>\w+)\s*=\s*'(?<data>data:image\/[^']+)'\s*;/.freeze
+        # ids_regex = /var\s+(?<ids_variable>\w+)\s*=\s*\[(?<ids>[^\]]*)\]\s*;/.freeze
+        # image_setter_regex = /(?<function_name>\w+)\((#{source_variable}|#{ids_variable}),\s*(#{source_variable}|#{ids_variable}).{,12}\z/.freeze
         next if desired_variables.nil?
 
         source_regex = /var\s+#{desired_variables[:source]}\s*=\s*'(?<data>data:image\/[^']+)'\s*;/.freeze
         ids_regex = /var\s+#{desired_variables[:id]}\s*=\s*\[(?<ids>[^\]]*)\]\s*;/.freeze
-        
+
         source_result = body.match(source_regex)
         ids_result = body.match(ids_regex)
 

spec/carousel_spec.rb

@@ -5,69 +5,25 @@ def doc_for(html)
     Nokolexbor::HTML(html)
   end
 
-  context "when sizes tie" do
-    it "prioritizes image elements with alt attributes over elements with title attributes" do
-      doc = doc_for(<<~HTML)
-        <html><body>
-          <div id="title">
-            <div><div title="one"></div><img></a></div>
-            <div><div title="two"></div><img></a></div>
-            <div><div title="three"></div><img></a></div>
-          </div>
-          <div id="alt">
-            <div><img alt="S1"></div>
-            <div><img alt="S2"></div>
-            <div><img alt="S3"></div>
-          </div>
-        </body></html>
-      HTML
-
-      tiles = described_class.tiles(doc)
-      expect(tiles.size).to eq(3)
-      expect(tiles.first.parent["id"]).to eq("alt")
-    end
-
-    it "prioritizes elements with title attributes over aria-labels" do
-      doc = doc_for(<<~HTML)
-        <html><body>
-          <div id="aria">
-            <div><div aria-label="one"></div><img></a></div>
-            <div><div aria-label="two"></div><img></a></div>
-            <div><div aria-label="three"></div><img></a></div>
-          </div>
-          <div id="title">
-            <div><div title="one"></div><img></a></div>
-            <div><div title="two"></div><img></a></div>
-            <div><div title="three"></div><img></a></div>
-          </div>
-        </body></html>
-      HTML
-
-      tiles = described_class.tiles(doc)
-      expect(tiles.size).to eq(3)
-      expect(tiles.first.parent["id"]).to eq("title")
-    end
-
-    it "is deterministic on exact ties by picking the first group in DOM order" do
-      doc = doc_for(<<~HTML)
-        <html><body>
-          <div id="first">
-            <div><a href="/search?stick=f1"><img alt="F1"></a></div>
-            <div><a href="/search?stick=f2"><img alt="F2"></a></div>
-            <div><a href="/search?stick=f3"><img alt="F3"></a></div>
-          </div>
-          <div id="second">
-            <div><a href="/search?stick=s1"><img alt="S1"></a></div>
-            <div><a href="/search?stick=s2"><img alt="S2"></a></div>
-            <div><a href="/search?stick=s3"><img alt="S3"></a></div>
-          </div>
-        </body></html>
-      HTML
+  it "is deterministic on exact ties by picking the first group in DOM order when sizes tie" do
+    doc = doc_for(<<~HTML)
+      <html><body>
+        <div id="first">
+          <div><a href="/search?stick=f1"><img alt="F1"></a></div>
+          <div><a href="/search?stick=f2"><img alt="F2"></a></div>
+          <div><a href="/search?stick=f3"><img alt="F3"></a></div>
+        </div>
+        <div id="second">
+          <div><a href="/search?stick=s1"><img alt="S1"></a></div>
+          <div><a href="/search?stick=s2"><img alt="S2"></a></div>
+          <div><a href="/search?stick=s3"><img alt="S3"></a></div>
+        </div>
+      </body></html>
+    HTML
 
-      tiles = described_class.tiles(doc)
-      expect(tiles.size).to eq(3)
-      expect(tiles.first.parent["id"]).to eq("first")
-    end
+    tiles = described_class.tiles(doc)
+    expect(tiles.size).to eq(3)
+    expect(tiles.first.parent["id"]).to eq("first")
   end
 
   describe "group score & quality weights" do