final tests & changes

Author: Judahmeek

lib/extractor/carousel.rb

@@ -24,10 +24,10 @@ def initialize(document)
     end
 
     def tiles
-      # scrapeMemo psuedocode: create empty scrapeMemo hash, which will serve as an index for future parsing of the same search result structure (data-attrid, tile grid container class, tile root class, tile count, name_attribute, image_script_variable_names)
+      # scrapeMemo psuedocode: create empty scrapeMemo hash, which will serve as an index for future parsing of the same search result structure (data-attrid, whether target grid is inside div#search or not, tile grid container class, tile root class, tile count, name_attribute, image_script_variable_names)
       target_section = @document.at_css('#search') || @document
       # scrapeMemo psuedocode: if '#search' can't be found, add that to scrapeMemo hash
-      target_section = target_section.css('div').find { |d| d['data-attrid'] } || target_section
+      target_section = target_section.css('div').find { |d| d['data-attrid'] } || @document.css('div').find { |d| d['data-attrid'] } || target_section
       # scrapeMemo psuedocode: if div['data-attrid'] can't be found, add that to scrapeMemo hash
       # scrapeMemo psuedocode: check database for any records containing the same ['data-attrid'] value
       # scrapeMemo psuedocode: if one or more record(s) exist, scan for the tile grid container class, prioritizing the record most recently created

lib/extractor/item.rb

@@ -31,7 +31,7 @@ def to_h
         "extensions" => extensions,
         "link" => link,
         "image" => image,
-      }
+      }.compact
     end
 
     private

spec/extractor_spec.rb

@@ -7,41 +7,30 @@
     end
     let(:result) { Extractor.call(File.join(FIXTURES_DIR, "van-gogh-paintings.html")) }
 
-    it "returns the same number of items as the SerpApi reference output" do
-      expect(result.size).to eq(expected.size)
+    it "matches SerpApi reference output" do
+      expect(result).to eq(expected)
     end
+  end
 
-    it "matches name, extensions and link byte-for-byte across all items" do
-      mismatches = result.each_with_index.reject do |item, i|
-        %w[name extensions link].all? { |f| item[f] == expected[i][f] }
-      end
-      expect(mismatches).to be_empty, -> {
-        mismatches.first(3).map { |it, i|
-          "row #{i}: got=#{it.reject { |k,_| k == 'image' }.inspect} " \
-            "exp=#{expected[i].reject { |k,_| k == 'image' }.inspect}"
-        }.join("\n")
-      }
+  describe ".call with U.S. Presidents fixture" do
+    let(:expected) do
+      JSON.parse(File.read((File.expand_path("fixtures/u.s._presidents.json", __dir__))))["artworks"]
     end
+    let(:result) { Extractor.call(File.expand_path("fixtures/u.s._presidents.html", __dir__)) }
 
-    it "extracts every inline thumbnail present in the HTML exactly" do
-      # The page only ships the first N base64 thumbnails inline. The rest
-      # are URL thumbnails in in-file attributes (e.g. data-src). Those are
-      # still part of the page snapshot and should be surfaced as-is.
-      inline_expected = expected.each_with_index.select { |e, _| e["image"].to_s.start_with?("data:") }
+    it "matches U.S. Presidents reference output" do
+      expect(result).to eq(expected)
+    end
+  end
 
-      inline_expected.each do |e, i|
-        expect(result[i]["image"]).to eq(e["image"]),
-          "mismatch on row #{i} (#{e['name']})"
-      end
+  describe ".call with Tom Cruise movies fixture" do
+    let(:expected) do
+      JSON.parse(File.read((File.expand_path("fixtures/tom_cruise_movies.json", __dir__))))["artworks"]
     end
+    let(:result) { Extractor.call(File.expand_path("fixtures/tom_cruise_movies.html", __dir__)) }
 
-    it "matches image output byte-for-byte against expected array" do
-      mismatches = result.each_with_index.reject { |item, i| item["image"] == expected[i]["image"] }
-      expect(mismatches).to be_empty, -> {
-        mismatches.first(3).map { |(item, i)|
-          "row #{i}: got=#{item['image'].inspect} exp=#{expected[i]['image'].inspect}"
-        }.join("\n")
-      }
+    it "matches Tom Cruise movies reference output" do
+      expect(result).to eq(expected)
     end
   end
 end

spec/item_spec.rb

@@ -53,7 +53,7 @@ def tile_from(html)
 
   it "returns nil when there is no anchor (malformed tile)" do
     node = tile_from("<div><span>just text</span></div>")
-    expected = {"extensions" => nil, "image" => nil, "link" => nil, "name" => "just text"}
+    expected = {"name" => "just text"}
     expect(described_class.parse(node, thumbnails: {})).to eq(expected)
   end