wip

Author: gregors

Gemfile.lock

@@ -0,0 +1,47 @@
+PATH
+  remote: .
+  specs:
+    boilerpipe-ruby (0.4.3)
+      nokogiri (~> 1.10)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.4.4)
+    docile (1.3.2)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rake (13.0.1)
+    rickshaw (0.5.0)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  boilerpipe-ruby!
+  bundler (~> 2.0)
+  rake (>= 12.3.3)
+  rickshaw (~> 0.5.0)
+  rspec (~> 3.9)
+  simplecov (~> 0.18.5)
+
+BUNDLED WITH
+   2.1.4

lib/boilerpipe/sax/html_content_handler.rb

@@ -31,13 +31,15 @@ def start_element(name, attrs = [])
       tag = name.upcase.intern
 
       tag_action = @tag_actions[tag]
+      org = @tag_level
       if tag_action
         @tag_level += 1 if tag_action.changes_tag_level?
         @flush = tag_action.start(self, name, attrs) | @flush
       else
         @tag_level += 1
         @flush = true
       end
+      puts "before: #{org}, after: #{@tag_level}"
 
       @last_event = :START_TAG
       @last_start_tag = tag
@@ -64,7 +66,11 @@ def characters(text)
       end
 
       # set block levels
-      @block_tag_level = @tag_level if @block_tag_level == -1
+      if @block_tag_level == -1
+        puts "-1 setting block level tag_level: #{@tag_level}"
+       @block_tag_level = @tag_level
+      end
+      puts "block_tag_level: #{@block_tag_level}"
 
       append_space if started_with_whitespace
       append_text(text)

spec/filters/block_proximity_fusion_spec.rb

@@ -2,10 +2,10 @@
 
 module Boilerpipe::Filters
   describe BlockProximityFusion do
-    let(:text_block1) { Boilerpipe::Document::TextBlock.new('one',   0, 0, 0, 0, 0) }
-    let(:text_block2) { Boilerpipe::Document::TextBlock.new('two',   0, 0, 0, 0, 1) }
-    let(:text_block3) { Boilerpipe::Document::TextBlock.new('three', 0, 0, 0, 0, 2) }
-    let(:text_block4) { Boilerpipe::Document::TextBlock.new('four',  0, 0, 0, 0, 3) }
+    let(:text_block1) { Boilerpipe::Document::TextBlock.new('one',   0, 0, 0, 1, 0) }
+    let(:text_block2) { Boilerpipe::Document::TextBlock.new('two',   0, 0, 0, 1, 1) }
+    let(:text_block3) { Boilerpipe::Document::TextBlock.new('three', 0, 0, 0, 1, 2) }
+    let(:text_block4) { Boilerpipe::Document::TextBlock.new('four',  0, 0, 0, 1, 3) }
 
     let(:text_blocks) { [text_block1, text_block2, text_block3, text_block4] }
     let!(:doc) { Boilerpipe::Document::TextDocument.new('', text_blocks) }
@@ -18,6 +18,7 @@ module Boilerpipe::Filters
 
     describe '#process' do
       context 'where blocks exceed distance' do
+        # only_content: true, same_tag_level: false
         it 'doesnt change blocks' do
           expect(doc.text_blocks.size).to eq 4
           filter = BlockProximityFusion.new(1, true, false)
@@ -27,10 +28,15 @@ module Boilerpipe::Filters
       end
 
       context 'where blocks do not exceed distance' do
+        # only_content: false, same_tag_level: false
         it 'Fuses adjacent blocks' do
+          puts doc.text_blocks.map(&:text).inspect
+          puts doc.debug_s
           expect(doc.text_blocks.last.text.size).to eq 4
           filter = BlockProximityFusion.new(1, false, false)
           filter.process(doc)
+          puts doc.text_blocks.map(&:text).inspect
+          puts doc.debug_s
           expect(doc.text_blocks.last.text).to eq "three\nfour"
         end