corpus_inventory.md

C++ Mega Corpus Inventory

Full scan of all shards under /Users/dave/sources/parquet (no sampling).

Generated: 2026-06-23. padded@4k = rows * 4096 (4k context budget).

clang_semantic_4k_v10 (CODE)

metric	value
kind	CODE
shards	66
rows	3,311,989
real_tokens (sum actual_token_count)	3,126,953,915
padded@4k (rows*4096)	13,565,906,944
size_on_disk	6,394,456,082 bytes (5.96 GiB)
distinct_repos	0
note	No repo recoverable: 'repo'/'commit' columns empty, constituent_provenance empty, no 'Repository:' text header. distinct_repos=0.

No repo distribution (see note).

clang_commits_4k_v1 (COMMITS)

metric	value
kind	COMMITS
shards	104
rows	5,246,129
real_tokens (sum actual_token_count)	10,061,088,043
padded@4k (rows*4096)	21,488,144,384
size_on_disk	18,993,478,088 bytes (17.69 GiB)
distinct_repos	415
note	Repository: parsed from text in 5246129/5246129 rows (100.00%).

Full repo distribution (sorted by count desc, then name)

repo	count
freebsd-src	209,415
mesa	194,232
src	181,764
netbsd	181,683
openbsd	180,669
intel-llvm-dpcpp	179,454
rocm-llvm	174,840
llvm-project	172,877
blender	139,683
xemu	119,313
qemu	118,935
wine	112,837
gstreamer	100,043
ffmpeg	99,788
zephyr	74,471
u-boot	68,801
vlc	66,915
systemd	66,511
gtk	65,966
retroarch	53,182
coreboot	52,066
binutils-gdb	51,300
php-src	49,925
tensorflow	49,255
wireshark	47,280
webkit	45,350
mariadb	43,169
xen	42,346
ruby	41,305
reactos	39,562
git-src	38,641
scummvm	38,127
cpython	37,808
dpdk	37,661
swift	37,139
radare2	36,703
postgres	36,536
mpv	35,507
mysql	33,952
dragonflybsd	32,631
mame	31,651
haiku	30,167
runtime	29,998
dotnet-runtime	29,939
petsc	28,454
nuttx	27,638
openssl	26,985
clickhouse	26,218
pytorch	26,072
tor	25,719
ladybird	25,238
mono	25,107
serenity	22,798
skia	22,473
ppsspp	22,092
openxla-xla	21,992
wiredtiger	21,587
illumos-gate	20,677
duckdb	19,683
ceph	19,600
riot	19,572
riot-os	19,512
mindspore	18,869
haproxy	18,036
trilinos	17,460
root	17,330
root-cern	17,324
esp-idf	17,098
glib	16,784
strongswan	16,707
sqlite	16,419
qtbase	16,241
scylladb	15,820
util-linux	15,610
open-watcom	15,292
open-watcom-v2	15,284
suricata	15,085
cgal	14,775
glibc	14,722
dolphin-emu	14,323
paddle	13,821
edk2	13,428
httpd	13,304
grpc	13,170
px4-autopilot	13,118
aosp-system-core	12,888
mruby	12,251
gromacs	12,065
tmux	11,729
arm-trusted-firmware	11,692
bitcoin	11,551
curl	11,396
squid	11,315
obs-studio	10,927
telegram-desktop	10,610
iree	10,193
contiki-ng	10,148
VTK	10,111
harfbuzz	9,963
john	9,944
gnupg	9,926
julia	9,414
v8	9,393
bionic	9,392
rt-thread	9,370
valkey	9,120
nginx	9,077
opencv	8,977
pdns	8,882
keydb	8,870
redis	8,854
pcsx2	8,820
openssh	8,658
h2o	8,641
lua	8,495
openzfs	8,404
qmk-firmware	8,231
rethinkdb	8,163
boringssl	8,000
optee	7,833
micropython	7,769
fio	7,725
wxWidgets	7,656
arangodb	7,503
rpcs3	7,477
chibios	7,361
strace	7,253
openjdk	7,231
neovim	7,148
clamav	7,130
hpx	7,118
onednn	6,974
dawn	6,936
protobuf	6,903
openvino	6,890
node	6,713
hdf5	6,428
unbound	6,391
aosp-frameworks-native	6,230
cppcheck	6,106
mbed-os	6,036
betaflight	5,885
erlang-otp	5,804
libuv	5,707
valgrind	5,596
SDL	5,595
usd	5,317
musl	5,173
clr	5,134
nghttp2	5,071
minix	5,044
mlpack	5,043
cmake	4,965
CMake	4,961
vim	4,900
zstd	4,892
intel-graphics-compiler	4,887
tcpdump	4,845
dealii	4,778
handbrake	4,761
mold	4,680
aosp-frameworks-av	4,569
arrow	4,462
dart-sdk	4,392
imagemagick	4,309
botan	4,279
mfem	4,277
aseprite	4,207
krita	4,183
seastar	4,138
opus	4,033
paraview	4,004
aosp-hardware-interfaces	3,971
foundationdb	3,881
samsung-one	3,824
apache-tvm	3,817
tvm	3,812
nmap	3,688
x64dbg	3,660
glfw	3,627
amrex	3,609
kodi	3,586
lvgl	3,572
s2n-tls	3,539
audacity	3,447
ardupilot	3,416
raylib	3,380
onnxruntime	3,291
mosquitto	3,260
hashcat	3,244
miopen	3,204
plan9	3,176
apollo	3,164
librdkafka	3,109
godot	3,056
duckstation	3,052
xnnpack	3,033
freecad	3,027
libevent	3,009
ntfs-3g	2,934
rocksdb	2,875
imhex	2,814
o3de	2,792
libpcap	2,761
hip	2,730
wolfssl	2,689
kokkos	2,651
executorch	2,647
ndpi	2,588
openblas	2,565
libsodium	2,559
opensbi	2,545
ITK	2,514
glow	2,500
openscad	2,482
mingw-w64	2,475
FreeRTOS	2,474
envoy	2,458
fftw3	2,451
nng	2,446
tinyusb	2,439
vulkan-validation-layers	2,386
carbon-lang	2,274
openvpn	2,241
tinycc	2,189
taskflow	2,158
flac	2,153
dragonflydb	2,136
folly	2,130
freetype	2,117
zeek	2,116
dragonfly	2,094
yara	2,058
chrono	2,032
spirv-tools	2,003
intel-compute-runtime	1,927
snort3	1,921
sel4	1,913
llama.cpp	1,875
triton	1,865
memcached	1,853
taichi	1,791
wicked-engine	1,766
z3	1,747
pytorch-xla	1,683
Catch2	1,670
lwip	1,669
mimalloc	1,666
assimp	1,628
bazel	1,609
c-ares	1,603
libusb	1,602
bgfx	1,571
imgui	1,527
pcl	1,521
kde-kio	1,507
kate	1,500
bullet3	1,492
mujoco	1,484
fbgemm	1,476
filament	1,434
DirectXShaderCompiler	1,422
entt	1,404
arm-compute-library	1,389
lammps	1,387
rocblas	1,382
shadps4	1,377
mediapipe	1,355
openh264	1,350
magnum	1,341
tesseract	1,310
arm-nn	1,301
nanomsg	1,284
thrift	1,268
jax	1,262
cntk	1,255
aria2	1,238
abseil-cpp	1,233
cryptopp	1,180
mxnet	1,163
jq	1,144
iperf3	1,104
lz4	1,096
ebpf-for-windows	1,094
ipex	1,084
rocr-runtime	1,074
marlin	1,070
apple-mlx	1,054
dlib	1,032
caffe	1,025
aosp-frameworks-ml	989
hiredis	972
xz	959
qbittorrent	956
whisper-cpp-latest	932
SFML	929
capnproto	920
whisper.cpp	907
carla	890
dali	886
openexr	871
proxygen	868
rccl	868
ggml	837
spdlog	819
luajit	808
ncnn	807
stablehlo	806
xiaomi-mace	789
mbedtls	788
composable-kernel	786
xgboost	781
cmsis5	764
iwyu	748
osl	715
meshlab	704
msgpack-c	704
rocfft	698
rocprim	695
ceres-solver	682
vulkan-loader	675
cuda-quantum	672
asio	665
cereal	652
benchmark	639
cjson	628
oceanbase	622
openfoam	619
onnx	612
glslang	602
glslang-khronos	602
react-native-windows	601
flatbuffers	586
terminal	582
winget-cli	582
windows-terminal	581
notepad-plus-plus	572
keepassxc	570
libzmq	570
pico-sdk	545
flashlight	519
ogre	516
rocsparse	507
mnn	489
veracrypt	477
msquic	470
tnn	454
littlefs	446
range-v3	432
gperftools	423
simdjson	418
eigen	413
faiss	413
gemma-cpp	404
mongodb-cxx	402
miniaudio	395
materialx	389
lmdb	380
pbrt-v4	374
re2	369
DirectXTex	366
googletest	349
libjpeg-turbo	337
verona	324
kde-kconfig	322
poco	312
llamafile	306
llamafile-latest	305
kde-kcoreaddons	304
openvdb	303
SEAL	289
cmsis-nn	283
cpuinfo	279
kde-kwidgetsaddons	269
cpprestsdk	266
box2d	260
spirv-cross	254
spirv-cross-khronos	253
vulkan-hpp	245
kde-kxmlgui	244
tensorrt-llm	244
json	243
transformer-engine	243
gpt4all	238
stlport	238
LightGBM	229
fmt	204
STL	200
stl	200
cpp-httplib	194
crow	181
rocdbgapi	174
libpng	169
pybind11	163
vulkan-examples	161
zlib	151
DirectXTK	149
directxtk	149
cachelib	148
cccl	144
glm	113
jolt-physics	101
rapidjson	90
DirectXTK12	87
DirectX-Graphics-Samples	65
directx-graphics-samples	65
flash-attention	57
thrust	55
stb	53
snmalloc	35
SymCrypt	33
microsoft-ui-xaml	19
:FetchStatus:	2
add	2
DirectXMath	1

treesitter_compilable_4k_v9 (CODE)

metric	value
kind	CODE
shards	60
rows	2,988,710
real_tokens (sum actual_token_count)	9,571,197,296
padded@4k (rows*4096)	12,241,756,160
size_on_disk	28,550,400,212 bytes (26.59 GiB)
distinct_repos	0
note	No repo recoverable: 'repo'/'commit' columns empty, constituent_provenance empty, no 'Repository:' text header. distinct_repos=0.

No repo distribution (see note).

enriched_commits_4k_v9 (COMMITS)

metric	value
kind	COMMITS
shards	34
rows	1,673,589
real_tokens (sum actual_token_count)	4,551,836,515
padded@4k (rows*4096)	6,855,020,544
size_on_disk	16,708,882,713 bytes (15.56 GiB)
distinct_repos	94
note	Repository: parsed from text in 1541265/1673589 rows (92.09%).

Full repo distribution (sorted by count desc, then name)

repo	count
blender	236,624
ceph	119,993
binutils-gdb	116,455
VTK	99,233
cgal	98,301
ardupilot	94,625
arangodb	45,257
cmake	44,638
CMake	44,621
bitcoin	36,625
audacity	34,941
ITK	32,489
aosp-system-core	29,196
aosp-frameworks-native	28,231
amrex	27,810
aosp-frameworks-av	26,824
SDL	22,710
chrono	21,058
arm-trusted-firmware	20,848
apollo	20,253
betaflight	19,314
chibios	18,859
botan	18,561
contiki-ng	17,833
boringssl	16,193
aosp-hardware-interfaces	15,934
clr	15,789
aseprite	15,152
clamav	14,489
arrow	13,627
bionic	10,721
apache-tvm	10,493
bullet3	9,761
assimp	9,300
cntk	9,003
aria2	7,279
carla	7,158
arm-compute-library	7,088
bgfx	6,939
cccl	6,878
capnproto	5,712
FreeRTOS	5,289
carbon-lang	5,259
Catch2	5,204
DirectXShaderCompiler	4,814
arm-nn	4,773
SFML	4,741
abseil-cpp	4,310
asio	3,912
composable-kernel	3,330
caffe	3,271
bazel	3,006
c-ares	2,795
ceres-solver	2,553
aosp-frameworks-ml	2,482
cmsis5	2,473
apple-mlx	2,445
LightGBM	2,076
box2d	1,918
benchmark	1,685
cachelib	1,672
SEAL	1,647
STL	1,309
cjson	1,248
DirectXTK	1,234
cereal	1,227
DirectXTex	1,062
calculator	1,028
DirectXTK12	965
cmsis-dsp	855
WindowsAppSDK	774
cmsis-nn	746
argon2	708
DirectX-Graphics-Samples	698
cpp-httplib	698
SymCrypt	500
apex	297
SPTAG	289
DirectXMath	269
boost	214
GSL	177
Detours	173
DirectX-Headers	110
Windows-classic-samples	53
ProcMon-for-Linux	33
apple-commoncrypto	30
apple-dyld	28
:updateEffectMetaList()	25
apple-objc4	14
apple-libdispatch	11
apple-security	8
apple-libc	7
Windows-Driver-Frameworks	4
:preInit()	1

Totals

metric	value
total rows	13,220,417
total real_tokens	27,311,075,769