Implementation part 1 (Tasks 0, 1, 2) (#1)

Author: lfoppiano

.editorconfig

@@ -0,0 +1,20 @@
+# see http://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# LF: not sure about this
+# [*.java]
+# charset = utf-8
+# indent_style = space
+# indent_size = 4
+
+[Makefile]
+indent_style = tab
+
+[*.yaml]
+indent_style = space
+indent_size = 2

.github/ci-build-unstable.yml

@@ -0,0 +1,21 @@
+name: Build unstable
+
+on: [ push ]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: 'mvn'
+      - name: Build with Gradle
+        run: mvn -B clean compile
+      - name: Check with spotless
+        run: mvn spotless:check

.gitignore

@@ -24,4 +24,19 @@ Icon
 Network Trash Folder
 Temporary Items
 .apdisk
-.idea
+
+# Borrowed from whirlwind-python
+*.cdxj
+construction/*.gz
+**/CC-MAIN*files
+**/TEST*.gz
+extraction.*
+testing.*
+**/whirlwind.parquet
+**/collinfo.json
+**/*.jar
+.venv/
+.vscode/
+.idea/
+target/
+build/

Makefile

@@ -0,0 +1,90 @@
+build:
+	mvn clean package
+
+iterate: build
+	@echo iterating over all of the local warcs:
+	@echo
+	@echo warc:
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.ReadWARC -Dexec.args="data/whirlwind.warc.gz"
+	@echo
+	@echo wet:
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.ReadWARC -Dexec.args="data/whirlwind.warc.wet.gz"
+	@echo
+	@echo wat:
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.ReadWARC -Dexec.args="data/whirlwind.warc.wat.gz"
+	@echo
+
+# cdxj:
+# 	@echo "creating *.cdxj index files from the local warcs"
+# 	cdxj-indexer whirlwind.warc.gz > whirlwind.warc.cdxj
+# 	cdxj-indexer --records conversion whirlwind.warc.wet.gz > whirlwind.warc.wet.cdxj
+# 	cdxj-indexer whirlwind.warc.wat.gz > whirlwind.warc.wat.cdxj
+
+# extract:
+# 	@echo "creating extraction.* from local warcs, the offset numbers are from the cdxj index"
+# 	warcio extract --payload whirlwind.warc.gz 1023 > extraction.html
+# 	warcio extract --payload whirlwind.warc.wet.gz 466 > extraction.txt
+# 	warcio extract --payload whirlwind.warc.wat.gz 443 > extraction.json
+# 	@echo "hint: python -m json.tool extraction.json"
+#
+# cdx_toolkit:
+# 	@echo demonstrate that we have this entry in the index
+# 	cdxt --crawl CC-MAIN-2024-22 --from 20240518015810 --to 20240518015810 iter an.wikipedia.org/wiki/Escopete
+# 	@echo
+# 	@echo cleanup previous work
+# 	rm -f TEST-000000.extracted.warc.gz
+# 	@echo retrieve the content from the commoncrawl s3 bucket
+# 	cdxt --crawl CC-MAIN-2024-22 --from 20240518015810 --to 20240518015810 warc an.wikipedia.org/wiki/Escopete
+# 	@echo
+# 	@echo index this new warc
+# 	cdxj-indexer TEST-000000.extracted.warc.gz  > TEST-000000.extracted.warc.cdxj
+# 	cat TEST-000000.extracted.warc.cdxj
+# 	@echo
+# 	@echo iterate this new warc
+# 	python ./warcio-iterator.py TEST-000000.extracted.warc.gz
+# 	@echo
+#
+# download_collinfo:
+# 	@echo "downloading collinfo.json so we can find out the crawl name"
+# 	curl -O https://index.commoncrawl.org/collinfo.json
+#
+# CC-MAIN-2024-22.warc.paths.gz:
+# 	@echo "downloading the list from s3, requires s3 auth even though it is free"
+# 	@echo "note that this file should be in the repo"
+# 	 aws s3 ls s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-22/subset=warc/ | awk '{print $$4}' | gzip -9 > CC-MAIN-2024-22.warc.paths.gz
+#
+# duck_local_files:
+# 	@echo "warning! 300 gigabyte download"
+# 	python duck.py local_files
+#
+# duck_ccf_local_files:
+# 	@echo "warning! only works on Common Crawl Foundadtion's development machine"
+# 	python duck.py ccf_local_files
+#
+# duck_cloudfront:
+# 	@echo "warning! this might take 1-10 minutes"
+# 	python duck.py cloudfront
+#
+wreck_the_warc: build
+	@echo
+	@echo we will break and then fix this warc
+	cp data/whirlwind.warc.gz data/testing.warc.gz
+	rm -f data/testing.warc
+	gzip -d data/testing.warc.gz  # windows gunzip no work-a
+	@echo
+	@echo iterate over this uncompressed warc: works
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.ReadWARC -Dexec.args="data/testing.warc"
+	@echo
+	@echo compress it the wrong way
+	gzip data/testing.warc
+	@echo
+	@echo iterating over this compressed warc fails
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.ReadWARC -Dexec.args="data/testing.warc.gz" || /usr/bin/true
+	@echo
+	@echo "now let's do it the right way"
+	gzip -d data/testing.warc.gz
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.RecompressWARC -Dexec.args="data/testing.warc data/testing.warc.gz"
+	@echo
+	@echo and now iterating works
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.ReadWARC -Dexec.args="data/testing.warc.gz"
+	@echo