fix: Malte's review: add test -s for logs. add fsspec to recs. grammar fix.

Author: handecelikkanat

Makefile

@@ -41,17 +41,21 @@ cdxj:
 cdxj-remote-https:
 	@echo "indexing End-of-Term-2024 Internet Archive WARC over HTTPS (File size ~1GB, showing first 10 records):"
 	cdxj-indexer $(EOT_IA_WARC_HTTPS) 2>/dev/null | head -n 10 | tee eot-ia.cdxj
+	@test -s eot-ia.cdxj || { echo "ERROR: no records indexed from $(EOT_IA_WARC_HTTPS) -- check network connectivity"; exit 1; }
 	@echo
 	@echo "indexing End-of-Term-2024 Common Crawl repackage WARC over HTTPS (File size ~1GB, showing first 10 records):"
 	cdxj-indexer $(EOT_CC_WARC_HTTPS) 2>/dev/null | head -n 10 | tee eot-cc.cdxj
+	@test -s eot-cc.cdxj || { echo "ERROR: no records indexed from $(EOT_CC_WARC_HTTPS) -- check network connectivity"; exit 1; }
 
 cdxj-remote-s3:
 	@echo "!! this step requires authentication via S3 credentials (even though it is free)"
 	@echo "indexing End-of-Term-2024 Internet Archive WARC over S3 (File size ~1GB, showing first 10 records):"
 	cdxj-indexer $(EOT_IA_WARC_S3) 2>/dev/null | head -n 10 | tee eot-ia.cdxj
+	@test -s eot-ia.cdxj || { echo "ERROR: no records indexed from $(EOT_IA_WARC_S3) -- check network connectivity and S3 credentials"; exit 1; }
 	@echo
 	@echo "indexing End-of-Term-2024 Common Crawl repackage WARC over S3 (File size ~1GB, showing first 10 records):"
 	cdxj-indexer $(EOT_CC_WARC_S3) 2>/dev/null | head -n 10 | tee eot-cc.cdxj
+	@test -s eot-cc.cdxj || { echo "ERROR: no records indexed from $(EOT_CC_WARC_S3) -- check network connectivity and S3 credentials"; exit 1; }
 
 extract:
 	@echo "creating extraction.* from local warcs, the offset numbers are from the cdxj index"

README.md

@@ -268,7 +268,7 @@ cdxj-indexer https://eotarchive.s3.amazonaws.com/crawl-data/EOT-2024/segments/CC
 ```
 </details>
 
-The first command fetches and indexes these two WARC over HTTPS. Since they are both around 1GB each, so we display and save only the first 10 records.
+The first command fetches and indexes these two WARCs over HTTPS. Since they are both around 1GB each, we display and save only the first 10 records.
 
 If you have AWS credentials configured, you can also access the same files over S3, which is faster when running on AWS. Even though you will need AWS credentials for authentication purposes, this process is still free of charge since these are public buckets.
 If you do not have AWS credentials, you can access the same information over HTTPS as described above.

requirements.txt

@@ -1,6 +1,7 @@
 warcio[s3]>=1.8.0
 cdx_toolkit
 duckdb
+fsspec
 pyarrow
 pandas
 polars