The Parxyval quest begins!

Author: avvertix

.env.example

@@ -0,0 +1,6 @@
+
+## Configure Parxy drivers with environment variables
+
+PARXY_LLAMAPARSE_API_KEY=
+
+PARXY_LLMWHISPERER_API_KEY=

.github/CONTRIBUTING.md

@@ -0,0 +1,54 @@
+# Contributing
+
+Contributions are **welcome** and will be fully **credited**.
+
+Please read and understand the contribution guide before creating an issue or pull request.
+
+## Etiquette
+
+This project is open source, and as such, the maintainers give their free time to build and maintain the source code held within. They make the code freely available in the hope that it will be of use to other developers. It would be extremely unfair for them to suffer abuse or anger for their hard work.
+
+Please be considerate towards maintainers when raising issues or presenting pull requests. Let's show the
+world that developers are civilized and selfless people.
+
+It's the duty of the maintainer to ensure that all submissions to the project are of sufficient
+quality to benefit the project. Many developers have different skillsets, strengths, and weaknesses. Respect the maintainer's decision, and do not be upset or abusive if your submission is not used.
+
+## Viability
+
+When requesting or submitting new features, first consider whether it might be useful to others. Open
+source projects are used by many developers, who may have entirely different needs to your own. Think about
+whether or not your feature is likely to be used by other users of the project.
+
+## Procedure
+
+> [!NOTE]
+> Issue tracking is not currently enabled for this repository. We are organising it.
+
+Before filing an issue:
+
+- Attempt to replicate the problem, to ensure that it wasn't a coincidental incident.
+- Check to make sure your feature suggestion isn't already present within the project.
+- Check the pull requests tab to ensure that the bug doesn't have a fix in progress.
+- Check the pull requests tab to ensure that the feature isn't already in progress.
+
+Before submitting a pull request:
+
+- Check the codebase to ensure that your feature doesn't already exist.
+- Check the pull requests to ensure that another person hasn't already submitted the feature or fix.
+
+## Requirements
+
+If the project maintainer has any additional requirements, you will find them listed here.
+
+- **Add tests!** - Your patch won't be accepted if it doesn't have tests.
+
+- **Document any change in behaviour** - Make sure the `README.md` and any other relevant documentation are kept up-to-date.
+
+- **Consider our release cycle** - We try to follow [SemVer v2.0.0](https://semver.org/). Randomly breaking public APIs is not an option.
+
+- **One pull request per feature** - If you want to do more than one thing, send multiple pull requests.
+
+- **Send coherent history** - Make sure each individual commit in your pull request is meaningful. If you had to make multiple intermediate commits while developing, please [squash them](https://www.git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Changing-Multiple-Commit-Messages) before submitting.
+
+**Happy coding**!

.github/SECURITY.md

@@ -0,0 +1,3 @@
+# Security Policy
+
+If you discover any security related issues, please email security@oneofftech.xyz instead of using the discussions or the issue tracker.

.gitignore

@@ -0,0 +1,16 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+
+.env
+
+data/
+

.python-version

@@ -0,0 +1 @@
+3.12

LICENCE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2025-present OneOff-Tech (UG)
+Copyright (c) 2025-present Alessio Vertemati
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,164 @@
+![pypi](https://img.shields.io/pypi/v/parxyval.svg)
+![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json) [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+
+# Parxyval
+
+
+Parxyval – The Developer's Knight of the Parsing Table.
+
+An evaluation framework for document parsing, inspired by the quest for the Holy Grail. 🏰⚔️
+
+In a world of imperfect parsers, Parxyval helps you measure, compare, and discover which tool truly preserves the meaning of your documents. Benchmark precision, recall, structure, and reliability across multiple parsing services — and let your pipeline find its Grail.
+
+
+**Requirements**
+
+- Python 3.12 or above.
+- A Hugging Face account for downloading datasets that requires a login
+
+
+**Next steps**
+
+- [Getting started](#getting-started)
+  - [Available commands](#command-reference)
+- [Benchmarks](#benchmarks)
+- [Supported evaluations](#evaluations)
+
+
+
+## Getting Started
+
+Parxyval is an unstructured document processing evaluation framework offering a CLI interface to benchmark PDF parsing solutions. Follow these steps to get started:
+
+Parxyval is available on Pypi. You can try it using `uv`.
+
+```bash
+uvx parxyval --help
+```
+
+
+
+1. **Download the Dataset**
+```bash
+# Download sample documents from DocLayNet dataset
+parxyval download --limit 100 --include-pdf
+```
+
+The ground truth is stored in `./data/doclaynet/json` while pdf files are stored in `./data/doclaynet/pdf`
+
+
+2. **Parse Documents**
+```bash
+# Parse PDFs using your chosen driver (default: pymupdf)
+parxyval parse --driver pymupdf 
+
+# you can personalize input and output locations
+# --input data/doclaynet/pdf --output data/doclaynet/processed
+```
+
+Parxyval supports all drivers available in [Parxy](https://github.com/OneOffTech/parxy).
+
+Pdf files are read from `./data/doclaynet/pdf` and the parser outputs is written in `./data/doclaynet/processed/{driver}`, e.g. `./data/doclaynet/processed/pymupdf`
+
+3. **Evaluate Results**
+
+```bash
+# Run evaluation with selected metrics
+parxyval evaluate --metric sequence_matcher --metric bleu_score --input ./data/doclaynet/processed/pymupdf
+```
+
+### Command Reference
+
+#### `parxyval download`
+Download documents from the DocLayNet dataset.
+
+Options:
+- `--limit, -l`: Number of entries to download (default: 100)
+- `--skip, -s`: Skip specified number of entries
+- `--output, -o`: Output folder path (default: data/doclaynet)
+- `--include-pdf`: Download PDF files (default: False)
+
+#### `parxyval parse`
+Parse PDF documents using specified driver.
+
+Options:
+- `--driver, -d`: Parser driver to use (default: pymupdf)
+- `--limit, -l`: Maximum documents to process (default: 100)
+- `--skip, -s`: Skip specified number of documents
+- `--input, -i`: Input folder with PDFs (default: data/doclaynet/pdf)
+- `--output, -o`: Output folder for results (default: data/doclaynet/processed)
+
+#### `parxyval evaluate`
+Evaluate parsing results against ground truth.
+
+Arguments:
+- `driver`: Parser driver to evaluate (default: pymupdf)
+
+Options:
+- `--metric, -m`: Metrics to use (can be specified multiple times)
+- `--golden, -g`: Ground truth folder (default: data/doclaynet/json)
+- `--input, -i`: Parsed documents folder (default: data/doclaynet/processed/pymupdf)
+- `--output, -o`: Results output folder (default: data/doclaynet/results)
+
+
+## Benchmarks
+
+Parxyval supports various benchmarks for the evaluation of document processing services.
+
+- [DocLayNet](https://huggingface.co/datasets/ds4sd/DocLayNet-v1.2): Evaluate text and layout using the DocLayNet v1.2 dataset.
+
+
+_Datasets we are evaluating to support:_
+
+- [DP-Bench: Document Parsing Benchmark](https://huggingface.co/datasets/upstage/dp-bench)
+- [OmniDocBench](https://huggingface.co/datasets/opendatalab/OmniDocBench)
+
+## Evaluations
+
+Parxyval provides a comprehensive suite of text evaluation metrics to assess the quality of PDF parsing results. Each metric focuses on different aspects of text similarity and accuracy:
+
+### Text Similarity Metrics
+
+- **Sequence Matcher**: Measures the similarity between two texts using Python's difflib sequence matcher. Ideal for detecting overall textual similarities and differences.
+
+- **Jaccard Similarity**: Computes the similarity between page contents by measuring the intersection over union of their token sets. Perfect for assessing vocabulary overlap between parsed and reference texts.
+
+- **Edit Distance**: Calculates the normalized Levenshtein distance between texts, measuring the minimum number of single-character edits required to change one text into another. Useful for identifying character-level parsing accuracy.
+
+### Natural Language Processing Metrics
+
+- **BLEU Score**: A precision-based metric that compares n-grams between the parsed and reference texts. Particularly effective for evaluating the preservation of word sequences and phrases.
+
+- **METEOR Score**: Advanced metric that considers stemming, synonymy, and paraphrasing. Provides a more nuanced evaluation of semantic similarity between parsed and reference texts.
+
+### Information Retrieval Metrics
+
+- **Precision**: Measures the accuracy of the parsed text by calculating the proportion of correctly parsed tokens relative to all tokens in the parsed text.
+
+- **Recall**: Evaluates completeness by calculating the proportion of reference tokens that were correctly captured in the parsed text.
+
+- **F1 Score**: The harmonic mean of precision and recall, providing a balanced measure of parsing accuracy.
+
+All metrics are computed page-wise and then averaged across the entire document, ensuring a comprehensive evaluation of parsing quality at both local and global levels.
+
+
+## Security Vulnerabilities
+
+Please review our [security policy](./.github/SECURITY.md) on how to report security vulnerabilities.
+
+
+## Supporters
+
+The project is provided and supported by OneOff-Tech (UG) and Alessio Vertemati.
+
+<p align="left"><a href="https://oneofftech.de" target="_blank"><img src="https://raw.githubusercontent.com/OneOffTech/.github/main/art/oneofftech-logo.svg" width="200"></a></p>
+
+
+## Licence and Copyright
+
+Parxy is licensed under the [MIT licence](./LICENCE).
+
+- Copyright (c) 2025-present Alessio Vertemati, @avvertix
+- Copyright (c) 2025-present Oneoff-tech UG, www.oneofftech.de
+- All contributors
+

docker-compose.yml

@@ -0,0 +1,12 @@
+
+services:
+    pdfact:
+        image: "ghcr.io/data-house/pdfact:main"
+        ports:
+            - "4567:4567"
+        networks:
+            - parxyval
+
+networks:
+    parxyval:
+        driver: bridge

pyproject.toml

@@ -0,0 +1,65 @@
+[project]
+name = "parxyval"
+version = "0.1.0" # DO NOT EDIT, updated automatically
+description = "An evaluation framework for document parsing."
+license = "MIT"
+keywords= ["parxy", "evaluation", "evaluation-framework", "convert", "document", "document-parsing", "parser-benchmark", "pdf", "docx", "html", "markdown", "layout model", "rag", "llm", "document-performance-monitoring", "document-ai","text-extraction","data-quality","rag","parsing-tools"]
+readme = "README.md"
+authors = [
+    { name = "Alessio Vertemati", email = "alessio@oneofftech.xyz" }
+]
+classifiers = [
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Microsoft :: Windows",
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+]
+requires-python = ">=3.12,<4.0"
+dependencies = [
+    "datasets>=4.1.1",
+    "nltk>=3.9.2",
+    "pandas>=2.3.3",
+    "parxy[all]>=0.10.0",
+    "pydantic>=2.11.9",
+    "pydantic-settings>=2.11.0",
+    "pymupdf>=1.26.4",
+    "python-dotenv>=1.1.1",
+    "rich>=14.1.0",
+    "tqdm>=4.67.1",
+    "typer>=0.19.2",
+]
+
+[project.urls]
+homepage = "https://github.com/OneOffTech/parxyval"
+repository = "https://github.com/OneOffTech/parxyval"
+issues = "https://github.com/OneOffTech/parxyval/issues"
+
+[project.scripts]
+parxyval = "parxyval.cli.main:app"
+
+[build-system]
+requires = ["uv_build>=0.8.11,<0.9.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.2",
+    "ruff>=0.13.2",
+]
+
+[tool.uv]
+default-groups = "all"
+
+[tool.pytest.ini_options]
+addopts = [
+    "--import-mode=importlib",
+]
+
+[tool.ruff.format]
+quote-style = "single"
+docstring-code-format = true

src/parxyval/__init__.py

@@ -0,0 +1 @@
+