diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1c9ffdd..636012e 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -11,15 +11,12 @@ jobs: with: fetch-depth: 1 + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.10.12" # Replace with the version you need + run: uv sync - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install mypy>=1.7.1 - name: Type Check Source Code - run: mypy src + run: uv run mypy src + \ No newline at end of file diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 845909a..023055e 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,36 +1,21 @@ -name: Pylint +name: Pylint Linting on: [pull_request] jobs: - build: + linting: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10.12"] + steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + - uses: actions/checkout@v2 with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - pip install -r requirements.txt - # - name: Analysing the code with pylint - # run: | - # pylint $(git ls-files '*.py') - - name: Run Pylint - run: | - PYLINT_OUTPUT=$(pylint $(git ls-files '*.py') || true) - PYLINT_SCORE=$(echo "$PYLINT_OUTPUT" | grep 'rated at' | sed 's/.*rated at \([0-9.]*\)\/10.*/\1/') - echo "PYLINT_SCORE=$PYLINT_SCORE" >> $GITHUB_ENV - echo "$PYLINT_OUTPUT" + fetch-depth: 1 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + run: uv sync - - name: Fail if below threshold (8) - run: | - if (( $(echo "$PYLINT_SCORE < 8.0" |bc -l) )); then - exit 1 - fi + - name: Lint Source Code + run: uv run pylint src \ No newline at end of file diff --git a/.github/workflows/unitttest.yml b/.github/workflows/unitttest.yml index 30406d7..f23191e 100644 --- a/.github/workflows/unitttest.yml +++ b/.github/workflows/unitttest.yml @@ -11,20 +11,11 @@ jobs: with: fetch-depth: 1 + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.10.12" # Replace with the version you need + run: uv sync - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - name: Run Unit Tests - run: | - source ./env.sh - python3 tests/test_ast.py - python3 tests/test_polymorphism.py - python3 tests/test_dependency.py - python3 tests/test_rewrite.py - python3 tests/test_alpha_equiv.py + run: uv run pytest diff --git a/.gitignore b/.gitignore index bf9b522..9d7ac05 100644 --- a/.gitignore +++ b/.gitignore @@ -173,6 +173,8 @@ result/ evaluation_log.txt banned.txt +alpharewrite/ + *.pdf *.png *.csv \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 57ca757..81676c9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -39,7 +39,7 @@ extension-pkg-whitelist= fail-on= # Specify a score threshold under which the program will exit with error. -fail-under=10 +fail-under=9 # Interpret the stdin as a python script, whose filename needs to be passed as # the module_or_package argument. @@ -340,7 +340,7 @@ indent-after-paren=4 indent-string=' ' # Maximum number of characters on a single line. -max-line-length=80 +max-line-length=100 # Maximum number of lines in a module. max-module-lines=1000 diff --git a/README.md b/README.md index 670b44a..e8285f1 100644 --- a/README.md +++ b/README.md @@ -16,20 +16,12 @@ uv sync # create a virtual environment, and install dependencies ## Building TF-Bench From Scratch (Optional) -### Getting Required Data - -```sh -./scripts/init.sh -``` - -This script will download raw data from [Hackage](https://hackage.haskell.org/). - ### TF-Bench This script will build the benchmark (Prelude with NL) from the raw data. ```sh -uv run reprocess_benchmark.py -i benchmark/ -o TF-Bench.json +uv run --project . scripts/preprocess_benchmark.py ``` ### TF-Bench_pure @@ -40,6 +32,8 @@ cd alpharewrite stack build stack exec alpharewrite-exe 1 TF-Bench.json > TF-Bench.pure.json + +cd .. ``` For details, please refer to the README of [alpharewrite](https://github.com/SecurityLab-UCD/alpharewrite). diff --git a/pyproject.toml b/pyproject.toml index 113885b..035a974 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "ollama==0.2.1", "openai==1.30.5", "pathos>=0.3.3", + "pylint>=3.3.6", "pytest>=8.0.0", "python-dotenv==1.0.1", "requests==2.32.3", @@ -31,3 +32,6 @@ dependencies = [ "tree-sitter-haskell==0.21.0", "types-requests>=2.31.0", ] +[tool.pytest.ini_options] +pythonpath = ["."] + diff --git a/preprocess_benchmark.py b/scripts/preprocess_benchmark.py similarity index 96% rename from preprocess_benchmark.py rename to scripts/preprocess_benchmark.py index a561b33..8e0dfe3 100644 --- a/preprocess_benchmark.py +++ b/scripts/preprocess_benchmark.py @@ -8,7 +8,7 @@ from funcy import lmap -def main(input_raw_benchmark_path: str = "benchmark", output_path: str = "Benchmark-F.json"): +def main(input_raw_benchmark_path: str = "benchmark", output_path: str = "tfb.json"): # read in all files ending with .md in the input_raw_benchmark_path tasks: list[BenchmarkTask] = [] diff --git a/src/hoogle.py b/src/hoogle.py index 4551b0f..f5ed3fb 100644 --- a/src/hoogle.py +++ b/src/hoogle.py @@ -244,7 +244,7 @@ def main( # Also transform them back into dictionaries for json format filtered = ( (tasks_w_dep) - .filter(lambda d: d.dependencies != None) + .filter(lambda d: d.dependencies is not None) .map(lambda x: x.__dict__) .value ) diff --git a/tests/test_rewrite.py b/tests/test_rewrite.py deleted file mode 100644 index 2a87cd7..0000000 --- a/tests/test_rewrite.py +++ /dev/null @@ -1,83 +0,0 @@ -from dacite import from_dict -from src.common import BenchmarkTask -from typing import Callable -from src.type_rewrite import ( - preprocess, - manual_change, - convert_upper_to_lower, - remove_string_content, - reverse_process, - rewrite, -) -from src.postprocessing import postprocess - - -def test_rewrite(): - task_data = { - "task_id": "data/repos/ghc-internal-9.1001.0/src/GHC/Internal/Real.hs--fromRational", - "signature": "fromRational :: Rational -> a", - "code": "fromRational (x:%y) = fromInteger x % fromInteger y", - "poly_type": "Parametric", - "dependencies": [ - "(:%) :: a -> a -> Ratio a", - "(%) :: Integral a => a -> a -> Ratio a", - "fromInteger :: Num a => Integer -> a", - ], - } - - task = from_dict(data_class=BenchmarkTask, data=task_data) - - dependencies = task.dependencies if task.dependencies is not None else [] - signature = task.signature if task.signature is not None else "" - code = task.code if task.code is not None else "" - - # put all code together - combined_code = ( - "\n".join(dependencies) - + "\n" - + "-" * 20 - + "\n" - + signature - + "\n" - + "-" * 20 - + "\n" - + code - ) - - # process the raw code - process_strategy: list[Callable[[str], str]] = [ - preprocess, - manual_change, - convert_upper_to_lower, - remove_string_content, - reverse_process, - ] - combined_code = postprocess(combined_code, process_strategy) - - rewritten_code = rewrite(combined_code) - - rewritten_parts = rewritten_code.split("\n" + "-" * 20 + "\n") - task.dependencies = rewritten_parts[0].split("\n") - task.signature = rewritten_parts[1] - task.code = rewritten_parts[2] - - valid_result = { - "task_id": "data/repos/ghc-internal-9.1001.0/src/GHC/Internal/Real.hs--fromRational", - "signature": "f2::E -> a", - "code": "f2 (v1:%v2) = f1 v1 % f1 v2", - "poly_type": "Parametric", - "dependencies": [ - "(:%)::a -> a -> A a", - "(%)::B a => a -> a -> A a", - "f1::C a => D -> a", - ], - } - - assert task.signature == valid_result["signature"] - assert task.code == valid_result["code"] - assert task.poly_type == valid_result["poly_type"] - assert task.dependencies[0] == valid_result["dependencies"][0] - - -if __name__ == "__main__": - test_rewrite() diff --git a/uv.lock b/uv.lock index 222bbde..9721063 100644 --- a/uv.lock +++ b/uv.lock @@ -46,6 +46,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916 }, ] +[[package]] +name = "astroid" +version = "3.3.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/33/536530122a22a7504b159bccaf30a1f76aa19d23028bd8b5009eb9b2efea/astroid-3.3.9.tar.gz", hash = "sha256:622cc8e3048684aa42c820d9d218978021c3c3d174fb03a9f0d615921744f550", size = 398731 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/80/c749efbd8eef5ea77c7d6f1956e8fbfb51963b7f93ef79647afd4d9886e3/astroid-3.3.9-py3-none-any.whl", hash = "sha256:d05bfd0acba96a7bd43e222828b7d9bc1e138aaeb0649707908d3702a9831248", size = 275339 }, +] + [[package]] name = "attrs" version = "25.3.0" @@ -395,6 +404,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 }, ] +[[package]] +name = "isort" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/21/1e2a441f74a653a144224d7d21afe8f4169e6c7c20bb13aec3a2dc3815e0/isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450", size = 821955 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/11/114d0a5f4dabbdcedc1125dee0888514c3c3b16d3e9facad87ed96fad97c/isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615", size = 94186 }, +] + [[package]] name = "jiter" version = "0.9.0" @@ -560,6 +578,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/c2/0d5aae823bdcc42cc99327ecdd4d28585e15ccd5218c453b7bcd827f3421/matplotlib-3.10.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bc411ebd5889a78dabbc457b3fa153203e22248bfa6eedc6797be5df0164dbf9", size = 8134832 }, ] +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350 }, +] + [[package]] name = "multiprocess" version = "0.70.17" @@ -801,6 +828,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/6c/41c21c6c8af92b9fea313aa47c75de49e2f9a467964ee33eb0135d47eb64/pillow-11.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:67cd427c68926108778a9005f2a04adbd5e67c442ed21d95389fe1d595458756", size = 2377651 }, ] +[[package]] +name = "platformdirs" +version = "4.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/2d/7d512a3913d60623e7eb945c6d1b4f0bddf1d0b7ada5225274c87e5b53d1/platformdirs-4.3.7.tar.gz", hash = "sha256:eb437d586b6a0986388f0d6f74aa0cde27b48d0e3d66843640bfb6bdcdb6e351", size = 21291 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/45/59578566b3275b8fd9157885918fcd0c4d74162928a5310926887b856a51/platformdirs-4.3.7-py3-none-any.whl", hash = "sha256:a03875334331946f13c549dbd8f4bac7a13a50a895a0eb1e8c6a8ace80d40a94", size = 18499 }, +] + [[package]] name = "pluggy" version = "1.5.0" @@ -929,6 +965,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/f8/66f328e411f1c9574b13c2c28ab01f308b53688bbbe6ca8fb981e6cabc42/pydantic_core-2.33.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:4b6d77c75a57f041c5ee915ff0b0bb58eabb78728b69ed967bc5b780e8f701b8", size = 2082099 }, ] +[[package]] +name = "pylint" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astroid" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "dill" }, + { name = "isort" }, + { name = "mccabe" }, + { name = "platformdirs" }, + { name = "tomlkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/a7/113d02340afb9dcbb0c8b25454e9538cd08f0ebf3e510df4ed916caa1a89/pylint-3.3.6.tar.gz", hash = "sha256:b634a041aac33706d56a0d217e6587228c66427e20ec21a019bc4cdee48c040a", size = 1519586 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/21/9537fc94aee9ec7316a230a49895266cf02d78aa29b0a2efbc39566e0935/pylint-3.3.6-py3-none-any.whl", hash = "sha256:8b7c2d3e86ae3f94fb27703d521dd0b9b6b378775991f504d7c3a6275aa0a6a6", size = 522462 }, +] + [[package]] name = "pyparsing" version = "3.2.3" @@ -1158,6 +1212,7 @@ dependencies = [ { name = "ollama" }, { name = "openai" }, { name = "pathos" }, + { name = "pylint" }, { name = "pytest" }, { name = "python-dotenv" }, { name = "requests" }, @@ -1187,6 +1242,7 @@ requires-dist = [ { name = "ollama", specifier = "==0.2.1" }, { name = "openai", specifier = "==1.30.5" }, { name = "pathos", specifier = ">=0.3.3" }, + { name = "pylint", specifier = ">=3.3.6" }, { name = "pytest", specifier = ">=8.0.0" }, { name = "python-dotenv", specifier = "==1.0.1" }, { name = "requests", specifier = "==2.32.3" }, @@ -1226,6 +1282,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/4b/48ca098cb580c099b5058bf62c4cb5e90ca6130fa43ef4df27088536245b/tiktoken-0.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:0bc603c30b9e371e7c4c7935aba02af5994a909fc3c0fe66e7004070858d3f8f", size = 799281 }, ] +[[package]] +name = "tomlkit" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/09/a439bec5888f00a54b8b9f05fa94d7f901d6735ef4e55dcec9bc37b5d8fa/tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79", size = 192885 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/b6/a447b5e4ec71e13871be01ba81f5dfc9d0af7e473da256ff46bc0e24026f/tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde", size = 37955 }, +] + [[package]] name = "tqdm" version = "4.67.1"