Migrate setup.py to pyproject.toml

Author: sakgoyal

.pre-commit-config.yaml

@@ -19,12 +19,11 @@ ci:
   autofix_prs: true
   autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
   autoupdate_schedule: quarterly
-  # skip all hooks that can change the files, use GitHub Action "code-formatting.yml" for this
-  skip: [black,isort]
+  skip: [ruff-check, ruff-format]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v6.0.0
     hooks:
       - id: check-yaml
       - id: check-case-conflict
@@ -33,20 +32,9 @@ repos:
         args: ['--maxkb=1000']
       - id: requirements-txt-fixer
 
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.8
     hooks:
-      - id: isort
-        name: Format imports
-        exclude: docs/
-
-  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.3.0
-    hooks:
-      - id: black
-        # It is recommended to specify the latest version of Python
-        # supported by your project here, or alternatively use
-        # pre-commit's default_language_version, see
-        # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.10
+      - id: ruff-check
+        args: [ --fix ]
+      - id: ruff-format

pyproject.toml

@@ -18,7 +18,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nemo-toolkit"
-dynamic = ["dependencies", "optional-dependencies", "version"]
+dynamic = ["version"]
 description = "NeMo - a toolkit for Conversational AI"
 readme = "README.md"
 license = {file = "LICENSE"}
@@ -32,7 +32,6 @@ keywords = [
     "gpu",
     "language",
     "learning",
-    "learning",
     "machine",
     "nvidia",
     "pytorch",
@@ -51,6 +50,10 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Scientific/Engineering :: Image Recognition",
     "Topic :: Scientific/Engineering :: Mathematics",
@@ -59,66 +62,50 @@ classifiers = [
     "Topic :: Software Development :: Libraries",
     "Topic :: Utilities",
 ]
+dependencies = [
+    "cuda-bindings ; sys_platform != 'darwin'",
+    "fsspec>=2024.12.0",
+    "huggingface-hub>=0.24",
+    "numba ; sys_platform == 'darwin'",
+    "numexpr<2.14.0", # WAR for attempted use of nonexistent numpy.typing
+    "numpy>=1.22",
+    "onnx>=1.7.0",
+    "python-dateutil", # Align with upstream PyTorch requirements
+    "ruamel.yaml",
+    "scikit-learn",
+    "tensorboard",
+    "text-unidecode",
+    "torch>=2.6.0",
+    "tqdm>=4.41.0",
+    "wget",
+    "wrapt",
+]
 
 [tool.setuptools]
-py-modules = ["nemo"]
+zip-safe = false
+include-package-data = true
+
+[tool.setuptools.packages.find]
+exclude = ["tools", "tests"]
+
+[tool.setuptools.package-data]
+"*" = ["*.tsv", "*.txt", "*.far", "*.fst", "*.cpp", "Makefile"]
+
 
 [project.entry-points."nemo_run.cli"]
 llm = "nemo.collections.llm"
 
 [project.urls]
 Download = "https://github.com/NVIDIA/NeMo/releases"
 Homepage = "https://github.com/nvidia/nemo"
+Repository = "https://github.com/NVIDIA-NeMo/NeMo"
 
-[tool.isort]
-profile = "black"  # black-compatible
-line_length = 119  # should match black parameters
-ignore_whitespace = true  # ignore whitespace for compatibility with the initial style
-py_version = 310  # python 3.10 as a target version
-known_first_party = ["nemo"]  # FIRSTPARTY section
-known_third_party = ["nemo_text_processing", "examples", "scripts"]  # THIRDPARTY section
-sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
-default_section = "THIRDPARTY"
-extend_skip = ["setup.py", "docs/source/conf.py"]
-
-
-[tool.black]
-line_length = 119
-skip_string_normalization = true
-# major year version is stable, see details in
-# https://black.readthedocs.io/en/stable/the_black_code_style/index.html
-# `required_version` is necessary for consistency (other `black` versions will fail to reformat files)
-required_version = "24"
-target-version = ['py310', 'py311', 'py312']
-extend-exclude = '''
-# A regex preceded with ^/ will apply only to files and directories
-# in the root of the project.
-# include here only current collections, new collections should not be ignored
-# exclude the collection once it is reformatted (due to changes in PRs)
-(
-  ^\/docs\/
-  | ^\/external\/
-  | ^\/examples\/
-  | ^\/nemo\/collections\/asr\/
-  | ^\/nemo\/collections\/common\/
-  | ^\/nemo\/collections\/multimodal\/
-  | ^\/nemo\/collections\/nlp\/
-  | ^\/nemo\/collections\/tts\/
-  | ^\/nemo\/collections\/vision\/
-  | ^\/nemo\/core\/
-  | ^\/nemo\/utils\/
-  | ^\/scripts\/
-  | ^\/tests\/
-  | ^\/tools\/
-  | ^\/tutorials\/
-  | ^\/setup.py
-)
-'''
-
-[tool.pytest.ini_options]
+
+[tool.pytest]
+strict = true
 # durations=0 will display all tests execution time, sorted in ascending order starting from from the slowest one.
 # -vv will also display tests with durration = 0.00s
-addopts = "--verbose --pyargs --durations=0 --strict-markers"  # always add these arguments to pytest
+addopts = ["--verbose", "--pyargs", "--durations=0", "--strict-markers"] # always add these arguments to pytest
 testpaths = ["tests"]
 pythonpath = ["tests/e2e_nightly"]
 # directories to ignore when discovering tests
@@ -160,6 +147,192 @@ conflicts = [
     ],
 ]
 
+[tool.ruff]
+line-length = 119
+extend-exclude = [
+    "docs",
+    "external",
+    "examples",
+    "nemo/collections/asr",
+    "nemo/collections/common",
+    "nemo/collections/multimodal",
+    "nemo/collections/nlp",
+    "nemo/collections/tts",
+    "nemo/collections/vision",
+    "nemo/core",
+    "nemo/utils",
+    "scripts",
+    "tests",
+    "tools",
+    "tutorials",
+]
+[tool.ruff.lint]
+extend-select = [
+    "E", # pycodestyle
+    "F", # Pyflakes
+    "UP", # pyupgrade
+    "B", # flake8-bugbear
+    "SIM", # flake8-simplify
+    "I", # isort
+    "F541", # f-string without any placeholders
+    "F841", # local variable 'x' is assigned to but never used
+    "F401", # 'x' imported but unused
+    "E741", # ambiguous variable name 'l'
+    "F821", # undefined name 'x'
+    # "E266", # too many leading '#' for block comment
+]
+extend-ignore = [
+    "SIM108",
+    "E722",
+]
+[tool.ruff.lint.isort]
+known-first-party = ["nemo"]  # FIRSTPARTY section
+known-third-party = ["nemo_text_processing", "examples", "scripts"]  # THIRDPARTY section
+
+
+[project.optional-dependencies]
+all = [
+    "nemo-toolkit[test]",
+    "nemo-toolkit[run]",
+    "nemo-toolkit[core]",
+    "nemo-toolkit[lightning]",
+    "nemo-toolkit[common]",
+    "nemo-toolkit[asr]",
+    "nemo-toolkit[tts]",
+    "nemo-toolkit[slu]",
+    "nemo-toolkit[audio]",
+]
+slu = [
+    "jiwer>=2.0.0,<4.0.0",
+    "progress>=1.5",
+    "tabulate>=0.8.7",
+    "textdistance>=4.1.5",
+    "tqdm",
+    "nemo-toolkit[asr]",
+]
+test = [
+    "click>=8.1",
+    "coverage",
+    "parameterized",
+    "pytest",
+    "pytest-httpserver",
+    "pytest-mock",
+    "pytest-runner",
+    "ruamel.yaml",
+    "sphinx",
+    "sphinxcontrib-bibtex",
+    "wandb",
+    "wget",
+    "wrapt",
+    "nemo-toolkit[tts]",
+    "nemo-toolkit[common]",
+]
+audio = [
+    "einops",
+    "lhotse>=1.32.2",
+    "librosa>=0.10.0",
+    "matplotlib",
+    "pesq; (platform_machine != 'x86_64' or platform_system != 'Darwin')",
+    "pystoi",
+    "scipy>=0.14",
+    "soundfile",
+    "nemo-toolkit[common]",
+]
+core = [
+    "nemo-toolkit[lightning]"
+]
+lightning = [
+    "cloudpickle",
+    "fiddle",
+    "hydra-core>1.3,<=1.3.2",
+    "lightning>2.2.1,<=2.4.0",
+    "nv_one_logger_core>=2.3.1",
+    "nv_one_logger_pytorch_lightning_integration>=2.3.1",
+    "nv_one_logger_training_telemetry>=2.3.1",
+    "omegaconf<=2.3",
+    "peft",
+    "torchmetrics>=0.11.0",
+    "transformers",
+    "wandb",
+    "webdataset>=0.2.86",
+]
+
+common-only = [
+    "datasets>=3.2.0",
+    "einops",
+    "inflect",
+    "mediapy==1.1.6",
+    "pandas",
+    "sacremoses>=0.0.43",
+    "sentencepiece<1.0.0",
+]
+
+common = [
+    "nemo-toolkit[common-only]",
+    "nemo-toolkit[core]",
+]
+
+asr-only = [
+    "braceexpand",
+    "diskcache",
+    "editdistance",
+    "einops",
+    "jiwer>=3.1.0,<4.0.0",
+    "kaldi-python-io",
+    "kaldialign",
+    "lhotse>=1.32.2",
+    "librosa>=0.10.1",
+    "marshmallow",
+    "optuna",
+    "packaging",
+    "pyannote.core",
+    "pyannote.metrics",
+    "pydub",
+    "pyloudnorm",
+    "resampy",
+    "ruamel.yaml",
+    "sacrebleu",
+    "scipy>=0.14",
+    "soundfile",
+    "sox<=1.5.0",
+    "whisper_normalizer",
+]
+
+asr = [
+    "nemo-toolkit[asr-only]",
+    "nemo-toolkit[common]",
+]
+tts = [
+    "attrdict",
+    "cdifflib==1.2.6",
+    "einops",
+    "janome",
+    "jieba",
+    "kornia",
+    "librosa",
+    "matplotlib",
+    # pynini does not currently support aarch, disable nemo_text_processing for now
+    "nemo_text_processing; 'arm' not in platform_machine and 'aarch' not in platform_machine and sys_platform != 'darwin'",
+    "nltk",
+    "pandas",
+    "pyopenjtalk",
+    "pypinyin",
+    "pypinyin-dict",
+    "seaborn",
+    "nemo-toolkit[asr]",
+    "nemo-toolkit[common]",
+]
+cu12 = [
+    "cuda-python>=12.6.0,<13 ; platform_system != 'Darwin'",
+    "numba-cuda[cu12] ; platform_system != 'Darwin'",
+]
+cu13 = [
+    "cuda-python>=13,<14 ; platform_system != 'Darwin'",
+    "numba-cuda[cu13] ; platform_system != 'Darwin'",
+]
+run = ["nemo_run"]
+
+
 [dependency-groups]
 docs = [
     "myst-parser>=4.0.1",