Add pre-commit hooks

.github/workflows/nestbuildmatrix.yml

@@ -697,8 +697,9 @@ jobs:
       - name: "Install Python test dependencies"
         # These could go to a separate venv…
         run: |
-          python -m pip install 'junitparser>=2' pytest pytest-timeout pytest-xdist terminaltables
-          python -m pip install numpy scipy pandas matplotlib
+          python -m pip install -r requirements_testing.txt
+          # 'junitparser>=2' pytest pytest-timeout pytest-xdist terminaltables
+          # numpy scipy pandas matplotlib
           test \! -e "=2"   # assert junitparser is correctly quoted and '>' is not interpreted as shell redirect
           python -c "import pytest; print('package location:', pytest.__file__)"
 
@@ -888,7 +889,8 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools
           python -c "import setuptools; print('package location:', setuptools.__file__)"
-          python -m pip install --force-reinstall --upgrade scipy 'junitparser>=2' numpy pytest pytest-timeout pytest-xdist mpi4py h5py cython matplotlib terminaltables pandoc pandas
+          python -m pip install --force-reinstall --upgrade -r requirements_testing.txt
+          # scipy 'junitparser>=2' numpy pytest pytest-timeout pytest-xdist mpi4py h5py cython matplotlib terminaltables pandoc pandas
           test \! -e "=2"   # assert junitparser is correctly quoted and '>' is not interpreted as shell redirect
           python -c "import pytest; print('package location:', pytest.__file__)"
           pip list

.pre-commit-config.yaml

@@ -1,31 +1,39 @@
+fail_fast: true
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
     hooks:
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
       - id: check-yaml
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
       - id: forbid-submodules
       - id: mixed-line-ending
-      - id: fix-byte-order-marker
+        args: [--fix=lf]
+      - id: trailing-whitespace
 
   - repo: https://github.com/gitleaks/gitleaks
     rev: v8.16.3
     hooks:
       - id: gitleaks
 
-  - repo: https://github.com/pycqa/isort
-    rev: 8.0.1
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v22.1.0
     hooks:
-      - id: isort
-        args: ["--profile", "black", "--thirdparty", "nest"]
+      - id: clang-format
+        types_or: [c++]
 
   - repo: https://github.com/psf/black
     rev: 26.1.0
     hooks:
       - id: black
         language_version: python3
 
+  - repo: https://github.com/pycqa/isort
+    rev: 8.0.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--thirdparty", "nest"]
+
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.11.0.1
     hooks:
@@ -47,10 +55,55 @@ repos:
           "testsuite/junit_xml.sh",
         ]
 
-  - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v22.1.0
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.3.0
     hooks:
-      - id: clang-format
-        types_or: [c++]
+      - id: flake8
+
+  - repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.3.0
+    hooks:
+      - id: pydocstyle
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.19.1
+    hooks:
+      - id: mypy
+        additional_dependencies: &TESTDEPS
+          - attrs
+          #- black
+          - csa
+          #- cython
+          - data-science-types
+          - h5py
+          #- isort
+          #- junitparser >= 2
+          #- jupyterlab
+          - matplotlib
+          - mpi4py
+          #- mypy >= 0.8
+          - notebook
+          - numpy
+          - pandas
+          #- pre-commit
+          #- pycodestyle
+          - pydocstyle
+          - pydot
+          #- pytest
+          #- pytest-cov
+          #- pytest-mypy
+          #- pytest-pylint
+          #- pytest-timeout
+          #- pytest-xdist
+          #- rstcheck
+          #- scipy
+          #- terminaltables
+        args: [--python-version=3.11]
+
+  - repo: https://github.com/PyCQA/pylint
+    rev: v4.0.5
+    hooks:
+      - id: pylint
+        additional_dependencies: *TESTDEPS
 
-exclude: 'build/.*|thirdparty/.*'
+exclude: 'build/.*|thirdparty/.*|venv/.*'

.pylintrc

@@ -6,7 +6,7 @@
 options = unneeded-not, line-too-long, unnecessary-semicolon, trailing-whitespace, missing-final-newline, bad-indentation, multiple-statements, bare-except
 ignore = CVS .git conda env __pycache__ .pytest_cache .mypy_cache
 
-disable = no-member, redefined-outer-name, invalid-name, consider-using-f-string, wrong-import-order, missing-function-docstring, missing-method-docstring, missing-class-docstring, attribute-defined-outside-init, no-else-return, cell-var-from-loop, import-error, pointless-string-statement, unused-import, redefined-builtin, superfluous-parens, unused-variable, too-many-locals, consider-using-from-import, consider-using-enumerate, no-name-in-module, too-many-arguments, too-many-instance-attributes, too-many-return-statements, import-outside-toplevel, too-few-public-methods, cyclic-import, missing-module-docstring, unidiomatic-typecheck, dangerous-default-value, unused-argument, use-dict-literal, exec-used, no-self-use, too-many-statements, ungrouped-imports, consider-using-sys-exit, too-many-statements, redundant-u-string-prefix, protected-access, consider-using-dict-comprehension, no-else-raise, too-many-nested-blocks, use-a-generator, reimported, undefined-variable, too-many-branches, raise-missing-from, trailing-comma-tuple, unspecified-encoding, consider-using-with, f-string-without-interpolation, broad-except, unnecessary-pass, global-statement, too-many-lines, consider-merging-isinstance, redefined-argument-from-local, global-variable-undefined, use-implicit-booleaness-not-len, inconsistent-return-statements, consider-using-in, inconsistent-return-statements, keyword-arg-before-vararg, consider-using-dict-items, import-self, fixme, c-extension-no-member, too-many-public-methods, consider-iterating-dictionary, consider-using-max-builtin, super-with-arguments, expression-not-assigned, unnecessary-comprehension, no-self-argument, chained-comparison, undefined-loop-variable, empty-docstring, use-maxsplit-arg, pointless-statement, wrong-import-position, redundant-unittest-assert, eval-used, not-callable, invalid-unary-operand-type, consider-using-generator, R0801, unnecessary-dunder-call, logging-fstring-interpolation, consider-using-get, useless-object-inheritance, unrecognized-option, unknown-option-value, useless-option-value
+disable = no-member, redefined-outer-name, invalid-name, consider-using-f-string, wrong-import-order, missing-function-docstring, missing-method-docstring, missing-class-docstring, attribute-defined-outside-init, no-else-return, cell-var-from-loop, import-error, pointless-string-statement, unused-import, redefined-builtin, superfluous-parens, unused-variable, too-many-locals, consider-using-from-import, consider-using-enumerate, no-name-in-module, too-many-arguments, too-many-instance-attributes, too-many-return-statements, import-outside-toplevel, too-few-public-methods, cyclic-import, missing-module-docstring, unidiomatic-typecheck, dangerous-default-value, unused-argument, use-dict-literal, exec-used, no-self-use, too-many-statements, ungrouped-imports, consider-using-sys-exit, too-many-statements, redundant-u-string-prefix, protected-access, consider-using-dict-comprehension, no-else-raise, too-many-nested-blocks, too-many-positional-arguments, use-a-generator, reimported, undefined-variable, too-many-branches, raise-missing-from, trailing-comma-tuple, unspecified-encoding, consider-using-with, f-string-without-interpolation, broad-except, unnecessary-pass, global-statement, too-many-lines, consider-merging-isinstance, redefined-argument-from-local, global-variable-undefined, use-implicit-booleaness-not-len, inconsistent-return-statements, consider-using-in, inconsistent-return-statements, keyword-arg-before-vararg, consider-using-dict-items, import-self, fixme, c-extension-no-member, too-many-public-methods, consider-iterating-dictionary, consider-using-max-builtin, super-with-arguments, expression-not-assigned, unnecessary-comprehension, no-self-argument, chained-comparison, undefined-loop-variable, empty-docstring, use-maxsplit-arg, pointless-statement, wrong-import-position, redundant-unittest-assert, eval-used, not-callable, invalid-unary-operand-type, consider-using-generator, R0801, unnecessary-dunder-call, logging-fstring-interpolation, consider-using-get, useless-object-inheritance, unrecognized-option, unknown-option-value, useless-option-value
 
 const-naming-style = snake_case
 method-naming-style = PascalCase

bin/nest-server-mpi

@@ -13,69 +13,71 @@ Options:
 
 """
 
-from docopt import docopt
-from mpi4py import MPI
-
-if __name__ == "__main__":
-    opt = docopt(__doc__)
-
 import logging
 import os
 import time
 
-logger = logging.getLogger(__name__)
-logger.setLevel(os.getenv("NEST_SERVER_MPI_LOGGER_LEVEL", "INFO"))
-
 import nest
 import nest.server
+from docopt import docopt
+from mpi4py import MPI
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv("NEST_SERVER_MPI_LOGGER_LEVEL", "INFO"))
 
 HOST = os.getenv("NEST_SERVER_HOST", "127.0.0.1")
 PORT = os.getenv("NEST_SERVER_PORT", "52425")
 
-comm = MPI.COMM_WORLD.Clone()
-rank = comm.Get_rank()
 
+def main(opt):
+    "Start the NEST Server on rank 0 and spinwait everywhere else."
+    comm = MPI.COMM_WORLD.Clone()
+    rank = comm.Get_rank()
+
+    def log(call_name, msg):
+        "Format debug output with rank annotation."
+        logger.debug(f"==> WORKER {rank}/{time.time():.7f} ({call_name}): {msg}")
 
-def log(call_name, msg):
-    msg = f"==> WORKER {rank}/{time.time():.7f} ({call_name}): {msg}"
-    logger.debug(msg)
+    if rank == 0:
+        logger.info("==> Starting NEST Server Master on rank 0")
+        nest.server.set_mpi_comm(comm)
+        nest.server.run_mpi_app(host=opt.get("--host", HOST), port=opt.get("--port", PORT))
 
+    else:
+        logger.info(f"==> Starting NEST Server Worker on rank {rank}")
+        nest.server.set_mpi_comm(comm)
 
-if rank == 0:
-    logger.info("==> Starting NEST Server Master on rank 0")
-    nest.server.set_mpi_comm(comm)
-    nest.server.run_mpi_app(host=opt.get("--host", HOST), port=opt.get("--port", PORT))
+        while True:
+            log("spinwait", "waiting for call bcast")
+            call_name = comm.bcast(None, root=0)
 
-else:
-    logger.info(f"==> Starting NEST Server Worker on rank {rank}")
-    nest.server.set_mpi_comm(comm)
+            log(call_name, "received call bcast, waiting for data bcast")
+            data = comm.bcast(None, root=0)
 
-    while True:
-        log("spinwait", "waiting for call bcast")
-        call_name = comm.bcast(None, root=0)
+            log(call_name, f"received data bcast, data={data}")
+            args, kwargs = data
 
-        log(call_name, "received call bcast, waiting for data bcast")
-        data = comm.bcast(None, root=0)
+            if call_name == "exec":
+                response = nest.server.do_exec(args, kwargs)
+            else:
+                call, args, kwargs = nest.server.nestify(call_name, args, kwargs)
+                log(call_name, f"local call, args={args}, kwargs={kwargs}")
 
-        log(call_name, f"received data bcast, data={data}")
-        args, kwargs = data
+                # The following exception handler is useful if an error
+                # occurs simulataneously on all processes. If only a
+                # subset of processes raises an exception, a deadlock due
+                # to mismatching MPI communication calls is inevitable on
+                # the next call.
+                try:
+                    response = call(*args, **kwargs)
+                except Exception:
+                    logger.error("Failed to execute call")
+                    continue
 
-        if call_name == "exec":
-            response = nest.server.do_exec(args, kwargs)
-        else:
-            call, args, kwargs = nest.server.nestify(call_name, args, kwargs)
-            log(call_name, f"local call, args={args}, kwargs={kwargs}")
+            log(call_name, f"sending reponse gather, data={response}")
+            comm.gather(nest.serialize_data(response), root=0)
 
-            # The following exception handler is useful if an error
-            # occurs simulataneously on all processes. If only a
-            # subset of processes raises an exception, a deadlock due
-            # to mismatching MPI communication calls is inevitable on
-            # the next call.
-            try:
-                response = call(*args, **kwargs)
-            except Exception:
-                logger.error("Failed to execute call")
-                continue
 
-        log(call_name, f"sending reponse gather, data={response}")
-        comm.gather(nest.serialize_data(response), root=0)
+if __name__ == "__main__":
+    opt = docopt(__doc__)
+    main(opt)

doc/htmldoc/_ext/extract_api_functions.py

@@ -35,9 +35,9 @@
 
 def find_all_variables(file_path):
     """
-    This function gets the names of all functions listed in ``__all__``
-    in each of the PyNEST API files, along with the Kernel Attributes
-    found in ``__init__.py`` of ``pynest/nest/``.
+    Get the names of all functions listed in ``__all__`` in each of the PyNEST
+    API files, along with the Kernel Attributes found in ``__init__.py`` of
+    ``pynest/nest/``.
     """
     all_variables = None
 
@@ -79,6 +79,7 @@ def process_directory(directory):
     api_exception_list = ["raster_plot", "visualization", "voltage_trace"]
     files = glob.glob(directory + "**/*.py", recursive=True)
 
+    api_name = None
     for file in files:
         # ignoring the low level api and connection_helpers and helper modules
         if "helper" in file or "ll_api" in file:
@@ -100,7 +101,7 @@ def process_directory(directory):
                 api_name = f"nest.{module_name}"
 
         all_variables = find_all_variables(file)
-        if all_variables:
+        if all_variables and api_name:
             api_dict[api_name] = all_variables
 
     return api_dict

doc/htmldoc/_ext/list_examples.py

@@ -52,7 +52,7 @@ def ProcessExamples(app, doctree, docname):
                 # correct model name
                 if requested_list["name"] == docname:
                     if requested_list["model_name"] not in models_to_examples_map:
-                        log.info("No examples found for Model: " + requested_list["model_name"])
+                        log.info("No examples found for Model: %s", requested_list["model_name"])
                         bullet_list = nodes.Text("None")
                         break
 

doc/htmldoc/_ext/model_tag_setup.py

@@ -73,7 +73,7 @@ def extract_model_text():
         with open(file_path, "r", encoding="utf8") as file:
             match = userdoc_re.search(file.read())
         if not match:
-            log.info("No user documentation found in " + str(file_path))
+            log.info("No user documentation found in %s", str(file_path))
             continue
         yield match, file_path
 
@@ -101,7 +101,7 @@ def create_rst_files(app, config):
 
     outdir = "models/"
     if not os.path.exists(outdir):
-        log.info("creating output directory " + outdir)
+        log.info("creating output directory %s", outdir)
         os.mkdir(outdir)
     outnames = []
     for match, file_path in extract_model_text():

doc/htmldoc/developer_space/templates/pynest_example_template.py

@@ -125,7 +125,6 @@
 :Authors: D Adams, N Gaiman
 """
 
-# flake8: noqa
 import nest  # [[ begin code section with imports]]
 import scipy
 
@@ -141,6 +140,13 @@
 # Note that we do not need to set parameters for the neuron and the
 # spike recorder, since they have satisfactory defaults.
 
+noise = nest.Create("poisson_generator", 2)
+voltmeter = nest.Create("voltmeter")
+n_ex = 20
+r_ex = 10
+n_in = 10
+r_in = 20
+
 nest.SetStatus(noise, [{"rate": n_ex * r_ex}, {"rate": n_in * r_in}])
 nest.SetStatus(voltmeter, {"withgid": True, "withtime": True})
 

doc/htmldoc/networks/scripts/connections.py

@@ -59,7 +59,6 @@ def beautify_layer(
     ax.set_axisbelow(True)
     ax.set_xlabel(xlabel)
     ax.set_ylabel(ylabel)
-    return
 
 
 def conn_figure(