Merge pull request #263 from ncsa/feature/claude-assisted-tests

Feature/claude assisted tests

Author: joshfactorial

.gitignore

@@ -1,6 +1,31 @@
 # Ignore filetypes
 *.pyc
+*.pyo
+*.pyd
+__pycache__/
+
+# Virtual environments
 /python2env/
+/.venv/
+/venv/
+/env/
+
+# IDEs
 /.ipynb_checkpoints/
 /.vscode/
-/.idea/
+/.idea/
+
+# Test & coverage artifacts
+.coverage
+.coverage.*
+htmlcov/
+.pytest_cache/
+
+# NEAT log files
+*.log
+
+# Build / packaging
+dist/
+build/
+*.egg-info/
+*.egg

ChangeLog.md

@@ -1,6 +1,9 @@
 # NEAT has a new home
 NEAT is now a part of the NCSA github and active development will continue here. Please direct issues, comments, and requests to the NCSA issue tracker. Submit pull requests here insead of the old repo.
 
+# NEAT v4.3.6
+- Multiple bug fixes, fixes to outputs. See release for full notes.
+
 # NEAT v4.3.5
 - An improvement rather than a bug fix this time. We moved vcf processing into the threaded portion, as our speeds were better than single threaded, but very slow on the vcf writing portion. This sped things up considerably, so we tested and confirmed that it is working as desired and are updating to a new version with improved VCF production in parallel mode.
 

README.md

@@ -1,6 +1,6 @@
-# The NEAT Project v4.3.5
+# The NEAT Project v4.3.6
 
-Welcome to the NEAT project, the NExt-generation sequencing Analysis Toolkit, version 4.3.5. This release of NEAT 4.3.5 includes several fixes and a little bit of restructuring, including a parallel process for running `neat read-simulator`. Our tests show much improved performance. If the logs seem excessive, you might try using the `--log-level ERROR` to reduce the output from the logs. See the [ChangeLog](ChangeLog.md) for notes. NEAT 4.3.5 is the official release of NEAT 4.0. It represents a lot of hard work from several contributors at NCSA and beyond. With the addition of parallel processing, we feel that the code is ready for production, and future releases will focus on compatibility, bug fixes, and testing. Future releases for the time being will be enumerations of 4.3.X.
+Welcome to the NEAT project, the NExt-generation sequencing Analysis Toolkit, version 4.3.6. This release of NEAT 4.3.5 includes several fixes and a little bit of restructuring, including a parallel process for running `neat read-simulator`. Our tests show much improved performance. If the logs seem excessive, you might try using the `--log-level ERROR` to reduce the output from the logs. See the [ChangeLog](ChangeLog.md) for notes. NEAT 4.3.5 is the official release of NEAT 4.0. It represents a lot of hard work from several contributors at NCSA and beyond. With the addition of parallel processing, we feel that the code is ready for production, and future releases will focus on compatibility, bug fixes, and testing. Future releases for the time being will be enumerations of 4.3.X.
 
 ## NEAT v4.3.5
 
@@ -22,7 +22,7 @@ To cite this work, please use:
 
 ## Table of Contents
 
-* [The NEAT Project v4.3.5](#the-neat-project-v435)
+* [The NEAT Project v4.3.6](#the-neat-project-v436)
 * [NEAT v4.3.5](#neat-v435)
 * [Table of Contents](#table-of-contents)
   * [Prerequisites](#prerequisites)
@@ -77,7 +77,7 @@ To install NEAT, you must create a virtual environment using a tool such as `con
 First, clone the environment and move to the NEAT directory:
 
 ```bash
-$ git clone git@github.com:ncsa/NEAT.git 
+$ git clone https://github.com/ncsa/NEAT.git
 $ cd NEAT
 ```
 

environment.yml

@@ -1,8 +1,8 @@
 name: neat
 
 channels:
-  - bioconda
   - conda-forge
+  - bioconda
 
 dependencies:
   - python==3.11.*

neat/read_simulator/utils/vcf_func.py

@@ -161,7 +161,7 @@ def parse_input_vcf(
                         # Retrieve the GT from the first sample in the record
                         genotype = retrieve_genotype(record)
 
-                    elif "WP" in [x.split('=')[0] for x in record[7].split(';')]:
+                    elif "WP" in [x.split('=')[0] for x in record[7].split(';') if '=' in x]:
                         """
                         "WP" is the legacy code NEAT used for genotype it added. It was found in the INFO field.
                         We're just going to make a sample column in this version of NEAT
@@ -171,10 +171,13 @@ def parse_input_vcf(
                         format_column = f"GT:{record[8]}"
                         sample_field = record[9]
                         for info_item in record[7].split(';'):
-                            if info_item.startswith('WP'):
+                            if info_item.startswith('WP') and '=' in info_item:
                                 genotype = info_item.split('=')[1].replace('/', '|').split('|')
                                 genotype = np.array([int(x) for x in genotype])
                                 normal_sample_field = f"{get_genotype_string(genotype)}:{sample_field}"
+                            elif info_item.startswith('WP'):
+                                _LOG.error(f'Malformed WP field in INFO (missing value): {record[7]}')
+                                sys.exit(1)
 
                     else:
                         format_column = 'GT:' + record[8]
@@ -183,7 +186,7 @@ def parse_input_vcf(
                         gt_field = get_genotype_string(genotype)
                         normal_sample_field = f'{gt_field}:{record[9]}'
 
-                elif "WP" in [x.split('=')[0] for x in record[7].split(';')]:
+                elif "WP" in [x.split('=')[0] for x in record[7].split(';') if '=' in x]:
                     """
                     "WP" is the legacy code NEAT used for genotype it added. It was found in the INFO field.
                     We're just going to make a sample column in this version of NEAT
@@ -192,10 +195,13 @@ def parse_input_vcf(
                     """
                     format_column = "GT"
                     for info_item in record[7].split(';'):
-                        if info_item.startswith('WP'):
+                        if info_item.startswith('WP') and '=' in info_item:
                             genotype = info_item.split('=')[1].replace('/', '|').split('|')
                             genotype = np.array([int(x) for x in genotype])
                             normal_sample_field = get_genotype_string(genotype)
+                        elif info_item.startswith('WP'):
+                            _LOG.error(f'Malformed WP field in INFO (missing value): {record[7]}')
+                            sys.exit(1)
 
                 else:
                     # If there was no format column, there's no sample column, so we'll generate one

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "neat"
-version = "4.3.5"
+version = "4.3.6"
 description = "NGS Simulation toolkit"
 authors = ["Joshua Allen <jallen17@illinois.edu>"]
 license = "BSD 3-Clause License"

tests/conftest.py

@@ -0,0 +1,30 @@
+import logging
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _isolate_neat_logging():
+    """
+    Close and remove any FileHandlers attached to NEAT loggers before each test.
+    Prevents 'ValueError: I/O operation on closed file' errors when a FileHandler
+    from a previous test is still attached after its underlying file is closed.
+    Propagation is left intact so caplog can capture NEAT log output.
+    """
+    def _close_file_handlers(logger):
+        for h in list(logger.handlers):
+            if isinstance(h, logging.FileHandler):
+                logger.removeHandler(h)
+                try:
+                    h.close()
+                except Exception:
+                    pass
+
+    for name, logger in list(logging.Logger.manager.loggerDict.items()):
+        if (name == "neat" or name.startswith("neat.")) and isinstance(logger, logging.Logger):
+            _close_file_handlers(logger)
+
+    yield
+
+    for name, logger in list(logging.Logger.manager.loggerDict.items()):
+        if (name == "neat" or name.startswith("neat.")) and isinstance(logger, logging.Logger):
+            _close_file_handlers(logger)

tests/test_cli/test_basic_cli.py

@@ -36,6 +36,7 @@ def test_basic_cli():
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
+            cwd=str(td),
         )
         assert proc.returncode == 0, f"STDERR:\n{proc.stderr}"
         assert out.exists()

tests/test_models/test_error_and_mut_models.py

@@ -41,26 +41,8 @@ def test_mutation_model_generate_snv_trinuc():
     assert snv.alt in ["A", "C", "G", "T"]
 
 
-def test_sequencing_error_model_zero_error_returns_none_or_empty():
-    """
-    avg_seq_error == 0 should yield no errors.
-    """
-    rng = default_rng(4)
-    sem = SequencingErrorModel(avg_seq_error=0.0)
-    ref = SeqRecord(Seq("A" * 40), id="chr1")
-    quals = np.array([40] * 40, dtype=int)
-    result = sem.get_sequencing_errors(
-        padding=20,
-        reference_segment=ref,
-        quality_scores=quals,
-        rng=rng,
-    )
-    if isinstance(result, tuple):
-        introduced, pad = result
-        assert introduced == []
-        assert pad >= 0
-    else:
-        assert result == []
+    # test_sequencing_error_model_zero_error_returns_none_or_empty removed:
+    # duplicate of test_error_models.py::test_sem_zero_error_rate_returns_empty
 
 
 def test_traditional_quality_model_shapes_and_range():
@@ -135,16 +117,8 @@ def test_mutation_model_snv_does_not_keep_reference_base():
     assert snv.alt != central
 
 
-def test_traditional_quality_model_reproducible_with_seed():
-    """Quality model should be deterministic given the same RNG state."""
-    rng1 = default_rng(8)
-    rng2 = default_rng(8)
-    qm = TraditionalQualityModel(average_error=0.01)
-
-    qs1 = qm.get_quality_scores(model_read_length=151, length=100, rng=rng1)
-    qs2 = qm.get_quality_scores(model_read_length=151, length=100, rng=rng2)
-
-    assert np.array_equal(qs1, qs2)
+    # test_traditional_quality_model_reproducible_with_seed removed:
+    # duplicate of test_error_models.py::test_tqm_get_quality_scores_reproducible
 
 
 def test_sequencing_error_model_reproducible_with_seed():