switch to array, use mmap

skip an array access

dna attempt

Author: SamWilsn

rounds/1_histogram/solution.py

@@ -4,25 +4,32 @@
 passes out of the box. Replace the body of ``compute_histogram`` with your
 own faster implementation.
 """
+from collections import defaultdict
+from mmap import mmap, ACCESS_READ
 
+def b2i(low: int, high: int) -> int:
+    return high + (low << 8)
+
+def i2b(x: int) -> bytes:
+    return bytes([(x & 0xFF00) >> 8, x & 0xFF])
 
 def compute_histogram(path: str) -> dict[bytes, int]:
     """Frequency of every 2-byte bigram in the file at ``path``."""
     # Step 1: read the whole file into memory as a single bytes object.
-    with open(path, "rb") as f:
-        data = f.read()
+    counts = [0 for _ in range(2**16)]
 
-    # Create a 2D matrix to count bigrams
-    counts = [[0] * 256 for _ in range(256)]
+    source = open(path, "rb", buffering=0)
+    data = mmap(source.fileno(), 0, access=ACCESS_READ)
 
+    # Step 2: slide a 2-byte window across the buffer. For ``b"ABCD"`` the
+    # iterations produce ``b"AB"``, ``b"BC"``, then ``b"CD"``. For each window,
+    # bump the matching bucket in a ``dict`` keyed by the bigram itself.
+    previous = data[0]
     for i in range(len(data) - 1):
-        # Increment the count in each cell
-        counts[data[i]][data[i + 1]] += 1
+        current = data[i + 1]
+        counts[current + (previous << 8)] += 1
+        previous = current
 
-    # Convert the matrix to the original format
-    output = {}
-    for i in range(256):
-        for j in range(256):
-            if counts[i][j] > 0:
-                output[bytes([i, j])] = counts[i][j]
-    return output
+    return {
+        i2b(idx): value for idx, value in enumerate(counts) if value != 0
+    }

rounds/3_dna/solution.py

@@ -5,80 +5,58 @@
 own faster implementation.
 """
 
-from __future__ import annotations
+from mmap import mmap, ACCESS_READ
+from concurrent.futures import ThreadPoolExecutor, wait
 
-import os
-from concurrent.futures import ThreadPoolExecutor
+def _subsearch(raw, record_id_start: int, data_start: int, data_end: int, pattern: bytes):
+    plen = len(pattern)
+    data = bytes(raw[data_start : data_end - 1]).replace(b"\n", b"")
+    locations = []
+    loc = data.find(pattern)
+    while loc != -1:
+        locations.append(loc)
+        loc = data.find(pattern, loc + plen)
 
-_NL = 0x0A  # b"\n"
+    if not locations:
+        return None
 
+    record_id = raw[record_id_start : data_start - 1].decode("ascii")
+    return (record_id, locations)
 
 def find_matches(fasta_path: str, pattern: bytes) -> list[tuple[str, list[int]]]:
-    with open(fasta_path, "rb") as f:
-        data = f.read()
+    """Find every FASTA record whose sequence contains ``pattern``.
 
-    # Step 1: locate every record start. A record starts with ``>`` either at
-    # offset 0 or immediately after a ``\n``.
-    starts: list[int] = []
-    i = 0
-    while True:
-        p = data.find(b">", i)
-        if p == -1:
-            break
-        if p == 0 or data[p - 1] == _NL:
-            starts.append(p)
-        i = p + 1
-    starts.append(len(data))  # sentinel marking the end of the last record.
+    Returns ``[(record_id, [positions...]), ...]`` in file order.
+    """
+    source = open(fasta_path, "rb")
+    data = mmap(source.fileno(), 0, access=ACCESS_READ)
 
-    num_records = len(starts) - 1
-    if num_records <= 0:
-        return []
+    last = -1
 
-    # Step 2: parallel scan. Choose enough batches to keep workers balanced
-    # even when record sizes vary.
-    n_workers = max(1, os.cpu_count() or 1)
-    batches = max(1, n_workers * 4)
-    batch_size = max(1, (num_records + batches - 1) // batches)
+    data_end = len(data) - 1
+    while data[data_end] == b"\n":
+        data_end -= 1
 
-    def scan_batch(start_idx: int, end_idx: int) -> list[tuple[int, str, list[int]]]:
-        out: list[tuple[int, str, list[int]]] = []
-        for j in range(start_idx, end_idx):
-            rec_start = starts[j]
-            rec_end = starts[j + 1]
+    with ThreadPoolExecutor(max_workers=16) as executor:
+        records = []
+        while data_end > 0:
+            gt_pos = data.rfind(b">", 0, data_end)
+            if gt_pos == -1:
+                raise Exception("expected greater than")
 
-            # Locate the end of the header line within this record's slice.
-            nl = data.find(b"\n", rec_start, rec_end)
-            if nl <= rec_start:
-                continue  # Malformed or header-only.
+            record_id_start = gt_pos + 1
 
-            record_id = data[rec_start + 1 : nl].decode("ascii").strip()
+            nl_pos = data.find(b"\n", record_id_start)
+            if nl_pos == -1:
+                raise Exception("expected new line")
 
-            # Contiguous sequence: drop the newlines so matches that straddle
-            # line breaks are still found by ``bytes.find``.
-            sequence = data[nl + 1 : rec_end].replace(b"\n", b"")
+            data_start = nl_pos + 1
 
-            positions: list[int] = []
-            s = 0
-            while True:
-                p = sequence.find(pattern, s)
-                if p == -1:
-                    break
-                positions.append(p)
-                s = p + 1
+            records.append(
+                executor.submit(_subsearch, data, record_id_start, data_start, data_end, pattern)
+            )
+            data_end = gt_pos
 
-            if positions:
-                out.append((j, record_id, positions))
-        return out
-
-    with ThreadPoolExecutor(max_workers=n_workers) as pool:
-        futures = [
-            pool.submit(scan_batch, lo, min(lo + batch_size, num_records))
-            for lo in range(0, num_records, batch_size)
-        ]
-        chunks = [f.result() for f in futures]
-
-    # Step 3: flatten and restore file order (record index is monotonic per
-    # batch, but batches finish in arbitrary order).
-    flat = [item for chunk in chunks for item in chunk]
-    flat.sort(key=lambda triple: triple[0])
-    return [(rid, positions) for _, rid, positions in flat]
+        results = [d.result() for d in records if d.result() is not None]
+        results.reverse()
+        return results