feat/bep036 glob and IndexColumns

changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md

@@ -0,0 +1,48 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+### Added
+
+- Added glob to expression language
+- evalIndexColumns now checks for IndexColumns field in sidecar nad uses it for validation.
+
+<!--
+### Changed
+
+- A bullet item for the Changed category.
+
+-->
+<!--
+### Fixed
+
+- A bullet item for the Fixed category.
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category.
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category.
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category.
+
+-->
+<!--
+### Infrastructure
+
+- A bullet item for the Infrastructure category.
+
+-->

src/schema/expressionLanguage.test.ts

@@ -1,17 +1,29 @@
-import { assert, assertEquals } from '@std/assert'
+import { assert, assertEquals, equal } from '@std/assert'
 import {
   contextFunction,
+  createMatcher,
   expressionFunctions,
   formatter,
+  glob,
+  matchRecursive,
+  parsePattern,
   prepareContext,
 } from './expressionLanguage.ts'
 import { dataFile, rootFileTree } from './fixtures.test.ts'
 import type { BIDSContext } from './context.ts'
 import { makeBIDSContext } from './context.test.ts'
+import type { BIDSFile } from '../types/filetree.ts'
+import { pathsToTree } from '../files/filetree.test.ts'
 
 Deno.test('test expression functions', async (t) => {
   const context = await makeBIDSContext(dataFile, undefined, rootFileTree)
 
+  await t.step('glob function', () => {
+    assert(glob.bind(context)('sub-*').length === 1)
+    assert(glob.bind(context)('wat-*').length === 0)
+    assert(glob.bind(context)('**/*task-*').length === 6)
+    assert(glob.bind(context)('*/**/*task-*').length === 4)
+  })
   await t.step('index function', () => {
     const index = expressionFunctions.index
     assert(index([1, 2, 3], 2) === 1)
@@ -328,3 +340,170 @@ Deno.test('formatter test', async (t) => {
     }
   })
 })
+
+Deno.test('test parsePattern helper', async (t) => {
+  await t.step('parsePattern normalizes input', () => {
+    // Leading/trailing slashes removed
+    assert(equal(parsePattern('/sub-*'), ['sub-*']))
+    assert(equal(parsePattern('sub-*'), ['sub-*']))
+    assert(equal(parsePattern('sub-*/'), ['sub-*']))
+    assert(equal(parsePattern('/sub-*/'), ['sub-*']))
+  })
+
+  await t.step('parsePattern splits by /', () => {
+    assert(equal(parsePattern('sub-*/ses-*'), ['sub-*', 'ses-*']))
+  })
+
+  await t.step('parsePattern preserves ** as single component', () => {
+    assert(equal(parsePattern('**/*task-*'), ['**', '*task-*']))
+    assert(equal(parsePattern('**/sub-*/ses-*'), ['**', 'sub-*', 'ses-*']))
+    assert(equal(parsePattern('sub-*/**/*_T1w.nii.gz'), ['sub-*', '**', '*_T1w.nii.gz']))
+  })
+
+  await t.step('parsePattern handles edge cases', () => {
+    assert(equal(parsePattern(''), []))
+    assert(equal(parsePattern('/'), []))
+    assert(equal(parsePattern('///'), []))
+  })
+})
+
+Deno.test('test createMatcher helper', async (t) => {
+  await t.step('createMatcher ** matches everything', () => {
+    const matcher = createMatcher('**')
+    assert(matcher('anything') === true)
+    assert(matcher('sub-01') === true)
+    assert(matcher('') === true)
+  })
+
+  await t.step('createMatcher * matches any name', () => {
+    const matcher = createMatcher('*')
+    assert(matcher('sub-01') === true)
+    assert(matcher('anything') === true)
+    assert(matcher('a') === true)
+  })
+
+  await t.step('createMatcher ? matches single character', () => {
+    const matcher = createMatcher('?')
+    assert(matcher('a') === true)
+    assert(matcher('1') === true)
+    assert(matcher('ab') === false)
+    assert(matcher('') === false)
+  })
+
+  await t.step('createMatcher literal glob patterns', () => {
+    const subMatcher = createMatcher('sub-*')
+    assert(subMatcher('sub-01') === true)
+    assert(subMatcher('sub-02') === true)
+    assert(subMatcher('ses-01') === false)
+
+    const taskMatcher = createMatcher('*task-*')
+    assert(taskMatcher('task-rest') === true)
+    assert(taskMatcher('boldtask-rest') === true)
+    assert(taskMatcher('no-match') === false)
+  })
+
+  await t.step('createMatcher ? in patterns', () => {
+    const matcher = createMatcher('sub-?')
+    assert(matcher('sub-0') === true)
+    assert(matcher('sub-a') === true)
+    assert(matcher('sub-01') === false)
+  })
+})
+
+Deno.test('test matchRecursive helper', async (t) => {
+  const context = await makeBIDSContext(dataFile, undefined, rootFileTree)
+
+  await t.step('matchRecursive basic functionality', () => {
+    // Match at root level
+    const results = Array.from(
+      matchRecursive(context.dataset.tree, ['sub-*'], ''),
+    ) as string[]
+    assert(results.length == 1)
+    assert(results[0].startsWith('sub-'))
+  })
+
+  await t.step('matchRecursive with multiple components', () => {
+    // Match nested directories
+    const results = Array.from(
+      matchRecursive(context.dataset.tree, ['sub-*', 'ses-*'], ''),
+    ) as string[]
+    assert(results.every((r) => r.includes('sub-') && r.includes('ses-')))
+  })
+
+  await t.step('matchRecursive with ** at start', () => {
+    // ** at start should match sub-* at root level
+    const results = Array.from(
+      matchRecursive(context.dataset.tree, ['**', 'sub-*'], ''),
+    ) as string[]
+    assert(results.length > 1)
+    assert(results.some((r) => r.startsWith('sub-')))
+    assert(results.every((r) => r == 'sub-01' || r.includes('/sub-01_')))
+  })
+
+  await t.step('matchRecursive with no matches', () => {
+    const results = Array.from(
+      matchRecursive(context.dataset.tree, ['nonexistent-*'], ''),
+    ) as string[]
+    assert(results.length === 0)
+  })
+})
+
+Deno.test('test glob edge cases and root-level matching', async (t) => {
+  const context = await makeBIDSContext(dataFile, undefined, rootFileTree)
+
+  await t.step('glob with ** at start matches at root level', () => {
+    // **/sub-* should match sub-* directories at root
+    const results = glob.bind(context)('**/sub-*')
+    assert(results.length > 0)
+    assert(results.some((r) => r.startsWith('sub-')))
+  })
+
+  await t.step('glob with just ** returns all paths', () => {
+    const results = glob.bind(context)('**')
+    assert(results.length > 0)
+  })
+
+  await t.step('glob with just * returns root-level items', () => {
+    const results = glob.bind(context)('*')
+    assert(results.every((r) => !r.includes('/')))
+  })
+
+  await t.step('glob with ? matches single characters', () => {
+    let results = glob.bind(context)('?')
+    assert(results.length === 0)
+    results = glob.bind(context)('?ataset_description.json')
+    assert(results.length === 1)
+    results = glob.bind(context)('sub-??')
+    assert(results.length === 1)
+  })
+
+  await t.step('glob deep patterns work correctly', () => {
+    const results = glob.bind(context)('**/sub-*/ses-*/anat/*T1*')
+    assert(Array.isArray(results))
+    // Verify all results contain expected components
+    assert(
+      results.every(
+        (r) => r.includes('sub-') && r.includes('ses-') && r.includes('anat'),
+      ),
+    )
+  })
+
+  await t.step('** does not result in duplicate matches', async () => {
+    const tree = pathsToTree(['/a/b/c/d/e'])
+    const datafile = tree.get('a/b/c/d/e') as BIDSFile
+    const context = await makeBIDSContext(datafile, undefined, tree)
+
+    let results = glob.bind(context)('a/**/e')
+    assertEquals(results.length, 1)
+    results = glob.bind(context)('a/*/**/e')
+    assertEquals(results.length, 1)
+    results = glob.bind(context)('a/**/*/e')
+    assertEquals(results.length, 1)
+  })
+
+  await t.step('glob returns both files and directories', () => {
+    const results = glob.bind(context)('**/*')
+    assert(results.some((r) => r.endsWith('anat'))) // Directory
+    assert(results.some((r) => r.endsWith('T1w.nii.gz'))) // File
+  })
+})

src/schema/expressionLanguage.ts

@@ -1,4 +1,5 @@
 import type { BIDSContext } from './context.ts'
+import type { FileTree } from '../types/filetree.ts'
 import { memoize } from '../utils/memoize.ts'
 
 function exists(this: BIDSContext, list: string[], rule: string = 'dataset'): number {
@@ -36,6 +37,124 @@ function exists(this: BIDSContext, list: string[], rule: string = 'dataset'): nu
   }
 }
 
+/*
+ * Glob utility function. Breaks initial glob pattern into per directory components.
+ */
+export function parsePattern(pattern: string): string[] {
+  // Remove leading/trailing slashes
+  pattern = pattern.replace(/^\/+|\/+$/g, '')
+
+  if (!pattern) return []
+
+  // Split by '/', filter out empty strings
+  return pattern.split('/').filter((c) => c.length > 0)
+}
+
+/*
+ * Glob utility function. Generates function to match individual components of glob.
+ */
+export function createMatcher(component: string): (name: string) => boolean {
+  if (component === '**') {
+    return () => true // Special case, handled in recursion
+  }
+
+  if (component === '*') {
+    return (name) => true // Match any single name
+  }
+
+  if (component === '?') {
+    return (name) => name.length === 1
+  }
+
+  // Glob pattern with * and ? wildcards
+  // Convert to regex: sub-* → /^sub-.*$/
+  const pattern = '^' + component.replace(/\*/g, '.*').replace(/\?/g, '.') + '$'
+  const re = new RegExp(pattern)
+  return (name) => re.test(name)
+}
+
+/*
+ * Glob utility function. Recurses filetree applying match functions to the
+ * appropriate directories and files.
+ */
+export function* matchRecursive(
+  tree: FileTree,
+  components: string[],
+  currentPath: string,
+): Generator<string> {
+  if (components.length === 0) {
+    return
+  }
+
+  const component = components[0]
+  const remaining = components.slice(1)
+
+  // Special handling for '**': match zero or more directories
+  if (component === '**') {
+    if (remaining.length === 0) {
+      // Just '**' - match all files and directories recursively
+      yield* matchAll(tree, currentPath)
+      return
+    }
+
+    // '**' followed by more patterns
+    // Try to match the next pattern at this level (zero directories)
+    yield* matchRecursive(tree, remaining, currentPath)
+
+    // Also try deeper (one or more directories)
+    for (const dir of tree.directories) {
+      yield* matchRecursive(dir, components, `${currentPath}/${dir.name}`.replace(/^\//, ''))
+    }
+    return
+  }
+
+  const matcher = createMatcher(component)
+
+  // Check files at this level
+  if (remaining.length === 0) {
+    for (const file of tree.files) {
+      if (matcher(file.name)) {
+        yield currentPath ? `${currentPath}/${file.name}` : file.name
+      }
+    }
+  }
+
+  // Check directories at this level
+  for (const dir of tree.directories) {
+    if (matcher(dir.name)) {
+      if (remaining.length === 0) {
+        yield currentPath ? `${currentPath}/${dir.name}` : dir.name
+      } else {
+        // Recurse into matched directory
+        yield* matchRecursive(dir, remaining, `${currentPath}/${dir.name}`.replace(/^\//, ''))
+      }
+    }
+  }
+}
+
+/*
+ * Glob utility function. Used to handle '**' in glob pattern.
+ */
+function* matchAll(tree: FileTree, currentPath: string): Generator<string> {
+  for (const file of tree.files) {
+    yield currentPath ? `${currentPath}/${file.name}` : file.name
+  }
+  for (const dir of tree.directories) {
+    const newPath = `${currentPath}/${dir.name}`.replace(/^\//, '')
+    yield newPath
+    yield* matchAll(dir, newPath)
+  }
+}
+
+export function glob(this: BIDSContext, toMatch: string): string[] {
+  const components = parsePattern(toMatch)
+  if (components.length === 0) {
+    return []
+  }
+
+  return Array.from(matchRecursive(this.dataset.tree, components, ''))
+}
+
 export const expressionFunctions = {
   index: <T>(list: T[], item: T): number | null => {
     const index = list.indexOf(item)
@@ -136,6 +255,7 @@ export const expressionFunctions = {
   allequal: <T>(a: T[], b: T[]): boolean => {
     return (a != null && b != null) && a.length === b.length && a.every((v, i) => v === b[i])
   },
+  glob: glob,
 }
 
 /**