Skip to content

Commit f8d1606

Browse files
chalmerlowechalmerlowe
committed
feat(version-scanner): generalize and enhance regex rules for dependency scanning
1 parent 0258405 commit f8d1606

5 files changed

Lines changed: 430 additions & 249 deletions

File tree

‎scripts/version_scanner/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ python3 scripts/version_scanner/version_scanner.py -d <dependency> -v <version>
1717
* `-p`, `--path`: Root directory to scan (defaults to current directory)
1818
* `--package`: Specific subdirectory filter (useful for monorepos)
1919
* `--package-file`: Path to a file containing a list of package directories to scan (e.g., `scripts/version_scanner/small_package_list.txt`)
20-
* `--config`: Path to the regex configuration file (defaults to scripts/version_scanner/regex_config.yaml)
20+
* `--config`: Path to the regex configuration file (defaults to scripts/version_scanner/regex_pattern_config.yaml)
2121
* `-o`, `--output`: Path to the output CSV file (defaults to <dependency>-<version>-<timestamp>.csv)
2222
* `--github-repo`: GitHub repository URL base (defaults to https://github.com/googleapis/google-cloud-python)
2323
* `--branch`: GitHub branch for links (defaults to main)
@@ -43,7 +43,7 @@ pip install -r scripts/version_scanner/requirements.txt
4343

4444
## Configuration
4545

46-
The scanner uses a YAML configuration file (`regex_config.yaml`) to define rules and regex patterns.
46+
The scanner uses a YAML configuration file (`regex_pattern_config.yaml`) to define rules and regex patterns.
4747

4848
## Ignoring Directories
4949

scripts/version_scanner/regex_config.yaml renamed to scripts/version_scanner/regex_pattern_config.yaml

Lines changed: 112 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
11
description: Search rules for identifying dependency versions
2+
23
rules:
3-
- name: explicit_version_string
4-
description: Finds explicit version strings in code or configs.
5-
examples:
6-
- "'3.7'"
7-
- '"3.7.1"'
8-
- "'3.7.12'"
9-
- "Python 3.7"
10-
rules:
11-
- |
12-
\b{major}\.{minor}(\.\d+)?\b
4+
# ==========================================
5+
# 🐍 PYTHON RUNTIME SPECIFIC RULES
6+
# ==========================================
137

148
- name: python_requires
159
description: Finds various forms of python_requires declarations.
@@ -123,16 +117,119 @@ rules:
123117
- |
124118
Python{major}{minor}(?!\d)
125119
120+
# ==========================================
121+
# 📦 GENERIC RULES (WILL APPLY TO RUNTIMES OR PACKAGES)
122+
# ==========================================
123+
124+
- name: explicit_version_string
125+
description: Finds explicit version strings in code or configs.
126+
examples:
127+
- "'3.7'"
128+
- '"3.7.1"'
129+
- "'3.7.12'"
130+
- "Python 3.7"
131+
rules:
132+
- |
133+
\b{major}\.{minor}(\.\d+)?\b
134+
126135
- name: dependency_requirement
127-
description: Finds standard dependency requirement formats (e.g., protobuf==3.7).
136+
description: Finds standard dependency requirements (e.g., protobuf==3.7, pandas>=1.0).
128137
examples:
129138
- "protobuf==3.7"
130-
- "protobuf>=3.7"
131-
- "protobuf<=3.7"
132-
- "protobuf~=3.7"
133-
- "protobuf!=3.7"
139+
- "protobuf >=3.7"
140+
- "protobuf<= 3.7"
141+
- "protobuf == 3.7"
134142
rules:
135143
- |
136144
{name}\s*(?:==|>=|<=|~=|!=)\s*{version}(?!\d)
137145
146+
- name: dependency_version_constant
147+
description: Finds checks against custom version constants (e.g., PROTOBUF_VERSION, PANDAS_VERSION).
148+
examples:
149+
- "PROTOBUF_VERSION = '3.7'"
150+
- "PROTOBUF_VERSION= '3.7'"
151+
- "PROTOBUF_VERSION ='3.7'"
152+
- "PROTOBUF_VERSION='3.7'"
153+
- 'PROTOBUF_VERSION = "3.7"'
154+
rules:
155+
- |
156+
{name}_VERSION\s*=\s*['"]?{version}['"]?
157+
158+
- name: dependency_version_constant_membership
159+
description: Finds checks where a version constant is checked for membership in a collection.
160+
examples:
161+
- 'PROTOBUF_VERSION[0:2] in ["3.", "4."]'
162+
- 'PROTOBUF_VERSION[0:2] in {"3.", "4."}'
163+
- 'PROTOBUF_VERSION[0:2] in ("3.", "4.")'
164+
rules:
165+
- |
166+
{name}_VERSION(?:\[.*?\])?\s+in\s+[\(\[\{{].*?['"]?{major}\b.*?[\)\]\}}]
167+
168+
- name: dependency_wildcard_assignment
169+
description: Finds wildcard version specifications in code assignments or requirements.
170+
examples:
171+
- 'protobuf == "3.x"'
172+
- 'protobuf >= "3.*"'
173+
rules:
174+
- |
175+
{name}\s*(?:==|>=|<=|~=|!=)\s*['"]?{major}\.[x\*]['"]?
176+
177+
- name: dependency_wildcard_generic
178+
description: Finds wildcard version specifications in comments or documentation.
179+
examples:
180+
- "protobuf 3.x"
181+
- "protobuf 3.*"
182+
rules:
183+
- |
184+
{name}\s+.*{major}\.[x\*](?!\w)
185+
186+
- name: dependency_flexible_version
187+
description: Finds versions with varying levels of specificity with accompanying qualifier.
188+
examples:
189+
- "protobuf 3."
190+
- "protobuf 3.7"
191+
- "protobuf 3.7."
192+
- "protobuf 3.7.5"
193+
rules:
194+
- |
195+
{name}\s+{major}\.(?!\d)
196+
- |
197+
\b{major}\.{minor}(?!\d)
198+
- |
199+
\b{major}\.{minor}\.(?!\d)
200+
- |
201+
\b{major}\.{minor}\.{patch}(?!\d)
202+
203+
- name: generic_dependency_check_call
204+
description: Finds generic function or class calls checking dependency and version.
205+
examples:
206+
- 'DependencyConstraint("google.protobuf", minimum_fully_supported_version="3.7")'
207+
- 'CheckDep("protobuf", "3.7")'
208+
rules:
209+
- |
210+
\b[A-Za-z_]\w*\s*\([^)]*['"](?:google\.)?{name}['"][^)]*{version}[^)]*\)
211+
212+
- name: dependency_introspection
213+
description: Finds standard package introspection methods and libraries.
214+
examples:
215+
- "protobuf.__version__"
216+
- "importlib.metadata.version('protobuf')"
217+
- "pkg_resources.get_distribution('protobuf')"
218+
- "packaging.version"
219+
- "dist.metadata['Name'] == 'protobuf'"
220+
rules:
221+
- |
222+
{name}\.__version__
223+
- |
224+
importlib\.metadata\.version\s*\(\s*['"]{name}['"]\s*\)
225+
- |
226+
pkg_resources\.get_distribution\s*\(\s*['"]{name}['"]\s*\)
227+
- |
228+
packaging\.version
229+
- |
230+
dist\.metadata\s*\[\s*['"]Name['"]\s*\]\s*==\s*['"]{name}['"]
138231
232+
# ==========================================
233+
# 🎯 PACKAGE SPECIFIC RULES (I.E. SPECIFIC TO PROTOBUF, PANDAS, ETC)
234+
# ==========================================
235+
# Currently empty.

‎scripts/version_scanner/tests/integration/test_scanner_integration.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
def test_integration_scan(tmp_path):
2121
# Paths to real tools
2222
scanner_path = os.path.abspath("version_scanner.py")
23-
config_path = os.path.abspath("regex_config.yaml")
23+
config_path = os.path.abspath("regex_pattern_config.yaml")
2424

2525
# Static data directory
2626
data_dir = os.path.abspath("tests/data")

‎scripts/version_scanner/tests/unit/test_version_scanner.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -454,7 +454,7 @@ def test_upload_to_drive(mock_auth, mock_build):
454454

455455
def test_regex_examples_from_config():
456456
"""Test that examples in config match at least one rule in the group."""
457-
config_path = "regex_config.yaml"
457+
config_path = "regex_pattern_config.yaml"
458458

459459
try:
460460
with open(config_path, 'r') as f:
@@ -505,7 +505,7 @@ def test_regex_examples_from_config():
505505

506506
def test_regex_negative_cases():
507507
"""Verify regex patterns prevent false positives (lookaheads, patch bounds) and support whitespace."""
508-
config_path = "regex_config.yaml"
508+
config_path = "regex_pattern_config.yaml"
509509
with open(config_path, 'r') as f:
510510
config = yaml.safe_load(f)
511511

@@ -646,7 +646,7 @@ def test_scan_file_truncation_bug(tmp_path):
646646
from version_scanner import ConfigManager, scan_file
647647

648648
# Init config for 3.1
649-
config_manager = ConfigManager("regex_config.yaml", "python", "3.1")
649+
config_manager = ConfigManager("regex_pattern_config.yaml", "python", "3.1")
650650
rules = config_manager.load_config()
651651
import re
652652
compiled_rules = [{"name": r["name"], "pattern": re.compile(r["pattern"], re.IGNORECASE)} for r in rules]
@@ -828,7 +828,7 @@ def test_scan_repository_multi_targets(tmp_path):
828828
file2.write_text("protobuf==4.25.8\n")
829829

830830
# Let's mock a config file with rules for both python and protobuf
831-
config_file = tmp_path / "regex_config.yaml"
831+
config_file = tmp_path / "regex_pattern_config.yaml"
832832
config_file.write_text("""
833833
rules:
834834
- name: python_requires_check

0 commit comments

Comments
 (0)