Skip to content

Commit cd514f5

Browse files
author
praneeth_paikray-data
committed
Add skill authoring workflow and CI quality checks (#338, #330)
Evaluated Anthropic's skill-creator and cherry-picked its conversational authoring workflow for ai-dev-kit contributors. The existing .test/ framework already outperforms skill-creator on evaluation and optimization, so we focused on the one gap: a guided skill creation experience. What's included: - Contributor authoring skill (.skill-authoring/) with 6-phase workflow adapted from Anthropic's skill-creator (Apache 2.0, attributed) - Reference docs for skill format and test format specifications - Quick trigger validation script (.test/scripts/quick_trigger.py) - CI quality warnings in validate_skills.py (non-blocking): missing "Use when" triggers, body over 500 lines, untagged code blocks, broken reference file links - Upgraded TEMPLATE from 55-line placeholder to realistic template - Expanded CONTRIBUTING.md with quality checklist and full workflow - Added "Developing Skills" section to databricks-skills/README.md - Fixed "Use when" triggers for databricks-ai-functions and databricks-dbsql Co-authored-by: Isaac
1 parent 5cdb0c5 commit cd514f5

File tree

14 files changed

+1418
-83
lines changed

14 files changed

+1418
-83
lines changed

.github/scripts/validate_skills.py

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,20 @@
11
#!/usr/bin/env python3
22
"""Validate skill structure and frontmatter.
33
4-
Checks:
4+
Checks (errors — block PRs):
55
1. Every skill directory has a SKILL.md file
66
2. SKILL.md has valid YAML frontmatter per best practices:
77
- name: required, ≤64 chars, lowercase letters/numbers/hyphens only,
88
no XML tags, no reserved words ("anthropic", "claude")
99
- description: required, non-empty, ≤1024 chars, no XML tags
1010
3. Local skill directories are registered in install_skills.sh
1111
(skill-list variables are auto-discovered, not hardcoded)
12+
13+
Quality warnings (non-blocking):
14+
4. Description should contain "Use when" trigger phrases
15+
5. SKILL.md body should be under 500 lines (use reference files for overflow)
16+
6. Code blocks should have language tags
17+
7. Referenced files (markdown links) should exist
1218
"""
1319

1420
import re
@@ -26,6 +32,62 @@
2632
XML_TAG_RE = re.compile(r"<[^>]+>")
2733

2834

35+
CODE_BLOCK_RE = re.compile(r"^```(\w*)$", re.MULTILINE)
36+
MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
37+
MAX_BODY_LINES = 500
38+
39+
40+
def quality_warnings(skill_dir: Path, content: str, frontmatter: dict) -> list[str]:
41+
"""Run non-blocking quality checks and return warnings."""
42+
warnings = []
43+
44+
# Check description contains "Use when" trigger phrases
45+
desc = str(frontmatter.get("description", ""))
46+
if desc and "use when" not in desc.lower():
47+
warnings.append(
48+
f'{skill_dir.name}: description lacks "Use when" trigger phrases '
49+
f"(helps Claude decide when to activate the skill)"
50+
)
51+
52+
# Check body length (lines below frontmatter)
53+
body = re.sub(r"^---\n.+?\n---\n?", "", content, count=1, flags=re.DOTALL)
54+
body_lines = len(body.strip().splitlines())
55+
if body_lines > MAX_BODY_LINES:
56+
warnings.append(
57+
f"{skill_dir.name}: SKILL.md body is {body_lines} lines "
58+
f"(>{MAX_BODY_LINES}). Consider moving content to reference files."
59+
)
60+
61+
# Check code blocks have language tags
62+
# Every pair of ``` markers forms a block; even-indexed matches (0,2,4..)
63+
# are opening markers, odd-indexed are closing markers.
64+
all_fences = list(CODE_BLOCK_RE.finditer(content))
65+
opening_fences = [all_fences[i] for i in range(0, len(all_fences), 2)]
66+
untagged = sum(1 for m in opening_fences if not m.group(1))
67+
if untagged > 0:
68+
warnings.append(
69+
f"{skill_dir.name}: {untagged} code block(s) missing language tags "
70+
f"(use ```python, ```sql, ```yaml, etc.)"
71+
)
72+
73+
# Check referenced markdown files exist
74+
for match in MD_LINK_RE.finditer(content):
75+
link_target = match.group(2)
76+
# Only check relative .md links (not URLs, not anchors)
77+
if (
78+
not link_target.startswith("http")
79+
and not link_target.startswith("#")
80+
and link_target.endswith(".md")
81+
):
82+
ref_path = skill_dir / link_target
83+
if not ref_path.exists():
84+
warnings.append(
85+
f"{skill_dir.name}: referenced file '{link_target}' not found"
86+
)
87+
88+
return warnings
89+
90+
2991
def parse_frontmatter(content: str) -> dict | None:
3092
"""Extract YAML frontmatter from markdown content."""
3193
match = re.match(r"^---\n(.+?)\n---", content, re.DOTALL)
@@ -117,6 +179,7 @@ def get_local_skill_dirs() -> set[str]:
117179

118180
def main() -> int:
119181
errors: list[str] = []
182+
warnings: list[str] = []
120183
actual_skills = get_local_skill_dirs()
121184

122185
# --- Validate each skill directory's SKILL.md and frontmatter ---
@@ -153,6 +216,10 @@ def main() -> int:
153216
for err in validate_description(str(frontmatter["description"])):
154217
errors.append(f"{skill_dir.name}: {err}")
155218

219+
# Quality warnings (non-blocking)
220+
for warn in quality_warnings(skill_dir, content, frontmatter):
221+
warnings.append(warn)
222+
156223
# --- Cross-reference with install_skills.sh ---
157224
install_content = INSTALL_SCRIPT.read_text()
158225
skill_vars, composite_vars = parse_skill_variables(install_content)
@@ -182,6 +249,13 @@ def main() -> int:
182249
errors.append(f"Skills in {var_name} but no directory found: {sorted(missing)}")
183250

184251
# --- Report ---
252+
# Surface warnings (non-blocking) before errors
253+
if warnings:
254+
print(f"Quality warnings ({len(warnings)}):\n")
255+
for warning in warnings:
256+
print(f"::warning::{warning}")
257+
print()
258+
185259
if errors:
186260
print("Skill validation failed:\n")
187261
for error in errors:

0 commit comments

Comments
 (0)