aitools: derive experimental state from manifest repo_dir

Drop the parallel `experimental_skills` map and the per-skill `experimental`
bool from the wire format. The manifest now exposes a single `skills` map
where each entry carries `repo_dir` ("skills" or "experimental"), and the
cli derives experimental state from it via SkillMeta.IsExperimental.

Renames flattenManifest → normalizeManifest; the rest of the installer is
unchanged. Acceptance fixtures and unit tests updated to the new shape.

Co-authored-by: Isaac

Author: jamesbroadhead

acceptance/experimental/aitools/skills/install-experimental-empty/output.txt

@@ -2,5 +2,6 @@
 === --experimental against a manifest with no experimental skills logs a nudge
 >>> [CLI] experimental aitools install --global --experimental
 Installing Databricks AI skills for Claude Code...
+Using skills version test-ref
 Warn: --experimental was set but the manifest at test-ref exposes no experimental skills. Set DATABRICKS_SKILLS_REF to a release that includes them (or =main for the latest).
-Installed 1 skill (vtest-ref).
+Installed 1 skill.

acceptance/experimental/aitools/skills/install-experimental-empty/test.toml

@@ -8,7 +8,7 @@ Ignore = [
 [EnvMatrix]
 DATABRICKS_BUNDLE_ENGINE = []
 
-# Manifest with stable skills only — no experimental_skills section.
+# Manifest with stable skills only — no entries with repo_dir=experimental.
 # Simulates a release tag that pre-dates the experimental feature.
 [[Server]]
 Pattern = "GET /test-ref/manifest.json"

acceptance/experimental/aitools/skills/install-specific/output.txt

@@ -2,16 +2,19 @@
 === install only one specific stable skill via --skills
 >>> [CLI] experimental aitools install --global --skills test-stable-a
 Installing Databricks AI skills for Claude Code...
-Installed 1 skill (vtest-ref).
+Using skills version test-ref
+Installed 1 skill.
 
 === install a specific experimental skill (note the -experimental suffix)
 >>> [CLI] experimental aitools install --global --skills test-exp-experimental --experimental
 Installing Databricks AI skills for Claude Code...
-Installed 1 skill (vtest-ref).
+Using skills version test-ref
+Installed 1 skill.
 
 === asking for an experimental skill without --experimental flag errors out
 >>> [CLI] experimental aitools install --global --skills test-exp-experimental
 Installing Databricks AI skills for Claude Code...
+Using skills version test-ref
 Error: skill "test-exp-experimental" is experimental; use --experimental to install
 
 Exit code: 1

acceptance/experimental/aitools/skills/install-specific/test.toml

@@ -15,11 +15,9 @@ Response.Body = '''
   "version": "2",
   "updated_at": "2026-01-01T00:00:00Z",
   "skills": {
-    "test-stable-a": {"version": "1.0.0", "files": ["SKILL.md"]},
-    "test-stable-b": {"version": "1.0.0", "files": ["SKILL.md"]}
-  },
-  "experimental_skills": {
-    "test-exp": {"version": "0.0.1", "files": ["SKILL.md"]}
+    "test-stable-a": {"version": "1.0.0", "files": ["SKILL.md"], "repo_dir": "skills"},
+    "test-stable-b": {"version": "1.0.0", "files": ["SKILL.md"], "repo_dir": "skills"},
+    "test-exp":      {"version": "0.0.1", "files": ["SKILL.md"], "repo_dir": "experimental"}
   }
 }
 '''

acceptance/experimental/aitools/skills/install/output.txt

@@ -2,14 +2,17 @@
 === stable-only install (no --experimental flag) installs 1 skill
 >>> [CLI] experimental aitools install --global
 Installing Databricks AI skills for Claude Code...
-Installed 1 skill (vtest-ref).
+Using skills version test-ref
+Installed 1 skill.
 
 === re-run with --experimental installs the experimental one too
 >>> [CLI] experimental aitools install --global --experimental
 Installing Databricks AI skills for Claude Code...
-Installed 2 skills (vtest-ref).
+Using skills version test-ref
+Installed 2 skills.
 
 === no-op re-run is idempotent (no new fetches, no errors)
 >>> [CLI] experimental aitools install --global --experimental
 Installing Databricks AI skills for Claude Code...
-Installed 2 skills (vtest-ref).
+Using skills version test-ref
+Installed 2 skills.

acceptance/experimental/aitools/skills/install/test.toml

@@ -13,7 +13,9 @@ Ignore = [
 [EnvMatrix]
 DATABRICKS_BUNDLE_ENGINE = []
 
-# Mock the manifest. One stable skill, one experimental skill.
+# Mock the manifest. One stable skill, one experimental skill — the
+# experimental state is conveyed by the per-skill repo_dir field, not a
+# separate experimental_skills map.
 [[Server]]
 Pattern = "GET /test-ref/manifest.json"
 Response.Body = '''
@@ -24,14 +26,14 @@ Response.Body = '''
     "test-stable": {
       "version": "1.0.0",
       "description": "Stable test skill",
-      "files": ["SKILL.md"]
-    }
-  },
-  "experimental_skills": {
+      "files": ["SKILL.md"],
+      "repo_dir": "skills"
+    },
     "test-exp": {
       "version": "0.0.1",
       "description": "Experimental test skill",
-      "files": ["SKILL.md"]
+      "files": ["SKILL.md"],
+      "repo_dir": "experimental"
     }
   }
 }

experimental/aitools/cmd/list.go

@@ -102,7 +102,7 @@ func defaultListSkills(cmd *cobra.Command, scope string) error {
 		meta := manifest.Skills[name]
 
 		tag := ""
-		if meta.Experimental {
+		if meta.IsExperimental() {
 			tag = " [experimental]"
 		}
 

experimental/aitools/lib/installer/installer.go

@@ -31,11 +31,11 @@ const (
 	experimentalSuffix   = "-experimental"
 )
 
-// manifestHasExperimental reports whether the flattened manifest contains
-// at least one entry marked Experimental=true.
+// manifestHasExperimental reports whether the manifest contains at least one
+// experimental skill (sourced from the experimental/ directory upstream).
 func manifestHasExperimental(m *Manifest) bool {
 	for _, meta := range m.Skills {
-		if meta.Experimental {
+		if meta.IsExperimental() {
 			return true
 		}
 	}
@@ -87,39 +87,44 @@ func GetSkillsBaseURL(ctx context.Context) string {
 // Manifest describes the skills manifest fetched from the skills repo.
 //
 // The repo exposes stable skills under skills/ and experimental skills under
-// experimental/. Both are flattened into Skills during FetchManifest so the
-// rest of the installer can treat them uniformly; experimental entries carry
-// Experimental=true and RepoDir=experimentalRepoPath.
+// experimental/. Both appear in a single Skills map; each entry carries
+// RepoDir indicating which top-level directory holds its files, which is
+// also the source of truth for whether the skill is experimental.
 type Manifest struct {
-	Version            string               `json:"version"`
-	UpdatedAt          string               `json:"updated_at"`
-	Skills             map[string]SkillMeta `json:"skills"`
-	ExperimentalSkills map[string]SkillMeta `json:"experimental_skills,omitempty"`
+	Version   string               `json:"version"`
+	UpdatedAt string               `json:"updated_at"`
+	Skills    map[string]SkillMeta `json:"skills"`
 }
 
 // SkillMeta describes a single skill entry in the manifest.
 type SkillMeta struct {
-	Version      string   `json:"version"`
-	UpdatedAt    string   `json:"updated_at"`
-	Files        []string `json:"files"`
-	Experimental bool     `json:"experimental,omitempty"`
-	Description  string   `json:"description,omitempty"`
-	MinCLIVer    string   `json:"min_cli_version,omitempty"`
+	Version     string   `json:"version"`
+	UpdatedAt   string   `json:"updated_at"`
+	Files       []string `json:"files"`
+	Description string   `json:"description,omitempty"`
+	MinCLIVer   string   `json:"min_cli_version,omitempty"`
 
 	// RepoDir is the top-level repo subdirectory (skills/ or experimental/)
-	// that holds this skill's files. Populated after manifest parsing; not
-	// part of the wire format.
-	RepoDir string `json:"-"`
+	// that holds this skill's files. Provided by the manifest; serves as
+	// the source of truth for whether the skill is experimental.
+	RepoDir string `json:"repo_dir,omitempty"`
 
 	// SourceName is the directory name within RepoDir that holds this
 	// skill's files in the upstream repo. For stable skills this equals
 	// the manifest key. For experimental skills the manifest key has a
 	// "-experimental" suffix appended for collision-free install paths,
 	// but the upstream repo dir does not — SourceName preserves it for
-	// the fetch URL.
+	// the fetch URL. Populated during normalization; not part of the wire
+	// format.
 	SourceName string `json:"-"`
 }
 
+// IsExperimental reports whether the skill is sourced from the experimental/
+// directory of the upstream repo.
+func (s SkillMeta) IsExperimental() bool {
+	return s.RepoDir == experimentalRepoPath
+}
+
 // InstallOptions controls the behavior of InstallSkillsForAgents.
 type InstallOptions struct {
 	IncludeExperimental bool
@@ -188,7 +193,7 @@ func InstallSkillsForAgents(ctx context.Context, src ManifestSource, targetAgent
 	}
 
 	// Helpful nudge for users testing --experimental against a ref that
-	// pre-dates the experimental_skills manifest section.
+	// pre-dates the experimental/ directory upstream.
 	if opts.IncludeExperimental && !manifestHasExperimental(manifest) {
 		log.Warnf(ctx, "--experimental was set but the manifest at %s exposes no experimental skills. Set DATABRICKS_SKILLS_REF to a release that includes them (or =main for the latest).", ref)
 	}
@@ -369,7 +374,7 @@ func resolveSkills(ctx context.Context, skills map[string]SkillMeta, opts Instal
 
 	result := make(map[string]SkillMeta, len(candidates))
 	for name, meta := range candidates {
-		if meta.Experimental && !opts.IncludeExperimental {
+		if meta.IsExperimental() && !opts.IncludeExperimental {
 			if isSpecific {
 				return nil, fmt.Errorf("skill %q is experimental; use --experimental to install", name)
 			}

experimental/aitools/lib/installer/installer_test.go

@@ -29,7 +29,7 @@ func (m *mockManifestSource) FetchManifest(_ context.Context, _ string) (*Manife
 	if m.fetchErr != nil {
 		return nil, m.fetchErr
 	}
-	flattenManifest(m.manifest)
+	normalizeManifest(m.manifest)
 	return m.manifest, nil
 }
 
@@ -261,10 +261,10 @@ func TestExperimentalSkillsSkippedByDefault(t *testing.T) {
 	t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef)
 
 	manifest := testManifest()
-	manifest.Skills["databricks-experimental"] = SkillMeta{
-		Version:      "0.1.0",
-		Files:        []string{"SKILL.md"},
-		Experimental: true,
+	manifest.Skills["databricks-iceberg"] = SkillMeta{
+		Version: "0.1.0",
+		Files:   []string{"SKILL.md"},
+		RepoDir: experimentalRepoPath,
 	}
 
 	src := &mockManifestSource{manifest: manifest}
@@ -278,7 +278,7 @@ func TestExperimentalSkillsSkippedByDefault(t *testing.T) {
 	require.NoError(t, err)
 	// Only non-experimental skills should be installed.
 	assert.Len(t, state.Skills, 2)
-	assert.NotContains(t, state.Skills, "databricks-experimental")
+	assert.NotContains(t, state.Skills, "databricks-iceberg-experimental")
 
 	assert.Contains(t, stderr.String(), "Installed 2 skills.")
 }
@@ -290,10 +290,10 @@ func TestExperimentalSkillsIncludedWithFlag(t *testing.T) {
 	t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef)
 
 	manifest := testManifest()
-	manifest.Skills["databricks-experimental"] = SkillMeta{
-		Version:      "0.1.0",
-		Files:        []string{"SKILL.md"},
-		Experimental: true,
+	manifest.Skills["databricks-iceberg"] = SkillMeta{
+		Version: "0.1.0",
+		Files:   []string{"SKILL.md"},
+		RepoDir: experimentalRepoPath,
 	}
 
 	src := &mockManifestSource{manifest: manifest}
@@ -308,7 +308,7 @@ func TestExperimentalSkillsIncludedWithFlag(t *testing.T) {
 	state, err := LoadState(globalDir)
 	require.NoError(t, err)
 	assert.Len(t, state.Skills, 3)
-	assert.Contains(t, state.Skills, "databricks-experimental")
+	assert.Contains(t, state.Skills, "databricks-iceberg-experimental")
 	assert.True(t, state.IncludeExperimental)
 
 	assert.Contains(t, stderr.String(), "Installed 3 skills.")
@@ -754,8 +754,8 @@ func TestInstallReplacesAlternateVariant(t *testing.T) {
 	// Now flip to experimental upstream. New install run.
 	experimentalManifest := &Manifest{
 		Version: "1",
-		ExperimentalSkills: map[string]SkillMeta{
-			"databricks-jobs": {Version: "0.2.0", Files: []string{"SKILL.md"}},
+		Skills: map[string]SkillMeta{
+			"databricks-jobs": {Version: "0.2.0", Files: []string{"SKILL.md"}, RepoDir: experimentalRepoPath},
 		},
 	}
 	require.NoError(t, InstallSkillsForAgents(

experimental/aitools/lib/installer/source.go

@@ -49,32 +49,33 @@ func (s *GitHubManifestSource) FetchManifest(ctx context.Context, ref string) (*
 		return nil, fmt.Errorf("failed to parse manifest: %w", err)
 	}
 
-	flattenManifest(&manifest)
+	normalizeManifest(&manifest)
 	return &manifest, nil
 }
 
-// flattenManifest merges ExperimentalSkills into Skills and stamps each entry
-// with its RepoDir + SourceName so the rest of the installer can treat both
-// kinds uniformly.
-//
-// Every experimental skill's manifest key gets a "-experimental" suffix so
-// the on-disk install path is unambiguously separate from any stable skill
+// normalizeManifest stamps SourceName, defaults a missing RepoDir to the
+// stable directory, and re-keys experimental skills with a "-experimental"
+// suffix so their on-disk install paths can't collide with stable skills
 // of the same name. SourceName preserves the upstream repo directory name
 // (which does not carry the suffix) for fetch URLs.
-func flattenManifest(m *Manifest) {
+//
+// RepoDir is provided by the manifest and is the source of truth for
+// whether a skill is experimental — see SkillMeta.IsExperimental.
+func normalizeManifest(m *Manifest) {
 	if m.Skills == nil {
 		m.Skills = map[string]SkillMeta{}
 	}
+	out := make(map[string]SkillMeta, len(m.Skills))
 	for name, meta := range m.Skills {
-		meta.RepoDir = stableSkillsRepoPath
-		meta.SourceName = name
-		m.Skills[name] = meta
-	}
-	for name, meta := range m.ExperimentalSkills {
-		meta.Experimental = true
-		meta.RepoDir = experimentalRepoPath
+		if meta.RepoDir == "" {
+			meta.RepoDir = stableSkillsRepoPath
+		}
 		meta.SourceName = name
-		m.Skills[name+"-experimental"] = meta
+		if meta.IsExperimental() {
+			out[name+experimentalSuffix] = meta
+		} else {
+			out[name] = meta
+		}
 	}
-	m.ExperimentalSkills = nil
+	m.Skills = out
 }