Fix background npm install with file dependencies in apps init` flow (#5045)

## Summary

When using `databricks apps init --template` with a template that has
`file:` protocol dependencies (e.g., `"@databricks/appkit":
"file:./databricks-appkit-0.24.0.tgz"`), the background `npm ci` fails
with exit 254 (ENOENT):

```
npm error path /private/tmp/dest-dir/databricks-appkit-ui-0.24.0.tgz
npm error errno -2
npm error enoent ENOENT: no such file or directory
```

**Root cause:** `startBackgroundNpmInstall()` copies only `package.json`
and `package-lock.json` to the destination directory before running `npm
ci`. The `.tgz` tarball files referenced by `file:` protocol are not
copied — they only arrive later during `copyTemplate()`.

**Fix:** Add `copyFileDeps()` that parses the rendered `package.json`,
finds `file:` protocol dependencies in both `dependencies` and
`devDependencies`, and copies the referenced files from the template
source to the destination directory before `npm ci` starts.

The retry after `copyTemplate()` still succeeds (all files present), so
the issue was cosmetic (confusing warning + ~10s delay), but this
eliminates the unnecessary failure and retry.

## Demo


https://github.com/user-attachments/assets/2de06bf0-30ef-49e6-baac-3d86d873d3ed

## Test plan

- [x] Added `TestCopyFileDeps` — verifies file: deps are copied,
registry deps are ignored, missing files are skipped gracefully
- [x] Added `TestCopyFileDepsInvalidJSON` — verifies no panic on
malformed input
- [x] Added `TestCopyFileDepsNoDeps` — verifies no side effects when no
file: deps exist
- [x] CI builds and passes all tests

---------

Signed-off-by: Pawel Kosiec <pawel.kosiec@databricks.com>

Author: pkosiec

cmd/apps/init.go

@@ -3,6 +3,7 @@ package apps
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -664,6 +665,14 @@ func startBackgroundNpmInstall(ctx context.Context, srcProjectDir, destDir, proj
 		return nil
 	}
 
+	// Copy any file: protocol dependencies (e.g., local .tgz tarballs) so npm ci can resolve them.
+	pkgData, err := os.ReadFile(filepath.Join(destDir, "package.json"))
+	if err != nil {
+		log.Warnf(ctx, "Failed to read package.json for file dep copy: %v", err)
+	} else {
+		copyFileDeps(ctx, pkgData, srcProjectDir, destDir)
+	}
+
 	// Copy package-lock.json raw (never has template vars).
 	lockData, err := os.ReadFile(lockFile)
 	if err != nil {
@@ -688,6 +697,41 @@ func startBackgroundNpmInstall(ctx context.Context, srcProjectDir, destDir, proj
 	return ch
 }
 
+// copyFileDeps copies local file: protocol dependencies (e.g., .tgz tarballs)
+// from srcDir to destDir so that npm ci can resolve them.
+func copyFileDeps(ctx context.Context, pkgJSON []byte, srcDir, destDir string) {
+	var pkg struct {
+		Dependencies    map[string]string `json:"dependencies"`
+		DevDependencies map[string]string `json:"devDependencies"`
+	}
+	if err := json.Unmarshal(pkgJSON, &pkg); err != nil {
+		log.Debugf(ctx, "Failed to parse package.json for file dep copy: %v", err)
+		return
+	}
+	for _, deps := range []map[string]string{pkg.Dependencies, pkg.DevDependencies} {
+		for _, v := range deps {
+			if !strings.HasPrefix(v, "file:") {
+				continue
+			}
+			relPath := filepath.Clean(strings.TrimPrefix(v, "file:"))
+			src := filepath.Join(srcDir, relPath)
+			data, err := os.ReadFile(src)
+			if err != nil {
+				log.Debugf(ctx, "Skipping file dep %s: %v", relPath, err)
+				continue
+			}
+			dst := filepath.Join(destDir, relPath)
+			if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+				log.Debugf(ctx, "Failed to create dir for file dep %s: %v", relPath, err)
+				continue
+			}
+			if err := os.WriteFile(dst, data, 0o644); err != nil {
+				log.Debugf(ctx, "Failed to copy file dep %s: %v", relPath, err)
+			}
+		}
+	}
+}
+
 // awaitBackgroundNpmInstall waits for the background npm install to complete.
 // Shows an instant checkmark if already done, or a spinner for the remainder.
 func awaitBackgroundNpmInstall(ctx context.Context, ch <-chan error) error {

cmd/apps/init_test.go

@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"errors"
 	"io"
+	"io/fs"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 
@@ -687,3 +689,171 @@ func TestRunManifestOnlyUsesTemplatePathEnvVar(t *testing.T) {
 	out := buf.String()
 	assert.Equal(t, content, out)
 }
+
+func TestCopyFileDeps(t *testing.T) {
+	ctx := t.Context()
+
+	srcDir := t.TempDir()
+	destDir := t.TempDir()
+
+	// Create a fake tarball in srcDir
+	tgzContent := []byte("fake-tarball-content")
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "my-pkg-1.0.0.tgz"), tgzContent, 0o644))
+
+	// package.json with file: dep, a registry dep, and a devDep with file:
+	pkgJSON := []byte(`{
+		"dependencies": {
+			"my-pkg": "file:./my-pkg-1.0.0.tgz",
+			"lodash": "4.17.21"
+		},
+		"devDependencies": {
+			"missing-pkg": "file:./nonexistent.tgz"
+		}
+	}`)
+
+	copyFileDeps(ctx, pkgJSON, srcDir, destDir)
+
+	// The file: dep should be copied
+	copied, err := os.ReadFile(filepath.Join(destDir, "my-pkg-1.0.0.tgz"))
+	require.NoError(t, err)
+	assert.Equal(t, tgzContent, copied)
+
+	// The registry dep should NOT create any file
+	_, err = os.Stat(filepath.Join(destDir, "4.17.21"))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// The missing file: dep should be skipped gracefully (no panic, no error)
+	_, err = os.Stat(filepath.Join(destDir, "nonexistent.tgz"))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
+
+func TestCopyFileDepsInvalidJSON(t *testing.T) {
+	ctx := t.Context()
+	srcDir := t.TempDir()
+	destDir := t.TempDir()
+
+	// Should not panic on invalid JSON
+	copyFileDeps(ctx, []byte("not json"), srcDir, destDir)
+
+	// destDir should remain empty
+	entries, err := os.ReadDir(destDir)
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}
+
+func TestCopyFileDepsNoDeps(t *testing.T) {
+	ctx := t.Context()
+	srcDir := t.TempDir()
+	destDir := t.TempDir()
+
+	// package.json with no file: deps
+	pkgJSON := []byte(`{"dependencies": {"react": "19.0.0"}}`)
+	copyFileDeps(ctx, pkgJSON, srcDir, destDir)
+
+	entries, err := os.ReadDir(destDir)
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}
+
+func skipIfNoNpm(t *testing.T) {
+	t.Helper()
+	if _, err := exec.LookPath("npm"); err != nil {
+		t.Skip("npm not found in PATH, skipping")
+	}
+}
+
+func TestStartBackgroundNpmInstall_NoLockFile(t *testing.T) {
+	srcDir := t.TempDir()
+	destDir := t.TempDir()
+
+	// Only package.json, no lock file
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package.json"), []byte(`{"name":"test"}`), 0o644))
+
+	ch := startBackgroundNpmInstall(t.Context(), srcDir, destDir, "test-app")
+	assert.Nil(t, ch)
+}
+
+func TestStartBackgroundNpmInstall_NoPackageJSON(t *testing.T) {
+	srcDir := t.TempDir()
+	destDir := t.TempDir()
+
+	// Only lock file, no package.json
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package-lock.json"), []byte(`{}`), 0o644))
+
+	ch := startBackgroundNpmInstall(t.Context(), srcDir, destDir, "test-app")
+	assert.Nil(t, ch)
+}
+
+func TestStartBackgroundNpmInstall_CopiesFiles(t *testing.T) {
+	skipIfNoNpm(t)
+
+	srcDir := t.TempDir()
+	destDir := filepath.Join(t.TempDir(), "output")
+
+	pkgJSON := []byte(`{"name":"{{.projectName}}","version":"1.0.0"}`)
+	lockJSON := []byte(`{"lockfileVersion":3,"packages":{}}`)
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package.json"), pkgJSON, 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package-lock.json"), lockJSON, 0o644))
+
+	ch := startBackgroundNpmInstall(t.Context(), srcDir, destDir, "my-app")
+	require.NotNil(t, ch)
+
+	// Drain the channel to avoid goroutine leak (npm ci will fail on fake data)
+	<-ch
+
+	// package.json should be written with template substitution
+	got, err := os.ReadFile(filepath.Join(destDir, "package.json"))
+	require.NoError(t, err)
+	assert.Contains(t, string(got), `"my-app"`)
+	assert.NotContains(t, string(got), "{{.projectName}}")
+
+	// package-lock.json should be copied verbatim
+	gotLock, err := os.ReadFile(filepath.Join(destDir, "package-lock.json"))
+	require.NoError(t, err)
+	assert.Equal(t, lockJSON, gotLock)
+}
+
+func TestStartBackgroundNpmInstall_CopiesFileDeps(t *testing.T) {
+	skipIfNoNpm(t)
+
+	srcDir := t.TempDir()
+	destDir := filepath.Join(t.TempDir(), "output")
+
+	tgzContent := []byte("fake-tarball")
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "my-pkg-1.0.0.tgz"), tgzContent, 0o644))
+
+	pkgJSON := []byte(`{"name":"test","dependencies":{"my-pkg":"file:./my-pkg-1.0.0.tgz"}}`)
+	lockJSON := []byte(`{"lockfileVersion":3,"packages":{}}`)
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package.json"), pkgJSON, 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package-lock.json"), lockJSON, 0o644))
+
+	ch := startBackgroundNpmInstall(t.Context(), srcDir, destDir, "test-app")
+	require.NotNil(t, ch)
+	<-ch
+
+	// The file: dep tarball should be copied to destDir
+	copied, err := os.ReadFile(filepath.Join(destDir, "my-pkg-1.0.0.tgz"))
+	require.NoError(t, err)
+	assert.Equal(t, tgzContent, copied)
+}
+
+func TestStartBackgroundNpmInstall_TemplateSubstitution(t *testing.T) {
+	skipIfNoNpm(t)
+
+	srcDir := t.TempDir()
+	destDir := filepath.Join(t.TempDir(), "output")
+
+	pkgJSON := []byte(`{"name":"{{.projectName}}","description":"{{.appDescription}}"}`)
+	lockJSON := []byte(`{"lockfileVersion":3}`)
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package.json"), pkgJSON, 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "package-lock.json"), lockJSON, 0o644))
+
+	ch := startBackgroundNpmInstall(t.Context(), srcDir, destDir, "cool-project")
+	require.NotNil(t, ch)
+	<-ch
+
+	got, err := os.ReadFile(filepath.Join(destDir, "package.json"))
+	require.NoError(t, err)
+	assert.Contains(t, string(got), `"cool-project"`)
+	assert.NotContains(t, string(got), "{{.projectName}}")
+}