Client/Server mode: sending configuration profile to a remote server (#377)

Author: creativeprojects

.github/workflows/build.yml

@@ -12,7 +12,6 @@ on:
       - 'docs/**'
 
 jobs:
-
   build:
     name: Build and test
     runs-on: ${{ matrix.os }}
@@ -25,7 +24,6 @@ jobs:
       GO: ${{ matrix.go_version }}
 
     steps:
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v6
 
@@ -45,6 +43,7 @@ jobs:
       - name: Test
         run: make test-ci
         env:
+          TEST_FUSE: 1
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build
@@ -53,8 +52,9 @@ jobs:
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v6
         with:
+          name: code-coverage-report-${{ matrix.os }}
           env_vars: OS,GO
-          files: ./coverage.out,./unit-tests.xml
+          files: ./coverage.out
           flags: unittests
           fail_ci_if_error: false
           verbose: true
@@ -68,3 +68,31 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           report_type: test_results
           files: ./unit-tests.xml
+
+  test-ssh:
+    name: Test SSH client
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: 1.26
+
+      - name: Run tests
+        run: |
+          make start-ssh-server
+          make ssh-test
+          make stop-ssh-server
+
+      - name: Upload code coverage to Codecov
+        uses: codecov/codecov-action@v6
+        with:
+          name: code-coverage-ssh-report
+          files: ./coverage-ssh.out
+          fail_ci_if_error: false
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -15,6 +15,7 @@
 
 # test output
 /coverage.out
+/coverage-ssh.out
 /unit-tests.xml
 
 # mock binaries

.vscode/settings.json

@@ -19,5 +19,6 @@
     "githubPullRequests.ignoredPullRequestBranches": [
         "master"
     ],
-    "makefile.configureOnOpen": false
+    "makefile.configureOnOpen": false,
+    "go.buildTags": "ssh"
 }

Makefile

@@ -28,6 +28,7 @@ README=README.md
 
 TESTS=./...
 COVERAGE_FILE=coverage.out
+COVERAGE_SSH_FILE=coverage-ssh.out
 JUNIT_FILE=unit-tests.xml
 
 BUILD=build/
@@ -55,6 +56,9 @@ ifeq ($(UNAME),Darwin)
 	TMP_MOUNT=${TMP_MOUNT_DARWIN}
 endif
 
+TMPDIR ?= /tmp
+SSH_TESTS_TMPDIR=$(shell echo "$(TMPDIR)/resticprofile-ssh-tests" | tr -s /)
+
 TOC_START=<\!--ts-->
 TOC_END=<\!--te-->
 TOC_PATH=toc.md
@@ -190,20 +194,22 @@ coverage: ## Generate coverage report
 clean: ## Clean up the build artifacts
 	@echo "[*] $@"
 	$(GOCLEAN)
-	rm -rf $(BINARY) \
-	       $(BINARY_DARWIN_AMD64) \
-	       $(BINARY_DARWIN_ARM64) \
-	       $(BINARY_LINUX_AMD64) \
-	       $(BINARY_LINUX_ARM64) \
-	       $(BINARY_PI) \
-	       $(BINARY_WINDOWS_AMD64) \
-	       $(BINARY_WINDOWS_ARM64) \
-	       $(COVERAGE_FILE) \
-		   $(JUNIT_FILE) \
-	       restic_*_linux_amd64* \
-	       ${BUILD}restic* \
-	       ${BUILD}rclone* \
-	       dist/*
+	rm -rf \
+		$(BINARY) \
+		$(BINARY_DARWIN_AMD64) \
+		$(BINARY_DARWIN_ARM64) \
+		$(BINARY_LINUX_AMD64) \
+		$(BINARY_LINUX_ARM64) \
+		$(BINARY_PI) \
+		$(BINARY_WINDOWS_AMD64) \
+		$(BINARY_WINDOWS_ARM64) \
+		$(COVERAGE_FILE) \
+		$(COVERAGE_SSH_FILE) \
+		$(JUNIT_FILE) \
+		restic_*_linux_amd64* \
+		${BUILD}restic* \
+		${BUILD}rclone* \
+		dist/*
 	find . -path "*/mocks/*" -exec rm {} \;
 	restic cache --cleanup
 
@@ -355,3 +361,27 @@ deploy-current: build-linux build-pi
 		rsync -avz --progress $(BINARY_PI) $$server: ; \
 		ssh -t $$server "sudo -S install $(BINARY_PI) /usr/local/bin/resticprofile" ; \
 	done
+
+.PHONY: start-ssh-server
+start-ssh-server: ## Start the SSH server for testing
+	@echo "[*] $@"
+	@mkdir -p $(SSH_TESTS_TMPDIR) && rm -f $(SSH_TESTS_TMPDIR)/id_* || echo "Failed to create temporary directory"
+	@ssh-keygen -t rsa -b 2048 -f $(SSH_TESTS_TMPDIR)/id_rsa -N "" -C "resticprofile@$(shell hostname)"
+	@ssh-keygen -t ecdsa -b 521 -f $(SSH_TESTS_TMPDIR)/id_ecdsa -N "" -C "resticprofile@$(shell hostname)"
+	@ssh-keygen -t ed25519 -f $(SSH_TESTS_TMPDIR)/id_ed25519 -N "" -C "resticprofile@$(shell hostname)"
+	@cd ./ssh/test && \
+		USER_ID=$(shell id -u) GROUP_ID=$(shell id -g) SSH_TESTS_TMPDIR=$(SSH_TESTS_TMPDIR) \
+		docker compose up -d --force-recreate
+	@sleep 1
+	@ssh-keyscan -p 2222 -H localhost > $(SSH_TESTS_TMPDIR)/known_hosts
+
+.PHONY: stop-ssh-server
+stop-ssh-server: ## Stop the SSH server and clean up temporary files
+	@echo "[*] $@"
+	cd ./ssh/test && SSH_TESTS_TMPDIR=$(SSH_TESTS_TMPDIR) docker compose down --remove-orphans
+	@test -d "$(SSH_TESTS_TMPDIR)" && rm -rf "$(SSH_TESTS_TMPDIR)" || echo "temporary directory not found, nothing to remove"
+
+.PHONY: ssh-test
+ssh-test: ## Run SSH client tests
+	@echo "[*] $@"
+	@go test -run TestSSHClient -v -tags ssh -coverprofile='$(COVERAGE_SSH_FILE)' ./ssh

codecov.yml

@@ -18,10 +18,10 @@ ignore:
 
 codecov:
   notify:
-    after_n_builds: 3
+    after_n_builds: 4
 
 comment:
-  after_n_builds: 3
+  after_n_builds: 4
 
 coverage:
   round: nearest

commands.go

@@ -168,6 +168,24 @@ func getOwnCommands() []ownCommand {
 			needConfiguration: false,
 			hide:              true,
 		},
+		{
+			name:              "send",
+			description:       "send a configuration profile to a remote client and execute a command",
+			action:            sendProfileCommand,
+			needConfiguration: true,
+			noProfile:         true,
+			hide:              true,
+			experimental:      true,
+		},
+		{
+			name:              "serve",
+			description:       "serve configuration profiles to remote clients",
+			action:            serveCommand,
+			needConfiguration: true,
+			noProfile:         true,
+			hide:              true,
+			experimental:      true,
+		},
 	}
 }
 

config/config.go

@@ -711,6 +711,22 @@ func (c *Config) getProfilePath(key string) string {
 	return c.flatKey(constants.SectionConfigurationProfiles, key)
 }
 
+// HasRemote returns true if the remote exists in the configuration
+func (c *Config) HasRemote(remoteName string) bool {
+	return c.IsSet(c.flatKey(constants.SectionConfigurationRemotes, remoteName))
+}
+
+func (c *Config) GetRemote(remoteName string) (*Remote, error) {
+	// we don't need to check the file version: the remotes can be in a separate configuration file
+
+	remote := NewRemote(c, remoteName)
+	err := c.unmarshalKey(c.flatKey(constants.SectionConfigurationRemotes, remoteName), remote)
+
+	rootPath := filepath.Dir(c.GetConfigFile())
+	remote.SetRootPath(rootPath)
+	return remote, err
+}
+
 // unmarshalConfig returns the decoder config options depending on the configuration version and format
 func (c *Config) unmarshalConfig() viper.DecoderConfigOption {
 	if c.GetVersion() == Version01 {

config/config_test.go

@@ -367,15 +367,6 @@ x=0
 }
 
 func TestIncludes(t *testing.T) {
-	files := []string{}
-	cleanFiles := func() {
-		for _, file := range files {
-			os.Remove(file)
-		}
-		files = files[:0]
-	}
-	defer cleanFiles()
-
 	createFile := func(t *testing.T, suffix, content string) string {
 		t.Helper()
 		name := ""
@@ -384,7 +375,9 @@ func TestIncludes(t *testing.T) {
 			defer file.Close()
 			_, err = file.WriteString(content)
 			name = file.Name()
-			files = append(files, name)
+			t.Cleanup(func() {
+				_ = os.Remove(name)
+			})
 		}
 		require.NoError(t, err)
 		return name
@@ -400,7 +393,6 @@ func TestIncludes(t *testing.T) {
 	testID := fmt.Sprintf("%d", time.Now().Unix())
 
 	t.Run("multiple-includes", func(t *testing.T) {
-		defer cleanFiles()
 		content := fmt.Sprintf(`includes=['*%[1]s.inc.toml','*%[1]s.inc.yaml','*%[1]s.inc.json']`, testID)
 
 		configFile := createFile(t, "profiles.conf", content)
@@ -416,8 +408,6 @@ func TestIncludes(t *testing.T) {
 	})
 
 	t.Run("overrides", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.conf", `
 includes = "*`+testID+`.inc.toml"
 [default]
@@ -436,8 +426,6 @@ repository = "overridden-repo"`)
 	})
 
 	t.Run("mixins", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.conf", `
 version = 2
 includes = "*`+testID+`.inc.toml"
@@ -464,8 +452,6 @@ use = "another-run-before2"`)
 	})
 
 	t.Run("hcl-includes-only-hcl", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.hcl", `includes = "*`+testID+`.inc.*"`)
 		createFile(t, "pass-"+testID+".inc.hcl", `one { }`)
 
@@ -474,13 +460,11 @@ use = "another-run-before2"`)
 
 		createFile(t, "fail-"+testID+".inc.toml", `[two]`)
 		_, err := LoadFile(configFile, "")
-		assert.Error(t, err)
+		require.Error(t, err)
 		assert.Regexp(t, ".+ is in hcl format, includes must use the same format", err.Error())
 	})
 
 	t.Run("non-hcl-include-no-hcl", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.toml", `includes = "*`+testID+`.inc.*"`)
 		createFile(t, "pass-"+testID+".inc.toml", "[one]\nk='v'")
 
@@ -489,12 +473,11 @@ use = "another-run-before2"`)
 
 		createFile(t, "fail-"+testID+".inc.hcl", `one { }`)
 		_, err := LoadFile(configFile, "")
-		assert.Error(t, err)
+		require.Error(t, err)
 		assert.Regexp(t, "hcl format .+ cannot be used in includes from toml", err.Error())
 	})
 
 	t.Run("cannot-load-different-versions", func(t *testing.T) {
-		defer cleanFiles()
 		content := fmt.Sprintf(`includes=['*%s.inc.json']`, testID)
 
 		configFile := createFile(t, "profiles.conf", content)
@@ -506,7 +489,6 @@ use = "another-run-before2"`)
 	})
 
 	t.Run("cannot-load-different-versions", func(t *testing.T) {
-		defer cleanFiles()
 		content := fmt.Sprintf(`{"version": 2, "includes":["*%s.inc.json"]}`, testID)
 
 		configFile := createFile(t, "profiles.json", content)

config/error.go

@@ -3,5 +3,6 @@ package config
 import "errors"
 
 var (
-	ErrNotFound = errors.New("not found")
+	ErrNotFound               = errors.New("not found")
+	ErrNotSupportedInVersion1 = errors.New("not supported in configuration version 1")
 )

config/remote.go

@@ -0,0 +1,39 @@
+package config
+
+type Remote struct {
+	name              string
+	config            *Config
+	Connection        string   `mapstructure:"connection" default:"ssh" enum:"ssh;openssh" description:"Connection type to use to connect to the remote client"`
+	Host              string   `mapstructure:"host" description:"Address of the remote client (without port)."`
+	Port              int      `mapstructure:"port" description:"Port to connect to on the remote client. If not specified, the default SSH port (22) will be used."`
+	Username          string   `mapstructure:"username" description:"User to connect to the remote client"`
+	PrivateKeyPaths   []string `mapstructure:"private-keys" description:"Path to the private key(s) to use for authentication"`
+	KnownHostsPath    string   `mapstructure:"known-hosts" description:"Path to the known hosts file"`
+	BinaryPath        string   `mapstructure:"binary-path" description:"Path to the resticprofile binary to use on the remote client"`
+	ConfigurationFile string   `mapstructure:"configuration-file" description:"Path to the configuration file to transfer to the remote client"`
+	ProfileName       string   `mapstructure:"profile-name" description:"Name of the profile to use on the remote client"`
+	SendFiles         []string `mapstructure:"send-files" description:"Other configuration files to transfer to the remote client"`
+	SSHConfig         string   `mapstructure:"ssh-config" description:"Path to the OpenSSH config file to use for the connection"`
+}
+
+func NewRemote(config *Config, name string) *Remote {
+	remote := &Remote{
+		name:   name,
+		config: config,
+	}
+	return remote
+}
+
+// SetRootPath changes the path of all the relative paths and files in the configuration
+func (r *Remote) SetRootPath(rootPath string) {
+	for i := range r.PrivateKeyPaths {
+		r.PrivateKeyPaths[i] = fixPath(r.PrivateKeyPaths[i], expandEnv, absolutePrefix(rootPath))
+	}
+	r.KnownHostsPath = fixPath(r.KnownHostsPath, expandEnv, absolutePrefix(rootPath))
+	r.ConfigurationFile = fixPath(r.ConfigurationFile, expandEnv, absolutePrefix(rootPath))
+	r.SSHConfig = fixPath(r.SSHConfig, expandEnv, absolutePrefix(rootPath))
+
+	for i := range r.SendFiles {
+		r.SendFiles[i] = fixPath(r.SendFiles[i], expandEnv, absolutePrefix(rootPath))
+	}
+}