Skip to content

feat: Add Node File Exchange #8407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wanghe-fit2cloud merged 1 commit into from
Apr 16, 2025
Merged

feat: Add Node File Exchange #8407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agent/app/service/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ func (f *FileService) GetPathByType(pathType string) string {
if pathType == "websiteDir" {
value, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
if value == "" {
return path.Join(global.Dir.BaseDir, "www")
return path.Join(global.Dir.BaseDir, "1panel", "www")
}
return value
}
Expand Down
2 changes: 1 addition & 1 deletion agent/init/dir/dir.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func Init() {
global.Dir.BaseDir, _ = fileOp.CreateDirWithPath(true, baseDir)
global.Dir.DataDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel"))
global.Dir.DbDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/db"))
global.Dir.LogDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/log"))
global.Dir.LogDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/log/task"))
global.Dir.TmpDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/tmp"))

global.Dir.AppDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/apps"))
Expand Down
16 changes: 9 additions & 7 deletions agent/utils/files/fileinfo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,6 @@ func sortFileList(list []FileSearchInfo, sortBy, sortOrder string) {
}

func (f *FileInfo) listChildren(option FileOption) error {
afs := &afero.Afero{Fs: f.Fs}
var (
files []FileSearchInfo
err error
Expand All @@ -230,7 +229,7 @@ func (f *FileInfo) listChildren(option FileOption) error {
return err
}
} else {
files, err = f.getFiles(afs, option)
files, err = f.getFiles(option)
if err != nil {
return err
}
Expand Down Expand Up @@ -261,21 +260,24 @@ func (f *FileInfo) listChildren(option FileOption) error {
return nil
}

func (f *FileInfo) getFiles(afs *afero.Afero, option FileOption) ([]FileSearchInfo, error) {
dirFiles, err := afs.ReadDir(f.Path)
func (f *FileInfo) getFiles(option FileOption) ([]FileSearchInfo, error) {
infos, err := os.ReadDir(f.Path)
if err != nil {
return nil, err
}

var (
dirs []FileSearchInfo
fileList []FileSearchInfo
)

for _, file := range dirFiles {
for _, file := range infos {
fileInfo, err := file.Info()
if err != nil {
continue
}
info := FileSearchInfo{
Path: f.Path,
FileInfo: file,
FileInfo: fileInfo,
}
if file.IsDir() {
dirs = append(dirs, info)
Expand Down
9 changes: 3 additions & 6 deletions core/app/task/task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,16 +48,13 @@ const (
TaskUpgrade = "TaskUpgrade"
TaskAddNode = "TaskAddNode"
TaskSync = "TaskSync"
TaskRsync = "TaskRsync"
)

const (
TaskScopeSystem = "System"
TaskScopeScript = "Script"
)

const (
TaskSuccess = "Success"
TaskFailed = "Failed"
TaskScopeNodeFile = "NodeFile"
)

func GetTaskName(resourceName, operate, scope string) string {
Expand All @@ -72,7 +69,7 @@ func NewTask(name, operate, taskScope, taskID string, resourceID uint) (*Task, e
if taskID == "" {
taskID = uuid.New().String()
}
logItem := path.Join(global.CONF.Base.InstallDir, "1panel/log")
logItem := path.Join(global.CONF.Base.InstallDir, "1panel/log/task")
logDir := path.Join(logItem, taskScope)
if _, err := os.Stat(logDir); os.IsNotExist(err) {
if err = os.MkdirAll(logDir, constant.DirPerm); err != nil {
Expand Down
10 changes: 9 additions & 1 deletion core/i18n/lang/zh.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,4 +172,12 @@ UserInfoAddr: "面板地址："
UserInfoPassHelp: "提示：修改密码可执行命令："
DBConnErr: "错误：初始化数据库连接失败，{{ .err }}"
SystemVersion: "版本："
SystemMode: "模式："
SystemMode: "模式："

#exchange
LocalNodeIpFailed: "无法获取主节点 IP ，请编辑主节点增加 IP 地址和 SSH 认证信息"
HandlePrivateKey: "处理节点私钥"
HandlePublicKey: "处理节点公钥"
ExchangeFile: "开始从 {{ .source }} 节点同步 {{ .sourcePath }} 到 {{ .dest }} 节点 {{ .destPath }}"
TaskRsync: "同步"
NodeFile: "节点文件"
1 change: 1 addition & 0 deletions core/init/migration/helper/menu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func LoadMenus() string {
{ID: "111", Disabled: false, Title: "xpack.node.nodeManagement", IsShow: true, Label: "Node", Path: "/xpack/node"},
{ID: "113", Disabled: false, Title: "xpack.monitor.name", IsShow: true, Label: "MonitorDashboard", Path: "/xpack/monitor/dashboard"},
{ID: "114", Disabled: false, Title: "xpack.tamper.tamper", IsShow: true, Label: "Tamper", Path: "/xpack/tamper"},
{ID: "117", Disabled: false, Title: "xpack.exchange.exchange", IsShow: true, Label: "FileExange", Path: "/xpack/exchange/file"},
{ID: "116", Disabled: false, Title: "xpack.alert.alert", IsShow: true, Label: "XAlertDashboard", Path: "/xpack/alert/dashboard"},
{ID: "115", Disabled: false, Title: "xpack.setting.setting", IsShow: true, Label: "XSetting", Path: "/xpack/setting"},
}},
Expand Down
6 changes: 3 additions & 3 deletions core/init/migration/migrations/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ var InitSetting = &gormigrate.Migration{
return err
}
val := `{"id":"1","label":"/xpack","isCheck":true,"title":"xpack.menu","children":[{"id":"2","label":"Dashboard","isCheck":true,"title":"xpack.waf.name","path":"/xpack/waf/dashboard"},{"id":"3","label":"Tamper","isCheck":true,"title":"xpack.tamper.tamper","path":"/xpack/tamper"},{"id":"4","label":"GPU","isCheck":true,"title":"xpack.gpu.gpu","path":"/xpack/gpu"},{"id":"5","label":"XSetting","isCheck":true,"title":"xpack.setting.setting","path":"/xpack/setting"},{"id":"6","label":"MonitorDashboard","isCheck":true,"title":"xpack.monitor.name","path":"/xpack/monitor/dashboard"},{"id":"7","label":"XAlertDashboard","isCheck":true,"title":"xpack.alert.alert","path":"/xpack/alert/dashboard"},{"id":"8","label":"Node","isCheck":true,"title":"xpack.node.nodeManagement","path":"/xpack/node"}]}`
if err := tx.Create(&model.Setting{Key: "XpackHideMenu", Value: val}).Error; err != nil {
if err := tx.Create(&model.Setting{Key: "HideMenu", Value: val}).Error; err != nil {
return err
}

Expand Down Expand Up @@ -303,9 +303,9 @@ var AddMFAInterval = &gormigrate.Migration{
}

var UpdateXpackHideMemu = &gormigrate.Migration{
ID: "20250227-update-xpack-hide-menu",
ID: "20250414-update-xpack-hide-menu",
Migrate: func(tx *gorm.DB) error {
if err := tx.Model(&model.Setting{}).Where("key = ?", "XpackHideMenu").Updates(map[string]interface{}{"key": "HideMenu", "value": helper.LoadMenus()}).Error; err != nil {
if err := tx.Model(&model.Setting{}).Where("key = ?", "HideMenu").Updates(map[string]interface{}{"key": "HideMenu", "value": helper.LoadMenus()}).Error; err != nil {
return err
}
return nil
Expand Down
11 changes: 2 additions & 9 deletions core/init/router/proxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,6 @@ import (
"github.com/gin-gonic/gin"
)

var wsUrl = map[string]struct{}{
"/api/v2/process/ws": {},
"/api/v2/files/wget/process": {},

"/api/v2/containers/search/log": {},
}

func Proxy() gin.HandlerFunc {
return func(c *gin.Context) {
reqPath := c.Request.URL.Path
Expand All @@ -38,8 +31,8 @@ func Proxy() gin.HandlerFunc {
return
}
var currentNode string
if _, ok := wsUrl[reqPath]; ok {
currentNode = c.Query("currentNode")
if c.Query("operateNode") != "" {
currentNode = c.Query("operateNode")
} else {
currentNode = c.Request.Header.Get("CurrentNode")
}
Expand Down
36 changes: 36 additions & 0 deletions core/utils/cmd/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package cmd

import (
"bufio"
"fmt"
"io"
"os/exec"
"strings"
)
Expand All @@ -20,3 +23,36 @@ func Which(name string) bool {
}
return true
}

func ExecWithStreamOutput(command string, outputCallback func(string)) error {
cmd := exec.Command("bash", "-c", command)

stdout, err := cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("failed to get stdout: %w", err)
}

stderr, err := cmd.StderrPipe()
if err != nil {
return fmt.Errorf("failed to get stderr: %w", err)
}

if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start command: %w", err)
}

go streamReader(stdout, outputCallback)
go streamReader(stderr, outputCallback)

if err := cmd.Wait(); err != nil {
return fmt.Errorf("command finished with error: %w", err)
}
return nil
}

func streamReader(reader io.ReadCloser, callback func(string)) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
callback(scanner.Text())
}
}
Copy link
Copy Markdown
Member

@wanghe-fit2cloud wanghe-fit2cloud Apr 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The provided code snippet includes several improvements over its initial version:

  1. Imports: The import statements were separated into sections for readability and clarity.

  2. Functionality Enhancement:

    • Added ExecWithStreamOutput function that takes a command string and an output callback function.
    • Implements streaming of standard output and error using bufio.Scanner.

  3. Error Handling:

    • Used proper error handling techniques across different parts of the functions to ensure robustness.

  4. Code Structure:

    • Cleaned up comments for better documentation and added spacing between lines for clearer separation of concerns.

Here's a summary of the changes with minor formatting adjustments:

package cmd

import (
	" bufio"
	"fmt"
	"io"
	"os/exec"
)

// Which checks if a given program is available in the system path.
func Which(name string) bool {
	out, err := exec.LookPath(name)
	if err == os.ErrNotExist {
		return false
	}
	return true
}

// ExecWithStreamOutput executes a shell command and streams both standard output and standard error to the specified callback function.
func ExecWithStreamOutput(command string, outputCallback func(string)) error {
	cmd := exec.Command("bash", "-c", command)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("failed to get stdout: %w", err)
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("failed to get stderr: %w", err)
	}

	if err := cmd.Start(); err != nil {
		return fmt.Errorf("failed to start command: %w", err)
	}

	go streamReader(stdout, outputCallback)
	go streamReader(stderr, outputCallback)

	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("command finished with error: %w", err)
	}

	return nil
}

func streamReader(reader io.ReadCloser, callback func(string)) {
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		callback(scanner.Text())
	}
}

These enhancements improve the functionality, maintainability, and performance of the cmd module while addressing potential issues related to command execution and output processing.

91 changes: 91 additions & 0 deletions core/utils/ssh/ssh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,94 @@ func makePrivateKeySigner(privateKey []byte, passPhrase []byte) (gossh.Signer, e
}
return gossh.ParsePrivateKey(privateKey)
}

func (c *SSHClient) RunWithStreamOutput(command string, outputCallback func(string)) error {
session, err := c.Client.NewSession()
if err != nil {
return fmt.Errorf("failed to create SSH session: %w", err)
}
defer session.Close()

stdout, err := session.StdoutPipe()
if err != nil {
return fmt.Errorf("failed to set up stdout pipe: %w", err)
}

stderr, err := session.StderrPipe()
if err != nil {
return fmt.Errorf("failed to set up stderr pipe: %w", err)
}

if err := session.Start(command); err != nil {
return fmt.Errorf("failed to start command: %w", err)
}

stdoutCh := make(chan string, 100)
stderrCh := make(chan string, 100)
doneCh := make(chan struct{})

go func() {
buffer := make([]byte, 1024)
for {
n, err := stdout.Read(buffer)
if err != nil {
close(stdoutCh)
return
}
if n > 0 {
stdoutCh <- string(buffer[:n])
}
}
}()

go func() {
buffer := make([]byte, 1024)
for {
n, err := stderr.Read(buffer)
if err != nil {
close(stderrCh)
return
}
if n > 0 {
stderrCh <- string(buffer[:n])
}
}
}()

go func() {
for {
select {
case stdoutOutput, ok := <-stdoutCh:
if !ok {
stdoutCh = nil
if stderrCh == nil {
close(doneCh)
return
}
continue
}
if outputCallback != nil {
outputCallback(stdoutOutput)
}

case stderrOutput, ok := <-stderrCh:
if !ok {
stderrCh = nil
if stdoutCh == nil {
close(doneCh)
return
}
continue
}
if outputCallback != nil {
outputCallback(stderrOutput)
}
}
}
}()

err = session.Wait()
<-doneCh

return err
}
Copy link
Copy Markdown
Member

@wanghe-fit2cloud wanghe-fit2cloud Apr 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The provided code modifications are mostly clean and optimized. Here are some minor points:

  1. The function makePrivateKeySigner remains unchanged, so no specific review is needed.

Regarding the RunWithStreamOutput method:
2. It creates an SSH session and pipes its standard input/output/error streams to separate goroutines for reading output.
3. A buffered channel (stdoutCh, stderrCh) is used to collect output data from both streams concurrently.
4. The output callback function is called with any text read from either stream whenever available.
5. The session waits for completion before closing the sessions, ensuring all processes and resources are properly finalized.
6. Error handling has been added around each step of setting up and running the command, providing clear failure messages if something goes wrong.

Overall, this method efficiently handles streaming output from an SSH command, making it suitable for applications that need to process or monitor command outputs.

4 changes: 4 additions & 0 deletions frontend/src/api/interface/file.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ export namespace File {
isDetail?: boolean;
}

export interface ReqNodeFile extends ReqFile {
node: string;
}

export interface SearchUploadInfo extends ReqPage {
path: string;
}
Expand Down
12 changes: 12 additions & 0 deletions frontend/src/api/modules/files.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ export const getFilesList = (params: File.ReqFile) => {
return http.post<File.File>('files/search', params, TimeoutEnum.T_5M);
};

export const getFilesListByNode = (params: File.ReqNodeFile) => {
return http.post<File.File>('files/search?operateNode=' + params.node, params, TimeoutEnum.T_5M);
};

export const getUploadList = (params: File.SearchUploadInfo) => {
return http.post<ResPage<File.UploadInfo>>('files/upload/search', params);
};
Expand All @@ -25,6 +29,10 @@ export const deleteFile = (form: File.FileDelete) => {
return http.post<File.File>('files/del', form);
};

export const deleteFileByNode = (form: File.FileDelete, node: string) => {
return http.post<File.File>('files/del?operateNode=' + node, form);
};

export const batchDeleteFile = (form: File.FileBatchDelete) => {
return http.post('files/batch/del', form);
};
Expand Down Expand Up @@ -129,6 +137,10 @@ export const getRecycleStatus = () => {
return http.get<string>('files/recycle/status');
};

export const getRecycleStatusByNode = (node: string) => {
return http.get<string>('files/recycle/status?operateNode=' + node);
};

export const getPathByType = (pathType: string) => {
return http.get<string>(`files/path/${pathType}`);
};
4 changes: 4 additions & 0 deletions frontend/src/api/modules/setting.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ export const listNodeOptions = () => {
return http.get<Array<Setting.NodeItem>>(`/core/nodes/list`);
};

export const listAllNodes = () => {
return http.get<Array<Setting.NodeItem>>(`/core/nodes/all`);
};

// agent
export const loadBaseDir = () => {
return http.get<string>(`/settings/basedir`);
Expand Down
4 changes: 2 additions & 2 deletions frontend/src/components/log/container/index.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ const searchLogs = async () => {
}
logs.value = [];
let currentNode = globalStore.currentNode;
let url = `/api/v2/containers/search/log?container=${logSearch.container}&since=${logSearch.mode}&tail=${logSearch.tail}&follow=${logSearch.isWatch}&currentNode=${currentNode}`;
let url = `/api/v2/containers/search/log?container=${logSearch.container}&since=${logSearch.mode}&tail=${logSearch.tail}&follow=${logSearch.isWatch}&operateNode=${currentNode}`;
if (logSearch.compose !== '') {
url = `/api/v2/containers/search/log?compose=${logSearch.compose}&since=${logSearch.mode}&tail=${logSearch.tail}&follow=${logSearch.isWatch}&currentNode=${currentNode}`;
url = `/api/v2/containers/search/log?compose=${logSearch.compose}&since=${logSearch.mode}&tail=${logSearch.tail}&follow=${logSearch.isWatch}&operateNode=${currentNode}`;
}
eventSource = new EventSource(url);
eventSource.onmessage = (event: MessageEvent) => {
Expand Down
4 changes: 3 additions & 1 deletion frontend/src/components/log/task/index.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
@close="handleClose"
:width="width"
>
<div>
<div v-if="open">
<LogFile :config="config" :showTail="showTail"></LogFile>
</div>
</el-dialog>
Expand All @@ -33,6 +33,8 @@ defineProps({
});

const config = reactive({
id: 0,
name: '',
taskID: '',
type: 'task',
taskOperate: '',
Expand Down
Loading
Loading