Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ Usage:
Flags:
-C, --cookie strings Pre-set these cookies
-h, --help help for goclone
-f, --offline Serve previously cloned files
-o, --open Automatically open project in default browser
-p, --proxy_string string Proxy connection string. Support http and socks5 https://pkg.go.dev/github.com/gocolly/colly#Collector.SetProxy
-s, --serve Serve the generated files using Echo.
Expand Down
53 changes: 30 additions & 23 deletions cmd/clone.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,33 +50,36 @@ func cloneSite(ctx context.Context, args []string) error {

var firstProject string
for _, u := range args {
isValid, isValidDomain := parser.ValidateURL(u), parser.ValidateDomain(u)
if !isValid && !isValidDomain {
return fmt.Errorf("%q is not valid", u)
}
name := u
if isValidDomain {
u = parser.CreateURL(name)
} else {
name = parser.GetDomain(u)
}
projectPath := file.CreateProject(name)
if firstProject == "" {
firstProject = projectPath
}
if !Offline {
isValid, isValidDomain := parser.ValidateURL(u), parser.ValidateDomain(u)
if !isValid && !isValidDomain {
return fmt.Errorf("%q is not valid", u)
}
name := u
if isValidDomain {
u = parser.CreateURL(name)
} else {
name = parser.GetDomain(u)
}
projectPath := file.CreateProject(name)
if firstProject == "" {
firstProject = projectPath
}

if err := crawler.Crawl(ctx, u, projectPath, jar, ProxyString, UserAgent); err != nil {
return fmt.Errorf("%q: %w", u, err)
}
// Restructure html
if err := html.LinkRestructure(projectPath); err != nil {
return fmt.Errorf("%q: %w", projectPath, err)
if err := crawler.Crawl(ctx, u, projectPath, jar, ProxyString, UserAgent); err != nil {
return fmt.Errorf("%q: %w", u, err)
}
// Restructure html
if err := html.LinkRestructure(projectPath); err != nil {
return fmt.Errorf("%q: %w", projectPath, err)
}
} else {
firstProject = u
}

}
if Serve {
serverUrl := fmt.Sprintf("http://localhost:%d", ServePort)
cmd := exec.Command("open", serverUrl)
cmd := open(serverUrl)
if err := cmd.Start(); err != nil {
return fmt.Errorf("%v: %w", cmd.Args, err)
}
Expand All @@ -103,7 +106,11 @@ func open(url string) *exec.Cmd {
case "darwin":
cmd = "open"
default: // "linux", "freebsd", "openbsd", "netbsd"
cmd = "xdg-open"
if _, err := exec.LookPath("xdg-open"); err != nil {
cmd = "echo"
} else {
cmd = "xdg-open"
}
}
args = append(args, url)
return exec.Command(cmd, args...)
Expand Down
2 changes: 2 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
var (
Open bool
Serve bool
Offline bool
ServePort int
UserAgent string
ProxyString string
Expand Down Expand Up @@ -50,6 +51,7 @@ func Execute() {
pf.BoolVarP(&Open, "open", "o", false, "Automatically open project in deafult browser")
// rootCmd.PersistentFlags().BoolVarP(&Login, "login", "l", false, "Wether to use a username or password")
pf.BoolVarP(&Serve, "serve", "s", false, "Serve the generated files using Echo.")
pf.BoolVarP(&Offline, "offline", "f", false, "Serve previously cloned files")
pf.IntVarP(&ServePort, "servePort", "P", 5000, "Serve port number.")
pf.StringVarP(&ProxyString, "proxy_string", "p", "", "Proxy connection string. Support http and socks5 https://pkg.go.dev/github.com/gocolly/colly#Collector.SetProxy")
pf.StringVarP(&UserAgent, "user_agent", "u", "", "Custom User Agent")
Expand Down