✨ feat: making js rendered requests

Author: shurco

README.md

@@ -45,13 +45,30 @@ Usage:
   goclone <url> [flags]
 
 Flags:
-  -c, --cookie                if set true, cookies won't send
-  -h, --help                  help for goclone
-  -o, --open                  automatically open project in default browser
-  -p, --proxy_string string   proxy connection string
-  -r, --robots                disable robots.txt checks
-  -s, --serve                 serve the generated files using gofiber
-  -P, --servePort int         serve port number (default 8088)
-  -u, --user_agent string     custom User-Agent (default "goclone")
-  -v, --version               version for goclone
+  -b, --browser_endpoint string   chrome headless browser WS endpoint
+  -c, --cookie                    if set true, cookies won't send
+  -h, --help                      help for goclone
+  -o, --open                      automatically open project in default browser
+  -p, --proxy_string string       proxy connection string
+  -r, --robots                    disable robots.txt checks
+  -s, --serve                     serve the generated files using gofiber
+  -P, --servePort int             serve port number (default 8088)
+  -u, --user_agent string         custom User-Agent (default "goclone")
+  -v, --version                   version for goclone
 ```
+
+## Making JS Rendered Requests
+
+JS Rendered requests can be made using ```-b``` flag. For example start image :  
+
+
+``` bash
+docker run -d -p 9222:9222 --rm --name headless-shell chromedp/headless-shell
+```
+
+then run goclone: 
+
+```bash
+goclone -b "ws://localhost:9222" https://domain.com
+```
+

cmd/main.go

@@ -53,6 +53,7 @@ func main() {
 	pf.StringVarP(&flags.UserAgent, "user_agent", "u", "goclone", "custom User-Agent")
 	pf.BoolVarP(&flags.Cookies, "cookie", "c", false, "if set true, cookies won't send")
 	pf.BoolVarP(&flags.Robots, "robots", "r", false, "disable robots.txt checks")
+	pf.StringVarP(&flags.BrowserEndpoint, "browser_endpoint", "b", "", "chrome headless browser WS endpoint")
 
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)

pkg/crawler/crawler.go

@@ -21,13 +21,14 @@ import (
 )
 
 type Flags struct {
-	Open        bool
-	Serve       bool
-	ServePort   int
-	UserAgent   string
-	ProxyString string
-	Cookies     bool
-	Robots      bool
+	Open            bool
+	Serve           bool
+	ServePort       int
+	UserAgent       string
+	ProxyString     string
+	Cookies         bool
+	Robots          bool
+	BrowserEndpoint string
 }
 
 type filesBase struct {
@@ -72,6 +73,12 @@ func CloneSite(ctx context.Context, args []string, flag Flags) error {
 	if flag.ProxyString != "" {
 		geziyorOptions.ProxyFunc = client.RoundRobinProxy(flag.ProxyString)
 	}
+	if flag.BrowserEndpoint != "" {
+		geziyorOptions.BrowserEndpoint = flag.BrowserEndpoint
+		geziyorOptions.StartRequestsFunc = func(g *geziyor.Geziyor) {
+			g.GetRendered(domain, g.Opt.ParseFunc)
+		}
+	}
 
 	geziyor.NewGeziyor(geziyorOptions).Start()
 
@@ -119,8 +126,7 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) {
 				fmt.Println("Css found", "-->", parsedURL)
 				if !files.css.Contains(parsedURL.Path) {
 					files.css = append(files.css, parsedURL.Path)
-					netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
-
+					go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
 					g.Get(r.JoinURL(projectURL.String()+parsedURL.Path), parseCSS)
 				}
 
@@ -142,7 +148,7 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) {
 				fmt.Println("Js found", "-->", parsedURL)
 				if !files.js.Contains(parsedURL.Path) {
 					files.js = append(files.js, parsedURL.Path)
-					netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
+					go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
 				}
 
 				body = strings.Replace(body, data, "/assets/js/"+filepath.Base(data), -1)
@@ -162,7 +168,7 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) {
 				fmt.Println("Js found", "-->", parsedURL)
 				if !files.js.Contains(parsedURL.Path) {
 					files.js = append(files.js, parsedURL.Path)
-					netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
+					go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
 				}
 
 				body = strings.Replace(body, data, "/assets/js/"+filepath.Base(data), -1)
@@ -186,7 +192,7 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) {
 				fmt.Println("Img found", "-->", parsedURL)
 				if !files.img.Contains(parsedURL.Path) {
 					files.img = append(files.img, parsedURL.Path)
-					netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
+					go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath)
 				}
 
 				body = strings.Replace(body, data, "/assets/img/"+filepath.Base(data), -1)