From 07244416f7922980b12b096bc05a3e5c733ac3bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 23:38:07 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=94=A7=20build(deps):=20bump=20golang?= =?UTF-8?q?.org/x/net=20from=200.8.0=20to=200.17.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [golang.org/x/net](https://github.com/golang/net) from 0.8.0 to 0.17.0. - [Commits](https://github.com/golang/net/compare/v0.8.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 6eddd59..d9a7d11 100644 --- a/go.mod +++ b/go.mod @@ -31,9 +31,9 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect google.golang.org/protobuf v1.28.0 // indirect ) diff --git a/go.sum b/go.sum index 98a8ab8..c745469 100644 --- a/go.sum +++ b/go.sum @@ -531,8 +531,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -616,8 +616,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -631,8 +631,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 7980f4f535b8ad58957a42158f2b3f60d440ee05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:42:59 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=94=A7=20build(deps):=20bump=20github?= =?UTF-8?q?.com/spf13/cobra=20from=201.7.0=20to=201.8.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/spf13/cobra/releases) - [Commits](https://github.com/spf13/cobra/compare/v1.7.0...v1.8.0) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index d9a7d11..2df0698 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/geziyor/geziyor v0.0.0-20230315135110-a242b58aaa65 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.0 ) require ( diff --git a/go.sum b/go.sum index c745469..092b3fa 100644 --- a/go.sum +++ b/go.sum @@ -96,7 +96,7 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -392,8 +392,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= From 9369451dc03458ee435310cb4077ad1b7a45499e Mon Sep 17 00:00:00 2001 From: Dmitry Shurco Date: Fri, 16 Feb 2024 15:54:28 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9E=20fix:=20first=20path=20segmen?= =?UTF-8?q?t=20in=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 6 +- go.mod | 34 +++---- go.sum | 70 ++++++++----- pkg/arrutil/arrutil.go | 20 ---- pkg/crawler/base.go | 39 +++++++ pkg/crawler/crawler.go | 226 ++++++++++++++++++++++++++++------------- pkg/crawler/css.go | 28 +++-- pkg/crawler/img.go | 25 +++++ pkg/crawler/js.go | 25 +++++ pkg/fsutil/file.go | 2 +- pkg/netutil/netutil.go | 59 ++++++----- 11 files changed, 365 insertions(+), 169 deletions(-) delete mode 100644 pkg/arrutil/arrutil.go create mode 100644 pkg/crawler/base.go create mode 100644 pkg/crawler/img.go create mode 100644 pkg/crawler/js.go diff --git a/.vscode/launch.json b/.vscode/launch.json index a66f7b8..b34e110 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,7 @@ "request": "launch", "mode": "auto", "program": "${workspaceFolder}/cmd/", - "args": ["-s","-r", "https://www.hellokuya.co"] - }, + "args": ["-s", "-r", "https://www.hellokuya.co"] + } ] -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 2df0698..e0db4e4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/shurco/goclone -go 1.20 +go 1.22 require ( github.com/PuerkitoBio/goquery v1.8.1 @@ -10,30 +10,28 @@ require ( require ( github.com/VividCortex/gohistogram v1.0.0 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chromedp/cdproto v0.0.0-20220428002153-285dfb42699c // indirect - github.com/chromedp/chromedp v0.8.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chromedp/cdproto v0.0.0-20240214232516-ad4608604e9e // indirect + github.com/chromedp/chromedp v0.9.5 // indirect github.com/chromedp/sysutil v1.0.0 // indirect - github.com/go-kit/kit v0.12.0 // indirect + github.com/go-kit/kit v0.13.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.1.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/gobwas/ws v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/prometheus/client_golang v1.12.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.34.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.46.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect - google.golang.org/protobuf v1.28.0 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect ) diff --git a/go.sum b/go.sum index 092b3fa..eea23f3 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,9 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -75,13 +76,17 @@ github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chromedp/cdproto v0.0.0-20220321060548-7bc2623472b3/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0= -github.com/chromedp/cdproto v0.0.0-20220428002153-285dfb42699c h1:9VfguIWKAk161/xCyWLV23cJQmL3pDXKJHUSySQxT3s= github.com/chromedp/cdproto v0.0.0-20220428002153-285dfb42699c/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0= -github.com/chromedp/chromedp v0.8.0 h1:+Cufl+QWWfbvyylGCtAUt34A2EI/kqxRM3wGHXMabU4= +github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/cdproto v0.0.0-20240214232516-ad4608604e9e h1:kXEolCWQZzuEFcuaTzfqXToX+e29OcvK87BcBiBBJ1c= +github.com/chromedp/cdproto v0.0.0-20240214232516-ad4608604e9e/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.8.0/go.mod h1:odCVV9o9i7HUKwHMFz9Y7T6s4Kbcz4GOyPlwKWopI9Q= +github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg= +github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -134,8 +139,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= +github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= +github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -149,8 +155,9 @@ github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= +github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q= +github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -183,7 +190,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -200,8 +206,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -281,6 +288,8 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -293,7 +302,6 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -334,8 +342,9 @@ github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je4 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= -github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5 h1:1SoBaSPudixRecmlHXb/GxmaD3fLMtHIDN13QujwQuc= github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= @@ -355,28 +364,32 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= @@ -486,6 +499,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -530,9 +544,11 @@ golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -552,6 +568,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -616,11 +633,15 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -631,15 +652,17 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -690,10 +713,10 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= @@ -783,8 +806,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/arrutil/arrutil.go b/pkg/arrutil/arrutil.go deleted file mode 100644 index 2c48198..0000000 --- a/pkg/arrutil/arrutil.go +++ /dev/null @@ -1,20 +0,0 @@ -package arrutil - -// Strings type -type Strings []string - -// Contains given element -func (ss Strings) Contains(sub string) bool { - for _, s := range ss { - if s == sub { - return true - } - } - return false -} - -// Length element -func (ss Strings) Length() int { - return len(ss) - -} diff --git a/pkg/crawler/base.go b/pkg/crawler/base.go new file mode 100644 index 0000000..63b33ac --- /dev/null +++ b/pkg/crawler/base.go @@ -0,0 +1,39 @@ +package crawler + +import "net/url" + +type Flags struct { + Open bool + Serve bool + ServePort int + UserAgent string + ProxyString string + Cookies bool + Robots bool + BrowserEndpoint string +} + +var ( + files filesBase + + projectURL *url.URL + projectPath string + domain string +) + +type filesBase struct { + pages []string + css []string + js []string + img []string + font []string +} + +func contains(slice []string, sub string) bool { + for _, s := range slice { + if s == sub { + return true + } + } + return false +} diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go index d7dda38..87766da 100644 --- a/pkg/crawler/crawler.go +++ b/pkg/crawler/crawler.go @@ -15,38 +15,10 @@ import ( "github.com/geziyor/geziyor" "github.com/geziyor/geziyor/client" - "github.com/shurco/goclone/pkg/arrutil" "github.com/shurco/goclone/pkg/fsutil" "github.com/shurco/goclone/pkg/netutil" ) -type Flags struct { - Open bool - Serve bool - ServePort int - UserAgent string - ProxyString string - Cookies bool - Robots bool - BrowserEndpoint string -} - -type filesBase struct { - pages arrutil.Strings - css arrutil.Strings - js arrutil.Strings - img arrutil.Strings - font arrutil.Strings -} - -var ( - files filesBase - - projectURL *url.URL - projectPath string - domain string -) - func CloneSite(ctx context.Context, args []string, flag Flags) error { if netutil.IsValidDomain(args[0]) { return fmt.Errorf("%q is not valid", args[0]) @@ -59,8 +31,6 @@ func CloneSite(ctx context.Context, args []string, flag Flags) error { return err } - projectPath = filepath.Join(fsutil.Workdir(), projectURL.Host) - geziyorOptions := &geziyor.Options{ AllowedDomains: []string{projectURL.Host}, StartURLs: []string{domain}, @@ -82,11 +52,13 @@ func CloneSite(ctx context.Context, args []string, flag Flags) error { geziyor.NewGeziyor(geziyorOptions).Start() - fmt.Printf("Pages: %v\n", files.pages.Length()) - fmt.Printf("CSS files: %v\n", files.css.Length()) - fmt.Printf("JS files: %v\n", files.js.Length()) - fmt.Printf("Img files: %v\n", files.img.Length()) - fmt.Printf("Font files: %v\n", files.font.Length()) + fmt.Printf("Pages: %v\n", len(files.pages)) + fmt.Printf("CSS files: %v\n", len(files.css)) + fmt.Printf("JS files: %v\n", len(files.js)) + fmt.Printf("Img files: %v\n", len(files.img)) + fmt.Printf("Font files: %v\n", len(files.font)) + + projectPath = filepath.Join(fsutil.Workdir(), projectURL.Host) if flag.Open { url := projectPath + "/index.html" @@ -109,9 +81,9 @@ func CloneSite(ctx context.Context, args []string, flag Flags) error { func quotesParse(g *geziyor.Geziyor, r *client.Response) { files = filesBase{} - body := string(r.Body) - fmt.Printf("page: %s://%s%s\n", projectURL.Scheme, projectURL.Host, r.Response.Request.URL.Path) + urlPath := r.Response.Request.URL.Path + fmt.Printf("page: %s://%s%s\n", projectURL.Scheme, projectURL.Host, urlPath) // search for all link tags that have a rel attribute that is equal to stylesheet - CSS r.HTMLDoc.Find("link[rel='stylesheet']").Each(func(i int, s *goquery.Selection) { @@ -120,39 +92,75 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) { parsedURL, err := url.Parse(data) if err != nil { fmt.Println("Error parsing URL:", err) + return } if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { - fmt.Println("Css found", "-->", parsedURL) - if !files.css.Contains(parsedURL.Path) { - files.css = append(files.css, parsedURL.Path) - go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath) - g.Get(r.JoinURL(projectURL.String()+parsedURL.Path), parseCSS) + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + + if !contains(files.css, link) { + fmt.Println("Css found", "-->", link) + files.css = append(files.css, link) + go netutil.Extractor(link, projectPath) + g.Get(r.JoinURL(link), parseCSS) } - body = strings.Replace(body, data, "/assets/css/"+filepath.Base(data), -1) + newLink := "/" + netutil.Folders["css"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + body = strings.Replace(body, data, newLink, -1) } } }) // search for all script tags with src attribute -- JS - r.HTMLDoc.Find("script[src]").Each(func(i int, s *goquery.Selection) { + r.HTMLDoc.Find("script[src],script[data-rocket-src]").Each(func(i int, s *goquery.Selection) { data, exists := s.Attr("src") if exists { parsedURL, err := url.Parse(data) if err != nil { fmt.Println("Error parsing URL:", err) + return } + body = saveJS(parsedURL, body) - if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { - fmt.Println("Js found", "-->", parsedURL) - if !files.js.Contains(parsedURL.Path) { - files.js = append(files.js, parsedURL.Path) - go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath) + /* + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + + if !files.js.Contains(link) { + fmt.Println("Js found", "-->", link) + files.js = append(files.js, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/" + netutil.Folders["js"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + body = strings.Replace(body, data, newLink, -1) } + */ + } - body = strings.Replace(body, data, "/assets/js/"+filepath.Base(data), -1) + data1, exists1 := s.Attr("data-rocket-src") + if exists1 { + parsedURL, err := url.Parse(data1) + if err != nil { + fmt.Println("Error parsing URL:", err) + return } + body = saveJS(parsedURL, body) + + /* + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + + if !files.js.Contains(link) { + fmt.Println("Js found", "-->", link) + files.js = append(files.js, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/" + netutil.Folders["js"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + body = strings.Replace(body, data1, newLink, -1) + } + */ } }) @@ -162,41 +170,113 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) { parsedURL, err := url.Parse(data) if err != nil { fmt.Println("Error parsing URL:", err) + return } + body = saveJS(parsedURL, body) - if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { - fmt.Println("Js found", "-->", parsedURL) - if !files.js.Contains(parsedURL.Path) { - files.js = append(files.js, parsedURL.Path) - go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath) - } + /* + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") - body = strings.Replace(body, data, "/assets/js/"+filepath.Base(data), -1) - } + if !files.js.Contains(link) { + fmt.Println("Js found", "-->", link) + files.js = append(files.js, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/" + netutil.Folders["js"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + body = strings.Replace(body, data, newLink, -1) + } + */ } }) // search for all img tags with src attribute -- Images - r.HTMLDoc.Find("img[src]").Each(func(i int, s *goquery.Selection) { + r.HTMLDoc.Find("img[src],img[data-lazy-src],img[data-lazy-srcset]").Each(func(i int, s *goquery.Selection) { + data1, exists1 := s.Attr("data-lazy-srcset") + if exists1 { + urls := strings.Split(data1, ", ") + for _, line := range urls { + linkTmp := strings.Split(line, " ") + parsedURL, err := url.Parse(linkTmp[0]) + if err != nil { + fmt.Println("Error parsing URL:", err) + return + } + body = saveIMG(parsedURL, body) + + /* + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + + if !files.img.Contains(link) { + fmt.Println("Img found", "-->", link) + files.img = append(files.img, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/" + netutil.Folders["img"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + body = strings.Replace(body, linkTmp[0], newLink, -1) + } + */ + } + } + + data2, exists2 := s.Attr("data-lazy-src") + if exists2 { + urls := strings.Split(data2, ", ") + for _, line := range urls { + linkTmp := strings.Split(line, " ") + parsedURL, err := url.Parse(linkTmp[0]) + if err != nil { + fmt.Println("Error parsing URL:", err) + return + } + body = saveIMG(parsedURL, body) + + /* + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + + if !files.img.Contains(link) { + fmt.Println("Img found", "-->", link) + files.img = append(files.img, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/" + netutil.Folders["img"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + body = strings.Replace(body, linkTmp[0], newLink, -1) + } + */ + } + } + data, exists := s.Attr("src") if exists { parsedURL, err := url.Parse(data) if err != nil { fmt.Println("Error parsing URL:", err) + return } - if strings.HasPrefix(projectURL.String()+parsedURL.Path, "data:image") || strings.HasPrefix(projectURL.String()+parsedURL.Path, "blob:") { + if parsedURL.Scheme == "data" || parsedURL.Scheme == "blob" { return } + body = saveIMG(parsedURL, body) - if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { - fmt.Println("Img found", "-->", parsedURL) - if !files.img.Contains(parsedURL.Path) { - files.img = append(files.img, parsedURL.Path) - go netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath) - } + /* + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") - body = strings.Replace(body, data, "/assets/img/"+filepath.Base(data), -1) - } + if !files.img.Contains(link) { + fmt.Println("Img found", "-->", link) + files.img = append(files.img, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/assets/img/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + body = strings.Replace(body, data, newLink, -1) + } + */ } }) @@ -206,23 +286,29 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) { body = readCSS(data, body) }) + // search all links pages r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) { data, exists := s.Attr("href") if exists { parsedURL, err := url.Parse(data) if err != nil { fmt.Println("Error parsing URL:", err) + return } if (parsedURL.Host == projectURL.Host || parsedURL.Host == "") && parsedURL.Path != "/" { - if !files.pages.Contains(parsedURL.Path) { + if !contains(files.pages, parsedURL.Path) { files.pages = append(files.pages, parsedURL.Path) } } } }) - index, err := fsutil.OpenFile(projectPath+r.Response.Request.URL.Path+"/index.html", fsutil.FsCWFlags, 0666) + if urlPath == "" && !contains(files.pages, urlPath) { + files.pages = append(files.pages, urlPath) + } + + index, err := fsutil.OpenFile(projectPath+urlPath+"/index.html", fsutil.FsCWFlags, 0o666) if err != nil { log.Fatal(err) } diff --git a/pkg/crawler/css.go b/pkg/crawler/css.go index b69c680..e763ccb 100644 --- a/pkg/crawler/css.go +++ b/pkg/crawler/css.go @@ -19,7 +19,7 @@ func parseCSS(g *geziyor.Geziyor, r *client.Response) { body := string(r.Body) base := path.Base(r.Request.URL.Path) - index, err := fsutil.OpenFile(projectPath+"/assets/css/"+base, fsutil.FsCWFlags, 0666) + index, err := fsutil.OpenFile(projectPath+"/assets/css/"+base, fsutil.FsCWFlags, 0o666) if err != nil { log.Fatal(err) } @@ -41,27 +41,33 @@ func readCSS(data, body string) string { matches := regExp.FindAllStringSubmatch(line, -1) for _, match := range matches { - parsedURL, err := url.Parse(match[1]) + link := strings.ReplaceAll(match[1], `'`, "") + link = strings.ReplaceAll(link, `"`, "") + parsedURL, err := url.Parse(link) if err != nil { fmt.Println("Error parsing URL:", err) } if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { folder := netutil.GetAssetDir(parsedURL.Path) + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + switch folder { - case "assets/font": - if !files.font.Contains(parsedURL.Path) { - files.font = append(files.font, parsedURL.Path) - netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath) + case netutil.Folders["font"]: + if !contains(files.font, link) { + fmt.Println("Font found", "-->", link) + files.font = append(files.font, link) + netutil.Extractor(link, projectPath) } - case "assets/img": - if !files.img.Contains(parsedURL.Path) { - files.img = append(files.img, parsedURL.Path) - netutil.Extractor(projectURL.String()+parsedURL.Path, projectPath) + case netutil.Folders["img"]: + if !contains(files.img, link) { + fmt.Println("Img found", "-->", link) + files.img = append(files.img, link) + netutil.Extractor(link, projectPath) } } - body = strings.Replace(body, match[1], "/"+folder+"/"+filepath.Base(match[1]), -1) + body = strings.Replace(body, link, "/"+folder+"/"+filepath.Base(link), -1) } } } diff --git a/pkg/crawler/img.go b/pkg/crawler/img.go new file mode 100644 index 0000000..ed1d71b --- /dev/null +++ b/pkg/crawler/img.go @@ -0,0 +1,25 @@ +package crawler + +import ( + "fmt" + "net/url" + "strings" + + "github.com/shurco/goclone/pkg/netutil" +) + +func saveIMG(parsedURL *url.URL, body string) string { + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + + if !contains(files.img, link) { + fmt.Println("Img found", "-->", link) + files.img = append(files.img, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/" + netutil.Folders["img"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + return strings.Replace(body, link, newLink, -1) + } + return body +} diff --git a/pkg/crawler/js.go b/pkg/crawler/js.go new file mode 100644 index 0000000..501d617 --- /dev/null +++ b/pkg/crawler/js.go @@ -0,0 +1,25 @@ +package crawler + +import ( + "fmt" + "net/url" + "strings" + + "github.com/shurco/goclone/pkg/netutil" +) + +func saveJS(parsedURL *url.URL, body string) string { + if parsedURL.Host == projectURL.Host || parsedURL.Host == "" { + link := domain + strings.ReplaceAll("/"+parsedURL.Path, "//", "/") + + if !contains(files.js, link) { + fmt.Println("Js found", "-->", link) + files.js = append(files.js, link) + go netutil.Extractor(link, projectPath) + } + + newLink := "/" + netutil.Folders["js"] + "/" + netutil.ReplaceSlashWithDash(parsedURL.Path) + return strings.Replace(body, link, newLink, -1) + } + return body +} diff --git a/pkg/fsutil/file.go b/pkg/fsutil/file.go index f002b22..22b772f 100644 --- a/pkg/fsutil/file.go +++ b/pkg/fsutil/file.go @@ -17,7 +17,7 @@ const ( // OpenFile like os.OpenFile, but will auto create dir. func OpenFile(filepath string, flag int, perm os.FileMode) (*os.File, error) { fileDir := path.Dir(filepath) - if err := os.MkdirAll(fileDir, 0775); err != nil { + if err := os.MkdirAll(fileDir, 0o775); err != nil { return nil, err } diff --git a/pkg/netutil/netutil.go b/pkg/netutil/netutil.go index d552808..41bc7a8 100644 --- a/pkg/netutil/netutil.go +++ b/pkg/netutil/netutil.go @@ -7,46 +7,50 @@ import ( "path" "path/filepath" "regexp" + "strings" "github.com/shurco/goclone/pkg/fsutil" ) -var ( - extensionDir = map[string]string{ - ".css": "assets/css", - ".js": "assets/js", - ".jpg": "assets/img", - ".jpeg": "assets/img", - ".gif": "assets/img", - ".png": "assets/img", - ".svg": "assets/img", - ".eot": "assets/font", - ".otf": "assets/font", - ".ttf": "assets/font", - ".woff": "assets/font", - ".woff2": "assets/font", - } -) +var Folders = map[string]string{ + "css": "assets/css", + "js": "assets/js", + "img": "assets/img", + "font": "assets/font", +} + +var Extensions = map[string]string{ + ".css": "assets/css", + ".js": "assets/js", + ".jpg": "assets/img", + ".jpeg": "assets/img", + ".gif": "assets/img", + ".png": "assets/img", + ".svg": "assets/img", + ".eot": "assets/font", + ".otf": "assets/font", + ".ttf": "assets/font", + ".woff": "assets/font", + ".woff2": "assets/font", +} // Extractor visits a link determines if its a page or sublink // downloads the contents to a correct directory in project folder -func Extractor(link string, projectPath string) { +func Extractor(link, projectPath string) { resp, err := http.Get(link) if err != nil { panic(err) } defer resp.Body.Close() - base := path.Base(resp.Request.URL.Path) - oldExt := filepath.Ext(base) ext := urlExtension(resp.Request.URL.Path) if ext != "" { - dirPath := extensionDir[ext] + dirPath := "/" + Extensions[ext] + "/" if dirPath != "" { - name := base[0:len(base)-len(oldExt)] + ext + name := ReplaceSlashWithDash(resp.Request.URL.Path) - file, err := fsutil.OpenFile(filepath.Join(projectPath, dirPath, name), fsutil.FsCWFlags, 0666) + file, err := fsutil.OpenFile(filepath.Join(projectPath, dirPath, name), fsutil.FsCWFlags, 0o666) if err != nil { log.Fatal(err) } @@ -59,7 +63,7 @@ func Extractor(link string, projectPath string) { // GetAssetDir is ... func GetAssetDir(filename string) string { - dirPath := extensionDir[urlExtension(filename)] + dirPath := "/" + Extensions[urlExtension(filename)] + "/" if dirPath != "" { return dirPath } @@ -92,3 +96,12 @@ func IsValidDomain(domain string) bool { return true } + +// ReplaceSlashWithDash is ... +func ReplaceSlashWithDash(input string) string { + if strings.HasPrefix(input, "/") { + input = input[1:] + } + + return strings.ReplaceAll(input, "/", "-") +}