Skip to content

Commit 4984bfb

Browse files
Mzack9999Mzack9999
authored
Merge pull request #1699 from projectdiscovery/feature/request-tracking-stats
Add request tracking to -stats flag
2 parents a2615fa + 4bba544 commit 4984bfb

54 files changed

Lines changed: 248 additions & 25 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pkg/runner/stats.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) {
2424
if sourceStats.Skipped {
2525
skipped = append(skipped, fmt.Sprintf(" %s", source))
2626
} else {
27-
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors))
27+
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors))
2828
}
2929
}
3030

3131
if len(lines) > 0 {
32-
gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56))
32+
gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68))
3333
gologger.Print().Msg(strings.Join(lines, "\n"))
3434
gologger.Print().Msgf("\n")
3535
}

‎pkg/subscraping/sources/alienvault/alienvault.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type Source struct {
2323
timeTaken time.Duration
2424
results int
2525
errors int
26+
requests int
2627
apiKeys []string
2728
skipped bool
2829
}
@@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3233
results := make(chan subscraping.Result)
3334
s.errors = 0
3435
s.results = 0
36+
s.requests = 0
3537

3638
go func() {
3739
defer func(startTime time.Time) {
@@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4547
return
4648
}
4749

50+
s.requests++
4851
resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "",
4952
map[string]string{"Authorization": "Bearer " + randomApiKey})
5053
if err != nil && resp == nil {
@@ -114,6 +117,7 @@ func (s *Source) Statistics() subscraping.Statistics {
114117
return subscraping.Statistics{
115118
Errors: s.errors,
116119
Results: s.results,
120+
Requests: s.requests,
117121
TimeTaken: s.timeTaken,
118122
Skipped: s.skipped,
119123
}

‎pkg/subscraping/sources/anubis/anubis.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,23 @@ type Source struct {
1717
timeTaken time.Duration
1818
errors int
1919
results int
20+
requests int
2021
}
2122

2223
// Run function returns all subdomains found with the service
2324
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
2425
results := make(chan subscraping.Result)
2526
s.errors = 0
2627
s.results = 0
28+
s.requests = 0
2729

2830
go func() {
2931
defer func(startTime time.Time) {
3032
s.timeTaken = time.Since(startTime)
3133
close(results)
3234
}(time.Now())
3335

36+
s.requests++
3437
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain))
3538
if err != nil {
3639
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -98,6 +101,7 @@ func (s *Source) Statistics() subscraping.Statistics {
98101
return subscraping.Statistics{
99102
Errors: s.errors,
100103
Results: s.results,
104+
Requests: s.requests,
101105
TimeTaken: s.timeTaken,
102106
}
103107
}

‎pkg/subscraping/sources/bevigil/bevigil.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ type Source struct {
2121
timeTaken time.Duration
2222
errors int
2323
results int
24+
requests int
2425
skipped bool
2526
}
2627

2728
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
2829
results := make(chan subscraping.Result)
2930
s.errors = 0
3031
s.results = 0
32+
s.requests = 0
3133

3234
go func() {
3335
defer func(startTime time.Time) {
@@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4345

4446
getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain)
4547

48+
s.requests++
4649
resp, err := session.Get(ctx, getUrl, "", map[string]string{
4750
"X-Access-Token": randomApiKey, "User-Agent": "subfinder",
4851
})
@@ -110,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics {
110113
return subscraping.Statistics{
111114
Errors: s.errors,
112115
Results: s.results,
116+
Requests: s.requests,
113117
TimeTaken: s.timeTaken,
114118
Skipped: s.skipped,
115119
}

‎pkg/subscraping/sources/bufferover/bufferover.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type Source struct {
2727
timeTaken time.Duration
2828
errors int
2929
results int
30+
requests int
3031
skipped bool
3132
}
3233

@@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3536
results := make(chan subscraping.Result)
3637
s.errors = 0
3738
s.results = 0
39+
s.requests = 0
3840

3941
go func() {
4042
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
5557
}
5658

5759
func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) {
60+
s.requests++
5861
resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey})
5962

6063
if err != nil && resp == nil {
@@ -135,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics {
135138
return subscraping.Statistics{
136139
Errors: s.errors,
137140
Results: s.results,
141+
Requests: s.requests,
138142
TimeTaken: s.timeTaken,
139143
Skipped: s.skipped,
140144
}

‎pkg/subscraping/sources/builtwith/builtwith.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ type Source struct {
3535
timeTaken time.Duration
3636
errors int
3737
results int
38+
requests int
3839
skipped bool
3940
}
4041

@@ -43,6 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4344
results := make(chan subscraping.Result)
4445
s.errors = 0
4546
s.results = 0
47+
s.requests = 0
4648

4749
go func() {
4850
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
5557
return
5658
}
5759

60+
s.requests++
5861
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain))
5962
if err != nil {
6063
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -116,6 +119,7 @@ func (s *Source) Statistics() subscraping.Statistics {
116119
return subscraping.Statistics{
117120
Errors: s.errors,
118121
Results: s.results,
122+
Requests: s.requests,
119123
TimeTaken: s.timeTaken,
120124
Skipped: s.skipped,
121125
}

‎pkg/subscraping/sources/c99/c99.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ type Source struct {
1818
timeTaken time.Duration
1919
errors int
2020
results int
21+
requests int
2122
skipped bool
2223
}
2324

@@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3637
results := make(chan subscraping.Result)
3738
s.errors = 0
3839
s.results = 0
40+
s.requests = 0
3941

4042
go func() {
4143
defer func(startTime time.Time) {
@@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
5052
}
5153

5254
searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain)
55+
s.requests++
5356
resp, err := session.SimpleGet(ctx, searchURL)
5457
if err != nil {
5558
session.DiscardHTTPResponse(resp)
@@ -123,6 +126,7 @@ func (s *Source) Statistics() subscraping.Statistics {
123126
return subscraping.Statistics{
124127
Errors: s.errors,
125128
Results: s.results,
129+
Requests: s.requests,
126130
TimeTaken: s.timeTaken,
127131
Skipped: s.skipped,
128132
}

‎pkg/subscraping/sources/censys/censys.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ type Source struct {
7575
timeTaken time.Duration
7676
errors int
7777
results int
78+
requests int
7879
skipped bool
7980
}
8081

@@ -83,6 +84,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
8384
results := make(chan subscraping.Result)
8485
s.errors = 0
8586
s.results = 0
87+
s.requests = 0
8688

8789
go func() {
8890
defer func(startTime time.Time) {
@@ -136,6 +138,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
136138
headers[orgIDHeader] = randomApiKey.orgID
137139
}
138140

141+
s.requests++
139142
resp, err := session.HTTPRequest(
140143
ctx,
141144
http.MethodPost,
@@ -224,6 +227,7 @@ func (s *Source) Statistics() subscraping.Statistics {
224227
return subscraping.Statistics{
225228
Errors: s.errors,
226229
Results: s.results,
230+
Requests: s.requests,
227231
TimeTaken: s.timeTaken,
228232
Skipped: s.skipped,
229233
}

‎pkg/subscraping/sources/certspotter/certspotter.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ type Source struct {
2222
timeTaken time.Duration
2323
errors int
2424
results int
25+
requests int
2526
skipped bool
2627
}
2728

@@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3031
results := make(chan subscraping.Result)
3132
s.errors = 0
3233
s.results = 0
34+
s.requests = 0
3335

3436
go func() {
3537
defer func(startTime time.Time) {
@@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4648
headers := map[string]string{"Authorization": "Bearer " + randomApiKey}
4749
cookies := ""
4850

51+
s.requests++
4952
resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers)
5053
if err != nil {
5154
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -88,6 +91,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
8891
}
8992
reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id)
9093

94+
s.requests++
9195
resp, err := session.Get(ctx, reqURL, cookies, headers)
9296
if err != nil {
9397
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -156,6 +160,7 @@ func (s *Source) Statistics() subscraping.Statistics {
156160
return subscraping.Statistics{
157161
Errors: s.errors,
158162
Results: s.results,
163+
Requests: s.requests,
159164
TimeTaken: s.timeTaken,
160165
Skipped: s.skipped,
161166
}

‎pkg/subscraping/sources/chaos/chaos.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ type Source struct {
1616
timeTaken time.Duration
1717
errors int
1818
results int
19+
requests int
1920
skipped bool
2021
}
2122

@@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session)
2425
results := make(chan subscraping.Result)
2526
s.errors = 0
2627
s.results = 0
28+
s.requests = 0
2729

2830
go func() {
2931
defer func(startTime time.Time) {
@@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session)
3840
}
3941

4042
chaosClient := chaos.New(randomApiKey)
43+
s.requests++
4144
for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{
4245
Domain: domain,
4346
}) {
@@ -90,6 +93,7 @@ func (s *Source) Statistics() subscraping.Statistics {
9093
return subscraping.Statistics{
9194
Errors: s.errors,
9295
Results: s.results,
96+
Requests: s.requests,
9397
TimeTaken: s.timeTaken,
9498
Skipped: s.skipped,
9599
}

0 commit comments

Comments
 (0)