Skip to content

Commit c846de7

Browse files
dogancanbakirdogancanbakir
authored
Merge pull request #1696 from projectdiscovery/dev
v2.12.0
2 parents 933918a + 4984bfb commit c846de7

60 files changed

Lines changed: 671 additions & 91 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pkg/passive/sources.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon"
4444
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
4545
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
46+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconeer"
4647
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs"
4748
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex"
4849
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud"
@@ -108,6 +109,7 @@ var AllSources = [...]subscraping.Source{
108109
&facebook.Source{},
109110
// &threatminer.Source{}, // failing api
110111
// &reconcloud.Source{}, // failing due to cloudflare bot protection
112+
&reconeer.Source{},
111113
&builtwith.Source{},
112114
&hudsonrock.Source{},
113115
&digitalyama.Source{},
@@ -182,9 +184,9 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup
182184
}
183185
}
184186

185-
// TODO: Consider refactoring this to avoid potential duplication issues
186187
for _, source := range sources {
187-
if source.NeedsKey() {
188+
keyReq := source.KeyRequirement()
189+
if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey {
188190
if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" {
189191
source.AddApiKeys([]string{apiKey})
190192
}

‎pkg/passive/sources_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ var (
5757
"facebook",
5858
// "threatminer",
5959
// "reconcloud",
60+
"reconeer",
6061
"builtwith",
6162
"hudsonrock",
6263
"digitalyama",
@@ -100,6 +101,7 @@ var (
100101
"facebook",
101102
// "threatminer",
102103
// "reconcloud",
104+
"reconeer",
103105
"builtwith",
104106
"digitalyama",
105107
"thc",

‎pkg/runner/banners.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ const banner = `
1717
const ToolName = `subfinder`
1818

1919
// Version is the current version of subfinder
20-
const version = `v2.11.0`
20+
const version = `v2.12.0`
2121

2222
// showBanner is used to show the banner to the user
2323
func showBanner() {

‎pkg/runner/config.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/projectdiscovery/gologger"
1010
"github.com/projectdiscovery/subfinder/v2/pkg/passive"
11+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
1112
fileutil "github.com/projectdiscovery/utils/file"
1213
)
1314

@@ -25,7 +26,8 @@ func createProviderConfigYAML(configFilePath string) error {
2526

2627
sourcesRequiringApiKeysMap := make(map[string][]string)
2728
for _, source := range passive.AllSources {
28-
if source.NeedsKey() {
29+
keyReq := source.KeyRequirement()
30+
if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey {
2931
sourceName := strings.ToLower(source.Name())
3032
sourcesRequiringApiKeysMap[sourceName] = []string{}
3133
}
@@ -46,7 +48,7 @@ func UnmarshalFrom(file string) error {
4648
for _, source := range passive.AllSources {
4749
sourceName := strings.ToLower(source.Name())
4850
apiKeys := sourceApiKeysMap[sourceName]
49-
if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 {
51+
if len(apiKeys) > 0 {
5052
gologger.Debug().Msgf("API key(s) found for %s.", sourceName)
5153
source.AddApiKeys(apiKeys)
5254
}

‎pkg/runner/options.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/projectdiscovery/gologger"
1616
"github.com/projectdiscovery/subfinder/v2/pkg/passive"
1717
"github.com/projectdiscovery/subfinder/v2/pkg/resolve"
18+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
1819
envutil "github.com/projectdiscovery/utils/env"
1920
fileutil "github.com/projectdiscovery/utils/file"
2021
folderutil "github.com/projectdiscovery/utils/folder"
@@ -223,16 +224,20 @@ func (options *Options) loadProvidersFrom(location string) {
223224

224225
func listSources(options *Options) {
225226
gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources))
226-
gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n")
227+
gologger.Info().Msgf("Sources marked with an * require key(s) or token(s) to work.\n")
228+
gologger.Info().Msgf("Sources marked with a ~ optionally support key(s) for better results.\n")
227229
gologger.Info().Msgf("You can modify %s to configure your keys/tokens.\n\n", options.ProviderConfig)
228230

229231
for _, source := range passive.AllSources {
230-
message := "%s\n"
231232
sourceName := source.Name()
232-
if source.NeedsKey() {
233-
message = "%s *\n"
233+
switch source.KeyRequirement() {
234+
case subscraping.RequiredKey:
235+
gologger.Silent().Msgf("%s *\n", sourceName)
236+
case subscraping.OptionalKey:
237+
gologger.Silent().Msgf("%s ~\n", sourceName)
238+
default:
239+
gologger.Silent().Msgf("%s\n", sourceName)
234240
}
235-
gologger.Silent().Msgf(message, sourceName)
236241
}
237242
}
238243

‎pkg/runner/stats.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) {
2424
if sourceStats.Skipped {
2525
skipped = append(skipped, fmt.Sprintf(" %s", source))
2626
} else {
27-
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors))
27+
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors))
2828
}
2929
}
3030

3131
if len(lines) > 0 {
32-
gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56))
32+
gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68))
3333
gologger.Print().Msg(strings.Join(lines, "\n"))
3434
gologger.Print().Msgf("\n")
3535
}

‎pkg/subscraping/sources/alienvault/alienvault.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type Source struct {
2323
timeTaken time.Duration
2424
results int
2525
errors int
26+
requests int
2627
apiKeys []string
2728
skipped bool
2829
}
@@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3233
results := make(chan subscraping.Result)
3334
s.errors = 0
3435
s.results = 0
36+
s.requests = 0
3537

3638
go func() {
3739
defer func(startTime time.Time) {
@@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4547
return
4648
}
4749

50+
s.requests++
4851
resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "",
4952
map[string]string{"Authorization": "Bearer " + randomApiKey})
5053
if err != nil && resp == nil {
@@ -98,8 +101,12 @@ func (s *Source) HasRecursiveSupport() bool {
98101
return true
99102
}
100103

104+
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
105+
return subscraping.RequiredKey
106+
}
107+
101108
func (s *Source) NeedsKey() bool {
102-
return true
109+
return s.KeyRequirement() == subscraping.RequiredKey
103110
}
104111

105112
func (s *Source) AddApiKeys(keys []string) {
@@ -110,6 +117,7 @@ func (s *Source) Statistics() subscraping.Statistics {
110117
return subscraping.Statistics{
111118
Errors: s.errors,
112119
Results: s.results,
120+
Requests: s.requests,
113121
TimeTaken: s.timeTaken,
114122
Skipped: s.skipped,
115123
}

‎pkg/subscraping/sources/anubis/anubis.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,23 @@ type Source struct {
1717
timeTaken time.Duration
1818
errors int
1919
results int
20+
requests int
2021
}
2122

2223
// Run function returns all subdomains found with the service
2324
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
2425
results := make(chan subscraping.Result)
2526
s.errors = 0
2627
s.results = 0
28+
s.requests = 0
2729

2830
go func() {
2931
defer func(startTime time.Time) {
3032
s.timeTaken = time.Since(startTime)
3133
close(results)
3234
}(time.Now())
3335

36+
s.requests++
3437
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain))
3538
if err != nil {
3639
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -82,8 +85,12 @@ func (s *Source) HasRecursiveSupport() bool {
8285
return false
8386
}
8487

88+
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
89+
return subscraping.NoKey
90+
}
91+
8592
func (s *Source) NeedsKey() bool {
86-
return false
93+
return s.KeyRequirement() == subscraping.RequiredKey
8794
}
8895

8996
func (s *Source) AddApiKeys(_ []string) {
@@ -94,6 +101,7 @@ func (s *Source) Statistics() subscraping.Statistics {
94101
return subscraping.Statistics{
95102
Errors: s.errors,
96103
Results: s.results,
104+
Requests: s.requests,
97105
TimeTaken: s.timeTaken,
98106
}
99107
}

‎pkg/subscraping/sources/bevigil/bevigil.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ type Source struct {
2121
timeTaken time.Duration
2222
errors int
2323
results int
24+
requests int
2425
skipped bool
2526
}
2627

2728
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
2829
results := make(chan subscraping.Result)
2930
s.errors = 0
3031
s.results = 0
32+
s.requests = 0
3133

3234
go func() {
3335
defer func(startTime time.Time) {
@@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4345

4446
getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain)
4547

48+
s.requests++
4649
resp, err := session.Get(ctx, getUrl, "", map[string]string{
4750
"X-Access-Token": randomApiKey, "User-Agent": "subfinder",
4851
})
@@ -94,8 +97,12 @@ func (s *Source) HasRecursiveSupport() bool {
9497
return false
9598
}
9699

100+
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
101+
return subscraping.RequiredKey
102+
}
103+
97104
func (s *Source) NeedsKey() bool {
98-
return true
105+
return s.KeyRequirement() == subscraping.RequiredKey
99106
}
100107

101108
func (s *Source) AddApiKeys(keys []string) {
@@ -106,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics {
106113
return subscraping.Statistics{
107114
Errors: s.errors,
108115
Results: s.results,
116+
Requests: s.requests,
109117
TimeTaken: s.timeTaken,
110118
Skipped: s.skipped,
111119
}

‎pkg/subscraping/sources/bufferover/bufferover.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type Source struct {
2727
timeTaken time.Duration
2828
errors int
2929
results int
30+
requests int
3031
skipped bool
3132
}
3233

@@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3536
results := make(chan subscraping.Result)
3637
s.errors = 0
3738
s.results = 0
39+
s.requests = 0
3840

3941
go func() {
4042
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
5557
}
5658

5759
func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) {
60+
s.requests++
5861
resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey})
5962

6063
if err != nil && resp == nil {
@@ -119,8 +122,12 @@ func (s *Source) HasRecursiveSupport() bool {
119122
return true
120123
}
121124

125+
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
126+
return subscraping.RequiredKey
127+
}
128+
122129
func (s *Source) NeedsKey() bool {
123-
return true
130+
return s.KeyRequirement() == subscraping.RequiredKey
124131
}
125132

126133
func (s *Source) AddApiKeys(keys []string) {
@@ -131,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics {
131138
return subscraping.Statistics{
132139
Errors: s.errors,
133140
Results: s.results,
141+
Requests: s.requests,
134142
TimeTaken: s.timeTaken,
135143
Skipped: s.skipped,
136144
}

0 commit comments

Comments
 (0)