Add more params to web/spider

Gemfile

@@ -15,8 +15,8 @@ end
 # gem 'ronin-support',  '~> 1.1', github: 'ronin-rb/ronin-support',
 #                                 branch: 'main'
 
-# gem 'ronin-core', '~> 0.2', github: 'ronin-rb/ronin-core',
-#                             branch: 'main'
+gem 'ronin-core', '~> 0.3', github: 'ronin-rb/ronin-core',
+                            branch: '0.3.0'
 
 # gem 'ronin-repos',    '~> 0.1', github: 'ronin-rb/ronin-repos',
 #                                 branch: 'main'

README.md

@@ -74,11 +74,12 @@ Arguments:
 
 Commands:
     completion
+    config
     help
     irb
     new
     run
-    test
+    run-worker, test
     worker
     workers
 ```
@@ -87,22 +88,27 @@ List all available recon workers:
 
 ```shell
 $ ronin-recon workers
-  api/crt_sh
-  dns/lookup
-  dns/mailservers
-  dns/nameservers
-  dns/reverse_lookup
-  dns/srv_enum
-  dns/subdomain_enum
-  dns/suffix_enum
-  net/cert_enum
-  net/cert_grab
-  net/ip_range_enum
-  net/port_scan
-  net/service_id
-  web/dir_enum
-  web/email_addresses
-  web/spider
+  api/built_with                    
+  api/crt_sh                        
+  api/hunter_io                     
+  api/security_trails               
+  api/zoom_eye                      
+  dns/lookup             [enabled]  
+  dns/mailservers        [enabled]  
+  dns/nameservers        [enabled]  
+  dns/reverse_lookup     [enabled]  
+  dns/srv_enum           [enabled]  
+  dns/subdomain_enum     [enabled]  
+  dns/suffix_enum        [enabled]  
+  net/ip_range_enum      [enabled]  
+  net/port_scan          [enabled]  
+  net/service_id         [enabled]  
+  ssl/cert_enum          [enabled]  
+  ssl/cert_grab          [enabled]  
+  web/dir_enum           [enabled]  
+  web/email_addresses    [enabled]  
+  web/screenshot                    
+  web/spider             [enabled]  
 ```
 
 Print info about a specific recon worker:
@@ -215,25 +221,41 @@ Save the recon results to a PDF image:
 $ ronin-recon run -o output.pdf example.com
 ```
 
+Enable an optional worker by default:
+
+```shell
+$ ronin-recon config enable api/hunter_io
+```
+
+Set the default concurrency for a worker:
+
+```shell
+$ ronin-recon config set --concurrency web/spider=4
+```
+
+Set the API key for a worker:
+
+```shell
+$ ronin-recon config set --param api/hunter_io.api_key=...
+```
+
 Generate a boilerplate recon worker file, with some custom information:
 
 ```shell
 $ ronin-recon new example_worker.rb \
-                  --name Example \
-                  --authors Postmodern \
+                  --author Postmodern \
                   --description "This is an example."
 ```
 
-Generate a ronin repository of your own payloads (or exploits):
+Generate a ronin repository of your own recon workers:
 
 ```shell
 $ ronin-repos new my-repo
 $ cd my-repo/
 $ mkdir recon
 $ ronin-recon new recon/my_recon.rb \
-                  --name MyRecon \
-                  --authors You \
-                  --description "This is my payload."
+                  --author You \
+                  --description "This is my recon worker."
 $ vim recon/my_recon.rb
 $ git add recon/my_recon.rb
 $ git commit
@@ -313,11 +335,12 @@ end
 * [async-http] ~> 0.60
 * [wordlist] ~> 1.0, >= 1.0.3
 * [ronin-support] ~> 1.1
-* [ronin-core] ~> 0.2
+* [ronin-core] ~> 0.3
 * [ronin-db] ~> 0.2
 * [ronin-repos] ~> 0.1
 * [ronin-nmap] ~> 0.1
 * [ronin-web-spider] ~> 0.2
+* [ronin-web-browser] ~> 0.1
 
 ## Install
 
@@ -394,3 +417,4 @@ along with ronin-recon.  If not, see <https://www.gnu.org/licenses/>.
 [ronin-masscan]: https://github.com/ronin-rb/ronin-masscan#readme
 [ronin-nmap]: https://github.com/ronin-rb/ronin-nmap#readme
 [ronin-web-spider]: https://github.com/ronin-rb/ronin-web-spider#readme
+[ronin-web-browser]: https://github.com/ronin-rb/ronin-web-browser#readme

data/templates/worker.rb.erb

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S ronin-recon test -f
+#!/usr/bin/env -S ronin-recon run-worker -f
 
 require 'ronin/recon/<%= @worker_type[:file] -%>'
 

gemspec.yml

@@ -27,11 +27,18 @@ generated_files:
  - data/completions/ronin-recon
  - man/ronin-recon.1
  - man/ronin-recon-completion.1
+ - man/ronin-recon-config.1
+ - man/ronin-recon-config-disable.1
+ - man/ronin-recon-config-enable.1
+ - man/ronin-recon-config-get.1
+ - man/ronin-recon-config-list.1
+ - man/ronin-recon-config-set.1
+ - man/ronin-recon-config-unset.1
  - man/ronin-recon-irb.1
  - man/ronin-recon-new.1
  - man/ronin-recon-workers.1
  - man/ronin-recon-worker.1
- - man/ronin-recon-test.1
+ - man/ronin-recon-run-worker.1
  - man/ronin-recon-run.1
  - data/wordlists/subdomains-1000.txt.gz
  - data/wordlists/raft-small-directories.txt.gz
@@ -46,11 +53,12 @@ dependencies:
   wordlist: ~> 1.0, >= 1.0.3
   # Ronin dependencies:
   ronin-support: ~> 1.1
-  ronin-core: ~> 0.2
+  ronin-core: ~> 0.3
   ronin-db: ~> 0.2
   ronin-repos: ~> 0.1
   ronin-nmap: ~> 0.1
   ronin-web-spider: ~> 0.2
+  ronin-web-browser: ~> 0.1
 
 development_dependencies:
   bundler: ~> 2.0

lib/ronin/recon/builtin/api/built_with.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+#
+# ronin-recon - A micro-framework and tool for performing reconnaissance.
+#
+# Copyright (c) 2023-2024 Hal Brodigan ([email protected])
+#
+# ronin-recon is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-recon is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-recon.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require_relative '../../worker'
+
+require 'async/http/internet/instance'
+require 'set'
+
+module Ronin
+  module Recon
+    module API
+      #
+      # A recon worker that queries https://api.builtwith.com and return
+      # informations for given domain
+      #
+      # ## Environment Variables
+      #
+      # * `BUILT_WITH_API_KEY` - Specifies the API key used for authorization.
+      #
+      class BuiltWith < Worker
+
+        register 'api/built_with'
+
+        summary "Queries the domain informations from https://api.builtwith.com"
+        description <<~DESC
+          Queriest the domain informations from https://api.builtwith.com.
+
+          The BuiltWith API key can be specified via the api/built_with.api_key
+          param or the BUILT_WITH_API_KEY environment variables.
+        DESC
+
+        accepts Domain
+        outputs Domain, EmailAddress
+        intensity :passive
+        concurrency 1
+
+        param :api_key, String, required: true,
+                                default:  ENV['BUILT_WITH_API_KEY'],
+                                desc:     'The API key for BuiltWith'
+
+        # The HTTP client for `https://api.builtwith.com`
+        #
+        # @return [Async::HTTP::Client]
+        #
+        # @api private
+        attr_reader :client
+
+        #
+        # Initializes the `api/built_with` worker.
+        #
+        # @param [Hash{Symbol => Object}] kwargs
+        #   Additional keyword arguments.
+        #
+        # @api private
+        #
+        def initialize(**kwargs)
+          super(**kwargs)
+
+          @client = Async::HTTP::Client.new(
+            Async::HTTP::Endpoint.for('https', 'api.builtwith.com')
+          )
+        end
+
+        #
+        # Returns all informations queried for given domain
+        #
+        # @param [Values::Domain] domain
+        #   The domain value to gather informations for.
+        #
+        # @yield [Value] value
+        #   The found value will be yielded
+        #
+        # @yieldparam [Values::Domain, Values::EmailAddress]
+        #   The found domains or email addresses
+        #
+        def process(domain)
+          path     = "/v21/api.json?KEY=#{params[:api_key]}&LOOKUP=#{domain}"
+          response = client.get(path)
+          body     = begin
+                       JSON.parse(response.read, symbolize_names: true)
+                      ensure
+                        response.close
+                     end
+
+          domains         = Set.new
+          email_addresses = Set.new
+
+          if (results = body[:Results])
+            results.each do |result|
+              if (paths = result.dig(:Result, :Paths))
+                paths.each do |result_path|
+                  if (sub_domain = result_path[:SubDomain])
+                    new_domain = "#{sub_domain}.#{domain}"
+
+                    yield Domain.new(new_domain) if domains.add?(new_domain)
+                  end
+                end
+              end
+
+              if (emails = result.dig(:Meta, :Emails))
+                emails.each do |email|
+                  yield EmailAddress.new(email) if email_addresses.add?(email)
+                end
+              end
+            end
+          end
+        end
+
+      end
+    end
+  end
+end