Merge pull request #147 from h4sh5/log-user-agent

Add option to log user agent

Author: Mzack9999

.gitignore

@@ -1,2 +1,3 @@
 *.exe
 cmd/simplehttpserver/simplehttpserver
+simplehttpserver

README.md

@@ -29,6 +29,7 @@ SimpleHTTPserver is a go enhanced version of the well known python simplehttpser
 - HTTP/S Web Server
 - File Server with arbitrary directory support
 - HTTP request/response dump
+- JSON request logging to file
 - Configurable ip address and listening port
 - Configurable HTTP/TCP server with customizable response via YAML template
 
@@ -72,6 +73,7 @@ This will display help for the tool. Here are all the switches it supports.
 | `-silent`        | Show only results                                       | `simplehttpserver -silent`                         |
 | `-py`            | Emulate Python Style                                    | `simplehttpserver -py`                             |
 | `-header`        | HTTP response header (can be used multiple times)       | `simplehttpserver -header 'X-Powered-By: Go'`      |
+| `-json-log`      | JSON log file path for request logging                  | `simplehttpserver -json-log /tmp/requests.json`    |
 
 ### Running simplehttpserver in the current folder  
 
@@ -121,12 +123,45 @@ To upload files use the following curl request with basic auth header:
 curl -v --user 'root:root' --upload-file file.txt http://localhost:8000/file.txt
 ```
 
+### Running simplehttpserver with JSON request logging
+
+This will run the tool and log all HTTP requests to a JSON file:
+
+```sh
+simplehttpserver -json-log /tmp/requests.json
+
+2021/01/11 21:40:48 Serving . on http://0.0.0.0:8000/...
+```
+
+The JSON log file will contain structured request data including:
+- Timestamp, remote address, HTTP method, URL, protocol
+- Status code, response size, user agent
+- Request headers, request body, response body
+
+Example JSON log entry:
+```json
+{
+  "timestamp": "2021-01-11T21:41:15Z",
+  "remote_addr": "127.0.0.1:50181",
+  "method": "GET",
+  "url": "/",
+  "proto": "HTTP/1.1",
+  "status_code": 200,
+  "size": 383,
+  "user_agent": "curl/7.68.0",
+  "headers": {
+    "Accept": "*/*",
+    "User-Agent": "curl/7.68.0"
+  }
+}
+```
+
 ### Running TCP server with custom responses
 
 This will run the tool as TLS TCP server and enable custom responses based on YAML templates:
 
 ```sh
-simplehttpserver -rule rules.yaml -tcp -tls -domain localhost
+simplehttpserver -rules rules.yaml -tcp -tls -domain localhost
 ```
 
 The rules are written as follows:

internal/runner/options.go

@@ -28,6 +28,7 @@ type Options struct {
 	Verbose         bool
 	EnableUpload    bool
 	EnableTCP       bool
+	LogUA           bool
 	RulesFile       string
 	TCPWithTLS      bool
 	Version         bool
@@ -39,6 +40,7 @@ type Options struct {
 	Python          bool
 	CORS            bool
 	HTTPHeaders     HTTPHeaders
+	JSONLogFile     string
 }
 
 // ParseOptions parses the command line options for application
@@ -76,11 +78,13 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.Python, "py", false, "Emulate Python Style"),
 		flagSet.BoolVar(&options.CORS, "cors", false, "Enable Cross-Origin Resource Sharing (CORS)"),
 		flagSet.Var(&options.HTTPHeaders, "header", "Add HTTP Response Header (name: value), can be used multiple times"),
+		flagSet.StringVar(&options.JSONLogFile, "json-log", "", "JSON log file path for request logging"),
 	)
 
 	flagSet.CreateGroup("debug", "Debug",
 		flagSet.BoolVar(&options.Version, "version", false, "Show version of the software"),
 		flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose"),
+		flagSet.BoolVar(&options.LogUA, "log-ua", false, "Log User Agent"),
 	)
 
 	if err := flagSet.Parse(); err != nil {

internal/runner/runner.go

@@ -59,6 +59,7 @@ func New(options *Options) (*Runner, error) {
 	httpServer, err := httpserver.New(&httpserver.Options{
 		Folder:            r.options.Folder,
 		EnableUpload:      r.options.EnableUpload,
+		LogUA:             r.options.LogUA,
 		ListenAddress:     r.options.ListenAddress,
 		TLS:               r.options.HTTPS,
 		Certificate:       r.options.TLSCertificate,
@@ -75,6 +76,7 @@ func New(options *Options) (*Runner, error) {
 		Python:            r.options.Python,
 		CORS:              r.options.CORS,
 		HTTPHeaders:       r.options.HTTPHeaders,
+		JSONLogFile:       r.options.JSONLogFile,
 	})
 	if err != nil {
 		return nil, err

pkg/httpserver/httpserver.go

@@ -14,6 +14,7 @@ import (
 type Options struct {
 	Folder            string
 	EnableUpload      bool
+	LogUA             bool // enable logging user agent
 	ListenAddress     string
 	TLS               bool
 	Certificate       string
@@ -30,12 +31,14 @@ type Options struct {
 	Python            bool
 	CORS              bool
 	HTTPHeaders       []HTTPHeader
+	JSONLogFile       string
 }
 
 // HTTPServer instance
 type HTTPServer struct {
-	options *Options
-	layers  http.Handler
+	options    *Options
+	layers     http.Handler
+	jsonLogger *JSONLogger
 }
 
 // LayerHandler is the interface of all layer funcs
@@ -46,6 +49,16 @@ func New(options *Options) (*HTTPServer, error) {
 	var h HTTPServer
 	EnableUpload = options.EnableUpload
 	EnableVerbose = options.Verbose
+	EnableLogUA = options.LogUA
+
+	// Initialize JSON logger if specified
+	if options.JSONLogFile != "" {
+		jsonLogger, err := NewJSONLogger(options.JSONLogFile)
+		if err != nil {
+			return nil, err
+		}
+		h.jsonLogger = jsonLogger
+	}
 	folder, err := filepath.Abs(options.Folder)
 	if err != nil {
 		return nil, err
@@ -127,5 +140,8 @@ func (t *HTTPServer) ListenAndServeTLS() error {
 
 // Close the service
 func (t *HTTPServer) Close() error {
+	if t.jsonLogger != nil {
+		return t.jsonLogger.Close()
+	}
 	return nil
 }

pkg/httpserver/jsonlogger.go

@@ -0,0 +1,93 @@
+package httpserver
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+)
+
+// JSONLogEntry represents a single HTTP request log entry in JSON format
+type JSONLogEntry struct {
+	Timestamp    string            `json:"timestamp"`
+	RemoteAddr   string            `json:"remote_addr"`
+	Method       string            `json:"method"`
+	URL          string            `json:"url"`
+	Proto        string            `json:"proto"`
+	StatusCode   int               `json:"status_code"`
+	Size         int               `json:"size"`
+	UserAgent    string            `json:"user_agent,omitempty"`
+	Headers      map[string]string `json:"headers,omitempty"`
+	RequestBody  string            `json:"request_body,omitempty"`
+	ResponseBody string            `json:"response_body,omitempty"`
+}
+
+// JSONLogger handles writing JSON log entries to a file
+type JSONLogger struct {
+	file   *os.File
+	mutex  sync.Mutex
+	enable bool
+}
+
+// NewJSONLogger creates a new JSON logger instance
+func NewJSONLogger(filePath string) (*JSONLogger, error) {
+	if filePath == "" {
+		return &JSONLogger{enable: false}, nil
+	}
+
+	file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open JSON log file: %w", err)
+	}
+
+	return &JSONLogger{
+		file:   file,
+		enable: true,
+	}, nil
+}
+
+// LogRequest logs an HTTP request to the JSON file
+func (jl *JSONLogger) LogRequest(r *http.Request, statusCode, size int, userAgent string, headers map[string]string, requestBody, responseBody string) error {
+	if !jl.enable {
+		return nil
+	}
+
+	entry := JSONLogEntry{
+		Timestamp:    time.Now().Format(time.RFC3339),
+		RemoteAddr:   r.RemoteAddr,
+		Method:       r.Method,
+		URL:          r.URL.String(),
+		Proto:        r.Proto,
+		StatusCode:   statusCode,
+		Size:         size,
+		UserAgent:    userAgent,
+		Headers:      headers,
+		RequestBody:  requestBody,
+		ResponseBody: responseBody,
+	}
+
+	jl.mutex.Lock()
+	defer jl.mutex.Unlock()
+
+	jsonData, err := json.Marshal(entry)
+	if err != nil {
+		return fmt.Errorf("failed to marshal JSON log entry: %w", err)
+	}
+
+	_, err = jl.file.Write(append(jsonData, '\n'))
+	if err != nil {
+		return fmt.Errorf("failed to write JSON log entry: %w", err)
+	}
+
+	return nil
+}
+
+// Close closes the JSON log file
+func (jl *JSONLogger) Close() error {
+	if jl.file != nil {
+		return jl.file.Close()
+	}
+	return nil
+}

pkg/httpserver/loglayer.go

@@ -5,13 +5,15 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"time"
+
 	"github.com/projectdiscovery/gologger"
 )
 
 // Convenience globals
 var (
 	EnableUpload  bool
 	EnableVerbose bool
+	EnableLogUA   bool
 )
 
 func (t *HTTPServer) shouldDumpBody(bodysize int64) bool {
@@ -29,12 +31,41 @@ func (t *HTTPServer) loglayer(handler http.Handler) http.Handler {
 		lrw := newLoggingResponseWriter(w, t.options.MaxDumpBodySize)
 		handler.ServeHTTP(lrw, r)
 
+		// Log to JSON file if JSON logger is enabled
+		if t.jsonLogger != nil {
+			// Extract headers
+			headers := make(map[string]string)
+			for name, values := range r.Header {
+				if len(values) > 0 {
+					headers[name] = values[0]
+				}
+			}
+
+			// Extract request body from fullRequest
+			requestBody := ""
+			if len(fullRequest) > 0 {
+				// Find the double CRLF that separates headers from body
+				bodyStart := bytes.Index(fullRequest, []byte("\r\n\r\n"))
+				if bodyStart != -1 && bodyStart+4 < len(fullRequest) {
+					requestBody = string(fullRequest[bodyStart+4:])
+				}
+			}
+
+			// Log to JSON file
+			_ = t.jsonLogger.LogRequest(r, lrw.statusCode, lrw.Size, r.UserAgent(), headers, requestBody, string(lrw.Data))
+		}
+
+		// Continue with existing console logging
 		if EnableVerbose {
 			headers := new(bytes.Buffer)
 			lrw.Header().Write(headers) //nolint
 			gologger.Print().Msgf("\n[%s]\nRemote Address: %s\n%s\n%s %d %s\n%s\n%s\n", time.Now().Format("2006-01-02 15:04:05"), r.RemoteAddr, string(fullRequest), r.Proto, lrw.statusCode, http.StatusText(lrw.statusCode), headers.String(), string(lrw.Data))
 		} else {
-			gologger.Print().Msgf("[%s] %s \"%s %s %s\" %d %d", time.Now().Format("2006-01-02 15:04:05"), r.RemoteAddr, r.Method, r.URL, r.Proto, lrw.statusCode, lrw.Size)
+			if EnableLogUA {
+				gologger.Print().Msgf("[%s] %s \"%s %s %s\" %d %d - %s", time.Now().Format("2006-01-02 15:04:05"), r.RemoteAddr, r.Method, r.URL, r.Proto, lrw.statusCode, lrw.Size, r.UserAgent())
+			} else {
+				gologger.Print().Msgf("[%s] %s \"%s %s %s\" %d %d", time.Now().Format("2006-01-02 15:04:05"), r.RemoteAddr, r.Method, r.URL, r.Proto, lrw.statusCode, lrw.Size)
+			}
 		}
 	})
 }

pkg/httpserver/pythonliststyle.go

@@ -17,6 +17,7 @@ const (
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Directory listing for %s</title>
 </head>
 <body>

test/pythonliststyle_test.go

@@ -15,6 +15,7 @@ func TestServePythonStyleHtmlPageForDirectories(t *testing.T) {
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Directory listing for /</title>
 </head>
 <body>