Skip to content

Commit 560e27c

Browse files
gaukasgaukas
authored
update: document and functional code (#23)
* update: add documentation to newly added contents Signed-off-by: Gaukas Wang <[email protected]> * update: minor improvements Signed-off-by: Gaukas Wang <[email protected]> * deps: bump up caddy version Signed-off-by: Gaukas Wang <[email protected]> * fix: always set header to disable QUIC for TLS Also updated some wording to make error messages sound consistent. Signed-off-by: Gaukas Wang <[email protected]> * docs: update comments and README [ci skip] Signed-off-by: Gaukas Wang <[email protected]> * fix: save QUIC visitor when H3 is enabled only otherwise the TLS-fallback will incorrectly overwrite the cached QUIC fingerprint's sender. Signed-off-by: Gaukas Wang <[email protected]> * logging: fix typo and add debugging printouts Signed-off-by: Gaukas Wang <[email protected]> * update: new Caddyfile Signed-off-by: Gaukas Wang <[email protected]> --------- Signed-off-by: Gaukas Wang <[email protected]>
1 parent 3f06ffa commit 560e27c

26 files changed

+680
-801
lines changed

‎.github/workflows/go.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ jobs:
1414
build:
1515
runs-on: ubuntu-latest
1616
steps:
17-
- uses: actions/checkout@v3
17+
- uses: actions/checkout@v4
1818

1919
- name: Set up Go
20-
uses: actions/setup-go@v3
20+
uses: actions/setup-go@v5
2121
with:
22-
go-version: "1.20"
22+
go-version: "1.22.x"
2323

2424
- name: Build
2525
run: go build -v ./...

‎README.md

+62-78
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,32 @@
1-
# clienthellod
1+
# `clienthellod`: TLS ClientHello/QUIC Initial Packet reflection service
22
![Go Build Status](https://github.com/gaukas/clienthellod/actions/workflows/go.yml/badge.svg)
33
[![Go Report Card](https://goreportcard.com/badge/github.com/gaukas/clienthellod)](https://goreportcard.com/report/github.com/gaukas/clienthellod)
4-
[![DeepSource](https://app.deepsource.com/gh/gaukas/clienthellod.svg/?label=active+issues&show_trend=true&token=GugDSBnYAxAF25QNpfyAO5d2)](https://app.deepsource.com/gh/gaukas/clienthellod/)
54
[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fgaukas%2Fclienthellod.svg?type=shield&issueType=license)](https://app.fossa.com/projects/git%2Bgithub.com%2Fgaukas%2Fclienthellod?ref=badge_shield&issueType=license)
5+
[![Go Doc](https://pkg.go.dev/badge/github.com/refraction-networking/water.svg)](https://pkg.go.dev/github.com/refraction-networking/water)
66

7-
ClientHello Parser/Resolver as a Service from [tlsfingerprint.io](https://tlsfingerprint.io).
7+
`clienthellod`, read as "client-hello-D", is a TLS ClientHello/QUIC Initial Packet reflection service. It can be used to parses TLS ClientHello messages and QUIC Initial Packets into human-readable and highly programmable formats such as JSON.
88

9-
## What does it do
10-
11-
`clienthellod`, read as "client hello DEE", is a service that parses and resolves the ClientHello message sent by the client to the server. It is a part of the TLS fingerprintability research project which spans [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](https://quic.tlsfingerprint.io). It parses the ClientHello messages sent by TLS clients and QUIC Client Initial Packets sent by QUIC clients and display the parsed information in a human-readable format with high programmability.
9+
Is is a part of the TLS fingerprintability research project which spans [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](https://quic.tlsfingerprint.io). It parses the ClientHello messages sent by TLS clients and QUIC Client Initial Packets sent by QUIC clients and display the parsed information in a human-readable format with high programmability.
1210

1311
See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](https://quic.tlsfingerprint.io) for more details about the project.
1412

15-
## How to use
13+
## Quick Start
14+
15+
`clienthellod` comes as a Go library, which can be used to parse both TLS and QUIC protocols.
16+
17+
### TLS/QUIC Fingerprinter
18+
19+
```go
20+
tlsFingerprinter := clienthellod.NewTLSFingerprinter()
21+
```
1622

17-
`clienthellod` is provided as a Go library in the root directory of this repository.
23+
```go
24+
quicFingerprinter := clienthellod.NewQUICFingerprinter()
25+
```
1826

19-
### Quick Start
27+
### TLS ClientHello
2028

21-
#### TLS ClientHello
29+
#### From a `net.Conn`
2230

2331
```go
2432
tcpLis, err := net.Listen("tcp", ":443")
@@ -30,7 +38,7 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
3038
}
3139
defer conn.Close()
3240

33-
ch, err := clienthellod.ReadClientHello(conn) // saves ClientHello
41+
ch, err := clienthellod.ReadClientHello(conn) // reads ClientHello from the connection
3442
if err != nil {
3543
panic(err)
3644
}
@@ -46,11 +54,24 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
4654
}
4755

4856
fmt.Println(string(jsonB))
49-
fmt.Println("ClientHello ID: " + ch.FingerprintID(false)) // prints ClientHello's original fingerprint ID, as TLS extension IDs in their provided order
50-
fmt.Println("ClientHello NormID: " + ch.FingerprintID(true)) // prints ClientHello's normalized fingerprint ID, as TLS extension IDs in a sorted order
57+
fmt.Println("ClientHello ID: " + ch.HexID) // prints ClientHello's original fingerprint ID calculated using observed TLS extension order
58+
fmt.Println("ClientHello NormID: " + ch.NormHexID) // prints ClientHello's normalized fingerprint ID calculated using sorted TLS extension list
5159
```
5260

53-
#### QUIC Client Initial Packet
61+
#### From raw `[]byte`
62+
63+
```go
64+
ch, err := clienthellod.UnmarshalClientHello(raw)
65+
if err != nil {
66+
panic(err)
67+
}
68+
69+
// err := ch.ParseClientHello() // no need to call again, UnmarshalClientHello automatically calls ParseClientHello
70+
```
71+
72+
### QUIC Initial Packets (Client-sourced)
73+
74+
#### Single packet
5475

5576
```go
5677
udpConn, err := net.ListenUDP("udp", ":443")
@@ -62,7 +83,7 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
6283
panic(err)
6384
}
6485

65-
cip, err := clienthellod.ParseQUICCIP(buf[:n]) // reads in and parses QUIC Client Initial Packet
86+
ci, err := clienthellod.UnmarshalQUICClientInitialPacket(buf[:n]) // decodes QUIC Client Initial Packet
6687
if err != nil {
6788
panic(err)
6889
}
@@ -75,79 +96,42 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
7596
fmt.Println(string(jsonB)) // including fingerprint IDs of: ClientInitialPacket, QUIC Header, QUIC ClientHello, QUIC Transport Parameters' combination
7697
```
7798

78-
#### Use with Caddy
79-
80-
`clienthellod` is also provided as a Caddy plugin, `modcaddy`, which can be used to capture ClientHello messages and QUIC Client Initial Packets. See Section [modcaddy](#modcaddy) for more details.
81-
82-
## modcaddy
83-
84-
`modcaddy` is a Caddy plugin that provides:
85-
- An caddy `app` that can be used to temporarily store captured ClientHello messages and QUIC Client Initial Packets.
86-
- A caddy `handler` that can be used to serve the ClientHello messages and QUIC Client Initial Packets to the client sending the request.
87-
- A caddy `listener` that can be used to capture ClientHello messages and QUIC Client Initial Packets.
99+
#### Multiple packets
88100

89-
You will need to use [xcaddy](https://github.com/caddyserver/xcaddy) to rebuild Caddy with `modcaddy` included.
101+
Implementations including Chrome/Chromium sends oversized Client Hello which does not fit into one single QUIC packet, in which case multiple QUIC Initial Packets are sent.
90102

91-
It is worth noting that some web browsers may not choose to switch to QUIC protocol in localhost environment, which may result in the QUIC Client Initial Packet not being sent and therefore not being captured/analyzed.
92-
93-
### Build
94-
95-
```bash
96-
xcaddy build --with github.com/gaukas/clienthellod/modcaddy
97-
```
98-
99-
#### When build locally with changes
103+
```go
104+
gci := GatherClientInitials() // Each GatherClientInitials reassembles one QUIC Client Initial Packets stream. Use a QUIC Fingerprinter for multiple potential senders, which automatically demultiplexes the packets based on the source address.
105+
106+
udpConn, err := net.ListenUDP("udp", ":443")
107+
defer udpConn.Close()
100108

101-
```bash
102-
xcaddy build --with github.com/gaukas/clienthellod/modcaddy --with github.com/gaukas/clienthellod/=./
103-
```
109+
for {
110+
buf := make([]byte, 65535)
111+
n, addr, err := udpConn.ReadFromUDP(buf)
112+
if err != nil {
113+
panic(err)
114+
}
104115

105-
### Caddyfile
116+
if addr != knownSenderAddr {
117+
continue
118+
}
106119

107-
A sample Caddyfile is provided below.
120+
ci, err := clienthellod.UnmarshalQUICClientInitialPacket(buf[:n]) // decodes QUIC Client Initial Packet
121+
if err != nil {
122+
panic(err)
123+
}
108124

109-
```Caddyfile
110-
{
111-
# debug # for debugging purpose
112-
# https_port 443 # currently, QUIC listener works only on port 443, otherwise you need to make changes to the code
113-
order clienthellod before file_server # make sure it hits handler before file_server
114-
clienthellod { # app (reservoir)
115-
validfor 120s 30s # params: validFor [cleanEvery] # increased for QUIC
116-
}
117-
servers {
118-
listener_wrappers {
119-
clienthellod { # listener
120-
tcp # listens for TCP and saves TLS ClientHello
121-
udp # listens for UDP and saves QUIC Client Initial Packet
122-
}
123-
tls
125+
err = gci.AddPacket(ci)
126+
if err != nil {
127+
panic(err)
124128
}
125-
# protocols h3
126129
}
127-
}
130+
```
128131

129-
1.mydomain.com {
130-
# tls internal
131-
clienthellod { # handler
132-
# quic # mutually exclusive with tls
133-
tls # listener_wrappers.clienthellod.tcp must be set
134-
}
135-
file_server {
136-
root /var/www/html
137-
}
138-
}
132+
### Use with Caddy
139133

140-
2.mydomain.com {
141-
# tls internal
142-
clienthellod { # handler
143-
quic # listener_wrappers.clienthellod.udp must be set
144-
# tls # mutually exclusive with quic
145-
}
146-
file_server {
147-
root /var/www/html
148-
}
149-
}
150-
```
134+
We also provide clienthellod as a Caddy Module in `modcaddy`, which you can use with Caddy to capture ClientHello messages and QUIC Client Initial Packets. See [modcaddy](https://github.com/gaukas/clienthellod/tree/master/modcaddy) for more details.
151135

152136
## License
153137

‎clienthello.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"golang.org/x/crypto/cryptobyte"
1616
)
1717

18+
// ClientHello represents a captured ClientHello message with all fingerprintable fields.
1819
type ClientHello struct {
1920
raw []byte
2021

@@ -53,7 +54,7 @@ type ClientHello struct {
5354
lengthPrefixedCertCompressAlgos []uint8
5455
keyshareGroupsWithLengths []uint16
5556

56-
// QUIC-only
57+
// QUIC-only, nil if not QUIC
5758
qtp *QUICTransportParameters
5859
}
5960

‎fingerprint_hash.go

+5-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"encoding/binary"
66
"encoding/hex"
77
"hash"
8-
"sort"
98

109
"github.com/gaukas/clienthellod/internal/utils"
1110
)
@@ -23,16 +22,17 @@ func updateU64(h hash.Hash, i uint64) {
2322
binary.Write(h, binary.BigEndian, i)
2423
}
2524

25+
// FingerprintID is the type of fingerprint ID.
2626
type FingerprintID int64
2727

28+
// AsHex returns the hex representation of this fingerprint ID.
2829
func (id FingerprintID) AsHex() string {
2930
hid := make([]byte, 8)
3031
binary.BigEndian.PutUint64(hid, uint64(id))
3132
return hex.EncodeToString(hid)
3233
}
3334

34-
// FingerprintNID calculates fingerprint Numeric ID of ClientHello.
35-
// Fingerprint is defined by
35+
// calcNumericID returns the numeric ID of this client hello.
3636
func (ch *ClientHello) calcNumericID() (orig, norm int64) {
3737
for _, normalized := range []bool{false, true} {
3838
h := sha1.New() // skipcq: GO-S1025, GSC-G401,
@@ -66,6 +66,7 @@ func (ch *ClientHello) calcNumericID() (orig, norm int64) {
6666
return
6767
}
6868

69+
// calcNumericID returns the numeric ID of this gathered client initial.
6970
func (gci *GatheredClientInitials) calcNumericID() uint64 {
7071
h := sha1.New() // skipcq: GO-S1025, GSC-G401
7172
updateArr(h, gci.Packets[0].Header.Version)
@@ -79,9 +80,6 @@ func (gci *GatheredClientInitials) calcNumericID() uint64 {
7980
allFrameIDs = append(allFrameIDs, p.frames.FrameTypesUint8()...)
8081
}
8182
dedupAllFrameIDs := utils.DedupIntArr(allFrameIDs)
82-
sort.Slice(dedupAllFrameIDs, func(i, j int) bool {
83-
return dedupAllFrameIDs[i] < dedupAllFrameIDs[j]
84-
})
8583
updateArr(h, dedupAllFrameIDs)
8684

8785
if gci.Packets[0].Header.HasToken {
@@ -93,7 +91,7 @@ func (gci *GatheredClientInitials) calcNumericID() uint64 {
9391
return binary.BigEndian.Uint64(h.Sum(nil)[0:8])
9492
}
9593

96-
// NID returns the numeric ID of this transport parameters combination.
94+
// calcNumericID returns the numeric ID of this transport parameters combination.
9795
func (qtp *QUICTransportParameters) calcNumericID() uint64 {
9896
h := sha1.New() // skipcq: GO-S1025, GSC-G401
9997
updateArr(h, qtp.MaxIdleTimeout)

0 commit comments

Comments
 (0)