From 914b88b4047879cd883fc25f4a9c3976b353f5ed Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Sun, 26 Jan 2025 16:40:42 +0100 Subject: [PATCH 01/10] Add nbnet --- client/embed/embed.go | 299 +++++++++++++++++++++ client/iface/device.go | 3 + client/iface/device/device_kernel_unix.go | 5 + client/iface/device/device_netstack.go | 24 +- client/iface/device/device_usp_unix.go | 5 + client/iface/iface.go | 9 + client/iface/iface_moc.go | 6 + client/iface/iwginterface.go | 2 + client/iface/netstack/env.go | 4 +- client/iface/netstack/tun.go | 42 ++- client/internal/dns/service_memory.go | 24 +- client/internal/dns/service_memory_test.go | 4 +- client/internal/engine.go | 36 ++- client/internal/routemanager/manager.go | 2 +- util/net/net.go | 20 ++ 15 files changed, 446 insertions(+), 39 deletions(-) create mode 100644 client/embed/embed.go diff --git a/client/embed/embed.go b/client/embed/embed.go new file mode 100644 index 00000000000..023d689ec56 --- /dev/null +++ b/client/embed/embed.go @@ -0,0 +1,299 @@ +package embed + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "os" + "sync" + + "github.com/sirupsen/logrus" + wgnetstack "golang.zx2c4.com/wireguard/tun/netstack" + + "github.com/netbirdio/netbird/client/iface/netstack" + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/peer" + "github.com/netbirdio/netbird/client/system" +) + +var ErrClientAlreadyStarted = errors.New("client already started") +var ErrClientNotStarted = errors.New("client not started") + +// Client manages a netbird embedded client instance +type Client struct { + deviceName string + config *internal.Config + mu sync.Mutex + cancel context.CancelFunc + setupKey string + connect *internal.ConnectClient +} + +// Options configures a new Client +type Options struct { + // DeviceName is this peer's name in the network + DeviceName string + // SetupKey is used for authentication + SetupKey string + // ManagementURL overrides the default management server URL + ManagementURL string + // PreSharedKey is the pre-shared key for the WireGuard interface + PreSharedKey string + // LogOutput is the output destination for logs (defaults to os.Stderr if nil) + LogOutput io.Writer + // LogLevel sets the logging level (defaults to info if empty) + LogLevel string + // NoUserspace disables the userspace networking mode. Needs admin/root privileges + NoUserspace bool + // ConfigPath is the path to the netbird config file. If empty, the config will be stored in memory and not persisted. + ConfigPath string + // StatePath is the path to the netbird state file + StatePath string + // DisableClientRoutes disables the client routes + DisableClientRoutes bool +} + +// New creates a new netbird embedded client +func New(opts Options) (*Client, error) { + if opts.LogOutput != nil { + logrus.SetOutput(opts.LogOutput) + } + + if opts.LogLevel != "" { + level, err := logrus.ParseLevel(opts.LogLevel) + if err != nil { + return nil, fmt.Errorf("parse log level: %w", err) + } + logrus.SetLevel(level) + } + + if !opts.NoUserspace { + if err := os.Setenv(netstack.EnvUseNetstackMode, "true"); err != nil { + return nil, fmt.Errorf("setenv: %w", err) + } + if err := os.Setenv(netstack.EnvSkipProxy, "true"); err != nil { + return nil, fmt.Errorf("setenv: %w", err) + } + } + + if opts.StatePath != "" { + // TODO: Disable state if path not provided + if err := os.Setenv("NB_DNS_STATE_FILE", opts.StatePath); err != nil { + return nil, fmt.Errorf("setenv: %w", err) + } + } + + t := true + var config *internal.Config + var err error + if opts.ConfigPath != "" { + config, err = internal.UpdateOrCreateConfig(internal.ConfigInput{ + ConfigPath: opts.ConfigPath, + ManagementURL: opts.ManagementURL, + PreSharedKey: &opts.PreSharedKey, + DisableServerRoutes: &t, + DisableClientRoutes: &opts.DisableClientRoutes, + }) + } else { + config, err = internal.CreateInMemoryConfig(internal.ConfigInput{ + ManagementURL: opts.ManagementURL, + PreSharedKey: &opts.PreSharedKey, + DisableServerRoutes: &t, + }) + } + if err != nil { + return nil, fmt.Errorf("create config: %w", err) + } + + return &Client{ + deviceName: opts.DeviceName, + setupKey: opts.SetupKey, + config: config, + }, nil +} + +// Start begins client operation and blocks until the engine has been started successfully or a startup error occurs. +// Pass a context with a deadline to limit the time spent waiting for the engine to start. +func (c *Client) Start(startCtx context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.cancel != nil { + return ErrClientAlreadyStarted + } + + ctx := internal.CtxInitState(context.Background()) + // nolint:staticcheck + ctx = context.WithValue(ctx, system.DeviceNameCtxKey, c.deviceName) + if err := internal.Login(ctx, c.config, c.setupKey, ""); err != nil { + return fmt.Errorf("login: %w", err) + } + + recorder := peer.NewRecorder(c.config.ManagementURL.String()) + client := internal.NewConnectClient(ctx, c.config, recorder) + + // either startup error (permanent backoff err) or nil err (successful engine up) + // TODO: make after-startup backoff err available + run := make(chan error, 1) + go func() { + if err := client.RunWithProbes(nil, run); err != nil { + run <- err + } + }() + + select { + case <-startCtx.Done(): + if stopErr := client.Stop(); stopErr != nil { + return fmt.Errorf("stop error after context done. Stop error: %w. Context done: %w", stopErr, startCtx.Err()) + } + return startCtx.Err() + case err := <-run: + if err != nil { + if stopErr := client.Stop(); stopErr != nil { + return fmt.Errorf("stop error after failed to startup. Stop error: %w. Start error: %w", stopErr, err) + } + return fmt.Errorf("startup: %w", err) + } + } + + c.connect = client + + return nil +} + +// Stop gracefully stops the client. +// Pass a context with a deadline to limit the time spent waiting for the engine to stop. +func (c *Client) Stop(ctx context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.connect == nil { + return ErrClientNotStarted + } + + done := make(chan error, 1) + go func() { + done <- c.connect.Stop() + }() + + select { + case <-ctx.Done(): + c.cancel = nil + return ctx.Err() + case err := <-done: + c.cancel = nil + if err != nil { + return fmt.Errorf("stop: %w", err) + } + return nil + } +} + +// Dial dials a network address in the netbird network. +// Not applicable if the userspace networking mode is disabled. +func (c *Client) Dial(ctx context.Context, network, address string) (net.Conn, error) { + c.mu.Lock() + connect := c.connect + if connect == nil { + c.mu.Unlock() + return nil, ErrClientNotStarted + } + c.mu.Unlock() + + engine := connect.Engine() + if engine == nil { + return nil, errors.New("engine not started") + } + + nsnet, err := engine.GetNet() + if err != nil { + return nil, fmt.Errorf("get net: %w", err) + } + + return nsnet.DialContext(ctx, network, address) +} + +// ListenTCP listens on the given address in the netbird network +// Not applicable if the userspace networking mode is disabled. +func (c *Client) ListenTCP(address string) (net.Listener, error) { + nsnet, addr, err := c.getNet() + if err != nil { + return nil, err + } + + _, port, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("split host port: %w", err) + } + listenAddr := fmt.Sprintf("%s:%s", addr, port) + + tcpAddr, err := net.ResolveTCPAddr("tcp", listenAddr) + if err != nil { + return nil, fmt.Errorf("resolve: %w", err) + } + return nsnet.ListenTCP(tcpAddr) +} + +// ListenUDP listens on the given address in the netbird network +// Not applicable if the userspace networking mode is disabled. +func (c *Client) ListenUDP(address string) (net.PacketConn, error) { + nsnet, addr, err := c.getNet() + if err != nil { + return nil, err + } + + _, port, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("split host port: %w", err) + } + listenAddr := fmt.Sprintf("%s:%s", addr, port) + + udpAddr, err := net.ResolveUDPAddr("udp", listenAddr) + if err != nil { + return nil, fmt.Errorf("resolve: %w", err) + } + + return nsnet.ListenUDP(udpAddr) +} + +// NewHTTPClient returns a configured http.Client that uses the netbird network for requests. +// Not applicable if the userspace networking mode is disabled. +func (c *Client) NewHTTPClient() (*http.Client, error) { + transport := &http.Transport{ + DialContext: c.Dial, + } + + return &http.Client{ + Transport: transport, + }, nil +} + +func (c *Client) getNet() (*wgnetstack.Net, netip.Addr, error) { + c.mu.Lock() + connect := c.connect + if connect == nil { + c.mu.Unlock() + return nil, netip.Addr{}, errors.New("client not started") + } + c.mu.Unlock() + + engine := connect.Engine() + if engine == nil { + return nil, netip.Addr{}, errors.New("engine not started") + } + + addr, err := engine.Address() + if err != nil { + return nil, netip.Addr{}, fmt.Errorf("engine address: %w", err) + } + + nsnet, err := engine.GetNet() + if err != nil { + return nil, netip.Addr{}, fmt.Errorf("get net: %w", err) + } + + return nsnet, addr, nil +} diff --git a/client/iface/device.go b/client/iface/device.go index 0d4e6914554..ba30941021b 100644 --- a/client/iface/device.go +++ b/client/iface/device.go @@ -3,6 +3,8 @@ package iface import ( + "golang.zx2c4.com/wireguard/tun/netstack" + "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/device" ) @@ -15,4 +17,5 @@ type WGTunDevice interface { DeviceName() string Close() error FilteredDevice() *device.FilteredDevice + GetNet() *netstack.Net } diff --git a/client/iface/device/device_kernel_unix.go b/client/iface/device/device_kernel_unix.go index 0dfed4d9071..91ec8526fec 100644 --- a/client/iface/device/device_kernel_unix.go +++ b/client/iface/device/device_kernel_unix.go @@ -9,6 +9,7 @@ import ( "github.com/pion/transport/v3" log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/configurer" @@ -159,3 +160,7 @@ func (t *TunKernelDevice) FilteredDevice() *FilteredDevice { func (t *TunKernelDevice) assignAddr() error { return t.link.assignAddr(t.address) } + +func (t *TunKernelDevice) GetNet() *netstack.Net { + return nil +} diff --git a/client/iface/device/device_netstack.go b/client/iface/device/device_netstack.go index f5d39e9e074..e6dd69d567f 100644 --- a/client/iface/device/device_netstack.go +++ b/client/iface/device/device_netstack.go @@ -8,10 +8,12 @@ import ( log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/configurer" - "github.com/netbirdio/netbird/client/iface/netstack" + nbnetstack "github.com/netbirdio/netbird/client/iface/netstack" + nbnet "github.com/netbirdio/netbird/util/net" ) type TunNetstackDevice struct { @@ -25,9 +27,11 @@ type TunNetstackDevice struct { device *device.Device filteredDevice *FilteredDevice - nsTun *netstack.NetStackTun + nsTun *nbnetstack.NetStackTun udpMux *bind.UniversalUDPMuxDefault configurer WGConfigurer + + net *netstack.Net } func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, mtu int, iceBind *bind.ICEBind, listenAddress string) *TunNetstackDevice { @@ -43,13 +47,19 @@ func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, m } func (t *TunNetstackDevice) Create() (WGConfigurer, error) { - log.Info("create netstack tun interface") - t.nsTun = netstack.NewNetStackTun(t.listenAddress, t.address.IP.String(), t.mtu) - tunIface, err := t.nsTun.Create() + log.Info("create nbnetstack tun interface") + + // TODO: get from service listener runtime IP + dnsAddr := nbnet.GetLastIPFromNetwork(t.address.Network, 1) + log.Debugf("netstack using address: %s", t.address.IP) + t.nsTun = nbnetstack.NewNetStackTun(t.listenAddress, t.address.IP, dnsAddr, t.mtu) + log.Debugf("netstack using dns address: %s", dnsAddr) + tunIface, net, err := t.nsTun.Create() if err != nil { return nil, fmt.Errorf("error creating tun device: %s", err) } t.filteredDevice = newDeviceFilter(tunIface) + t.net = net t.device = device.NewDevice( t.filteredDevice, @@ -117,3 +127,7 @@ func (t *TunNetstackDevice) DeviceName() string { func (t *TunNetstackDevice) FilteredDevice() *FilteredDevice { return t.filteredDevice } + +func (t *TunNetstackDevice) GetNet() *netstack.Net { + return t.net +} diff --git a/client/iface/device/device_usp_unix.go b/client/iface/device/device_usp_unix.go index 3562f312ded..ed8c4f8f04e 100644 --- a/client/iface/device/device_usp_unix.go +++ b/client/iface/device/device_usp_unix.go @@ -8,6 +8,7 @@ import ( log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/tun" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/configurer" @@ -130,3 +131,7 @@ func (t *USPDevice) assignAddr() error { return link.assignAddr(t.address) } + +func (t *USPDevice) GetNet() *netstack.Net { + return nil +} diff --git a/client/iface/iface.go b/client/iface/iface.go index 1fb9c269179..f0df8ac10dd 100644 --- a/client/iface/iface.go +++ b/client/iface/iface.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/pion/transport/v3" log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/errors" @@ -234,3 +235,11 @@ func (w *WGIface) waitUntilRemoved() error { } } } + +// GetNet returns the netstack.Net for the netstack device +func (w *WGIface) GetNet() *netstack.Net { + w.mu.Lock() + defer w.mu.Unlock() + + return w.tun.GetNet() +} diff --git a/client/iface/iface_moc.go b/client/iface/iface_moc.go index d91a7224ff2..078705d4ea8 100644 --- a/client/iface/iface_moc.go +++ b/client/iface/iface_moc.go @@ -4,6 +4,7 @@ import ( "net" "time" + "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/iface/bind" @@ -32,6 +33,7 @@ type MockWGIface struct { GetStatsFunc func(peerKey string) (configurer.WGStats, error) GetInterfaceGUIDStringFunc func() (string, error) GetProxyFunc func() wgproxy.Proxy + GetNetFunc func() *netstack.Net } func (m *MockWGIface) GetInterfaceGUIDString() (string, error) { @@ -110,3 +112,7 @@ func (m *MockWGIface) GetProxy() wgproxy.Proxy { //TODO implement me panic("implement me") } + +func (m *MockWGIface) GetNet() *netstack.Net { + return m.GetNetFunc() +} diff --git a/client/iface/iwginterface.go b/client/iface/iwginterface.go index f5ab2953905..efd386ddc8b 100644 --- a/client/iface/iwginterface.go +++ b/client/iface/iwginterface.go @@ -6,6 +6,7 @@ import ( "net" "time" + "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/iface/bind" @@ -33,4 +34,5 @@ type IWGIface interface { GetFilter() device.PacketFilter GetDevice() *device.FilteredDevice GetStats(peerKey string) (configurer.WGStats, error) + GetNet() *netstack.Net } diff --git a/client/iface/netstack/env.go b/client/iface/netstack/env.go index 09889a57e55..cdbf975b102 100644 --- a/client/iface/netstack/env.go +++ b/client/iface/netstack/env.go @@ -8,9 +8,11 @@ import ( log "github.com/sirupsen/logrus" ) +const EnvUseNetstackMode = "NB_USE_NETSTACK_MODE" + // IsEnabled todo: move these function to cmd layer func IsEnabled() bool { - return os.Getenv("NB_USE_NETSTACK_MODE") == "true" + return os.Getenv(EnvUseNetstackMode) == "true" } func ListenAddr() string { diff --git a/client/iface/netstack/tun.go b/client/iface/netstack/tun.go index c180e4ef553..01f19875e3d 100644 --- a/client/iface/netstack/tun.go +++ b/client/iface/netstack/tun.go @@ -1,15 +1,22 @@ package netstack import ( + "fmt" + "net" "net/netip" + "os" + "strconv" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/tun" "golang.zx2c4.com/wireguard/tun/netstack" ) +const EnvSkipProxy = "NB_NETSTACK_SKIP_PROXY" + type NetStackTun struct { //nolint:revive - address string + address net.IP + dnsAddress net.IP mtu int listenAddress string @@ -17,29 +24,48 @@ type NetStackTun struct { //nolint:revive tundev tun.Device } -func NewNetStackTun(listenAddress string, address string, mtu int) *NetStackTun { +func NewNetStackTun(listenAddress string, address net.IP, dnsAddress net.IP, mtu int) *NetStackTun { return &NetStackTun{ address: address, + dnsAddress: dnsAddress, mtu: mtu, listenAddress: listenAddress, } } -func (t *NetStackTun) Create() (tun.Device, error) { +func (t *NetStackTun) Create() (tun.Device, *netstack.Net, error) { + addr, ok := netip.AddrFromSlice(t.address) + if !ok { + return nil, nil, fmt.Errorf("convert address to netip.Addr: %v", t.address) + } + + dnsAddr, ok := netip.AddrFromSlice(t.dnsAddress) + if !ok { + return nil, nil, fmt.Errorf("convert dns address to netip.Addr: %v", t.dnsAddress) + } + nsTunDev, tunNet, err := netstack.CreateNetTUN( - []netip.Addr{netip.MustParseAddr(t.address)}, - []netip.Addr{}, + []netip.Addr{addr.Unmap()}, + []netip.Addr{dnsAddr.Unmap()}, t.mtu) if err != nil { - return nil, err + return nil, nil, err } t.tundev = nsTunDev + skipProxy, err := strconv.ParseBool(os.Getenv(EnvSkipProxy)) + if err != nil { + log.Errorf("failed to parse NB_ETSTACK_SKIP_PROXY: %s", err) + } + if skipProxy { + return nsTunDev, tunNet, nil + } + dialer := NewNSDialer(tunNet) t.proxy, err = NewSocks5(dialer) if err != nil { _ = t.tundev.Close() - return nil, err + return nil, nil, err } go func() { @@ -49,7 +75,7 @@ func (t *NetStackTun) Create() (tun.Device, error) { } }() - return nsTunDev, nil + return nsTunDev, tunNet, nil } func (t *NetStackTun) Close() error { diff --git a/client/internal/dns/service_memory.go b/client/internal/dns/service_memory.go index 729b90cc027..250f3ab2e04 100644 --- a/client/internal/dns/service_memory.go +++ b/client/internal/dns/service_memory.go @@ -2,7 +2,6 @@ package dns import ( "fmt" - "math/big" "net" "sync" @@ -10,6 +9,8 @@ import ( "github.com/google/gopacket/layers" "github.com/miekg/dns" log "github.com/sirupsen/logrus" + + nbnet "github.com/netbirdio/netbird/util/net" ) type ServiceViaMemory struct { @@ -27,7 +28,7 @@ func NewServiceViaMemory(wgIface WGIface) *ServiceViaMemory { wgInterface: wgIface, dnsMux: dns.NewServeMux(), - runtimeIP: getLastIPFromNetwork(wgIface.Address().Network, 1), + runtimeIP: nbnet.GetLastIPFromNetwork(wgIface.Address().Network, 1).String(), runtimePort: defaultPort, } return s @@ -118,22 +119,3 @@ func (s *ServiceViaMemory) filterDNSTraffic() (string, error) { return filter.AddUDPPacketHook(false, net.ParseIP(s.runtimeIP), uint16(s.runtimePort), hook), nil } - -func getLastIPFromNetwork(network *net.IPNet, fromEnd int) string { - // Calculate the last IP in the CIDR range - var endIP net.IP - for i := 0; i < len(network.IP); i++ { - endIP = append(endIP, network.IP[i]|^network.Mask[i]) - } - - // convert to big.Int - endInt := big.NewInt(0) - endInt.SetBytes(endIP) - - // subtract fromEnd from the last ip - fromEndBig := big.NewInt(int64(fromEnd)) - resultInt := big.NewInt(0) - resultInt.Sub(endInt, fromEndBig) - - return net.IP(resultInt.Bytes()).String() -} diff --git a/client/internal/dns/service_memory_test.go b/client/internal/dns/service_memory_test.go index bea4f4ce8fb..244adfaef4b 100644 --- a/client/internal/dns/service_memory_test.go +++ b/client/internal/dns/service_memory_test.go @@ -3,6 +3,8 @@ package dns import ( "net" "testing" + + nbnet "github.com/netbirdio/netbird/util/net" ) func TestGetLastIPFromNetwork(t *testing.T) { @@ -23,7 +25,7 @@ func TestGetLastIPFromNetwork(t *testing.T) { return } - lastIP := getLastIPFromNetwork(ipnet, 1) + lastIP := nbnet.GetLastIPFromNetwork(ipnet, 1).String() if lastIP != tt.ip { t.Errorf("wrong IP address, expected %s: got %s", tt.ip, lastIP) } diff --git a/client/internal/engine.go b/client/internal/engine.go index b3689c91153..da43e2496d8 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -20,6 +20,7 @@ import ( "github.com/pion/ice/v3" "github.com/pion/stun/v2" log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "google.golang.org/protobuf/proto" @@ -29,7 +30,7 @@ import ( "github.com/netbirdio/netbird/client/iface" "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/device" - "github.com/netbirdio/netbird/client/iface/netstack" + nbnetstack "github.com/netbirdio/netbird/client/iface/netstack" "github.com/netbirdio/netbird/client/internal/acl" "github.com/netbirdio/netbird/client/internal/dns" "github.com/netbirdio/netbird/client/internal/dnsfwd" @@ -752,7 +753,7 @@ func (e *Engine) updateSSH(sshConf *mgmProto.SSHConfig) error { // start SSH server if it wasn't running if isNil(e.sshServer) { listenAddr := fmt.Sprintf("%s:%d", e.wgInterface.Address().IP.String(), nbssh.DefaultSSHPort) - if netstack.IsEnabled() { + if nbnetstack.IsEnabled() { listenAddr = fmt.Sprintf("127.0.0.1:%d", nbssh.DefaultSSHPort) } // nil sshServer means it has not yet been started @@ -1737,6 +1738,37 @@ func (e *Engine) updateDNSForwarder(enabled bool, domains []string) { } } +func (e *Engine) GetNet() (*netstack.Net, error) { + e.syncMsgMux.Lock() + intf := e.wgInterface + e.syncMsgMux.Unlock() + if intf == nil { + return nil, errors.New("wireguard interface not initialized") + } + + nsnet := intf.GetNet() + if nsnet == nil { + return nil, errors.New("failed to get netstack") + } + return nsnet, nil +} + +func (e *Engine) Address() (netip.Addr, error) { + e.syncMsgMux.Lock() + intf := e.wgInterface + e.syncMsgMux.Unlock() + if intf == nil { + return netip.Addr{}, errors.New("wireguard interface not initialized") + } + + addr := e.wgInterface.Address() + ip, ok := netip.AddrFromSlice(addr.IP) + if !ok { + return netip.Addr{}, errors.New("failed to convert address to netip.Addr") + } + return ip.Unmap(), nil +} + // isChecksEqual checks if two slices of checks are equal. func isChecksEqual(checks []*mgmProto.Checks, oChecks []*mgmProto.Checks) bool { for _, check := range checks { diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 6f73fb166c6..fff19aa302e 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -454,7 +454,7 @@ func (m *DefaultManager) initialClientRoutes(initialRoutes []*route.Route) []*ro } func isRouteSupported(route *route.Route) bool { - if !nbnet.CustomRoutingDisabled() || route.IsDynamic() { + if netstack.IsEnabled() || !nbnet.CustomRoutingDisabled() || route.IsDynamic() { return true } diff --git a/util/net/net.go b/util/net/net.go index 403aa87e7d1..7b43b952f02 100644 --- a/util/net/net.go +++ b/util/net/net.go @@ -1,6 +1,7 @@ package net import ( + "math/big" "net" "github.com/google/uuid" @@ -26,3 +27,22 @@ type RemoveHookFunc func(connID ConnectionID) error func GenerateConnID() ConnectionID { return ConnectionID(uuid.NewString()) } + +func GetLastIPFromNetwork(network *net.IPNet, fromEnd int) net.IP { + // Calculate the last IP in the CIDR range + var endIP net.IP + for i := 0; i < len(network.IP); i++ { + endIP = append(endIP, network.IP[i]|^network.Mask[i]) + } + + // convert to big.Int + endInt := big.NewInt(0) + endInt.SetBytes(endIP) + + // subtract fromEnd from the last ip + fromEndBig := big.NewInt(int64(fromEnd)) + resultInt := big.NewInt(0) + resultInt.Sub(endInt, fromEndBig) + + return resultInt.Bytes() +} From 33014983c5ff080fb36ad71ab5c5bdabd527cc18 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Wed, 29 Jan 2025 23:24:37 +0100 Subject: [PATCH 02/10] Add missing implementations --- client/iface/device/device_darwin.go | 5 +++++ client/iface/device/device_ios.go | 5 +++++ client/iface/device/device_windows.go | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/client/iface/device/device_darwin.go b/client/iface/device/device_darwin.go index b5a128bc1cc..fc44d30cba9 100644 --- a/client/iface/device/device_darwin.go +++ b/client/iface/device/device_darwin.go @@ -9,6 +9,7 @@ import ( log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/tun" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/configurer" @@ -138,3 +139,7 @@ func (t *TunDevice) assignAddr() error { } return nil } + +func (t *TunDevice) GetNet() *netstack.Net { + return nil +} diff --git a/client/iface/device/device_ios.go b/client/iface/device/device_ios.go index b9591e0b8c6..fe22c75515a 100644 --- a/client/iface/device/device_ios.go +++ b/client/iface/device/device_ios.go @@ -10,6 +10,7 @@ import ( "golang.org/x/sys/unix" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/tun" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/configurer" @@ -131,3 +132,7 @@ func (t *TunDevice) UpdateAddr(addr WGAddress) error { func (t *TunDevice) FilteredDevice() *FilteredDevice { return t.filteredDevice } + +func (t *TunDevice) GetNet() *netstack.Net { + return nil +} diff --git a/client/iface/device/device_windows.go b/client/iface/device/device_windows.go index 86968d06d7e..0a2550565c3 100644 --- a/client/iface/device/device_windows.go +++ b/client/iface/device/device_windows.go @@ -8,6 +8,7 @@ import ( "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/tun" + "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" "github.com/netbirdio/netbird/client/iface/bind" @@ -169,3 +170,7 @@ func (t *TunDevice) assignAddr() error { log.Debugf("adding address %s to interface: %s", t.address.IP, t.name) return luid.SetIPAddresses([]netip.Prefix{netip.MustParsePrefix(t.address.String())}) } + +func (t *TunDevice) GetNet() *netstack.Net { + return nil +} From cc15a86c16693d567a40de8be1e5a027ca73ab98 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Wed, 29 Jan 2025 23:38:10 +0100 Subject: [PATCH 03/10] Add missing interface method for android --- client/iface/device_android.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/iface/device_android.go b/client/iface/device_android.go index 3d15080fff4..4ca1604f017 100644 --- a/client/iface/device_android.go +++ b/client/iface/device_android.go @@ -1,6 +1,8 @@ package iface import ( + "golang.zx2c4.com/wireguard/tun/netstack" + "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/device" ) @@ -13,4 +15,5 @@ type WGTunDevice interface { DeviceName() string Close() error FilteredDevice() *device.FilteredDevice + GetNet() *netstack.Net } From 6d60d3d9ba7b72ab9ef5436f0e9e1267817a97b8 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Wed, 29 Jan 2025 23:39:15 +0100 Subject: [PATCH 04/10] Add missing interface method for windows --- client/iface/iwginterface_windows.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/iface/iwginterface_windows.go b/client/iface/iwginterface_windows.go index 96eec52a502..58c44dc3e56 100644 --- a/client/iface/iwginterface_windows.go +++ b/client/iface/iwginterface_windows.go @@ -4,6 +4,7 @@ import ( "net" "time" + "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/iface/bind" @@ -32,4 +33,5 @@ type IWGIface interface { GetDevice() *device.FilteredDevice GetStats(peerKey string) (configurer.WGStats, error) GetInterfaceGUIDString() (string, error) + GetNet() *netstack.Net } From 021f839c6a49ccdb03e3d1972a7605a107ff7f66 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Wed, 29 Jan 2025 23:47:59 +0100 Subject: [PATCH 05/10] Add missing android implementation --- client/iface/device/device_android.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/iface/device/device_android.go b/client/iface/device/device_android.go index fac2ba63df9..f398c108a3c 100644 --- a/client/iface/device/device_android.go +++ b/client/iface/device/device_android.go @@ -9,6 +9,7 @@ import ( "golang.org/x/sys/unix" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/tun" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/configurer" @@ -130,6 +131,10 @@ func (t *WGTunDevice) FilteredDevice() *FilteredDevice { return t.filteredDevice } +func (t *WGTunDevice) GetNet() *netstack.Net { + return nil +} + func routesToString(routes []string) string { return strings.Join(routes, ";") } From 2c40260859249e26a495c7ef6b6e7e50e42e07b6 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Thu, 30 Jan 2025 00:12:48 +0100 Subject: [PATCH 06/10] Use consistent config --- client/embed/embed.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/client/embed/embed.go b/client/embed/embed.go index 30ef5a892f3..93033cc460a 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -90,20 +90,17 @@ func New(opts Options) (*Client, error) { t := true var config *internal.Config var err error + input := internal.ConfigInput{ + ConfigPath: opts.ConfigPath, + ManagementURL: opts.ManagementURL, + PreSharedKey: &opts.PreSharedKey, + DisableServerRoutes: &t, + DisableClientRoutes: &opts.DisableClientRoutes, + } if opts.ConfigPath != "" { - config, err = internal.UpdateOrCreateConfig(internal.ConfigInput{ - ConfigPath: opts.ConfigPath, - ManagementURL: opts.ManagementURL, - PreSharedKey: &opts.PreSharedKey, - DisableServerRoutes: &t, - DisableClientRoutes: &opts.DisableClientRoutes, - }) + config, err = internal.UpdateOrCreateConfig(input) } else { - config, err = internal.CreateInMemoryConfig(internal.ConfigInput{ - ManagementURL: opts.ManagementURL, - PreSharedKey: &opts.PreSharedKey, - DisableServerRoutes: &t, - }) + config, err = internal.CreateInMemoryConfig(input) } if err != nil { return nil, fmt.Errorf("create config: %w", err) From ffc7d1b974fe3c792ffb821529db00fc548ef02f Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Wed, 5 Feb 2025 17:36:53 +0100 Subject: [PATCH 07/10] Add go doc --- client/embed/doc.go | 172 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 client/embed/doc.go diff --git a/client/embed/doc.go b/client/embed/doc.go new file mode 100644 index 00000000000..8f32e2a2630 --- /dev/null +++ b/client/embed/doc.go @@ -0,0 +1,172 @@ +package embed + +// Package embed provides a way to embed the NetBird client directly +// into Go programs without requiring a separate NetBird client installation. +// +// +// Basic Usage: +// +// client, err := embed.New(embed.Options{ +// DeviceName: "my-service", +// SetupKey: os.Getenv("NB_SETUP_KEY"), +// ManagementURL: os.Getenv("NB_MANAGEMENT_URL"), +// }) +// if err != nil { +// log.Fatal(err) +// } +// +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// if err := client.Start(ctx); err != nil { +// log.Fatal(err) +// } +// +// Complete HTTP Server Example: +// +// package main +// +// import ( +// "context" +// "fmt" +// "log" +// "net/http" +// "os" +// "os/signal" +// "syscall" +// "time" +// +// netbird "github.com/netbirdio/netbird/client/embed" +// ) +// +// func main() { +// // Create client with setup key and device name +// client, err := netbird.New(netbird.Options{ +// DeviceName: "http-server", +// SetupKey: os.Getenv("NB_SETUP_KEY"), +// ManagementURL: os.Getenv("NB_MANAGEMENT_URL"), +// LogOutput: io.Discard, +// }) +// if err != nil { +// log.Fatal(err) +// } +// +// // Start with timeout +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// if err := client.Start(ctx); err != nil { +// log.Fatal(err) +// } +// +// // Create HTTP server +// mux := http.NewServeMux() +// mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { +// fmt.Printf("Request from %s: %s %s\n", r.RemoteAddr, r.Method, r.URL.Path) +// fmt.Fprintf(w, "Hello from netbird!") +// }) +// +// // Listen on netbird network +// l, err := client.ListenTCP(":8080") +// if err != nil { +// log.Fatal(err) +// } +// +// server := &http.Server{Handler: mux} +// go func() { +// if err := server.Serve(l); !errors.Is(err, http.ErrServerClosed) { +// log.Printf("HTTP server error: %v", err) +// } +// }() +// +// log.Printf("HTTP server listening on netbird network port 8080") +// +// // Handle shutdown +// stop := make(chan os.Signal, 1) +// signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) +// <-stop +// +// shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +// defer cancel() +// +// if err := server.Shutdown(shutdownCtx); err != nil { +// log.Printf("HTTP shutdown error: %v", err) +// } +// if err := client.Stop(shutdownCtx); err != nil { +// log.Printf("Netbird shutdown error: %v", err) +// } +// } +// +// Complete HTTP Client Example: +// +// package main +// +// import ( +// "context" +// "fmt" +// "io" +// "log" +// "os" +// "time" +// +// netbird "github.com/netbirdio/netbird/client/embed" +// ) +// +// func main() { +// // Create client with setup key and device name +// client, err := netbird.New(netbird.Options{ +// DeviceName: "http-client", +// SetupKey: os.Getenv("NB_SETUP_KEY"), +// ManagementURL: os.Getenv("NB_MANAGEMENT_URL"), +// LogOutput: io.Discard, +// }) +// if err != nil { +// log.Fatal(err) +// } +// +// // Start with timeout +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// +// if err := client.Start(ctx); err != nil { +// log.Fatal(err) +// } +// +// // Create HTTP client that uses netbird network +// httpClient, err := client.NewHTTPClient() +// if err != nil { +// log.Fatal(err) +// } +// httpClient.Timeout = 10 * time.Second +// +// // Make request to server in netbird network +// target := os.Getenv("NB_TARGET") +// resp, err := httpClient.Get(target) +// if err != nil { +// log.Fatal(err) +// } +// defer resp.Body.Close() +// +// // Read and print response +// body, err := io.ReadAll(resp.Body) +// if err != nil { +// log.Fatal(err) +// } +// +// fmt.Printf("Response from server: %s\n", string(body)) +// +// // Clean shutdown +// shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +// defer cancel() +// +// if err := client.Stop(shutdownCtx); err != nil { +// log.Printf("Netbird shutdown error: %v", err) +// } +// } +// +// The package provides several methods for network operations: +// - Dial: Creates outbound connections +// - ListenTCP: Creates TCP listeners +// - ListenUDP: Creates UDP listeners +// +// By default, the embed package uses userspace networking mode, which doesn't +// require root/admin privileges. For production deployments, consider setting +// appropriate config and state paths for persistence. From d88c219a264fe4c3f124768e93da63505e30b2c2 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Thu, 6 Feb 2025 10:57:21 +0100 Subject: [PATCH 08/10] Fix package description position --- client/embed/doc.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/client/embed/doc.go b/client/embed/doc.go index 8f32e2a2630..a95206561fd 100644 --- a/client/embed/doc.go +++ b/client/embed/doc.go @@ -1,9 +1,7 @@ -package embed - // Package embed provides a way to embed the NetBird client directly // into Go programs without requiring a separate NetBird client installation. -// -// +package embed + // Basic Usage: // // client, err := embed.New(embed.Options{ From 6bea77d7132c92afa74d3cb52b4f917a9f036346 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Thu, 6 Feb 2025 11:48:11 +0100 Subject: [PATCH 09/10] Remove error from http client --- client/embed/doc.go | 5 +---- client/embed/embed.go | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/client/embed/doc.go b/client/embed/doc.go index a95206561fd..069d53ebfdc 100644 --- a/client/embed/doc.go +++ b/client/embed/doc.go @@ -129,10 +129,7 @@ package embed // } // // // Create HTTP client that uses netbird network -// httpClient, err := client.NewHTTPClient() -// if err != nil { -// log.Fatal(err) -// } +// httpClient := client.NewHTTPClient() // httpClient.Timeout = 10 * time.Second // // // Make request to server in netbird network diff --git a/client/embed/embed.go b/client/embed/embed.go index 93033cc460a..9ded618c550 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -258,14 +258,14 @@ func (c *Client) ListenUDP(address string) (net.PacketConn, error) { // NewHTTPClient returns a configured http.Client that uses the netbird network for requests. // Not applicable if the userspace networking mode is disabled. -func (c *Client) NewHTTPClient() (*http.Client, error) { +func (c *Client) NewHTTPClient() *http.Client { transport := &http.Transport{ DialContext: c.Dial, } return &http.Client{ Transport: transport, - }, nil + } } func (c *Client) getNet() (*wgnetstack.Net, netip.Addr, error) { From 82b4f61d6617e891666e8e3167c7d877957616f7 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Fri, 7 Feb 2025 15:56:35 +0100 Subject: [PATCH 10/10] Allow virtual listeners to pick up local traffic --- client/firewall/uspfilter/uspfilter.go | 45 +++++++++++++------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/client/firewall/uspfilter/uspfilter.go b/client/firewall/uspfilter/uspfilter.go index 889e4cbb1a9..1da43c5b005 100644 --- a/client/firewall/uspfilter/uspfilter.go +++ b/client/firewall/uspfilter/uspfilter.go @@ -149,17 +149,16 @@ func create(iface common.IFaceMapper, nativeFirewall firewall.Manager, disableSe return d }, }, - nativeFirewall: nativeFirewall, - outgoingRules: make(map[string]RuleSet), - incomingRules: make(map[string]RuleSet), - wgIface: iface, - localipmanager: newLocalIPManager(), - routingEnabled: false, - stateful: !disableConntrack, - logger: nblog.NewFromLogrus(log.StandardLogger()), - netstack: netstack.IsEnabled(), - // default true for non-netstack, for netstack only if explicitly enabled - localForwarding: !netstack.IsEnabled() || enableLocalForwarding, + nativeFirewall: nativeFirewall, + outgoingRules: make(map[string]RuleSet), + incomingRules: make(map[string]RuleSet), + wgIface: iface, + localipmanager: newLocalIPManager(), + routingEnabled: false, + stateful: !disableConntrack, + logger: nblog.NewFromLogrus(log.StandardLogger()), + netstack: netstack.IsEnabled(), + localForwarding: enableLocalForwarding, } if err := m.localipmanager.UpdateLocalIPs(iface); err != nil { @@ -612,11 +611,6 @@ func (m *Manager) dropFilter(packetData []byte) bool { // handleLocalTraffic handles local traffic. // If it returns true, the packet should be dropped. func (m *Manager) handleLocalTraffic(d *decoder, srcIP, dstIP net.IP, packetData []byte) bool { - if !m.localForwarding { - m.logger.Trace("Dropping local packet (local forwarding disabled): src=%s dst=%s", srcIP, dstIP) - return true - } - if m.peerACLsBlock(srcIP, packetData, m.incomingRules, d) { m.logger.Trace("Dropping local packet (ACL denied): src=%s dst=%s", srcIP, dstIP) @@ -625,22 +619,29 @@ func (m *Manager) handleLocalTraffic(d *decoder, srcIP, dstIP net.IP, packetData // if running in netstack mode we need to pass this to the forwarder if m.netstack { - m.handleNetstackLocalTraffic(packetData) - - // don't process this packet further - return true + return m.handleNetstackLocalTraffic(packetData) } return false } -func (m *Manager) handleNetstackLocalTraffic(packetData []byte) { + +func (m *Manager) handleNetstackLocalTraffic(packetData []byte) bool { + if !m.localForwarding { + // pass to virtual tcp/ip stack to be picked up by listeners + return false + } + if m.forwarder == nil { - return + m.logger.Trace("Dropping local packet (forwarder not initialized)") + return true } if err := m.forwarder.InjectIncomingPacket(packetData); err != nil { m.logger.Error("Failed to inject local packet: %v", err) } + + // don't process this packet further + return true } // handleRoutedTraffic handles routed traffic.