diff --git a/.gitmodules b/.gitmodules index 77a1d15e..a3efd487 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy"] path = ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy url = https://github.com/facebook/gnlpy.git +[submodule "server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy"] + path = server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy + url = https://github.com/inetaf/tcpproxy.git diff --git a/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2 b/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2 index c27d1cdf..028afeb8 100644 --- a/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2 +++ b/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2 @@ -40,8 +40,8 @@ virtual=2 {% endfor %} fallback=127.0.0.1 gate service=http - request="heartbeat/http?codename=ANY" - virtualhost="scripts.mit.edu" + request="__scripts/heartbeat/http?codename=ANY" + virtualhost="heartbeat.scripts.scripts.mit.edu" receive="1" checktype=negotiate checkport=80 diff --git a/ansible/roles/packages/meta/main.yml b/ansible/roles/packages/meta/main.yml new file mode 100644 index 00000000..2dec6cc9 --- /dev/null +++ b/ansible/roles/packages/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - rpm-repos diff --git a/ansible/roles/proxy-scripts-proxy/handlers/main.yml b/ansible/roles/proxy-scripts-proxy/handlers/main.yml new file mode 100644 index 00000000..66f30f7b --- /dev/null +++ b/ansible/roles/proxy-scripts-proxy/handlers/main.yml @@ -0,0 +1,5 @@ +- name: restart scripts-proxy + service: + name: scripts-proxy + state: restarted + enabled: yes diff --git a/ansible/roles/proxy-scripts-proxy/tasks/main.yml b/ansible/roles/proxy-scripts-proxy/tasks/main.yml new file mode 100644 index 00000000..2bbdabd5 --- /dev/null +++ b/ansible/roles/proxy-scripts-proxy/tasks/main.yml @@ -0,0 +1,39 @@ +--- +- name: Disable haproxy-related services + service: + name: "{{ item }}" + state: stopped + enabled: no + failed_when: no + loop: + - named-scripts-proxy + - haproxy +- name: Remove haproxy-related configuration + file: + path: "{{ item }}" + state: absent + loop: + - /etc/named.scripts-proxy.conf + - /etc/systemd/system/named-scripts-proxy.service + - /etc/haproxy/haproxy.cfg + - /etc/rsyslog.d/haproxy.conf + - /etc/systemd/system/haproxy.service.d/10-scripts.conf + - /usr/local/bin/hatop +- name: Remove haproxy-related packages + dnf: + name: + - bind + - bind-dlz-ldap + - haproxy + state: absent +- name: Install scripts-proxy + dnf: + name: + - scripts-proxy + state: present +- name: Configure scripts-proxy + copy: + dest: /etc/sysconfig/scripts-proxy + content: | + OPTIONS="-ldap_servers={{ groups['scripts-ldap'] | join(':389,') }}:389" + notify: restart scripts-proxy diff --git a/ansible/roles/rpm-repos/defaults/main.yml b/ansible/roles/rpm-repos/defaults/main.yml new file mode 100644 index 00000000..3a3dd0b4 --- /dev/null +++ b/ansible/roles/rpm-repos/defaults/main.yml @@ -0,0 +1,10 @@ +--- +rpm_repos: + - key: scripts + name: Scripts + baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}/ + enabled: yes + - key: scripts-testing + name: Scripts Testing + baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}-testing/ + enabled: "{{ enable_testing_repo | default(False) }}" diff --git a/ansible/roles/rpm-repos/tasks/main.yml b/ansible/roles/rpm-repos/tasks/main.yml new file mode 100644 index 00000000..c6d3fc25 --- /dev/null +++ b/ansible/roles/rpm-repos/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: Configure scripts RPM repos + copy: + dest: /etc/yum.repos.d/scripts.repo + content: | + {% for repo in rpm_repos %} + [{{ repo.key }}] + name={{ repo.name }} + baseurl={{ repo.baseurl }} + enabled={{ 1 if repo.enabled else 0 }} + gpgcheck=0 + {% endfor %} +- name: Configure dnf.conf + ini_file: + path: /etc/dnf/dnf.conf + section: main + option: "{{ item.option }}" + value: "{{ item.value }}" + loop: + - option: installonly_limit + value: 0 + - option: installonlypkgs + value: kernel kernel-devel kernel-modules kmod-openafs ghc-cgi ghc-cgi-devel + - option: excludepkgs + value: fedora-obsolete-packages php-fpm nfs-utils diff --git a/ansible/scripts-proxy.yml b/ansible/scripts-proxy.yml index 64536aa9..12842b3a 100644 --- a/ansible/scripts-proxy.yml +++ b/ansible/scripts-proxy.yml @@ -13,15 +13,17 @@ dest: /etc/munin/plugin-conf.d/ src: files/conntrack roles: + - role: rpm-repos + tags: [always] - ansible-config-me - k5login - syslog-client - root-aliases + - mock - proxy-munin-node - nrpe - dnf-automatic - - proxy-dns - - proxy-haproxy + - proxy-scripts-proxy - proxy-logrotate tasks: - package: diff --git a/ansible/scripts-real.yml b/ansible/scripts-real.yml index eb1d81db..895b0322 100644 --- a/ansible/scripts-real.yml +++ b/ansible/scripts-real.yml @@ -14,15 +14,6 @@ vars: ldap_server: "{{ use_local_ldap | default(True) | ternary('ldapi://%2fvar%2frun%2fslapd-scripts.socket/', 'ldap://scripts-ldap.mit.edu/') }}" ldap_server_tcp: "{{ use_local_ldap | default(True) | ternary('ldap://127.0.0.1/', 'ldap://scripts-ldap.mit.edu/') }}" - rpm_repos: - - key: scripts - name: Scripts - baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}/ - enabled: yes - - key: scripts-testing - name: Scripts Testing - baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}-testing/ - enabled: "{{ enable_testing_repo | default(False) }}" preferred_mta: postfix pre_tasks: - name: Block Ansible on legacy realservers @@ -39,33 +30,9 @@ state: absent - include_role: name: real-network - - name: Configure dnf - block: - - name: Configure scripts RPM repos - copy: - dest: /etc/yum.repos.d/scripts.repo - content: | - {% for repo in rpm_repos %} - [{{ repo.key }}] - name={{ repo.name }} - baseurl={{ repo.baseurl }} - enabled={{ 1 if repo.enabled else 0 }} - gpgcheck=0 - {% endfor %} - - name: Configure dnf.conf - ini_file: - path: /etc/dnf/dnf.conf - section: main - option: "{{ item.option }}" - value: "{{ item.value }}" - loop: - - option: installonly_limit - value: 0 - - option: installonlypkgs - value: kernel kernel-devel kernel-modules kmod-openafs ghc-cgi ghc-cgi-devel - - option: excludepkgs - value: fedora-obsolete-packages php-fpm nfs-utils roles: + - role: rpm-repos + tags: [always] - role: packages tags: [always] - role: syslog-client diff --git a/server/common/oursrc/scripts-proxy/ldap/conn.go b/server/common/oursrc/scripts-proxy/ldap/conn.go new file mode 100644 index 00000000..d4505d42 --- /dev/null +++ b/server/common/oursrc/scripts-proxy/ldap/conn.go @@ -0,0 +1,61 @@ +package ldap + +import ( + "fmt" + "log" + "sync" + "time" + + ldap "gopkg.in/ldap.v3" +) + +type conn struct { + server, baseDn string + // mu protects conn during reconnect cycles + // TODO: The ldap package supports multiple in-flight queries; + // by using a Mutex we are only going to issue one at a + // time. We should figure out how to do retry/reconnect + // behavior with parallel queries. + mu sync.Mutex + conn *ldap.Conn +} + +func (c *conn) reconnect() { + c.mu.Lock() + defer c.mu.Unlock() + if c.conn != nil { + c.conn.Close() + } + var err error + for { + log.Printf("connecting to %s", c.server) + c.conn, err = ldap.Dial("tcp", c.server) + if err == nil { + return + } + log.Printf("connecting to %s: %v", c.server, err) + time.Sleep(100 * time.Millisecond) + } +} + +func (c *conn) resolvePool(hostname string) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + + escapedHostname := ldap.EscapeFilter(hostname) + req := &ldap.SearchRequest{ + BaseDN: c.baseDn, + Scope: ldap.ScopeWholeSubtree, + Filter: fmt.Sprintf("(|(scriptsVhostName=%s)(scriptsVhostAlias=%s))", escapedHostname, escapedHostname), + Attributes: []string{"scriptsVhostPoolIPv4"}, + } + sr, err := c.conn.Search(req) + if err != nil { + return "", err + } + for _, entry := range sr.Entries { + return entry.GetAttributeValue("scriptsVhostPoolIPv4"), nil + } + // Not found is not an error + return "", nil +} diff --git a/server/common/oursrc/scripts-proxy/ldap/pool.go b/server/common/oursrc/scripts-proxy/ldap/pool.go new file mode 100644 index 00000000..f95bf2aa --- /dev/null +++ b/server/common/oursrc/scripts-proxy/ldap/pool.go @@ -0,0 +1,48 @@ +package ldap + +import "log" + +// Pool handles a concurrency-safe pool of connections to LDAP servers. +type Pool struct { + retries int + // connCh holds open connections to servers. + connCh chan *conn +} + +// NewPool constructs a connection pool that queries for baseDn from servers. +func NewPool(servers []string, baseDn string, retries int) *Pool { + p := &Pool{ + retries: retries, + connCh: make(chan *conn, len(servers)), + } + for _, s := range servers { + c := &conn{ + server: s, + baseDn: baseDn, + } + go p.reconnect(c) + } + return p +} + +func (p *Pool) reconnect(c *conn) { + c.reconnect() + p.connCh <- c +} + +// ResolvePool attempts to resolve the pool for hostname to an IP address, returned as a string. +func (p *Pool) ResolvePool(hostname string) (string, error) { + var ip string + var err error + for i := 0; i < p.retries; i++ { + c := <-p.connCh + ip, err = c.resolvePool(hostname) + if err == nil { + p.connCh <- c + return ip, err + } + log.Printf("resolving %q on %s: %v", hostname, c.server, err) + go p.reconnect(c) + } + return ip, err +} diff --git a/server/common/oursrc/scripts-proxy/main.go b/server/common/oursrc/scripts-proxy/main.go new file mode 100644 index 00000000..06a35706 --- /dev/null +++ b/server/common/oursrc/scripts-proxy/main.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "strings" + + "github.com/mit-scripts/scripts/server/common/oursrc/scripts-proxy/ldap" + "inet.af/tcpproxy" +) + +var ( + httpAddrs = flag.String("http_addrs", "0.0.0.0:80", "comma-separated addresses to listen for HTTP traffic on") + sniAddrs = flag.String("sni_addrs", "0.0.0.0:443,0.0.0.0:444", "comma-separated addresses to listen for SNI traffic on") + ldapServers = flag.String("ldap_servers", "scripts-ldap.mit.edu:389", "comma-spearated LDAP servers to query") + defaultHost = flag.String("default_host", "scripts.mit.edu", "default host to route traffic to if SNI/Host header cannot be parsed or cannot be found in LDAP") + baseDn = flag.String("base_dn", "ou=VirtualHosts,dc=scripts,dc=mit,dc=edu", "base DN to query for hosts") + localRange = flag.String("local_range", "18.4.86.0/24", "IP block for client IP spoofing. If the resolved destination address is in this subnet, the source IP address of the backend connection will be spoofed to match the client IP. This subnet needs to be local to the proxy.") +) + +const ldapRetries = 3 + +func always(context.Context, string) bool { + return true +} + +type ldapTarget struct { + localPoolRange *net.IPNet + ldap *ldap.Pool + statuszServer *HijackedServer + unavailableServer *HijackedServer +} + +// HandleConn is called by tcpproxy after receiving a connection and sniffing the host. +// If a host could be identified, netConn is an instance of *tcpproxy.Conn. +// If not, it is just an instance of the net.Conn interface. +func (l *ldapTarget) HandleConn(netConn net.Conn) { + var pool string + var err error + if conn, ok := netConn.(*tcpproxy.Conn); ok { + switch conn.HostName { + case "proxy.scripts.scripts.mit.edu": + // Special handling for proxy.scripts.scripts.mit.edu + l.statuszServer.HandleConn(netConn) + return + case "heartbeat.scripts.scripts.mit.edu": + if nolvsPresent() { + l.unavailableServer.HandleConn(netConn) + return + } + } + pool, err = l.ldap.ResolvePool(conn.HostName) + if err != nil { + log.Printf("resolving %q: %v", conn.HostName, err) + } + } + if pool == "" { + pool, err = l.ldap.ResolvePool(*defaultHost) + if err != nil { + log.Printf("resolving default pool: %v", err) + } + } + // TODO: Forward to sorry server on director? + if pool == "" { + l.unavailableServer.HandleConn(netConn) + return + } + laddr := netConn.LocalAddr().(*net.TCPAddr) + destAddrStr := net.JoinHostPort(pool, fmt.Sprintf("%d", laddr.Port)) + destAddr, err := net.ResolveTCPAddr("tcp", destAddrStr) + if err != nil { + netConn.Close() + log.Printf("parsing pool address %q: %v", pool, err) + return + } + dp := &tcpproxy.DialProxy{ + Addr: destAddrStr, + } + if l.localPoolRange.Contains(destAddr.IP) { + raddr := netConn.RemoteAddr().(*net.TCPAddr) + td := &TransparentDialer{ + SourceAddr: &net.TCPAddr{ + IP: raddr.IP, + }, + DestAddr: destAddr, + } + dp.DialContext = td.DialContext + } + dp.HandleConn(netConn) +} + +func main() { + flag.Parse() + + _, ipnet, err := net.ParseCIDR(*localRange) + if err != nil { + log.Fatal(err) + } + + ldapPool := ldap.NewPool(strings.Split(*ldapServers, ","), *baseDn, ldapRetries) + + var p tcpproxy.Proxy + t := &ldapTarget{ + localPoolRange: ipnet, + ldap: ldapPool, + statuszServer: NewHijackedServer(nil), + unavailableServer: NewUnavailableServer(), + } + for _, addr := range strings.Split(*httpAddrs, ",") { + p.AddHTTPHostMatchRoute(addr, always, t) + } + for _, addr := range strings.Split(*sniAddrs, ",") { + p.AddStopACMESearch(addr) + p.AddSNIMatchRoute(addr, always, t) + } + log.Fatal(p.Run()) +} diff --git a/server/common/oursrc/scripts-proxy/scripts-proxy.service b/server/common/oursrc/scripts-proxy/scripts-proxy.service new file mode 100644 index 00000000..13ea4965 --- /dev/null +++ b/server/common/oursrc/scripts-proxy/scripts-proxy.service @@ -0,0 +1,20 @@ +[Unit] +Description=Scripts HTTP/SNI proxy +After=nss-lookup.target +Wants=network-online.target +After=network-online.target + +[Service] +Restart=on-failure + +# Run as nobody but grant the ability to bind 80/443/444 +User=nobody +Group=nobody +AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN + +EnvironmentFile=-/etc/sysconfig/scripts-proxy + +ExecStart=/usr/sbin/scripts-proxy $OPTIONS + +[Install] +WantedBy=multi-user.target diff --git a/server/common/oursrc/scripts-proxy/statusz.go b/server/common/oursrc/scripts-proxy/statusz.go new file mode 100644 index 00000000..d35cbe90 --- /dev/null +++ b/server/common/oursrc/scripts-proxy/statusz.go @@ -0,0 +1,64 @@ +package main + +import ( + "errors" + "net" + "net/http" + _ "net/http/pprof" + "os" +) + +func nolvsPresent() bool { + if _, err := os.Stat("/etc/nolvs"); err == nil { + return true + } + return false +} + +// HijackedServer is an HTTP server that serves from connections hijacked from another server instead of a listening socket. +// (See net/http.Hijacker for the opposite direction.) +// Users can call HandleConn to handle any request(s) waiting on that net.Conn. +type HijackedServer struct { + connCh chan net.Conn +} + +// NewHijackedServer constructs a HijackedServer that handles incoming HTTP connections with handler. +func NewHijackedServer(handler http.Handler) *HijackedServer { + s := &HijackedServer{ + connCh: make(chan net.Conn), + } + go http.Serve(s, handler) + return s +} + +// Accept is called by http.Server to acquire a new connection. +func (s *HijackedServer) Accept() (net.Conn, error) { + c, ok := <-s.connCh + if ok { + return c, nil + } + return nil, errors.New("closed") +} + +// Close shuts down the server. +func (s *HijackedServer) Close() error { + close(s.connCh) + return nil +} + +// Addr must be present to implement net.Listener +func (s *HijackedServer) Addr() net.Addr { + return nil +} + +// HandleConn instructs the server to take control of c and handle any HTTP request(s) that are waiting. +func (s *HijackedServer) HandleConn(c net.Conn) { + s.connCh <- c +} + +// NewUnavailableServer constructs a HijackedServer that serves 500 Service Unavailable for all requests. +func NewUnavailableServer() *HijackedServer { + return NewHijackedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "0 proxy nolvs", http.StatusServiceUnavailable) + })) +} diff --git a/server/common/oursrc/scripts-proxy/tproxy.go b/server/common/oursrc/scripts-proxy/tproxy.go new file mode 100644 index 00000000..66388103 --- /dev/null +++ b/server/common/oursrc/scripts-proxy/tproxy.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "log" + "net" + "syscall" +) + +// TransparentDialer makes a connection to DestAddr using SourceAddr as the non-local source address. +type TransparentDialer struct { + SourceAddr net.Addr + DestAddr net.Addr +} + +func (t *TransparentDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + d := &net.Dialer{ + LocalAddr: t.SourceAddr, + Control: func(network, address string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + for _, opt := range []int{ + syscall.IP_TRANSPARENT, + syscall.IP_FREEBIND, + } { + err := syscall.SetsockoptInt(int(fd), syscall.SOL_IP, opt, 1) + if err != nil { + log.Printf("control: %s", err) + return + } + } + }) + }, + } + return d.DialContext(ctx, network, t.DestAddr.String()) +} diff --git a/server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy b/server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy new file mode 160000 index 00000000..b6bb9b5b --- /dev/null +++ b/server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy @@ -0,0 +1 @@ +Subproject commit b6bb9b5b82524122bcf27291ede32d1517a14ab8 diff --git a/server/fedora/Makefile b/server/fedora/Makefile index 85f52e97..34ff438d 100644 --- a/server/fedora/Makefile +++ b/server/fedora/Makefile @@ -24,7 +24,7 @@ gems = pony:1.8 fcgi:0.9.2.1 upstream_gems = rubygem-pony rubygem-fcgi upstream_eggs = python-authkit upstream = openafs hesiod $(upstream_yum) $(upstream_gems) $(upstream_eggs) moira zephyr zephyr.i686 python-zephyr python-afs python-moira python-hesiod athena-aclocal discuss fuse-python -oursrc = execsys tokensys accountadm httpdmods logview nss_nonlocal nss_nonlocal.i686 athrun php_scripts scripts-wizard scripts-base scripts-static-cat fuse-better-mousetrapfs scripts-munin-plugins scripts-krb5-localauth shackle +oursrc = execsys tokensys accountadm httpdmods logview nss_nonlocal nss_nonlocal.i686 athrun php_scripts scripts-wizard scripts-base scripts-static-cat fuse-better-mousetrapfs scripts-munin-plugins scripts-krb5-localauth shackle scripts-proxy allsrc = $(upstream) $(oursrc) oursrcdir = ${PWD}/../common/oursrc patches = ${PWD}/../common/patches diff --git a/server/fedora/specs/scripts-proxy.spec b/server/fedora/specs/scripts-proxy.spec new file mode 100644 index 00000000..88f28f98 --- /dev/null +++ b/server/fedora/specs/scripts-proxy.spec @@ -0,0 +1,57 @@ +# https://fedoraproject.org/wiki/PackagingDrafts/Go + +Name: scripts-proxy +Version: 0.0 +Release: 0.%{scriptsversion}%{?dist} +Summary: HTTP/SNI proxy for scripts.mit.edu + +License: GPL+ +URL: http://scripts.mit.edu/ +Source0: %{name}.tar.gz + +BuildRequires: (systemd-rpm-macros or systemd < 240) +BuildRequires: go-rpm-macros +BuildRequires: golang >= 1.6 +BuildRequires: golang(gopkg.in/ldap.v3) + +%description +scripts-proxy proxies HTTP and HTTPS+SNI requests to backend servers +based on LDAP. + +%global goipath github.com/mit-scripts/scripts/server/common/oursrc/scripts-proxy +%global extractdir %{name} + +%gometa +%gopkg + +%prep +%goprep -k + +%build +%gobuild -o %{gobuilddir}/bin/scripts-proxy %{goipath} + +%install +%gopkginstall +install -d %{buildroot}%{_sbindir} +install -p -m 0755 %{gobuilddir}/bin/scripts-proxy %{buildroot}%{_sbindir}/scripts-proxy +install -d %{buildroot}%{_unitdir} +install -p -m 0644 ./scripts-proxy.service %{buildroot}%{_unitdir}/scripts-proxy.service + +%files +%defattr(0644, root, root) +%{_unitdir}/scripts-proxy.service +%attr(755,root,root) %{_sbindir}/scripts-proxy +%gopkgfiles + +%post +%systemd_post scripts-proxy.service + +%preun +%systemd_preun scripts-proxy.service + +%postun +%systemd_postun_with_restart scripts-proxy.service + +%changelog +* Sun Mar 8 2020 Quentin Smith - 0.0-0 +- Initial packaging for scripts-proxy