From 31002af343b549516d1fac3a4bd08b36c5b595b0 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Sun, 8 Mar 2020 07:40:44 -0400
Subject: [PATCH 01/11] Initial proxy that parses SNI and Host headers

---
 server/common/oursrc/scripts-proxy/main.go | 75 ++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 server/common/oursrc/scripts-proxy/main.go

diff --git a/server/common/oursrc/scripts-proxy/main.go b/server/common/oursrc/scripts-proxy/main.go
new file mode 100644
index 00000000..4a2ab4ab
--- /dev/null
+++ b/server/common/oursrc/scripts-proxy/main.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"strings"
+
+	"inet.af/tcpproxy"
+)
+
+var (
+	httpAddrs   = flag.String("http_addrs", "0.0.0.0:80", "comma-separated addresses to listen for HTTP traffic on")
+	sniAddrs    = flag.String("sni_addrs", "0.0.0.0:443,0.0.0.0:444", "comma-separated addresses to listen for SNI traffic on")
+	defaultHost = flag.String("default_host", "scripts.mit.edu", "default host to route traffic to if SNI/Host header cannot be parsed or cannot be found in LDAP")
+)
+
+func always(context.Context, string) bool {
+	return true
+}
+
+type ldapTarget struct {
+}
+
+// HandleConn is called by tcpproxy after receiving a connection and sniffing the host.
+// If a host could be identified, netConn is an instance of *tcpproxy.Conn.
+// If not, it is just an instance of the net.Conn interface.
+func (l *ldapTarget) HandleConn(netConn net.Conn) {
+	var pool string
+	var err error
+	if conn, ok := netConn.(*tcpproxy.Conn); ok {
+		pool, err = l.resolvePool(conn.HostName)
+		if err != nil {
+			log.Printf("resolving %q: %v", conn.HostName, err)
+		}
+	}
+	if pool == "" {
+		pool, err = l.resolvePool(*defaultHost)
+		if err != nil {
+			log.Printf("resolving default pool: %v", err)
+		}
+	}
+	if pool == "" {
+		netConn.Close()
+		return
+	}
+	laddr := netConn.LocalAddr().(*net.TCPAddr)
+	dp := &tcpproxy.DialProxy{
+		Addr: fmt.Sprintf("%s:%d", pool, laddr.Port),
+		// TODO: Set DialContext to override the source address
+	}
+	dp.HandleConn(netConn)
+}
+
+func (l *ldapTarget) resolvePool(hostname string) (string, error) {
+	// TODO: Hardcoding F20 pool until we can resolve the pool.
+	return "18.4.86.22", nil
+}
+
+func main() {
+	flag.Parse()
+
+	var p tcpproxy.Proxy
+	t := &ldapTarget{}
+	for _, addr := range strings.Split(*httpAddrs, ",") {
+		p.AddHTTPHostMatchRoute(addr, always, t)
+	}
+	for _, addr := range strings.Split(*sniAddrs, ",") {
+		p.AddStopACMESearch(addr)
+		p.AddSNIMatchRoute(addr, always, t)
+	}
+	log.Fatal(p.Run())
+}

From f6cf9b7f35a5520adce65db80f7022efb6781b56 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Sun, 8 Mar 2020 07:54:36 -0400
Subject: [PATCH 02/11] Route connections based on LDAP

---
 server/common/oursrc/scripts-proxy/main.go | 32 ++++++++++++++++++++--
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/server/common/oursrc/scripts-proxy/main.go b/server/common/oursrc/scripts-proxy/main.go
index 4a2ab4ab..6493312e 100644
--- a/server/common/oursrc/scripts-proxy/main.go
+++ b/server/common/oursrc/scripts-proxy/main.go
@@ -9,12 +9,16 @@ import (
 	"strings"
 
 	"inet.af/tcpproxy"
+
+	ldap "gopkg.in/ldap.v3"
 )
 
 var (
 	httpAddrs   = flag.String("http_addrs", "0.0.0.0:80", "comma-separated addresses to listen for HTTP traffic on")
 	sniAddrs    = flag.String("sni_addrs", "0.0.0.0:443,0.0.0.0:444", "comma-separated addresses to listen for SNI traffic on")
+	ldapServer  = flag.String("ldap_server", "scripts-ldap.mit.edu:389", "LDAP server to query")
 	defaultHost = flag.String("default_host", "scripts.mit.edu", "default host to route traffic to if SNI/Host header cannot be parsed or cannot be found in LDAP")
+	baseDn      = flag.String("base_dn", "ou=VirtualHosts,dc=scripts,dc=mit,dc=edu", "base DN to query for hosts")
 )
 
 func always(context.Context, string) bool {
@@ -22,6 +26,7 @@ func always(context.Context, string) bool {
 }
 
 type ldapTarget struct {
+	ldap *ldap.Conn
 }
 
 // HandleConn is called by tcpproxy after receiving a connection and sniffing the host.
@@ -55,15 +60,36 @@ func (l *ldapTarget) HandleConn(netConn net.Conn) {
 }
 
 func (l *ldapTarget) resolvePool(hostname string) (string, error) {
-	// TODO: Hardcoding F20 pool until we can resolve the pool.
-	return "18.4.86.22", nil
+	escapedHostname := ldap.EscapeFilter(hostname)
+	req := ldap.NewSearchRequest(
+		*baseDn,
+		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+		fmt.Sprintf("(|(scriptsVhostName=%s)(scriptsVhostAlias=%s))", escapedHostname, escapedHostname),
+		[]string{"scriptsVhostPoolIPv4"},
+		nil,
+	)
+	sr, err := l.ldap.Search(req)
+	if err != nil {
+		return "", err
+	}
+	for _, entry := range sr.Entries {
+		return entry.GetAttributeValue("scriptsVhostPoolIPv4"), nil
+	}
+	// Not found is not an error
+	return "", nil
 }
 
 func main() {
 	flag.Parse()
 
+	l, err := ldap.Dial("tcp", *ldapServer)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer l.Close()
+
 	var p tcpproxy.Proxy
-	t := &ldapTarget{}
+	t := &ldapTarget{ldap: l}
 	for _, addr := range strings.Split(*httpAddrs, ",") {
 		p.AddHTTPHostMatchRoute(addr, always, t)
 	}

From 879aa1083ca72d96c7bbac70bde481ecb84fa67f Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Sun, 8 Mar 2020 08:21:03 -0400
Subject: [PATCH 03/11] Spoof client source IP

---
 .../common/oursrc/scripts-proxy/ldap/conn.go  | 61 +++++++++++++++++
 .../common/oursrc/scripts-proxy/ldap/pool.go  | 48 ++++++++++++++
 server/common/oursrc/scripts-proxy/main.go    | 65 ++++++++++---------
 3 files changed, 143 insertions(+), 31 deletions(-)
 create mode 100644 server/common/oursrc/scripts-proxy/ldap/conn.go
 create mode 100644 server/common/oursrc/scripts-proxy/ldap/pool.go

diff --git a/server/common/oursrc/scripts-proxy/ldap/conn.go b/server/common/oursrc/scripts-proxy/ldap/conn.go
new file mode 100644
index 00000000..d4505d42
--- /dev/null
+++ b/server/common/oursrc/scripts-proxy/ldap/conn.go
@@ -0,0 +1,61 @@
+package ldap
+
+import (
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	ldap "gopkg.in/ldap.v3"
+)
+
+type conn struct {
+	server, baseDn string
+	// mu protects conn during reconnect cycles
+	// TODO: The ldap package supports multiple in-flight queries;
+	// by using a Mutex we are only going to issue one at a
+	// time. We should figure out how to do retry/reconnect
+	// behavior with parallel queries.
+	mu   sync.Mutex
+	conn *ldap.Conn
+}
+
+func (c *conn) reconnect() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.conn != nil {
+		c.conn.Close()
+	}
+	var err error
+	for {
+		log.Printf("connecting to %s", c.server)
+		c.conn, err = ldap.Dial("tcp", c.server)
+		if err == nil {
+			return
+		}
+		log.Printf("connecting to %s: %v", c.server, err)
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func (c *conn) resolvePool(hostname string) (string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	escapedHostname := ldap.EscapeFilter(hostname)
+	req := &ldap.SearchRequest{
+		BaseDN:     c.baseDn,
+		Scope:      ldap.ScopeWholeSubtree,
+		Filter:     fmt.Sprintf("(|(scriptsVhostName=%s)(scriptsVhostAlias=%s))", escapedHostname, escapedHostname),
+		Attributes: []string{"scriptsVhostPoolIPv4"},
+	}
+	sr, err := c.conn.Search(req)
+	if err != nil {
+		return "", err
+	}
+	for _, entry := range sr.Entries {
+		return entry.GetAttributeValue("scriptsVhostPoolIPv4"), nil
+	}
+	// Not found is not an error
+	return "", nil
+}
diff --git a/server/common/oursrc/scripts-proxy/ldap/pool.go b/server/common/oursrc/scripts-proxy/ldap/pool.go
new file mode 100644
index 00000000..f95bf2aa
--- /dev/null
+++ b/server/common/oursrc/scripts-proxy/ldap/pool.go
@@ -0,0 +1,48 @@
+package ldap
+
+import "log"
+
+// Pool handles a concurrency-safe pool of connections to LDAP servers.
+type Pool struct {
+	retries int
+	// connCh holds open connections to servers.
+	connCh chan *conn
+}
+
+// NewPool constructs a connection pool that queries for baseDn from servers.
+func NewPool(servers []string, baseDn string, retries int) *Pool {
+	p := &Pool{
+		retries: retries,
+		connCh:  make(chan *conn, len(servers)),
+	}
+	for _, s := range servers {
+		c := &conn{
+			server: s,
+			baseDn: baseDn,
+		}
+		go p.reconnect(c)
+	}
+	return p
+}
+
+func (p *Pool) reconnect(c *conn) {
+	c.reconnect()
+	p.connCh <- c
+}
+
+// ResolvePool attempts to resolve the pool for hostname to an IP address, returned as a string.
+func (p *Pool) ResolvePool(hostname string) (string, error) {
+	var ip string
+	var err error
+	for i := 0; i < p.retries; i++ {
+		c := <-p.connCh
+		ip, err = c.resolvePool(hostname)
+		if err == nil {
+			p.connCh <- c
+			return ip, err
+		}
+		log.Printf("resolving %q on %s: %v", hostname, c.server, err)
+		go p.reconnect(c)
+	}
+	return ip, err
+}
diff --git a/server/common/oursrc/scripts-proxy/main.go b/server/common/oursrc/scripts-proxy/main.go
index 6493312e..b183ef2d 100644
--- a/server/common/oursrc/scripts-proxy/main.go
+++ b/server/common/oursrc/scripts-proxy/main.go
@@ -8,25 +8,28 @@ import (
 	"net"
 	"strings"
 
+	"github.com/mit-scripts/scripts/server/common/oursrc/scripts-proxy/ldap"
 	"inet.af/tcpproxy"
-
-	ldap "gopkg.in/ldap.v3"
 )
 
 var (
 	httpAddrs   = flag.String("http_addrs", "0.0.0.0:80", "comma-separated addresses to listen for HTTP traffic on")
 	sniAddrs    = flag.String("sni_addrs", "0.0.0.0:443,0.0.0.0:444", "comma-separated addresses to listen for SNI traffic on")
-	ldapServer  = flag.String("ldap_server", "scripts-ldap.mit.edu:389", "LDAP server to query")
+	ldapServers = flag.String("ldap_servers", "scripts-ldap.mit.edu:389", "comma-spearated LDAP servers to query")
 	defaultHost = flag.String("default_host", "scripts.mit.edu", "default host to route traffic to if SNI/Host header cannot be parsed or cannot be found in LDAP")
 	baseDn      = flag.String("base_dn", "ou=VirtualHosts,dc=scripts,dc=mit,dc=edu", "base DN to query for hosts")
+	localRange  = flag.String("local_range", "18.4.86.0/24", "IP block for client IP spoofing. If the resolved destination address is in this subnet, the source IP address of the backend connection will be spoofed to match the client IP. This subnet needs to be local to the proxy.")
 )
 
+const ldapRetries = 3
+
 func always(context.Context, string) bool {
 	return true
 }
 
 type ldapTarget struct {
-	ldap *ldap.Conn
+	localPoolRange *net.IPNet
+	ldap           *ldap.Pool
 }
 
 // HandleConn is called by tcpproxy after receiving a connection and sniffing the host.
@@ -36,60 +39,60 @@ func (l *ldapTarget) HandleConn(netConn net.Conn) {
 	var pool string
 	var err error
 	if conn, ok := netConn.(*tcpproxy.Conn); ok {
-		pool, err = l.resolvePool(conn.HostName)
+		pool, err = l.ldap.ResolvePool(conn.HostName)
 		if err != nil {
 			log.Printf("resolving %q: %v", conn.HostName, err)
 		}
 	}
 	if pool == "" {
-		pool, err = l.resolvePool(*defaultHost)
+		pool, err = l.ldap.ResolvePool(*defaultHost)
 		if err != nil {
 			log.Printf("resolving default pool: %v", err)
 		}
 	}
+	// TODO: Serve an error page? Forward to scripts-director?
 	if pool == "" {
 		netConn.Close()
 		return
 	}
 	laddr := netConn.LocalAddr().(*net.TCPAddr)
-	dp := &tcpproxy.DialProxy{
-		Addr: fmt.Sprintf("%s:%d", pool, laddr.Port),
-		// TODO: Set DialContext to override the source address
-	}
-	dp.HandleConn(netConn)
-}
-
-func (l *ldapTarget) resolvePool(hostname string) (string, error) {
-	escapedHostname := ldap.EscapeFilter(hostname)
-	req := ldap.NewSearchRequest(
-		*baseDn,
-		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
-		fmt.Sprintf("(|(scriptsVhostName=%s)(scriptsVhostAlias=%s))", escapedHostname, escapedHostname),
-		[]string{"scriptsVhostPoolIPv4"},
-		nil,
-	)
-	sr, err := l.ldap.Search(req)
+	destAddrStr := net.JoinHostPort(pool, fmt.Sprintf("%d", laddr.Port))
+	destAddr, err := net.ResolveTCPAddr("tcp", destAddrStr)
 	if err != nil {
-		return "", err
+		netConn.Close()
+		log.Printf("parsing pool address %q: %v", pool, err)
+		return
+	}
+	dp := &tcpproxy.DialProxy{
+		Addr: destAddrStr,
 	}
-	for _, entry := range sr.Entries {
-		return entry.GetAttributeValue("scriptsVhostPoolIPv4"), nil
+	if l.localPoolRange.Contains(destAddr.IP) {
+		raddr := netConn.RemoteAddr().(*net.TCPAddr)
+		sourceAddr := &net.TCPAddr{
+			IP: raddr.IP,
+		}
+		dp.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
+			return net.DialTCP(network, sourceAddr, destAddr)
+		}
 	}
-	// Not found is not an error
-	return "", nil
+	dp.HandleConn(netConn)
 }
 
 func main() {
 	flag.Parse()
 
-	l, err := ldap.Dial("tcp", *ldapServer)
+	_, ipnet, err := net.ParseCIDR(*localRange)
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer l.Close()
+
+	ldapPool := ldap.NewPool(strings.Split(*ldapServers, ","), *baseDn, ldapRetries)
 
 	var p tcpproxy.Proxy
-	t := &ldapTarget{ldap: l}
+	t := &ldapTarget{
+		localPoolRange: ipnet,
+		ldap:           ldapPool,
+	}
 	for _, addr := range strings.Split(*httpAddrs, ",") {
 		p.AddHTTPHostMatchRoute(addr, always, t)
 	}

From 3bbbd8a264c8a87b5ff4d0c554ad7eb9d33bc017 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Sun, 8 Mar 2020 22:55:56 -0400
Subject: [PATCH 04/11] Set IP_TRANSPARENT when binding a non-local address

---
 server/common/oursrc/scripts-proxy/main.go   | 11 +++---
 server/common/oursrc/scripts-proxy/tproxy.go | 35 ++++++++++++++++++++
 2 files changed, 41 insertions(+), 5 deletions(-)
 create mode 100644 server/common/oursrc/scripts-proxy/tproxy.go

diff --git a/server/common/oursrc/scripts-proxy/main.go b/server/common/oursrc/scripts-proxy/main.go
index b183ef2d..42d415e2 100644
--- a/server/common/oursrc/scripts-proxy/main.go
+++ b/server/common/oursrc/scripts-proxy/main.go
@@ -68,12 +68,13 @@ func (l *ldapTarget) HandleConn(netConn net.Conn) {
 	}
 	if l.localPoolRange.Contains(destAddr.IP) {
 		raddr := netConn.RemoteAddr().(*net.TCPAddr)
-		sourceAddr := &net.TCPAddr{
-			IP: raddr.IP,
-		}
-		dp.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
-			return net.DialTCP(network, sourceAddr, destAddr)
+		td := &TransparentDialer{
+			SourceAddr: &net.TCPAddr{
+				IP: raddr.IP,
+			},
+			DestAddr: destAddr,
 		}
+		dp.DialContext = td.DialContext
 	}
 	dp.HandleConn(netConn)
 }
diff --git a/server/common/oursrc/scripts-proxy/tproxy.go b/server/common/oursrc/scripts-proxy/tproxy.go
new file mode 100644
index 00000000..66388103
--- /dev/null
+++ b/server/common/oursrc/scripts-proxy/tproxy.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+	"context"
+	"log"
+	"net"
+	"syscall"
+)
+
+// TransparentDialer makes a connection to DestAddr using SourceAddr as the non-local source address.
+type TransparentDialer struct {
+	SourceAddr net.Addr
+	DestAddr   net.Addr
+}
+
+func (t *TransparentDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	d := &net.Dialer{
+		LocalAddr: t.SourceAddr,
+		Control: func(network, address string, c syscall.RawConn) error {
+			return c.Control(func(fd uintptr) {
+				for _, opt := range []int{
+					syscall.IP_TRANSPARENT,
+					syscall.IP_FREEBIND,
+				} {
+					err := syscall.SetsockoptInt(int(fd), syscall.SOL_IP, opt, 1)
+					if err != nil {
+						log.Printf("control: %s", err)
+						return
+					}
+				}
+			})
+		},
+	}
+	return d.DialContext(ctx, network, t.DestAddr.String())
+}

From 06ff98fb494162ef1ab25aa6ed5d78c513f83e77 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Mon, 9 Mar 2020 00:34:44 -0400
Subject: [PATCH 05/11] Refactor dnf configuration so it can be used by proxies

---
 ansible/roles/packages/meta/main.yml      |  2 ++
 ansible/roles/rpm-repos/defaults/main.yml | 10 ++++++
 ansible/roles/rpm-repos/tasks/main.yml    | 25 +++++++++++++++
 ansible/scripts-proxy.yml                 |  2 ++
 ansible/scripts-real.yml                  | 37 ++---------------------
 5 files changed, 41 insertions(+), 35 deletions(-)
 create mode 100644 ansible/roles/packages/meta/main.yml
 create mode 100644 ansible/roles/rpm-repos/defaults/main.yml
 create mode 100644 ansible/roles/rpm-repos/tasks/main.yml

diff --git a/ansible/roles/packages/meta/main.yml b/ansible/roles/packages/meta/main.yml
new file mode 100644
index 00000000..2dec6cc9
--- /dev/null
+++ b/ansible/roles/packages/meta/main.yml
@@ -0,0 +1,2 @@
+dependencies:
+  - rpm-repos
diff --git a/ansible/roles/rpm-repos/defaults/main.yml b/ansible/roles/rpm-repos/defaults/main.yml
new file mode 100644
index 00000000..3a3dd0b4
--- /dev/null
+++ b/ansible/roles/rpm-repos/defaults/main.yml
@@ -0,0 +1,10 @@
+---
+rpm_repos:
+  - key: scripts
+    name: Scripts
+    baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}/
+    enabled: yes
+  - key: scripts-testing
+    name: Scripts Testing
+    baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}-testing/
+    enabled: "{{ enable_testing_repo | default(False) }}"
diff --git a/ansible/roles/rpm-repos/tasks/main.yml b/ansible/roles/rpm-repos/tasks/main.yml
new file mode 100644
index 00000000..c6d3fc25
--- /dev/null
+++ b/ansible/roles/rpm-repos/tasks/main.yml
@@ -0,0 +1,25 @@
+---
+- name: Configure scripts RPM repos
+  copy:
+    dest: /etc/yum.repos.d/scripts.repo
+    content: |
+      {% for repo in rpm_repos %}
+      [{{ repo.key }}]
+      name={{ repo.name }}
+      baseurl={{ repo.baseurl }}
+      enabled={{ 1 if repo.enabled else 0 }}
+      gpgcheck=0
+      {% endfor %}
+- name: Configure dnf.conf
+  ini_file:
+    path: /etc/dnf/dnf.conf
+    section: main
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+  loop:
+    - option: installonly_limit
+      value: 0
+    - option: installonlypkgs
+      value: kernel kernel-devel kernel-modules kmod-openafs ghc-cgi ghc-cgi-devel
+    - option: excludepkgs
+      value: fedora-obsolete-packages php-fpm nfs-utils
diff --git a/ansible/scripts-proxy.yml b/ansible/scripts-proxy.yml
index 64536aa9..dd31c308 100644
--- a/ansible/scripts-proxy.yml
+++ b/ansible/scripts-proxy.yml
@@ -13,6 +13,8 @@
       dest: /etc/munin/plugin-conf.d/
       src: files/conntrack
   roles:
+    - role: rpm-repos
+      tags: [always]
     - ansible-config-me
     - k5login
     - syslog-client
diff --git a/ansible/scripts-real.yml b/ansible/scripts-real.yml
index eb1d81db..895b0322 100644
--- a/ansible/scripts-real.yml
+++ b/ansible/scripts-real.yml
@@ -14,15 +14,6 @@
   vars:
     ldap_server: "{{ use_local_ldap | default(True) | ternary('ldapi://%2fvar%2frun%2fslapd-scripts.socket/', 'ldap://scripts-ldap.mit.edu/') }}"
     ldap_server_tcp: "{{ use_local_ldap | default(True) | ternary('ldap://127.0.0.1/', 'ldap://scripts-ldap.mit.edu/') }}"
-    rpm_repos:
-      - key: scripts
-        name: Scripts
-        baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}/
-        enabled: yes
-      - key: scripts-testing
-        name: Scripts Testing
-        baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}-testing/
-        enabled: "{{ enable_testing_repo | default(False) }}"
     preferred_mta: postfix
   pre_tasks:
     - name: Block Ansible on legacy realservers
@@ -39,33 +30,9 @@
         state: absent
     - include_role:
         name: real-network
-    - name: Configure dnf
-      block:
-        - name: Configure scripts RPM repos
-          copy:
-            dest: /etc/yum.repos.d/scripts.repo
-            content: |
-              {% for repo in rpm_repos %}
-              [{{ repo.key }}]
-              name={{ repo.name }}
-              baseurl={{ repo.baseurl }}
-              enabled={{ 1 if repo.enabled else 0 }}
-              gpgcheck=0
-              {% endfor %}
-        - name: Configure dnf.conf
-          ini_file:
-            path: /etc/dnf/dnf.conf
-            section: main
-            option: "{{ item.option }}"
-            value: "{{ item.value }}"
-          loop:
-            - option: installonly_limit
-              value: 0
-            - option: installonlypkgs
-              value: kernel kernel-devel kernel-modules kmod-openafs ghc-cgi ghc-cgi-devel
-            - option: excludepkgs
-              value: fedora-obsolete-packages php-fpm nfs-utils
   roles:
+    - role: rpm-repos
+      tags: [always]
     - role: packages
       tags: [always]
     - role: syslog-client

From 9b92c070d8399c32c436fa5928bbf3b86345e381 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Sun, 8 Mar 2020 19:36:59 -0400
Subject: [PATCH 06/11] Install mock build environment on proxy servers

---
 ansible/scripts-proxy.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ansible/scripts-proxy.yml b/ansible/scripts-proxy.yml
index dd31c308..73b3d9c0 100644
--- a/ansible/scripts-proxy.yml
+++ b/ansible/scripts-proxy.yml
@@ -19,6 +19,7 @@
     - k5login
     - syslog-client
     - root-aliases
+    - mock
     - proxy-munin-node
     - nrpe
     - dnf-automatic

From 5c98021de6822d817fc9d8e7d4bb35b513d30f0d Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Mon, 9 Mar 2020 00:20:04 -0400
Subject: [PATCH 07/11] Package scripts-proxy for Fedora

---
 .../scripts-proxy/scripts-proxy.service       | 20 +++++++
 server/fedora/Makefile                        |  2 +-
 server/fedora/specs/scripts-proxy.spec        | 56 +++++++++++++++++++
 3 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 server/common/oursrc/scripts-proxy/scripts-proxy.service
 create mode 100644 server/fedora/specs/scripts-proxy.spec

diff --git a/server/common/oursrc/scripts-proxy/scripts-proxy.service b/server/common/oursrc/scripts-proxy/scripts-proxy.service
new file mode 100644
index 00000000..13ea4965
--- /dev/null
+++ b/server/common/oursrc/scripts-proxy/scripts-proxy.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=Scripts HTTP/SNI proxy
+After=nss-lookup.target
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Restart=on-failure
+
+# Run as nobody but grant the ability to bind 80/443/444
+User=nobody
+Group=nobody
+AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN
+
+EnvironmentFile=-/etc/sysconfig/scripts-proxy
+
+ExecStart=/usr/sbin/scripts-proxy $OPTIONS
+
+[Install]
+WantedBy=multi-user.target
diff --git a/server/fedora/Makefile b/server/fedora/Makefile
index 85f52e97..34ff438d 100644
--- a/server/fedora/Makefile
+++ b/server/fedora/Makefile
@@ -24,7 +24,7 @@ gems		= pony:1.8 fcgi:0.9.2.1
 upstream_gems	= rubygem-pony rubygem-fcgi
 upstream_eggs   = python-authkit
 upstream	= openafs hesiod $(upstream_yum) $(upstream_gems) $(upstream_eggs) moira zephyr zephyr.i686 python-zephyr python-afs python-moira python-hesiod athena-aclocal discuss fuse-python
-oursrc		= execsys tokensys accountadm httpdmods logview nss_nonlocal nss_nonlocal.i686 athrun php_scripts scripts-wizard scripts-base scripts-static-cat fuse-better-mousetrapfs scripts-munin-plugins scripts-krb5-localauth shackle
+oursrc		= execsys tokensys accountadm httpdmods logview nss_nonlocal nss_nonlocal.i686 athrun php_scripts scripts-wizard scripts-base scripts-static-cat fuse-better-mousetrapfs scripts-munin-plugins scripts-krb5-localauth shackle scripts-proxy
 allsrc		= $(upstream) $(oursrc)
 oursrcdir	= ${PWD}/../common/oursrc
 patches		= ${PWD}/../common/patches
diff --git a/server/fedora/specs/scripts-proxy.spec b/server/fedora/specs/scripts-proxy.spec
new file mode 100644
index 00000000..faf247a5
--- /dev/null
+++ b/server/fedora/specs/scripts-proxy.spec
@@ -0,0 +1,56 @@
+# https://fedoraproject.org/wiki/PackagingDrafts/Go
+
+Name:           scripts-proxy
+Version:        0.0
+Release:        0.%{scriptsversion}%{?dist}
+Summary:        HTTP/SNI proxy for scripts.mit.edu
+
+License:        GPL+
+URL:            http://scripts.mit.edu/
+Source0:        %{name}.tar.gz
+
+BuildRequires:  (systemd-rpm-macros or systemd < 240)
+BuildRequires:  go-rpm-macros
+BuildRequires:  golang >= 1.6
+
+%description
+scripts-proxy proxies HTTP and HTTPS+SNI requests to backend servers
+based on LDAP.
+
+%global goipath github.com/mit-scripts/scripts/server/common/oursrc/scripts-proxy
+%global extractdir %{name}
+
+%gometa
+%gopkg
+
+%prep
+%goprep -k
+
+%build
+%gobuild -o %{gobuilddir}/bin/scripts-proxy %{goipath}
+
+%install
+%gopkginstall
+install -d %{buildroot}%{_sbindir}
+install -p -m 0755 %{gobuilddir}/bin/scripts-proxy %{buildroot}%{_sbindir}/scripts-proxy
+install -d %{buildroot}%{_unitdir}
+install -p -m 0644 ./scripts-proxy.service %{buildroot}%{_unitdir}/scripts-proxy.service
+
+%files
+%defattr(0644, root, root)
+%{_unitdir}/scripts-proxy.service
+%attr(755,root,root) %{_sbindir}/scripts-proxy
+%gopkgfiles
+
+%post
+%systemd_post scripts-proxy.service
+
+%preun
+%systemd_preun scripts-proxy.service
+
+%postun
+%systemd_postun_with_restart scripts-proxy.service
+
+%changelog
+* Sun Mar 8 2020 Quentin Smith - 0.0-0
+- Initial packaging for scripts-proxy

From 33dc19adbe1be2bfa8b0c4464d1d985ae7492ca3 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Mon, 9 Mar 2020 02:50:30 -0400
Subject: [PATCH 08/11] Add dependencies for scripts-proxy

---
 .gitmodules                                                | 3 +++
 server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy | 1 +
 server/fedora/specs/scripts-proxy.spec                     | 1 +
 3 files changed, 5 insertions(+)
 create mode 160000 server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy

diff --git a/.gitmodules b/.gitmodules
index 77a1d15e..a3efd487 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -13,3 +13,6 @@
 [submodule "ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy"]
 	path = ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy
 	url = https://github.com/facebook/gnlpy.git
+[submodule "server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy"]
+	path = server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy
+	url = https://github.com/inetaf/tcpproxy.git
diff --git a/server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy b/server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy
new file mode 160000
index 00000000..b6bb9b5b
--- /dev/null
+++ b/server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy
@@ -0,0 +1 @@
+Subproject commit b6bb9b5b82524122bcf27291ede32d1517a14ab8
diff --git a/server/fedora/specs/scripts-proxy.spec b/server/fedora/specs/scripts-proxy.spec
index faf247a5..88f28f98 100644
--- a/server/fedora/specs/scripts-proxy.spec
+++ b/server/fedora/specs/scripts-proxy.spec
@@ -12,6 +12,7 @@ Source0:        %{name}.tar.gz
 BuildRequires:  (systemd-rpm-macros or systemd < 240)
 BuildRequires:  go-rpm-macros
 BuildRequires:  golang >= 1.6
+BuildRequires:  golang(gopkg.in/ldap.v3)
 
 %description
 scripts-proxy proxies HTTP and HTTPS+SNI requests to backend servers

From e0f8cfaee0576ee920bd7484c72c51d6dcfa6fa9 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Mon, 9 Mar 2020 03:44:33 -0400
Subject: [PATCH 09/11] Ansibilize scripts-proxy deployment

---
 .../proxy-scripts-proxy/handlers/main.yml     |  5 +++
 .../roles/proxy-scripts-proxy/tasks/main.yml  | 39 +++++++++++++++++++
 ansible/scripts-proxy.yml                     |  3 +-
 3 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 ansible/roles/proxy-scripts-proxy/handlers/main.yml
 create mode 100644 ansible/roles/proxy-scripts-proxy/tasks/main.yml

diff --git a/ansible/roles/proxy-scripts-proxy/handlers/main.yml b/ansible/roles/proxy-scripts-proxy/handlers/main.yml
new file mode 100644
index 00000000..66f30f7b
--- /dev/null
+++ b/ansible/roles/proxy-scripts-proxy/handlers/main.yml
@@ -0,0 +1,5 @@
+- name: restart scripts-proxy
+  service:
+    name: scripts-proxy
+    state: restarted
+    enabled: yes
diff --git a/ansible/roles/proxy-scripts-proxy/tasks/main.yml b/ansible/roles/proxy-scripts-proxy/tasks/main.yml
new file mode 100644
index 00000000..2bbdabd5
--- /dev/null
+++ b/ansible/roles/proxy-scripts-proxy/tasks/main.yml
@@ -0,0 +1,39 @@
+---
+- name: Disable haproxy-related services
+  service:
+    name: "{{ item }}"
+    state: stopped
+    enabled: no
+  failed_when: no
+  loop:
+    - named-scripts-proxy
+    - haproxy
+- name: Remove haproxy-related configuration
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/named.scripts-proxy.conf
+    - /etc/systemd/system/named-scripts-proxy.service
+    - /etc/haproxy/haproxy.cfg
+    - /etc/rsyslog.d/haproxy.conf
+    - /etc/systemd/system/haproxy.service.d/10-scripts.conf
+    - /usr/local/bin/hatop
+- name: Remove haproxy-related packages
+  dnf:
+    name:
+      - bind
+      - bind-dlz-ldap
+      - haproxy
+    state: absent
+- name: Install scripts-proxy
+  dnf:
+    name:
+      - scripts-proxy
+    state: present
+- name: Configure scripts-proxy
+  copy:
+    dest: /etc/sysconfig/scripts-proxy
+    content: |
+      OPTIONS="-ldap_servers={{ groups['scripts-ldap'] | join(':389,') }}:389"
+  notify: restart scripts-proxy
diff --git a/ansible/scripts-proxy.yml b/ansible/scripts-proxy.yml
index 73b3d9c0..12842b3a 100644
--- a/ansible/scripts-proxy.yml
+++ b/ansible/scripts-proxy.yml
@@ -23,8 +23,7 @@
     - proxy-munin-node
     - nrpe
     - dnf-automatic
-    - proxy-dns
-    - proxy-haproxy
+    - proxy-scripts-proxy
     - proxy-logrotate
   tasks:
     - package:

From 31d1588a380424a3c11a9eb541024dca0e63fa06 Mon Sep 17 00:00:00 2001
From: Quentin Smith <quentin@mit.edu>
Date: Tue, 10 Mar 2020 03:20:51 -0400
Subject: [PATCH 10/11] Add a special vhost for health checking that uses
 /etc/nolvs

Also adds another special vhost for debugging/monitoring
---
 server/common/oursrc/scripts-proxy/main.go    | 27 ++++++--
 server/common/oursrc/scripts-proxy/statusz.go | 64 +++++++++++++++++++
 2 files changed, 85 insertions(+), 6 deletions(-)
 create mode 100644 server/common/oursrc/scripts-proxy/statusz.go

diff --git a/server/common/oursrc/scripts-proxy/main.go b/server/common/oursrc/scripts-proxy/main.go
index 42d415e2..06a35706 100644
--- a/server/common/oursrc/scripts-proxy/main.go
+++ b/server/common/oursrc/scripts-proxy/main.go
@@ -28,8 +28,10 @@ func always(context.Context, string) bool {
 }
 
 type ldapTarget struct {
-	localPoolRange *net.IPNet
-	ldap           *ldap.Pool
+	localPoolRange    *net.IPNet
+	ldap              *ldap.Pool
+	statuszServer     *HijackedServer
+	unavailableServer *HijackedServer
 }
 
 // HandleConn is called by tcpproxy after receiving a connection and sniffing the host.
@@ -39,6 +41,17 @@ func (l *ldapTarget) HandleConn(netConn net.Conn) {
 	var pool string
 	var err error
 	if conn, ok := netConn.(*tcpproxy.Conn); ok {
+		switch conn.HostName {
+		case "proxy.scripts.scripts.mit.edu":
+			// Special handling for proxy.scripts.scripts.mit.edu
+			l.statuszServer.HandleConn(netConn)
+			return
+		case "heartbeat.scripts.scripts.mit.edu":
+			if nolvsPresent() {
+				l.unavailableServer.HandleConn(netConn)
+				return
+			}
+		}
 		pool, err = l.ldap.ResolvePool(conn.HostName)
 		if err != nil {
 			log.Printf("resolving %q: %v", conn.HostName, err)
@@ -50,9 +63,9 @@ func (l *ldapTarget) HandleConn(netConn net.Conn) {
 			log.Printf("resolving default pool: %v", err)
 		}
 	}
-	// TODO: Serve an error page? Forward to scripts-director?
+	// TODO: Forward to sorry server on director?
 	if pool == "" {
-		netConn.Close()
+		l.unavailableServer.HandleConn(netConn)
 		return
 	}
 	laddr := netConn.LocalAddr().(*net.TCPAddr)
@@ -91,8 +104,10 @@ func main() {
 
 	var p tcpproxy.Proxy
 	t := &ldapTarget{
-		localPoolRange: ipnet,
-		ldap:           ldapPool,
+		localPoolRange:    ipnet,
+		ldap:              ldapPool,
+		statuszServer:     NewHijackedServer(nil),
+		unavailableServer: NewUnavailableServer(),
 	}
 	for _, addr := range strings.Split(*httpAddrs, ",") {
 		p.AddHTTPHostMatchRoute(addr, always, t)
diff --git a/server/common/oursrc/scripts-proxy/statusz.go b/server/common/oursrc/scripts-proxy/statusz.go
new file mode 100644
index 00000000..d35cbe90
--- /dev/null
+++ b/server/common/oursrc/scripts-proxy/statusz.go
@@ -0,0 +1,64 @@
+package main
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+)
+
+func nolvsPresent() bool {
+	if _, err := os.Stat("/etc/nolvs"); err == nil {
+		return true
+	}
+	return false
+}
+
+// HijackedServer is an HTTP server that serves from connections hijacked from another server instead of a listening socket.
+// (See net/http.Hijacker for the opposite direction.)
+// Users can call HandleConn to handle any request(s) waiting on that net.Conn.
+type HijackedServer struct {
+	connCh chan net.Conn
+}
+
+// NewHijackedServer constructs a HijackedServer that handles incoming HTTP connections with handler.
+func NewHijackedServer(handler http.Handler) *HijackedServer {
+	s := &HijackedServer{
+		connCh: make(chan net.Conn),
+	}
+	go http.Serve(s, handler)
+	return s
+}
+
+// Accept is called by http.Server to acquire a new connection.
+func (s *HijackedServer) Accept() (net.Conn, error) {
+	c, ok := <-s.connCh
+	if ok {
+		return c, nil
+	}
+	return nil, errors.New("closed")
+}
+
+// Close shuts down the server.
+func (s *HijackedServer) Close() error {
+	close(s.connCh)
+	return nil
+}
+
+// Addr must be present to implement net.Listener
+func (s *HijackedServer) Addr() net.Addr {
+	return nil
+}
+
+// HandleConn instructs the server to take control of c and handle any HTTP request(s) that are waiting.
+func (s *HijackedServer) HandleConn(c net.Conn) {
+	s.connCh <- c
+}
+
+// NewUnavailableServer constructs a HijackedServer that serves 500 Service Unavailable for all requests.
+func NewUnavailableServer() *HijackedServer {
+	return NewHijackedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "0 proxy nolvs", http.StatusServiceUnavailable)
+	}))
+}

From b76722d40c58526fa81646765a033e24a2ddcf54 Mon Sep 17 00:00:00 2001
From: Miriam Rittenberg <mrittenb@mit.edu>
Date: Fri, 20 Mar 2020 02:38:13 -0400
Subject: [PATCH 11/11] Change heartbeat vhost and path.

Make ldirectord check proxy health on heartbeat.scripts.scripts.mit.edu
which triggers scripts-proxy to check nolvs.
---
 ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2 b/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2
index c27d1cdf..028afeb8 100644
--- a/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2
+++ b/ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2
@@ -40,8 +40,8 @@ virtual=2
 {% endfor %}
 	fallback=127.0.0.1 gate
 	service=http
-	request="heartbeat/http?codename=ANY"
-	virtualhost="scripts.mit.edu"
+	request="__scripts/heartbeat/http?codename=ANY"
+	virtualhost="heartbeat.scripts.scripts.mit.edu"
 	receive="1"
 	checktype=negotiate
 	checkport=80