IPv6 support

Makefile

@@ -16,6 +16,9 @@ appengine:
 cloudflare:
 	go run ./cmd/ipcat/main.go -cloudflare
 
+tor:
+	go run ./cmd/ipcat/main.go -tor -csvfile tor.csv -statsfile=""
+
 test:
 	find . -name '*.go' | xargs gofmt -w -s
 	find . -name '*.go' | xargs goimports -w

appengine.go

@@ -30,22 +30,31 @@ func lookupSPFRecord(name string, f func(dir string) error) error {
 // DownloadAppEngine downloads and returns raw bytes of the Google App Engine ip
 // range list
 func DownloadAppEngine() ([]string, error) {
-	var ranges []string
-	if err := lookupSPFRecord("_cloud-netblocks.googleusercontent.com", func(dir string) error {
-		inc := strings.TrimPrefix(dir, "include:")
-		if dir == inc {
-			return nil
-		}
+	ranges := []string{}
+	domainList := []string{"_cloud-netblocks.googleusercontent.com"}
 
-		return lookupSPFRecord(inc, func(dir string) error {
+	for len(domainList) > 0 {
+		// Dequeue a domain from this list
+		var domain string
+		domain, domainList = domainList[0], domainList[1:]
+		err := lookupSPFRecord(domain, func(dir string) error {
+			// Enqueue domain from this record
+			if inc := strings.TrimPrefix(dir, "include:"); dir != inc {
+				domainList = append(domainList, inc)
+			}
+			// Add IPv4 range
 			if ip4 := strings.TrimPrefix(dir, "ip4:"); dir != ip4 {
 				ranges = append(ranges, ip4)
 			}
-
+			// Add IPv6 range
+			if ip6 := strings.TrimPrefix(dir, "ip6:"); dir != ip6 {
+				ranges = append(ranges, ip6)
+			}
 			return nil
 		})
-	}); err != nil {
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return ranges, nil

aws.go

@@ -13,16 +13,18 @@ var (
 
 // AWSPrefix is AWS prefix in their IP ranges file
 type AWSPrefix struct {
-	IPPrefix string `json:"ip_prefix"`
-	Region   string `json:"region"`
-	Service  string `json:"service"`
+	IPPrefix   string `json:"ip_prefix"`
+	IPv6Prefix string `json:"ipv6_prefix"`
+	Region     string `json:"region"`
+	Service    string `json:"service"`
 }
 
 // AWS is main record for AWS IP info
 type AWS struct {
-	SyncToken  string      `json:"syncToken"`
-	CreateDate string      `json:"createDate"`
-	Prefixes   []AWSPrefix `json:"prefixes"`
+	SyncToken    string      `json:"syncToken"`
+	CreateDate   string      `json:"createDate"`
+	Prefixes     []AWSPrefix `json:"prefixes"`
+	IPv6Prefixes []AWSPrefix `json:"ipv6_prefixes"`
 }
 
 // DownloadAWS downloads the latest AWS IP ranges list
@@ -60,9 +62,23 @@ func UpdateAWS(ipmap *IntervalSet, body []byte) error {
 	ipmap.DeleteByName(awsName)
 
 	// and add back
-	for _, rec := range aws.Prefixes {
-		if rec.Service == "EC2" {
-			err := ipmap.AddCIDR(rec.IPPrefix, awsName, awsURL)
+	for _, prefixList := range []*[]AWSPrefix{&aws.Prefixes, &aws.IPv6Prefixes} {
+		for _, rec := range *prefixList {
+			if rec.Service != "AMAZON" {
+				// Service is the subset of IP address ranges. Specify AMAZON to get
+				// all IP address ranges (for example, the ranges in the EC2 subset
+				// are also in the AMAZON subset). Note that some IP address ranges
+				// are only in the AMAZON subset.
+				// <https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html>
+				continue
+			}
+
+			prefix := rec.IPPrefix
+			if prefix == "" {
+				prefix = rec.IPv6Prefix
+			}
+
+			err := ipmap.AddCIDR(prefix, awsName, awsURL)
 			if err != nil {
 				return err
 			}

aws_test.go

@@ -23,6 +23,16 @@ func TestAWS(t *testing.T) {
 				  "ip_prefix": "13.54.0.0/15",
 				  "region": "ap-southeast-2",
 				  "service": "EC2"
+				},
+				{
+				  "ipv6_prefix": "2a05:d016::/36",
+				  "region": "eu-north-1",
+				  "service": "AMAZON"
+				},
+				{
+				  "ipv6_prefix": "2a05:d018::/36",
+				  "region": "eu-west-1",
+				  "service": "EC2"
 				}
 			]
 		}`)
@@ -39,19 +49,27 @@ func TestAWS(t *testing.T) {
 	if err != nil {
 		t.Fatalf("UpdateAWS error: %v", err)
 	}
-	// UpdateAWS should only add EC2 IP ranges
-	rec, err := ipset.Contains("216.182.224.0")
-	if err != nil {
-		t.Fatalf("ipset.Contains(%q) error: %v", "216.182.224.0", err)
-	}
-	if rec != nil {
-		t.Errorf("ipset.Contains(%q) rec = %v, want nil", "216.182.224.0", rec)
-	}
-	rec, err = ipset.Contains("13.54.0.1")
-	if err != nil {
-		t.Fatalf("ipset.Contains(%q) error: %v", "13.54.0.0", err)
+
+	tests := []struct {
+		IP       string
+		Contains bool
+	}{
+		{IP: "216.182.224.0", Contains: true},
+		{IP: "2a05:d016::20", Contains: true},
+
+		// Only AMAZON services will be contained (superset of all)
+		{IP: "13.54.0.1", Contains: false},
+		{IP: "2a05:d018::20", Contains: false},
 	}
-	if rec == nil {
-		t.Errorf("ipset.Contains(%q) rec = nil, want exists", "13.54.0.0")
+
+	for _, test := range tests {
+		record, err := ipset.Contains(test.IP)
+		if err != nil {
+			t.Fatalf("ipset.Contains(%q) error: %v", test.IP, err)
+		}
+		if (record != nil) != test.Contains {
+			t.Errorf("ipset.Contains(%q) rec = %v, want rec != nil to be %v",
+				test.IP, record, test.Contains)
+		}
 	}
 }

azure.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"regexp"
 )
@@ -58,7 +57,6 @@ func DownloadAzure() ([]byte, error) {
 		return nil, fmt.Errorf("failed to find public IPs url during retry: %s", err)
 	}
 
-	log.Printf("Attempting ip range download with url %s...", url)
 	resp, err := http.Get(url)
 	if err != nil {
 		return nil, err

cloudflare.go

@@ -3,28 +3,44 @@ package ipcat
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 )
 
 var (
-	cloudflareDownload = "https://www.cloudflare.com/ips-v4"
+	cloudflareDownload = []string{
+		"https://www.cloudflare.com/ips-v4",
+		"https://www.cloudflare.com/ips-v6",
+	}
 )
 
 // DownloadCloudflare downloads the latest Cloudflare IP ranges list
 func DownloadCloudflare() ([]byte, error) {
-	resp, err := http.Get(cloudflareDownload)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("Failed to download Cloudflare ranges: status code %s", resp.Status)
+	readers := make([]io.Reader, 0, len(cloudflareDownload))
+
+	defer func() {
+		for _, reader := range readers {
+			reader.(io.ReadCloser).Close()
+		}
+	}()
+
+	for _, uri := range cloudflareDownload {
+		resp, err := http.Get(uri)
+		if err != nil {
+			return nil, err
+		}
+		readers = append(readers, resp.Body)
+
+		if resp.StatusCode != 200 {
+			return nil, fmt.Errorf("Failed to download Cloudflare ranges: status code %s", resp.Status)
+		}
 	}
-	body, err := ioutil.ReadAll(resp.Body)
+
+	body, err := ioutil.ReadAll(io.MultiReader(readers...))
 	if err != nil {
 		return nil, err
 	}
-	resp.Body.Close()
 
 	return bytes.TrimSpace(body), nil
 }

cmd/ipcat/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"net"
 	"os"
 	"strings"
 
@@ -16,6 +17,7 @@ func main() {
 	updateAzure := flag.Bool("azure", false, "update Azure records")
 	updateAppEngine := flag.Bool("appengine", false, "update AppEngine (Google App Engine) records")
 	updateCloudflare := flag.Bool("cloudflare", false, "update Cloudflare records")
+	updateTor := flag.Bool("tor", false, "update Tor records")
 	datafile := flag.String("csvfile", "datacenters.csv", "read/write from this file")
 	statsfile := flag.String("statsfile", "datacenters-stats.csv", "write statistics to this file")
 	addCIDR := flag.String("addcidr", "", "add this CIDR range to the data file [CIDR,name,url]")
@@ -41,7 +43,7 @@ func main() {
 		if rec == nil {
 			log.Fatalf("Not found: %s", *lookup)
 		}
-		fmt.Printf("[%s:%s] %s %s\n", rec.LeftDots, rec.RightDots, rec.Name, rec.URL)
+		fmt.Printf("[%s:%s] %s %s\n", net.IP(rec.Left[:]), net.IP(rec.Right[:]), rec.Name, rec.URL)
 		return
 	}
 
@@ -89,6 +91,13 @@ func main() {
 		}
 	}
 
+	if *updateTor {
+		err = ipcat.UpdateTor(&set)
+		if err != nil {
+			log.Fatalf("Unable to update Tor IP ranges: %s", err)
+		}
+	}
+
 	if *addCIDR != "" {
 		t := strings.Split(*addCIDR, ",")
 		if len(t) != 3 {

datacenters-stats.csv

@@ -1,10 +1,10 @@
 Datacenter Name, Total IPs
-Amazon AWS,29459456
+Amazon AWS,1542202570030687795110271325976
+Cloudflare Inc,1109194275199700726309617091584
+Google App Engine,9903520314283042199196141312
 Microsoft Azure,13024127
 Akamai,8145728
-Google App Engine,2431744
 SoftLayer,1903104
-Cloudflare Inc,1786880
 "ThePlanet.com Internet Services, Inc.",1638656
 PEER 1,1462288
 Ubiquity Server Solutions,1441792