resolve conflicts

Author: syywu

checks/block_lists.go

@@ -0,0 +1,128 @@
+package checks
+
+import (
+	"context"
+	"net"
+	"slices"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/xray-web/web-check-api/checks/clients/ip"
+)
+
+type dnsServer struct {
+	Name string
+	IP   string
+}
+
+var DNS_SERVERS = []dnsServer{
+	{Name: "AdGuard", IP: "176.103.130.130"},
+	{Name: "AdGuard Family", IP: "176.103.130.132"},
+	{Name: "CleanBrowsing Adult", IP: "185.228.168.10"},
+	{Name: "CleanBrowsing Family", IP: "185.228.168.168"},
+	{Name: "CleanBrowsing Security", IP: "185.228.168.9"},
+	{Name: "CloudFlare", IP: "1.1.1.1"},
+	{Name: "CloudFlare Family", IP: "1.1.1.3"},
+	{Name: "Comodo Secure", IP: "8.26.56.26"},
+	{Name: "Google DNS", IP: "8.8.8.8"},
+	{Name: "Neustar Family", IP: "156.154.70.3"},
+	{Name: "Neustar Protection", IP: "156.154.70.2"},
+	{Name: "Norton Family", IP: "199.85.126.20"},
+	{Name: "OpenDNS", IP: "208.67.222.222"},
+	{Name: "OpenDNS Family", IP: "208.67.222.123"},
+	{Name: "Quad9", IP: "9.9.9.9"},
+	{Name: "Yandex Family", IP: "77.88.8.7"},
+	{Name: "Yandex Safe", IP: "77.88.8.88"},
+}
+
+var knownBlockIPs = []string{
+	"146.112.61.106",
+	"185.228.168.10",
+	"8.26.56.26",
+	"9.9.9.9",
+	"208.69.38.170",
+	"208.69.39.170",
+	"208.67.222.222",
+	"208.67.222.123",
+	"199.85.126.10",
+	"199.85.126.20",
+	"156.154.70.22",
+	"77.88.8.7",
+	"77.88.8.8",
+	"::1",
+	"2a02:6b8::feed:0ff",
+	"2a02:6b8::feed:bad",
+	"2a02:6b8::feed:a11",
+	"2620:119:35::35",
+	"2620:119:53::53",
+	"2606:4700:4700::1111",
+	"2606:4700:4700::1001",
+	"2001:4860:4860::8888",
+	"2a0d:2a00:1::",
+	"2a0d:2a00:2::",
+}
+
+type Blocklist struct {
+	Server    string `json:"server"`
+	ServerIP  string `json:"serverIp"`
+	IsBlocked bool   `json:"isBlocked"`
+}
+
+type BlockList struct {
+	lookup ip.DNSLookup
+}
+
+func NewBlockList(lookup ip.DNSLookup) *BlockList {
+	return &BlockList{lookup: lookup}
+}
+
+func (b *BlockList) domainBlocked(ctx context.Context, domain, serverIP string) bool {
+	ips, err := b.lookup.DNSLookupIP(ctx, "ip4", domain, serverIP)
+	if err != nil {
+		// if there's an error, consider it not blocked
+		// TODO: return more detailed errors for each server
+		return false
+	}
+
+	return slices.ContainsFunc(ips, func(ip net.IP) bool {
+		return slices.Contains(knownBlockIPs, ip.String())
+	})
+}
+
+func (b *BlockList) BlockedServers(ctx context.Context, domain string) []Blocklist {
+	var lock sync.Mutex
+	var wg sync.WaitGroup
+	limit := make(chan struct{}, 5)
+
+	var results []Blocklist
+
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	for _, server := range DNS_SERVERS {
+		wg.Add(1)
+		go func(server dnsServer) {
+			limit <- struct{}{}
+			defer func() {
+				<-limit
+				wg.Done()
+			}()
+
+			isBlocked := b.domainBlocked(ctx, domain, server.IP)
+			lock.Lock()
+			defer lock.Unlock()
+			results = append(results, Blocklist{
+				Server:    server.Name,
+				ServerIP:  server.IP,
+				IsBlocked: isBlocked,
+			})
+		}(server)
+	}
+	wg.Wait()
+
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].Server < results[j].Server
+	})
+	return results
+}

checks/block_lists_test.go

@@ -0,0 +1,24 @@
+package checks
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/xray-web/web-check-api/checks/clients/ip"
+)
+
+func TestBlockList(t *testing.T) {
+	t.Parallel()
+
+	t.Run("blocked IP", func(t *testing.T) {
+		t.Parallel()
+
+		dnsLookup := ip.DNSLookupFunc(func(ctx context.Context, network, host, dns string) ([]net.IP, error) {
+			return []net.IP{net.ParseIP("146.112.61.106")}, nil
+		})
+		list := NewBlockList(dnsLookup).BlockedServers(context.Background(), "example.com")
+		assert.Contains(t, list, Blocklist{Server: "AdGuard", ServerIP: "176.103.130.130", IsBlocked: true})
+	})
+}

checks/checks.go

@@ -4,30 +4,37 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/xray-web/web-check-api/checks/clients/ip"
 	"github.com/xray-web/web-check-api/checks/store/legacyrank"
 )
 
 type Checks struct {
-	Carbon     *Carbon
-	IpAddress  *Ip
-	LegacyRank *LegacyRank
-	Rank       *Rank
-	SocialTags *SocialTags
-	Tls        *Tls
-	Hsts       *Hsts
+	BlockList   *BlockList
+	Carbon      *Carbon
+	Headers     *Headers
+	Hsts        *Hsts
+	IpAddress   *Ip
+	LegacyRank  *LegacyRank
+	LinkedPages *LinkedPages
+	Rank        *Rank
+	SocialTags  *SocialTags
+	Tls         *Tls
 }
 
 func NewChecks() *Checks {
 	client := &http.Client{
 		Timeout: 5 * time.Second,
 	}
 	return &Checks{
-		Carbon:     NewCarbon(client),
-		IpAddress:  NewIp(NewNetIp()),
-		LegacyRank: NewLegacyRank(legacyrank.NewInMemoryStore()),
-		Rank:       NewRank(client),
-		SocialTags: NewSocialTags(client),
-		Tls:        NewTls(client),
-		Hsts:       NewHsts(client),
+		BlockList:   NewBlockList(&ip.NetDNSLookup{}),
+		Carbon:      NewCarbon(client),
+		Headers:     NewHeaders(client),
+		Hsts:        NewHsts(client),
+		IpAddress:   NewIp(NewNetIp()),
+		LegacyRank:  NewLegacyRank(legacyrank.NewInMemoryStore()),
+		LinkedPages: NewLinkedPages(client),
+		Rank:        NewRank(client),
+		SocialTags:  NewSocialTags(client),
+		Tls:         NewTls(client),
 	}
 }

checks/clients/ip/ip.go

@@ -0,0 +1,54 @@
+package ip
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+)
+
+type Lookup interface {
+	LookupIP(ctx context.Context, network string, host string) ([]net.IP, error)
+}
+
+type LookupFunc func(ctx context.Context, network string, host string) ([]net.IP, error)
+
+func (fn LookupFunc) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) {
+	return fn(ctx, network, host)
+}
+
+// NetLookup is a client for looking up IP addresses using a net.Resolver.
+type NetLookup struct{}
+
+func (l *NetLookup) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) {
+	netResolver := &net.Resolver{
+		PreferGo: true,
+	}
+	return netResolver.LookupIP(ctx, network, host)
+}
+
+type DNSLookup interface {
+	DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error)
+}
+
+type DNSLookupFunc func(ctx context.Context, network, host, dns string) ([]net.IP, error)
+
+func (fn DNSLookupFunc) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) {
+	return fn(ctx, network, host, dns)
+}
+
+// DNSLookup is a client for looking up IP addresses with a custom DNS server.
+type NetDNSLookup struct{}
+
+func (l *NetDNSLookup) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) {
+	netResolver := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			d := net.Dialer{
+				Timeout: 3 * time.Second,
+			}
+			return d.DialContext(ctx, network, fmt.Sprintf("%s:%d", dns, 53))
+		},
+	}
+	return netResolver.LookupIP(ctx, network, host)
+}

checks/headers.go

@@ -0,0 +1,29 @@
+package checks
+
+import (
+	"context"
+	"net/http"
+)
+
+type Headers struct {
+	client *http.Client
+}
+
+func NewHeaders(client *http.Client) *Headers {
+	return &Headers{client: client}
+}
+
+func (h *Headers) List(ctx context.Context, url string) (http.Header, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := h.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return resp.Header, nil
+}

checks/headers_test.go

@@ -0,0 +1,28 @@
+package checks
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/xray-web/web-check-api/testutils"
+)
+
+func TestList(t *testing.T) {
+	t.Parallel()
+
+	c := testutils.MockClient(&http.Response{
+		Header: http.Header{
+			"Cache-Control":    {"private", "max-age=0"},
+			"X-Xss-Protection": {"0"},
+		},
+	})
+	h := NewHeaders(c)
+
+	actual, err := h.List(context.Background(), "example.com")
+	assert.NoError(t, err)
+
+	assert.Equal(t, []string{"private", "max-age=0"}, actual["Cache-Control"])
+	assert.Equal(t, []string{"0"}, actual["X-Xss-Protection"])
+}