diff --git a/Makefile b/Makefile index 06d9252..22c8015 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,8 @@ GIT_HASH := $(shell git rev-parse HEAD) BUILD_FLAGS := -ldflags "-X main.git_date=$(GIT_DATE) -X main.git_hash=$(GIT_HASH)" PLATFORMS := linux/amd64 linux/386 linux/arm darwin/amd64 windows/amd64 windows/386 openbsd/amd64 -SOURCES := certgraph.go google_ct.go +SOURCES := $(shell find . -maxdepth 1 -type f -name "*.go") +ALL_SOURCES = $(shell find . -type f -name '*.go' -not -path "./vendor/*") temp = $(subst /, ,$@) os = $(word 1, $(temp)) @@ -15,7 +16,7 @@ all: certgraph release: $(PLATFORMS) -certgraph: $(SOURCES) +certgraph: $(SOURCES) $(ALL_SOURCES) go build $(BUILD_FLAGS) -o $@ $(SOURCES) $(PLATFORMS): $(SOURCES) diff --git a/certgraph.go b/certgraph.go index 0e75dfc..958649e 100644 --- a/certgraph.go +++ b/certgraph.go @@ -1,7 +1,6 @@ package main import ( - "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/json" @@ -12,13 +11,14 @@ import ( "net/smtp" "os" "path" - "sort" "strconv" "strings" "sync" - "syscall" "time" - "regexp" + + "github.com/lanrat/certgraph/ct/google" + "github.com/lanrat/certgraph/graph" + "github.com/lanrat/certgraph/status" ) /* TODO @@ -28,7 +28,7 @@ follow http redirects // vars var conf = &tls.Config{InsecureSkipVerify: true} var markedDomains = make(map[string]bool) // TODO move to graph? -var graph = NewCertGraph() +var dgraph = graph.NewCertGraph() var depth uint var save bool var git_date = "none" @@ -50,283 +50,6 @@ var tls_connect bool var ver bool var skipCDN bool -// domain node conection status -type domainStatus int - -const ( - UNKNOWN = iota - GOOD = iota - TIMEOUT = iota - NO_HOST = iota - REFUSED = iota - ERROR = iota -) - -type fingerprint [sha256.Size]byte - -// print fingerprint as hex -func (fp fingerprint) HexString() string { - return fmt.Sprintf("%X", fp) -} - -// return domain status for printing -func (status domainStatus) String() string { - switch status { - case UNKNOWN: - return "Unknown" - case GOOD: - return "Good" - case TIMEOUT: - return "Timeout" - case NO_HOST: - return "No Host" - case REFUSED: - return "Refused" - case ERROR: - return "Error" - } - return "?" -} - -// structure to store a domain and its edges -type DomainNode struct { - Domain string - Depth uint - VisitedCert fingerprint - CTCerts []fingerprint - Status domainStatus - Root bool -} - -// constructor for DomainNode, converts domain to directDomain -func NewDomainNode(domain string, depth uint) *DomainNode { - node := new(DomainNode) - node.Domain = directDomain(domain) - node.Depth = depth - node.CTCerts = make([]fingerprint, 0, 0) - return node -} - -// get the string representation of a node -func (d *DomainNode) String() string { - if details { - cert := "" - if d.Status == GOOD { - cert = d.VisitedCert.HexString() - } - return fmt.Sprintf("%s\t%d\t%s\t%s", d.Domain, d.Depth, d.Status, cert) - } - return fmt.Sprintf("%s", d.Domain) -} - -func (d *DomainNode) AddCTFingerprint(fp fingerprint) { - d.CTCerts = append(d.CTCerts, fp) -} - -func (d *DomainNode) ToMap() map[string]string { - m := make(map[string]string) - m["type"] = "domain" - m["id"] = d.Domain - m["status"] = d.Status.String() - m["root"] = strconv.FormatBool(d.Root) - m["depth"] = strconv.FormatUint(uint64(d.Depth), 10) - return m -} - -type CertNode struct { - Fingerprint fingerprint - Domains []string - CT bool - HTTP bool - CDNCert bool -} - -func (c *CertNode) String() string { - //TODO Currently unused.. - ct := "" - if c.CT { - ct = "CT" - } - http := "" - if c.HTTP { - http = "HTTP" - } - return fmt.Sprintf("%s\t%s %s\t%v", c.Fingerprint.HexString(), http, ct, c.Domains) -} - -func (c *CertNode) ToMap() map[string]string { - m := make(map[string]string) - m["type"] = "certificate" - m["id"] = c.Fingerprint.HexString() - s := "" - if c.HTTP { - s = "HTTP " - } - if c.CT { - s = s + "CT" - } - m["status"] = strings.TrimSuffix(s, " ") - return m -} - -func NewCertNode(cert *x509.Certificate) *CertNode { - certnode := new(CertNode) - - // generate Fingerprint - certnode.Fingerprint = sha256.Sum256(cert.Raw) - - // domains - // used to ensure uniq entries in domains array - domainMap := make(map[string]bool) - // add the CommonName just to be safe - cn := strings.ToLower(cert.Subject.CommonName) - if len(cn) > 0 { - domainMap[cn] = true - } - for _, domain := range cert.DNSNames { - if len(domain) > 0 { - domain = strings.ToLower(domain) - domainMap[domain] = true - } - } - for domain := range domainMap { - certnode.Domains = append(certnode.Domains, domain) - } - sort.Strings(certnode.Domains) - - return certnode -} - -// main graph storage engine -type CertGraph struct { - domains sync.Map - certs sync.Map -} - -func NewCertGraph() *CertGraph { - graph := new(CertGraph) - return graph -} - -func (graph *CertGraph) LoadOrStoreCert(nodein *CertNode) (*CertNode, bool) { - nodeout, ok := graph.certs.LoadOrStore(nodein.Fingerprint, nodein) - return nodeout.(*CertNode), ok -} - -// TODO check for existing? -func (graph *CertGraph) AddCert(certnode *CertNode) { - graph.certs.Store(certnode.Fingerprint, certnode) -} - -// TODO check for existing? -func (graph *CertGraph) AddDomain(domainnode *DomainNode) { - graph.domains.Store(domainnode.Domain, domainnode) -} - -func (graph *CertGraph) GetCert(fp fingerprint) (*CertNode, bool) { - node, ok := graph.certs.Load(fp) - if ok { - return node.(*CertNode), true - } - return nil, false -} - -func (graph *CertGraph) GetDomain(domain string) (*DomainNode, bool) { - node, ok := graph.domains.Load(domain) - if ok { - return node.(*DomainNode), true - } - return nil, false -} - -func (graph *CertGraph) GetDomainNeighbors(domain string) []string { - neighbors := make(map[string]bool) - - //domain = directDomain(domain) - node, ok := graph.domains.Load(domain) - if ok { - domainnode := node.(*DomainNode) - // visited cert neighbors - node, ok := graph.certs.Load(domainnode.VisitedCert) - if ok { - certnode := node.(*CertNode) - if skipCDN && certnode.CDNCert { - v(domain, "-> CDN CERT") - } else { - for _, neighbor := range certnode.Domains { - neighbors[neighbor] = true - v(domain, "- CERT ->", neighbor) - } - - } - } - - // CT neighbors - for _, fp := range domainnode.CTCerts { - node, ok := graph.certs.Load(fp) - if ok { - certnode := node.(*CertNode) - if skipCDN && certnode.CDNCert { - v(domain, "-> CDN CERT") - }else { - for _, neighbor := range certnode.Domains { - neighbors[neighbor] = true - v(domain, "-- CT -->", neighbor) - } - } - - } - } - } - - //exclude domain from own neighbors list - neighbors[domain] = false - - // convert map to array - neighbor_list := make([]string, 0, len(neighbors)) - for key := range neighbors { - if neighbors[key] { - neighbor_list = append(neighbor_list, key) - } - } - return neighbor_list -} - -func (graph *CertGraph) GenerateMap() map[string]interface{} { - m := make(map[string]interface{}) - nodes := make([]map[string]string, 0, 2*len(markedDomains)) - links := make([]map[string]string, 0, 2*len(markedDomains)) - - - // add all domain nodes - graph.domains.Range(func(key, value interface{}) bool { - domainnode := value.(*DomainNode) - nodes = append(nodes, domainnode.ToMap()) - if domainnode.Status == GOOD { - links = append(links, map[string]string{"source": domainnode.Domain, "target": domainnode.VisitedCert.HexString(), "type": "uses"}) - } - return true - }) - - // add all cert nodes - graph.certs.Range(func(key, value interface{}) bool { - certnode := value.(*CertNode) - nodes = append(nodes, certnode.ToMap()) - for _, domain := range certnode.Domains { - domain := directDomain(domain) - _, ok := graph.GetDomain(domain) - if ok { - links = append(links, map[string]string{"source": certnode.Fingerprint.HexString(), "target": domain, "type": "sans"}) - }// TODO do something with alt-names that are not in graph like wildcards - } - return true - }) - - m["nodes"] = nodes - m["links"] = links - return m -} - func generateGraphMetadata() map[string]interface{} { data := make(map[string]interface{}) data["version"] = version() @@ -351,7 +74,6 @@ func version() string { } - func main() { var notls bool flag.BoolVar(&ver, "version", false, "print version and exit") @@ -394,6 +116,10 @@ func main() { flag.Usage() return } + + // set verbose loggin + graph.Verbose = verbose + port = strconv.FormatUint(uint64(*portPtr), 10) timeout = time.Duration(*timeoutPtr) * time.Second startDomains := flag.Args() @@ -430,43 +156,9 @@ func v(a ...interface{}) { } } -// Check for errors, print if network related -func checkNetErr(err error) domainStatus { - if err == nil { - return GOOD - } else if netError, ok := err.(net.Error); ok && netError.Timeout() { - return TIMEOUT - } else { - switch t := err.(type) { - case *net.OpError: - if t.Op == "dial" { - return NO_HOST - } else if t.Op == "read" { - return REFUSED - } - case syscall.Errno: - if t == syscall.ECONNREFUSED { - return REFUSED - } - } - } - return ERROR -} - -// given a domain returns the non-wildcard version of that domain -func directDomain(domain string) string { - if len(domain) < 3 { - return domain - } - if domain[0:2] == "*." { - domain = domain[2:] - } - return domain -} - // prnts the graph as a json object func printJSONGraph() { - jsonGraph := graph.GenerateMap() + jsonGraph := dgraph.GenerateMap() jsonGraph["certgraph"] = generateGraphMetadata() j, err := json.MarshalIndent(jsonGraph, "", "\t") @@ -480,8 +172,8 @@ func printJSONGraph() { // perform Breadth first search to build the graph func BFS(roots []string) { var wg sync.WaitGroup - domainChan := make(chan *DomainNode, 5) // input queue - domainGraphChan := make(chan *DomainNode, 5) // output queue + domainChan := make(chan *graph.DomainNode, 5) // input queue + domainGraphChan := make(chan *graph.DomainNode, 5) // output queue // thread limit code threadPass := make(chan bool, parallel) @@ -492,7 +184,7 @@ func BFS(roots []string) { // put root nodes/domains into queue for _, root := range roots { wg.Add(1) - n := NewDomainNode(root, 0) + n := graph.NewDomainNode(root, 0) n.Root = true domainChan <- n } @@ -513,8 +205,8 @@ func BFS(roots []string) { if !markedDomains[domainNode.Domain] { markedDomains[domainNode.Domain] = true - graph.AddDomain(domainNode) - go func(domainNode *DomainNode) { + dgraph.AddDomain(domainNode) + go func(domainNode *graph.DomainNode) { defer wg.Done() // wait for pass <-threadPass @@ -524,9 +216,9 @@ func BFS(roots []string) { v("Visiting", domainNode.Depth, domainNode.Domain) BFSVisit(domainNode) // visit domainGraphChan <- domainNode - for _, neighbor := range graph.GetDomainNeighbors(domainNode.Domain) { + for _, neighbor := range dgraph.GetDomainNeighbors(domainNode.Domain, skipCDN) { wg.Add(1) - domainChan <- NewDomainNode(neighbor, domainNode.Depth+1) + domainChan <- graph.NewDomainNode(neighbor, domainNode.Depth+1) } }(domainNode) } else { @@ -542,8 +234,12 @@ func BFS(roots []string) { domainNode, more := <-domainGraphChan if more { if !printJSON { - fmt.Fprintln(os.Stdout, domainNode) - }else if details { + if details { + fmt.Fprintln(os.Stdout, domainNode) + } else { + fmt.Fprintln(os.Stdout, domainNode.Domain) + } + } else if details { fmt.Fprintln(os.Stderr, domainNode) } } else { @@ -559,7 +255,7 @@ func BFS(roots []string) { } // visit each node and get and set its neighbors -func BFSVisit(node *DomainNode) { +func BFSVisit(node *graph.DomainNode) { if tls_connect { visitTLS(node) } @@ -569,37 +265,30 @@ func BFSVisit(node *DomainNode) { } // visit nodes by searching certificate transparancy logs -func visitCT(node *DomainNode) { +func visitCT(node *graph.DomainNode) { // perform ct search // TODO do pagnation in multiple threads to not block on long searches - search_result, err := QueryDomain(node.Domain, false, include_ct_sub) + fingerprints, err := google.QueryDomain(node.Domain, false, include_ct_sub) if err != nil { v(err) return } // add cert nodes to graph - for _, result := range search_result { + for _, fp := range fingerprints { // add certnode to graph - fp := result.GetFingerprint() - certnode, exists := graph.GetCert(fp) + certnode, exists := dgraph.GetCert(fp) if !exists { // get cert details - cert_result, err := QueryHash(result.Hash) + certnode, err = google.GetCert(fp) if err != nil { v(err) continue } - certnode = new(CertNode) - certnode.Fingerprint = fp - certnode.Domains = cert_result.DnsNames - - certnode.CDNCert = CDNCert(cert_result.DnsNames) - - graph.AddCert(certnode) + dgraph.AddCert(certnode) } certnode.CT = true @@ -608,7 +297,7 @@ func visitCT(node *DomainNode) { } // visit nodes by connecting to them -func visitTLS(node *DomainNode) { +func visitTLS(node *graph.DomainNode) { var certs []*x509.Certificate node.Status, certs = getPeerCerts(node.Domain) if save && len(certs) > 0 { @@ -619,44 +308,25 @@ func visitTLS(node *DomainNode) { return } - // TODO iterate over all certs, needs to also update graph.GetDomainNeighbors() too - certnode := NewCertNode(certs[0]) - certnode.CDNCert = CDNCert(certnode.Domains) + // TODO iterate over all certs, needs to also update dgraph.GetDomainNeighbors() too + certnode := graph.NewCertNode(certs[0]) - certnode, _ = graph.LoadOrStoreCert(certnode) + certnode, _ = dgraph.LoadOrStoreCert(certnode) certnode.HTTP = true node.VisitedCert = certnode.Fingerprint } -// TODO make method on cert object? -func CDNCert(domains []string) bool { - for _, domain := range domains { - // cloudflair - matched, _ := regexp.MatchString("ssl[0-9]*\\.cloudflaressl\\.com", domain) - if matched { - return true - } - - if domain == "i.ssl.fastly.net" { - return true - } - // TODO include other CDNs - } - return false -} - - // gets the certificats found for a given domain -func getPeerCerts(host string) (dStatus domainStatus, certs []*x509.Certificate) { +func getPeerCerts(host string) (dStatus status.DomainStatus, certs []*x509.Certificate) { addr := net.JoinHostPort(host, port) dialer := &net.Dialer{Timeout: timeout} - dStatus = ERROR + dStatus = status.ERROR if starttls { conn, err := dialer.Dial("tcp", addr) - dStatus = checkNetErr(err) - if dStatus != GOOD { + dStatus = status.CheckNetErr(err) + if dStatus != status.GOOD { v(dStatus, host) return } @@ -675,17 +345,17 @@ func getPeerCerts(host string) (dStatus domainStatus, certs []*x509.Certificate) if !ok { return } - return GOOD, connState.PeerCertificates + return status.GOOD, connState.PeerCertificates } else { conn, err := tls.DialWithDialer(dialer, "tcp", addr, conf) - dStatus = checkNetErr(err) - if dStatus != GOOD { + dStatus = status.CheckNetErr(err) + if dStatus != status.GOOD { v(dStatus, host) return } conn.Close() connState := conn.ConnectionState() - return GOOD, connState.PeerCertificates + return status.GOOD, connState.PeerCertificates } } diff --git a/ct/.gitignore b/ct/.gitignore new file mode 100644 index 0000000..ef616ac --- /dev/null +++ b/ct/.gitignore @@ -0,0 +1 @@ +ct diff --git a/ct/Makefile b/ct/Makefile new file mode 100644 index 0000000..049fc7a --- /dev/null +++ b/ct/Makefile @@ -0,0 +1,15 @@ +SOURCES := $(shell find . -maxdepth 1 -type f -name "*.go") +ALL_SOURCES = $(shell find . -type f -name '*.go' -not -path "./vendor/*") + +all: ct + +ct: $(SOURCES) $(ALL_SOURCES) + go build -o $@ $(SOURCES) + +fmt: + gofmt -s -w -l . + +clean: + rm ct + +.PHONY: all fmt clean diff --git a/ct/google/google.go b/ct/google/google.go new file mode 100644 index 0000000..4428b41 --- /dev/null +++ b/ct/google/google.go @@ -0,0 +1,191 @@ +package google + +/* + * This file implements an unofficial API client for Google's + * Certificate Transparency search + * https://www.google.com/transparencyreport/https/ct/ + * + * As the API is unofficial and has been reverse engineered it may stop working + * at any time and comes with no guarantees. + */ + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/lanrat/certgraph/graph" +) + +// BASE URLs for Googl'e CT API +const searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com" +const searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF" +const certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF" +const MAX_PAGES = 50 // TODO: something better than hard-coding + +// global vars +var jsonClient = &http.Client{Timeout: 10 * time.Second} + +// gets JSON from url and parses it into target object +func getJsonP(url string, target interface{}) error { + r, err := jsonClient.Get(url) + if err != nil { + return err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return errors.New("Got non OK HTTP status:" + r.Status + "on URL: " + url) + } + + respData, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + + respData = respData[5:] // this removes the leading ")]}'" from the response + + return json.Unmarshal(respData, target) +} + +func QueryDomain(domain string, include_expired bool, include_subdomains bool) ([]graph.Fingerprint, error) { + results := make([]graph.Fingerprint, 0, 5) + + u, err := url.Parse(searchURL1) + if err != nil { + return results, err + } + + // get page 1 + q := u.Query() + q.Set("include_expired", strconv.FormatBool(include_expired)) + q.Set("include_subdomains", strconv.FormatBool(include_subdomains)) + q.Set("domain", domain) + u.RawQuery = q.Encode() + + var raw [][]interface{} + nextURL := u.String() + currentPage := float64(1) + + // TODO allow for selective pagnation + + // iterate over results + for len(nextURL) > 1 && currentPage <= MAX_PAGES { + err = getJsonP(nextURL, &raw) + if err != nil { + return results, err + } + + // simple corectness checks + if raw[0][0] != "https.ct.cdsr" { + return results, errors.New("Got Unexpected Query output: " + raw[0][0].(string)) + } + if len(raw[0]) != 4 { + // result not correct length, likely no results + //fmt.Println(raw[0]) + break + } + if len(raw[0][3].([]interface{})) != 5 { + // pageinfo result not correct length, likely no results + //fmt.Println(raw[0]) + break + } + + // pageInfo: [prevToken, nextToken, ? currPage, totalPages] + pageInfo := raw[0][3].([]interface{}) + currentPage = pageInfo[3].(float64) + + foundCerts := raw[0][1].([]interface{}) + for _, foundCert := range foundCerts { + certHash := foundCert.([]interface{})[5].(string) + certFP := graph.FingerprintFromB64(certHash) + results = append(results, certFP) + } + + //fmt.Println("Page:", pageInfo[3]) + + // create url or next page + nextURL = "" + if pageInfo[1] != nil { + u, err := url.Parse(searchURL2) + if err != nil { + return results, err + } + + // get page n + q := u.Query() + q.Set("p", pageInfo[1].(string)) + u.RawQuery = q.Encode() + nextURL = u.String() + } + } + + return results, nil +} + +func GetCert(fp graph.Fingerprint) (*graph.CertNode, error) { + certnode := new(graph.CertNode) + certnode.Fingerprint = fp + certnode.Domains = make([]string, 0, 5) + certnode.CT = true + + u, err := url.Parse(certURL) + if err != nil { + return certnode, err + } + + q := u.Query() + q.Set("hash", fp.B64Encode()) + u.RawQuery = q.Encode() + + var raw [][]interface{} + + err = getJsonP(u.String(), &raw) + if err != nil { + return certnode, err + } + + // simple corectness checks + if raw[0][0] != "https.ct.chr" { + return certnode, errors.New("Got Unexpected Cert output: " + raw[0][0].(string)) + } + if len(raw[0]) != 3 { + // result not correct length, likely no results + //fmt.Println(raw[0]) + return certnode, errors.New("Cert Does not exist! output: " + raw[0][0].(string)) + } + + certInfo := raw[0][1].([]interface{}) + domains := certInfo[7].([]interface{}) + + for _, domain := range domains { + certnode.Domains = append(certnode.Domains, domain.(string)) + } + + return certnode, nil +} + +// example function to use Google's CT API +func CTexample(domain string) error { + s, err := QueryDomain(domain, false, false) + if err != nil { + return err + } + + for i := range s { + fmt.Println(s[i].HexString(), " ", s[i].B64Encode()) + cert, err := GetCert(s[i]) + if err != nil { + return err + } + for j := range cert.Domains { + fmt.Println("\t", cert.Domains[j]) + } + } + + return nil +} diff --git a/ct/main.go b/ct/main.go new file mode 100644 index 0000000..de6cecc --- /dev/null +++ b/ct/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + "github.com/lanrat/certgraph/ct/google" + "os" +) + +func main() { + if len(os.Args) != 2 { + fmt.Printf("Usage: %s example.com\n", os.Args[0]) + return + } + + err := google.CTexample(os.Args[1]) + if err != nil { + fmt.Println(err) + } +} diff --git a/google_ct.go b/google_ct.go deleted file mode 100644 index ee6c861..0000000 --- a/google_ct.go +++ /dev/null @@ -1,201 +0,0 @@ -package main - -/* - * This file implements an unofficial API client for Google's - * Certificate Transparency search - * https://www.google.com/transparencyreport/https/ct/ - * - * As the API is unofficial and has been reverse engineered it may stop working - * at any time and comes with no guarantees. - */ - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "regexp" - "strconv" - "time" -) - -// BASE URLs for Googl'e CT API -const searchURL = "https://www.google.com/transparencyreport/jsonp/ct/search?incl_exp=false&incl_sub=false&c=jsonp&token=CAA=&domain=example.com" -const certURL = "https://www.google.com/transparencyreport/jsonp/ct/cert?c=jsonp&hash=AAA" - -// global vars -var jsonClient = &http.Client{Timeout: 10 * time.Second} -var jsonPattern = regexp.MustCompile(`jsonp\((.*)\)`) - -// struct to hold the results of a hash search -type CTHashSearch struct { - Result CTCertResult `json:"result"` - Refrences []CTCertRefrence `json:"references"` -} - -// struct to hold details of a certificate -type CTCertResult struct { - SerialNumber string `json:"serialNumber"` - Subject string `json:"subject"` - DnsNames []string `json:"dnsNames"` - CertificateType string `json:"certificateType"` - Issuer string `json:"issuer"` - ValidFrom int64 `json:"validFrom"` - ValidTo int64 `json:"validTo"` - SignatureAlgorithm string `json:"signatureAlgorithm"` -} - -// struct to hold details about refrences to a certificate -type CTCertRefrence struct { - LogName string `json:"logName"` - LogId string `json:"logId"` - Index int `json:"index"` -} - -// struct to hold the results of a domain search -type CTDomainSearch struct { - Results []CTDomainSearchResult `json:"results"` - StartIndex int `json:"startIndex"` - NumResults int `json:"numResults"` - NextPageToken string `json:"nextPageToken"` - IssuanceSummary []CTDomainIssuanceSummary `json:"issuanceSummary"` - Expired bool // not part of the json API but added as part of the query - Subdomains bool // not part of the json API but added as part of the query -} - -// struct to hold a single certifiacate result from a domain search -type CTDomainSearchResult struct { - SerialNumber string `json:"serialNumber"` - Subject string `json:"subject"` - Issuer string `json:"issuer"` - ValidFrom int64 `json:"validFrom"` - ValidTo int64 `json:"validTo"` - NumLogs int `json:"numLogs"` - Hash string `json:"hash"` - FirstDnsName string `json:"firstDnsName"` - NumDnsNames int `json:"numDnsNames"` -} - -func (sr *CTDomainSearchResult) GetFingerprint() fingerprint { - var fp fingerprint - data, err := base64.StdEncoding.DecodeString(sr.Hash) - if err != nil { - v(err) - } - if len(data) != sha256.Size { - v("Hash is not correct SHA256 size", sr.Hash) - } - for i := 0; i < len(data) && i < len(fp); i++ { - fp[i] = data[i] - } - return fp -} - -// struct to hold CA information about a domain search -type CTDomainIssuanceSummary struct { - IssuerUid string `json:"issuerUid"` - IssuerPkHash string `json:"issuerPkHash"` - Subject string `json:"subject"` - numIssued string `json:"numIssued"` -} - -// gets JSON from url and parses it into target object -func getJsonP(url string, target interface{}) error { - r, err := jsonClient.Get(url) - if err != nil { - return err - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - return errors.New("Got non OK HTTP status:" + r.Status) - } - - respData, err := ioutil.ReadAll(r.Body) - if err != nil { - return err - } - - obj := jsonPattern.FindSubmatch(respData) - if len(obj) < 2 { - return errors.New("Unable to find JSONP in response") - } - - return json.Unmarshal(obj[1], target) -} - -// queries the CT logs for domain, gets all pages of results -func QueryDomain(domain string, include_expired bool, include_subdomains bool) ([]CTDomainSearchResult, error) { - results := make([]CTDomainSearchResult, 0, 5) - - u, err := url.Parse(searchURL) - if err != nil { - return results, err - } - - token := "CAA=" - // get every page - for token != "" { - q := u.Query() - q.Set("incl_exp", strconv.FormatBool(include_expired)) - q.Set("incl_sub", strconv.FormatBool(include_subdomains)) - q.Set("domain", domain) - q.Set("token", token) - u.RawQuery = q.Encode() - - search := new(CTDomainSearch) - search.Expired = include_expired - search.Subdomains = include_subdomains - err := getJsonP(u.String(), search) - if err != nil { - return results, err - } - token = search.NextPageToken - - results = append(results, search.Results...) - } - - return results, nil -} - -// queries the CT logs for the hash to get the cert details -func QueryHash(hash string) (CTCertResult, error) { - search := new(CTHashSearch) - - u, err := url.Parse(certURL) - if err != nil { - return search.Result, err - } - - q := u.Query() - q.Set("hash", hash) - u.RawQuery = q.Encode() - - err = getJsonP(u.String(), search) - return search.Result, err -} - -// example function to use Google's CT API -func ct_example(domain string) error { - s, err := QueryDomain(domain, false, false) - if err != nil { - return err - } - - for i := range s { - fmt.Print(s[i].Hash, " ") - h, err := QueryHash(s[i].Hash) - if err != nil { - return err - } - fmt.Println(h.SerialNumber) - for j := range h.DnsNames { - fmt.Println("\t", h.DnsNames[j]) - } - } - - return nil -} diff --git a/graph/fingerprint.go b/graph/fingerprint.go new file mode 100644 index 0000000..1f25410 --- /dev/null +++ b/graph/fingerprint.go @@ -0,0 +1,33 @@ +package graph + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" +) + +type Fingerprint [sha256.Size]byte + +// print fingerprint as hex +func (fp *Fingerprint) HexString() string { + return fmt.Sprintf("%X", fp) +} + +func FingerprintFromB64(hash string) Fingerprint { + var fp Fingerprint + data, err := base64.StdEncoding.DecodeString(hash) + if err != nil { + v(err) + } + if len(data) != sha256.Size { + v("Hash is not correct SHA256 size", hash) + } + for i := 0; i < len(data) && i < len(fp); i++ { + fp[i] = data[i] + } + return fp +} + +func (fp *Fingerprint) B64Encode() string { + return base64.StdEncoding.EncodeToString(fp[:]) +} diff --git a/graph/graph.go b/graph/graph.go new file mode 100644 index 0000000..f589ba2 --- /dev/null +++ b/graph/graph.go @@ -0,0 +1,142 @@ +package graph + +import ( + "sync" + + "github.com/lanrat/certgraph/status" +) + +// main graph storage engine +type CertGraph struct { + domains sync.Map + certs sync.Map + numDomains int +} + +func NewCertGraph() *CertGraph { + graph := new(CertGraph) + return graph +} + +func (graph *CertGraph) LoadOrStoreCert(nodein *CertNode) (*CertNode, bool) { + nodeout, ok := graph.certs.LoadOrStore(nodein.Fingerprint, nodein) + return nodeout.(*CertNode), ok +} + +// TODO check for existing? +func (graph *CertGraph) AddCert(certnode *CertNode) { + graph.certs.Store(certnode.Fingerprint, certnode) +} + +// TODO check for existing? +func (graph *CertGraph) AddDomain(domainnode *DomainNode) { + graph.numDomains++ + graph.domains.Store(domainnode.Domain, domainnode) +} + +func (graph *CertGraph) Len() int { + return graph.numDomains +} + +func (graph *CertGraph) GetCert(fp Fingerprint) (*CertNode, bool) { + node, ok := graph.certs.Load(fp) + if ok { + return node.(*CertNode), true + } + return nil, false +} + +func (graph *CertGraph) GetDomain(domain string) (*DomainNode, bool) { + node, ok := graph.domains.Load(domain) + if ok { + return node.(*DomainNode), true + } + return nil, false +} + +func (graph *CertGraph) GetDomainNeighbors(domain string, skipCDN bool) []string { + neighbors := make(map[string]bool) + + //domain = directDomain(domain) + node, ok := graph.domains.Load(domain) + if ok { + domainnode := node.(*DomainNode) + // visited cert neighbors + node, ok := graph.certs.Load(domainnode.VisitedCert) + if ok { + certnode := node.(*CertNode) + if skipCDN && certnode.CDNCert() { + v(domain, "-> CDN CERT") + } else { + for _, neighbor := range certnode.Domains { + neighbors[neighbor] = true + v(domain, "- CERT ->", neighbor) + } + + } + } + + // CT neighbors + for _, fp := range domainnode.CTCerts { + node, ok := graph.certs.Load(fp) + if ok { + certnode := node.(*CertNode) + if skipCDN && certnode.CDNCert() { + v(domain, "-> CDN CERT") + } else { + for _, neighbor := range certnode.Domains { + neighbors[neighbor] = true + v(domain, "-- CT -->", neighbor) + } + } + + } + } + } + + //exclude domain from own neighbors list + neighbors[domain] = false + + // convert map to array + neighbor_list := make([]string, 0, len(neighbors)) + for key := range neighbors { + if neighbors[key] { + neighbor_list = append(neighbor_list, key) + } + } + return neighbor_list +} + +func (graph *CertGraph) GenerateMap() map[string]interface{} { + m := make(map[string]interface{}) + nodes := make([]map[string]string, 0, 2*graph.numDomains) + links := make([]map[string]string, 0, 2*graph.numDomains) + + // add all domain nodes + graph.domains.Range(func(key, value interface{}) bool { + domainnode := value.(*DomainNode) + nodes = append(nodes, domainnode.ToMap()) + if domainnode.Status == status.GOOD { + links = append(links, map[string]string{"source": domainnode.Domain, "target": domainnode.VisitedCert.HexString(), "type": "uses"}) + } + return true + }) + + // add all cert nodes + graph.certs.Range(func(key, value interface{}) bool { + certnode := value.(*CertNode) + nodes = append(nodes, certnode.ToMap()) + for _, domain := range certnode.Domains { + domain := directDomain(domain) + _, ok := graph.GetDomain(domain) + if ok { + links = append(links, map[string]string{"source": certnode.Fingerprint.HexString(), "target": domain, "type": "sans"}) + } // TODO do something with alt-names that are not in graph like wildcards + } + return true + }) + + m["nodes"] = nodes + m["links"] = links + return m +} diff --git a/graph/misc.go b/graph/misc.go new file mode 100644 index 0000000..1d0ef25 --- /dev/null +++ b/graph/misc.go @@ -0,0 +1,25 @@ +package graph + +import ( + "fmt" + "os" +) + +var Verbose = false + +func v(a ...interface{}) { + if Verbose { + fmt.Fprintln(os.Stderr, a...) + } +} + +// given a domain returns the non-wildcard version of that domain +func directDomain(domain string) string { + if len(domain) < 3 { + return domain + } + if domain[0:2] == "*." { + domain = domain[2:] + } + return domain +} diff --git a/graph/nodes.go b/graph/nodes.go new file mode 100644 index 0000000..e5cf3ff --- /dev/null +++ b/graph/nodes.go @@ -0,0 +1,134 @@ +package graph + +import ( + "crypto/sha256" + "crypto/x509" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/lanrat/certgraph/status" +) + +// structure to store a domain and its edges +type DomainNode struct { + Domain string + Depth uint + VisitedCert Fingerprint + CTCerts []Fingerprint + Status status.DomainStatus + Root bool +} + +// constructor for DomainNode, converts domain to directDomain +func NewDomainNode(domain string, depth uint) *DomainNode { + node := new(DomainNode) + node.Domain = directDomain(domain) + node.Depth = depth + node.CTCerts = make([]Fingerprint, 0, 0) + return node +} + +// get the string representation of a node +func (d *DomainNode) String() string { + cert := "" + if d.Status == status.GOOD { + cert = d.VisitedCert.HexString() + } + return fmt.Sprintf("%s\t%d\t%s\t%s", d.Domain, d.Depth, d.Status, cert) +} + +func (d *DomainNode) AddCTFingerprint(fp Fingerprint) { + d.CTCerts = append(d.CTCerts, fp) +} + +func (d *DomainNode) ToMap() map[string]string { + m := make(map[string]string) + m["type"] = "domain" + m["id"] = d.Domain + m["status"] = d.Status.String() + m["root"] = strconv.FormatBool(d.Root) + m["depth"] = strconv.FormatUint(uint64(d.Depth), 10) + return m +} + +type CertNode struct { + Fingerprint Fingerprint + Domains []string + CT bool + HTTP bool +} + +func (c *CertNode) String() string { + //TODO Currently unused.. + ct := "" + if c.CT { + ct = "CT" + } + http := "" + if c.HTTP { + http = "HTTP" + } + return fmt.Sprintf("%s\t%s %s\t%v", c.Fingerprint.HexString(), http, ct, c.Domains) +} + +func (c *CertNode) CDNCert() bool { + for _, domain := range c.Domains { + // cloudflair + matched, _ := regexp.MatchString("([0-9][a-z])*\\.cloudflaressl\\.com", domain) + if matched { + return true + } + + if domain == "i.ssl.fastly.net" { + return true + } + // TODO include other CDNs + } + return false +} + +func (c *CertNode) ToMap() map[string]string { + m := make(map[string]string) + m["type"] = "certificate" + m["id"] = c.Fingerprint.HexString() + s := "" + if c.HTTP { + s = "HTTP " + } + if c.CT { + s = s + "CT" + } + m["status"] = strings.TrimSuffix(s, " ") + return m +} + +func NewCertNode(cert *x509.Certificate) *CertNode { + certnode := new(CertNode) + + // generate Fingerprint + certnode.Fingerprint = sha256.Sum256(cert.Raw) + + // domains + // used to ensure uniq entries in domains array + domainMap := make(map[string]bool) + // add the CommonName just to be safe + cn := strings.ToLower(cert.Subject.CommonName) + if len(cn) > 0 { + domainMap[cn] = true + } + for _, domain := range cert.DNSNames { + if len(domain) > 0 { + domain = strings.ToLower(domain) + domainMap[domain] = true + } + } + for domain := range domainMap { + certnode.Domains = append(certnode.Domains, domain) + } + sort.Strings(certnode.Domains) + + return certnode +} diff --git a/status/domain.go b/status/domain.go new file mode 100644 index 0000000..3deddd1 --- /dev/null +++ b/status/domain.go @@ -0,0 +1,60 @@ +package status + +import ( + "net" + "syscall" +) + +// domain node conection status +type DomainStatus int + +const ( + UNKNOWN = iota + GOOD = iota + TIMEOUT = iota + NO_HOST = iota + REFUSED = iota + ERROR = iota +) + +// return domain status for printing +func (status DomainStatus) String() string { + switch status { + case UNKNOWN: + return "Unknown" + case GOOD: + return "Good" + case TIMEOUT: + return "Timeout" + case NO_HOST: + return "No Host" + case REFUSED: + return "Refused" + case ERROR: + return "Error" + } + return "?" +} + +// Check for errors, print if network related +func CheckNetErr(err error) DomainStatus { + if err == nil { + return GOOD + } else if netError, ok := err.(net.Error); ok && netError.Timeout() { + return TIMEOUT + } else { + switch t := err.(type) { + case *net.OpError: + if t.Op == "dial" { + return NO_HOST + } else if t.Op == "read" { + return REFUSED + } + case syscall.Errno: + if t == syscall.ECONNREFUSED { + return REFUSED + } + } + } + return ERROR +}