Removed obsolete sumdbaudit command (#983)

This has been replaced with sumdbverify, which now has useful information that was in the sumdbaudit README incorporated

Author: mhutchinson

sumdbaudit/client/sumdb.go clone/cmd/sumdbclone/internal/client/sumdb.go

@@ -16,6 +16,7 @@
 package client
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"net/http"
@@ -72,11 +73,6 @@ func (c *SumDBClient) LatestCheckpoint() (*Checkpoint, error) {
 		return nil, fmt.Errorf("failed to get /latest Checkpoint; %w", err)
 	}
 
-	return c.ParseCheckpointNote(checkpoint)
-}
-
-// ParseCheckpointNote parses a previously acquired raw checkpoint note data.
-func (c *SumDBClient) ParseCheckpointNote(checkpoint []byte) (*Checkpoint, error) {
 	verifier, err := note.NewVerifier(c.vkey)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create verifier: %w", err)
@@ -126,46 +122,11 @@ func (c *SumDBClient) tilePath(offset int) string {
 }
 
 func dataToLeaves(data []byte) [][]byte {
-	result := make([][]byte, 0)
-	start := 0
-	for i, b := range data {
-		if b == '\n' {
-			if i > start && data[i-1] == '\n' {
-				result = append(result, data[start:i])
-				start = i + 1
-			}
-		}
+	leaves := bytes.Split(data, []byte{'\n', '\n'})
+	for i, l := range leaves {
+		leaves[i] = append(l, '\n')
 	}
-	result = append(result, data[start:])
-	return result
-}
-
-// TileHashes gets the hashes at the given level and offset.
-// If partial > 0 then a partial tile will be fetched with the number of hashes.
-// TODO(mhutchinson): Add better tests for this.
-func (c *SumDBClient) TileHashes(level, offset, partial int) ([]tlog.Hash, error) {
-	url := fmt.Sprintf("/tile/%d/%d/%s", c.height, level, c.tilePath(offset))
-	if partial > 0 {
-		url = fmt.Sprintf("%s.p/%d", url, partial)
-	}
-	data, err := c.fetcher.GetData(url)
-	if err != nil {
-		return nil, err
-	}
-	expectedHashes := 1 << c.height
-	if partial > 0 {
-		expectedHashes = partial
-	}
-	if got, want := len(data), HashLenBytes*expectedHashes; got != want {
-		return nil, fmt.Errorf("got %d bytes, expected %d", got, want)
-	}
-	hashes := make([]tlog.Hash, expectedHashes)
-	for i := 0; i < cap(hashes); i++ {
-		var h tlog.Hash
-		copy(h[:], data[HashLenBytes*i:HashLenBytes*(i+1)])
-		hashes[i] = h
-	}
-	return hashes, nil
+	return leaves
 }
 
 // HTTPFetcher gets the data over HTTP(S).
@@ -189,9 +150,5 @@ func (f *HTTPFetcher) GetData(path string) ([]byte, error) {
 	if resp.StatusCode != 200 {
 		return nil, fmt.Errorf("GET %v: %v", target, resp.Status)
 	}
-	data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
-	if err != nil {
-		return nil, err
-	}
-	return data, nil
+	return io.ReadAll(io.LimitReader(resp.Body, 1<<20))
 }

sumdbaudit/client/sumdb_test.go clone/cmd/sumdbclone/internal/client/sumdb_test.go

@@ -16,7 +16,6 @@ package client
 
 import (
 	"bytes"
-	"encoding/hex"
 	"fmt"
 	"testing"
 
@@ -43,8 +42,6 @@ kn9DgqDhXzoZMM8828SQsbuovr/WRn7QfFd5Qe1rpwA=
 
 — sum.golang.org Az3grunuggF5mKymPJeK/l9Pq71lOg/rAVkQVCzGkWRJcnS3ZFunzveHr9PAH8LFsuhpcCWzGDNrn9FFDyXm/66tBg8=
 `
-
-	tileHashData = `d7b9018cbad2a2fa3950dcd60411cd67ef9d8c1074043c0e033953ec510fd68413f83190fb460efeb65670f9298b4249b8b5fd2492a6cd486f1fe14bfa3eb545590ac0de6fc0f9b016875ba518353cc57654df733f2fa1d1f0dad66f84b66d9ea2744fc0a64bb00d9f286c52838284b4b76bcbe895854d4709c55df1c266b681`
 )
 
 func TestLeavesAtOffset(t *testing.T) {
@@ -100,27 +97,6 @@ func TestLatestCheckpoint(t *testing.T) {
 	}
 }
 
-func TestTileHashes(t *testing.T) {
-	hashData, err := hex.DecodeString(tileHashData)
-	if err != nil {
-		t.Fatalf("failed to decode hash data: %v", err)
-	}
-	sumdb := &SumDBClient{
-		vkey:   "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8",
-		height: 2,
-		fetcher: &FakeFetcher{
-			values: map[string]string{"/tile/2/0/000": string(hashData)},
-		},
-	}
-	hashes, err := sumdb.TileHashes(0, 0, 0)
-	if err != nil {
-		t.Fatalf("failed to get hashes: %v", err)
-	}
-	if got, want := len(hashes), 4; got != want {
-		t.Errorf("got, want = %d, %d", got, want)
-	}
-}
-
 type FakeFetcher struct {
 	values map[string]string
 }

clone/cmd/sumdbclone/sumdbclone.go

@@ -25,10 +25,10 @@ import (
 
 	"github.com/cenkalti/backoff/v4"
 	"github.com/golang/glog"
+	sdbclient "github.com/google/trillian-examples/clone/cmd/sumdbclone/internal/client"
 	"github.com/google/trillian-examples/clone/internal/cloner"
 	"github.com/google/trillian-examples/clone/internal/verify"
 	"github.com/google/trillian-examples/clone/logdb"
-	sdbclient "github.com/google/trillian-examples/sumdbaudit/client"
 	"github.com/transparency-dev/merkle/rfc6962"
 
 	_ "github.com/go-sql-driver/mysql"

clone/cmd/sumdbverify/README.md

@@ -4,6 +4,34 @@ This tool clones the log for sum.golang.org and ensures that:
  1. The cloned data matches the commitments made in the log checkpoint
  2. That no module + version is declared with different hashes in the log
 
+## Background
+
+This is a quick summary of https://blog.golang.org/module-mirror-launch but is not
+intended to replace this introduction.
+If you have no context on Go SumDB, read that intro first :-)
+
+Go SumDB is a Verifiable Log based on Trillian, which contains entries of the form:
+```
+github.com/google/trillian v1.3.11 h1:pPzJPkK06mvXId1LHEAJxIegGgHzzp/FUnycPYfoCMI=
+github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw=
+```
+Every module & version used in the Go ecosystem will have such an entry in this log,
+and the values are hashes which commit to the state of the repository and its `go.mod`
+file at that particular version.
+
+Clients can be assured that they have downloaded the same version of a module as
+everybody else provided all of the following are true:
+ 1. The hash of what they have downloaded matches an entry in the SumDB Log
+ 2. There is only one entry in the Log for the `module@version`
+ 3. Entries in the Log are immutable / the Log is append-only
+ 4. Everyone else sees the same Log as this user
+
+Verification of these is performed by different parties:
+ 1. The client checks an inclusion proof for their {module, version, hashes} in the Log
+ 2. This `sumdbverify` tool verifies there are no conflicting versions
+ 3. A [witness](https://github.com/transparency-dev/witness) verifies the log is append-only
+ 4. A [distributor](https://github.com/transparency-dev/distributor) aggregates multiple witness confirmations to verify consensus
+
 ## Running in Docker
 
 The `docker-compose` scripts in this directory allow for deployment of the `sumdbclone` tool in a single command: