Migrate storage implementations to Appender lifecycle model (#482)

cmd/conformance/aws/main.go

@@ -69,20 +69,20 @@ func main() {
 
 	// Create our Tessera storage backend:
 	awsCfg := storageConfigFromFlags()
-	driver, err := aws.New(ctx, awsCfg,
+	driver, err := aws.New(ctx, awsCfg)
+	if err != nil {
+		klog.Exitf("Failed to create new AWS storage: %v", err)
+	}
+	appender, _, err := tessera.NewAppender(ctx, driver,
 		tessera.WithCheckpointSigner(s, a...),
 		tessera.WithCheckpointInterval(*publishInterval),
 		tessera.WithBatching(1024, time.Second),
 		tessera.WithPushback(10*4096),
-	)
-	if err != nil {
-		klog.Exitf("Failed to create new AWS storage: %v", err)
-	}
-	appender, _, err := tessera.NewAppender(driver, tessera.WithAppendDeduplication(tessera.InMemoryDedupe(256)))
-	addFn := appender.Add
+		tessera.WithAppendDeduplication(tessera.InMemoryDedupe(256)))
 	if err != nil {
 		klog.Exit(err)
 	}
+	addFn := appender.Add
 
 	// Expose a HTTP handler for the conformance test writes.
 	// This should accept arbitrary bytes POSTed to /add, and return an ascii

cmd/conformance/gcp/main.go

@@ -59,12 +59,7 @@ func main() {
 
 	// Create our Tessera storage backend:
 	gcpCfg := storageConfigFromFlags()
-	driver, err := gcp.New(ctx, gcpCfg,
-		tessera.WithCheckpointSigner(s, a...),
-		tessera.WithCheckpointInterval(10*time.Second),
-		tessera.WithBatching(1024, time.Second),
-		tessera.WithPushback(10*4096),
-	)
+	driver, err := gcp.New(ctx, gcpCfg)
 	if err != nil {
 		klog.Exitf("Failed to create new GCP storage: %v", err)
 	}
@@ -92,11 +87,17 @@ func main() {
 		}()
 	}
 
-	appender, _, err := tessera.NewAppender(driver, tessera.WithAppendDeduplication(dedups...))
-	addFn := appender.Add
+	appender, _, err := tessera.NewAppender(ctx, driver,
+		tessera.WithCheckpointSigner(s, a...),
+		tessera.WithCheckpointInterval(10*time.Second),
+		tessera.WithBatching(1024, time.Second),
+		tessera.WithPushback(10*4096),
+		tessera.WithAppendDeduplication(dedups...),
+	)
 	if err != nil {
 		klog.Exit(err)
 	}
+	addFn := appender.Add
 
 	// Expose a HTTP handler for the conformance test writes.
 	// This should accept arbitrary bytes POSTed to /add, and return an ascii

cmd/conformance/mysql/main.go

@@ -62,15 +62,15 @@ func main() {
 	noteSigner, additionalSigners := createSignersOrDie()
 
 	// Initialise the Tessera MySQL storage
-	driver, err := mysql.New(ctx, db,
-		tessera.WithCheckpointSigner(noteSigner, additionalSigners...),
-		tessera.WithCheckpointInterval(*publishInterval),
-	)
+	driver, err := mysql.New(ctx, db)
 	if err != nil {
 		klog.Exitf("Failed to create new MySQL storage: %v", err)
 	}
 
-	appender, reader, err := tessera.NewAppender(driver, tessera.WithAppendDeduplication(tessera.InMemoryDedupe(256)))
+	appender, reader, err := tessera.NewAppender(ctx, driver,
+		tessera.WithCheckpointSigner(noteSigner, additionalSigners...),
+		tessera.WithCheckpointInterval(*publishInterval),
+		tessera.WithAppendDeduplication(tessera.InMemoryDedupe(256)))
 	addFn := appender.Add
 	if err != nil {
 		klog.Exit(err)

cmd/conformance/posix/README.md

@@ -19,7 +19,6 @@ Then, start the personality:
 ```shell
 go run ./cmd/conformance/posix \
   --storage_dir=${LOG_DIR} \
-  --initialise \
   --listen=:2025 \
   --v=2
 ```

cmd/conformance/posix/docker/compose.yaml

@@ -12,7 +12,6 @@ services:
       LOG_PUBLIC_KEY: "example.com/log/testdata+33d7b496+AeHTu4Q3hEIMHNqc6fASMsq3rKNx280NI+oO5xCFkkSx"
     command: [
       "--storage_dir=/tmp/tessera-posix-log",
-      "--initialise",
       "--listen=:2025",
       "--alsologtostderr",
       "--v=2",

cmd/conformance/posix/main.go

@@ -36,7 +36,6 @@ import (
 
 var (
 	storageDir                = flag.String("storage_dir", "", "Root directory to store log data.")
-	initialise                = flag.Bool("initialise", false, "Set when creating a new log to initialise the structure.")
 	listen                    = flag.String("listen", ":2025", "Address:port to listen on")
 	privKeyFile               = flag.String("private_key", "", "Location of private key file. If unset, uses the contents of the LOG_PRIVATE_KEY environment variable.")
 	additionalPrivateKeyFiles = []string{}
@@ -65,15 +64,18 @@ func main() {
 	s, a := getSignersOrDie()
 
 	// Create the Tessera POSIX storage, using the directory from the --storage_dir flag
-	driver, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s, a...), tessera.WithBatching(256, time.Second))
+	driver, err := posix.New(ctx, *storageDir)
 	if err != nil {
 		klog.Exitf("Failed to construct storage: %v", err)
 	}
-	appender, _, err := tessera.NewAppender(driver, tessera.WithAppendDeduplication(tessera.InMemoryDedupe(256)))
-	addFn := appender.Add
+	appender, _, err := tessera.NewAppender(ctx, driver,
+		tessera.WithCheckpointSigner(s, a...),
+		tessera.WithBatching(256, time.Second),
+		tessera.WithAppendDeduplication(tessera.InMemoryDedupe(256)))
 	if err != nil {
 		klog.Exit(err)
 	}
+	addFn := appender.Add
 
 	// Define a handler for /add that accepts POST requests and adds the POST body to the log
 	http.HandleFunc("POST /add", func(w http.ResponseWriter, r *http.Request) {

cmd/examples/posix-oneshot/README.md

@@ -11,10 +11,6 @@ The commands below create a new log and add entries to it, and then show a few a
 export LOG_PRIVATE_KEY="PRIVATE+KEY+example.com/log/testdata+33d7b496+AeymY/SZAX0jZcJ8enZ5FY1Dz+wTML2yWSkK+9DSF3eg"
 export LOG_PUBLIC_KEY="example.com/log/testdata+33d7b496+AeHTu4Q3hEIMHNqc6fASMsq3rKNx280NI+oO5xCFkkSx"
 
-# Initialize a new log
-export LOG_DIR=/tmp/mylog
-go run ./cmd/examples/posix-oneshot --storage_dir=${LOG_DIR} --initialise
-
 # Create files containing new leaves to add
 mkdir /tmp/stuff
 echo "foo" > /tmp/stuff/foo

cmd/examples/posix-oneshot/main.go

@@ -36,7 +36,6 @@ import (
 
 var (
 	storageDir  = flag.String("storage_dir", "", "Root directory to store log data.")
-	initialise  = flag.Bool("initialise", false, "Set when creating a new log to initialise the structure.")
 	entries     = flag.String("entries", "", "File path glob of entries to add to the log.")
 	privKeyFile = flag.String("private_key", "", "Location of private key file. If unset, uses the contents of the LOG_PRIVATE_KEY environment variable.")
 )
@@ -65,41 +64,34 @@ func main() {
 
 	// Gather the info needed for reading/writing checkpoints
 	s := getSignerOrDie()
-
-	// Handle the case where no entries are to be added.
-	if len(*entries) == 0 {
-		if *initialise {
-			_, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s))
-			if err != nil {
-				klog.Exitf("Failed to initialise storage: %v", err)
-			}
-		}
-		klog.Info("No entries provided to integrate; exiting")
-		os.Exit(0)
-	}
-
-	// Evaluate the glob provided by the --entries flag to determine the files containing leaves
-	filesToAdd := readEntriesOrDie()
-
 	// Construct a new Tessera POSIX log storage, anchored at the correct directory, and initialising it if requested.
 	// The options provide the checkpoint signer & verifier, and batch options.
 	// In this case, we want to create a single batch containing all of the leaves being added in order to
 	// add all of these leaves without creating any intermediate checkpoints.
 	driver, err := posix.New(
 		ctx,
 		*storageDir,
-		*initialise,
-		tessera.WithCheckpointSigner(s),
-		tessera.WithCheckpointInterval(checkpointInterval),
-		tessera.WithBatching(uint(len(filesToAdd)), time.Second))
+	)
 	if err != nil {
 		klog.Exitf("Failed to construct storage: %v", err)
 	}
-	appender, r, err := tessera.NewAppender(driver)
-	addFn := appender.Add
+
+	// Evaluate the glob provided by the --entries flag to determine the files containing leaves
+	filesToAdd := readEntriesOrDie()
+	batchSize := uint(len(filesToAdd))
+	if batchSize == 0 {
+		// batchSize can't be zero
+		batchSize = 1
+	}
+
+	appender, r, err := tessera.NewAppender(ctx, driver,
+		tessera.WithCheckpointSigner(s),
+		tessera.WithCheckpointInterval(checkpointInterval),
+		tessera.WithBatching(batchSize, time.Second))
 	if err != nil {
 		klog.Exit(err)
 	}
+	addFn := appender.Add
 
 	// We don't want to exit until our entries have been integrated into the tree, so we'll use Tessera's
 	// IntegrationAwaiter to help with that.
@@ -166,8 +158,5 @@ func readEntriesOrDie() []string {
 		klog.Exitf("Failed to glob entries %q: %q", *entries, err)
 	}
 	klog.V(1).Infof("toAdd: %v", toAdd)
-	if len(toAdd) == 0 {
-		klog.Exit("Sequence must be run with at least one valid entry")
-	}
 	return toAdd
 }

cmd/experimental/migrate/posix/main.go

@@ -65,7 +65,7 @@ func main() {
 	}
 
 	// Create our Tessera storage backend:
-	st, err := posix.NewMigrationTarget(ctx, *storageDir, *initialise, internal.BundleHasher)
+	st, err := posix.NewMigrationTarget(ctx, *storageDir, *initialise, internal.BundleHasher, nil)
 	if err != nil {
 		klog.Exitf("Failed to create new POSIX storage: %v", err)
 	}

ct_only.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/transparency-dev/trillian-tessera/api/layout"
 	"github.com/transparency-dev/trillian-tessera/ctonly"
-	"github.com/transparency-dev/trillian-tessera/internal/options"
 )
 
 // Storage described the expected functions from Tessera storage implementations.
@@ -63,8 +62,8 @@ func convertCTEntry(e *ctonly.Entry) *Entry {
 }
 
 // WithCTLayout instructs the underlying storage to use a Static CT API compatible scheme for layout.
-func WithCTLayout() func(*options.StorageOptions) {
-	return func(opts *options.StorageOptions) {
+func WithCTLayout() func(*AppendOptions) {
+	return func(opts *AppendOptions) {
 		opts.EntriesPath = ctEntriesPath
 	}
 }