rhp4: add append sectors rpc

go.mod

@@ -4,7 +4,7 @@ go 1.23.0
 
 require (
 	go.etcd.io/bbolt v1.3.11
-	go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf
+	go.sia.tech/core v0.4.8-0.20241002233523-418236837b5a
 	go.sia.tech/mux v1.3.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.27.0

go.sum

@@ -8,8 +8,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
 go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
-go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf h1:x/lM7Y8Rlo12rcpPXapLvSVNyrHZEKO6j4eLNccPMKw=
-go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf/go.mod h1:j2Ke8ihV8or7d2VDrFZWcCkwSVHO0DNMQJAGs9Qop2M=
+go.sia.tech/core v0.4.8-0.20241002233523-418236837b5a h1:Lz7h42eZCk8iI4/agdWNwlVneqaybxkV1lor+Eavzso=
+go.sia.tech/core v0.4.8-0.20241002233523-418236837b5a/go.mod h1:j2Ke8ihV8or7d2VDrFZWcCkwSVHO0DNMQJAGs9Qop2M=
 go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c=
 go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=

rhp/v4/rpc.go

@@ -126,6 +126,14 @@ type (
 		Cost     types.Currency       `json:"cost"`
 	}
 
+	// RPCAppendSectorResults contains the result of executing the append sectors
+	// RPC.
+	RPCAppendSectorsResult struct {
+		Revision types.V2FileContract `json:"revision"`
+		Cost     types.Currency       `json:"cost"`
+		Sectors  []types.Hash256      `json:"sectors"`
+	}
+
 	// RPCFundAccountResult contains the result of executing the fund accounts RPC.
 	RPCFundAccountResult struct {
 		Revision types.V2FileContract `json:"revision"`
@@ -287,7 +295,7 @@ func RPCVerifySector(ctx context.Context, t TransportClient, prices rhp4.HostPri
 }
 
 // RPCModifySectors modifies sectors on the host.
-func RPCModifySectors(ctx context.Context, t TransportClient, cs consensus.State, prices rhp4.HostPrices, sk types.PrivateKey, contract ContractRevision, actions []rhp4.WriteAction) (RPCModifySectorsResult, error) {
+func RPCModifySectors(ctx context.Context, t TransportClient, cs consensus.State, prices rhp4.HostPrices, sk types.PrivateKey, contract ContractRevision, actions []rhp4.ModifyAction) (RPCModifySectorsResult, error) {
 	req := rhp4.RPCModifySectorsRequest{
 		ContractID: contract.ID,
 		Prices:     prices,
@@ -308,8 +316,8 @@ func RPCModifySectors(ctx context.Context, t TransportClient, cs consensus.State
 	}
 
 	// TODO: verify proof
-
-	revision, err := rhp4.ReviseForModifySectors(contract.Revision, req, resp)
+	root := resp.Proof[len(resp.Proof)-1]
+	revision, err := rhp4.ReviseForModifySectors(contract.Revision, prices, root, actions)
 	if err != nil {
 		return RPCModifySectorsResult{}, fmt.Errorf("failed to revise contract: %w", err)
 	}
@@ -339,6 +347,62 @@ func RPCModifySectors(ctx context.Context, t TransportClient, cs consensus.State
 	}, nil
 }
 
+func RPCAppendSectors(ctx context.Context, t TransportClient, cs consensus.State, prices rhp4.HostPrices, sk types.PrivateKey, contract ContractRevision, roots []types.Hash256) (RPCAppendSectorsResult, error) {
+	req := rhp4.RPCAppendSectorsRequest{
+		Prices:     prices,
+		Sectors:    roots,
+		ContractID: contract.ID,
+	}
+	req.ChallengeSignature = sk.SignHash(req.ChallengeSigHash(contract.Revision.RevisionNumber + 1))
+
+	s := t.DialStream(ctx)
+	defer s.Close()
+
+	if err := rhp4.WriteRequest(s, rhp4.RPCAppendSectorsID, &req); err != nil {
+		return RPCAppendSectorsResult{}, fmt.Errorf("failed to write request: %w", err)
+	}
+
+	var resp rhp4.RPCAppendSectorsResponse
+	if err := rhp4.ReadResponse(s, &resp); err != nil {
+		return RPCAppendSectorsResult{}, fmt.Errorf("failed to read response: %w", err)
+	} else if len(resp.Accepted) != len(roots) {
+		return RPCAppendSectorsResult{}, errors.New("host returned less roots")
+	}
+	appended := make([]types.Hash256, 0, len(roots))
+	for i := range resp.Accepted {
+		if resp.Accepted[i] {
+			appended = append(appended, roots[i])
+		}
+	}
+
+	root := resp.Proof[len(resp.Proof)-1]
+	revision, err := rhp4.ReviseForAppendSectors(contract.Revision, prices, root, uint64(len(appended)))
+	if err != nil {
+		return RPCAppendSectorsResult{}, fmt.Errorf("failed to revise contract: %w", err)
+	}
+	sigHash := cs.ContractSigHash(revision)
+	revision.RenterSignature = sk.SignHash(sigHash)
+
+	signatureResp := rhp4.RPCAppendSectorsSecondResponse{
+		RenterSignature: revision.RenterSignature,
+	}
+	if err := rhp4.WriteResponse(s, &signatureResp); err != nil {
+		return RPCAppendSectorsResult{}, fmt.Errorf("failed to write signature response: %w", err)
+	}
+
+	var hostSignature rhp4.RPCAppendSectorsThirdResponse
+	if err := rhp4.ReadResponse(s, &hostSignature); err != nil {
+		return RPCAppendSectorsResult{}, fmt.Errorf("failed to read host signatures: %w", err)
+	} else if !contract.Revision.HostPublicKey.VerifyHash(sigHash, hostSignature.HostSignature) {
+		return RPCAppendSectorsResult{}, rhp4.ErrInvalidSignature
+	}
+	return RPCAppendSectorsResult{
+		Revision: revision,
+		Cost:     contract.Revision.RenterOutput.Value.Sub(revision.RenterOutput.Value),
+		Sectors:  appended,
+	}, nil
+}
+
 // RPCFundAccounts funds accounts on the host.
 func RPCFundAccounts(ctx context.Context, t TransportClient, cs consensus.State, signer ContractSigner, contract ContractRevision, deposits []rhp4.AccountDeposit) (RPCFundAccountResult, error) {
 	var total types.Currency

rhp/v4/rpc_test.go

@@ -820,6 +820,124 @@ func TestReadWriteSector(t *testing.T) {
 	}
 }
 
+func TestAppendSectors(t *testing.T) {
+	log := zaptest.NewLogger(t)
+	n, genesis := testutil.V2Network()
+	hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey()
+
+	cm, s, w := startTestNode(t, n, genesis, log)
+
+	// fund the wallet with two UTXOs
+	mineAndSync(t, cm, w.Address(), 146, w)
+
+	sr := testutil.NewEphemeralSettingsReporter()
+	sr.Update(proto4.HostSettings{
+		Release:             "test",
+		AcceptingContracts:  true,
+		WalletAddress:       w.Address(),
+		MaxCollateral:       types.Siacoins(10000),
+		MaxContractDuration: 1000,
+		MaxSectorDuration:   3 * 144,
+		MaxModifyActions:    100,
+		RemainingStorage:    100 * proto4.SectorSize,
+		TotalStorage:        100 * proto4.SectorSize,
+		Prices: proto4.HostPrices{
+			ContractPrice: types.Siacoins(1).Div64(5), // 0.2 SC
+			StoragePrice:  types.NewCurrency64(100),   // 100 H / byte / block
+			IngressPrice:  types.NewCurrency64(100),   // 100 H / byte
+			EgressPrice:   types.NewCurrency64(100),   // 100 H / byte
+			Collateral:    types.NewCurrency64(200),
+		},
+	})
+	ss := testutil.NewEphemeralSectorStore()
+	c := testutil.NewEphemeralContractor(cm)
+
+	transport := testRenterHostPair(t, hostKey, cm, s, w, c, sr, ss, log)
+
+	settings, err := rhp4.RPCSettings(context.Background(), transport)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fundAndSign := &fundAndSign{w, renterKey}
+	renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200)
+	formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{
+		RenterPublicKey: renterKey.PublicKey(),
+		RenterAddress:   w.Address(),
+		Allowance:       renterAllowance,
+		Collateral:      hostCollateral,
+		ProofHeight:     cm.Tip().Height + 50,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	revision := formResult.Contract
+
+	cs := cm.TipState()
+	account := proto4.Account(renterKey.PublicKey())
+
+	accountFundAmount := types.Siacoins(25)
+	fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{
+		{Account: account, Amount: accountFundAmount},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	revision.Revision = fundResult.Revision
+
+	token := proto4.AccountToken{
+		Account:    account,
+		ValidUntil: time.Now().Add(time.Hour),
+	}
+	tokenSigHash := token.SigHash()
+	token.Signature = renterKey.SignHash(tokenSigHash)
+
+	// store random sectors
+	roots := make([]types.Hash256, 0, 10)
+	for i := 0; i < 10; i++ {
+		var sector [proto4.SectorSize]byte
+		frand.Read(sector[:])
+		root := proto4.SectorRoot(&sector)
+
+		writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, NewReaderLen(sector[:]), 5)
+		if err != nil {
+			t.Fatal(err)
+		} else if writeResult.Root != root {
+			t.Fatal("root mismatch")
+		}
+		roots = append(roots, root)
+	}
+
+	// corrupt a random root
+	excludedIndex := frand.Intn(len(roots))
+	roots[excludedIndex] = frand.Entropy256()
+
+	// append the sectors to the contract
+	appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, roots)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(appendResult.Sectors) != len(roots)-1 {
+		t.Fatalf("expected %v, got %v", len(roots)-1, len(appendResult.Sectors))
+	}
+	roots = append(roots[:excludedIndex], roots[excludedIndex+1:]...)
+	if appendResult.Revision.FileMerkleRoot != proto4.MetaRoot(roots) {
+		t.Fatal("root mismatch")
+	}
+
+	// read the sectors back
+	buf := bytes.NewBuffer(make([]byte, 0, proto4.SectorSize))
+	for _, root := range roots {
+		buf.Reset()
+
+		_, err = rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, root, 0, proto4.SectorSize)
+		if err != nil {
+			t.Fatal(err)
+		} else if proto4.SectorRoot((*[proto4.SectorSize]byte)(buf.Bytes())) != root {
+			t.Fatal("data mismatch")
+		}
+	}
+}
+
 func TestVerifySector(t *testing.T) {
 	log := zaptest.NewLogger(t)
 	n, genesis := testutil.V2Network()
@@ -1013,26 +1131,19 @@ func TestRPCModifySectors(t *testing.T) {
 	}
 
 	// append all the sector roots to the contract
-	var actions []proto4.WriteAction
-	for _, root := range roots {
-		actions = append(actions, proto4.WriteAction{
-			Type: proto4.ActionAppend,
-			Root: root,
-		})
-	}
-	modifyResult, err := rhp4.RPCModifySectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, actions)
+	appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, roots)
 	if err != nil {
 		t.Fatal(err)
 	}
-	assertRevision(t, modifyResult.Revision, roots)
-	revision.Revision = modifyResult.Revision
+	assertRevision(t, appendResult.Revision, roots)
+	revision.Revision = appendResult.Revision
 
 	// swap two random sectors
 	swapA, swapB := frand.Uint64n(uint64(len(roots)/2)), frand.Uint64n(uint64(len(roots)/2))+uint64(len(roots)/2)
-	actions = []proto4.WriteAction{
+	actions := []proto4.ModifyAction{
 		{Type: proto4.ActionSwap, A: swapA, B: swapB},
 	}
-	modifyResult, err = rhp4.RPCModifySectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, actions)
+	modifyResult, err := rhp4.RPCModifySectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, actions)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1041,7 +1152,7 @@ func TestRPCModifySectors(t *testing.T) {
 	revision.Revision = modifyResult.Revision
 
 	// delete the last 10 sectors
-	actions = []proto4.WriteAction{
+	actions = []proto4.ModifyAction{
 		{Type: proto4.ActionTrim, N: uint64(len(roots) / 2)},
 	}
 	modifyResult, err = rhp4.RPCModifySectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, actions)
@@ -1055,7 +1166,7 @@ func TestRPCModifySectors(t *testing.T) {
 	// update a random sector with one of the trimmed sectors
 	updateIdx := frand.Intn(len(roots))
 	trimmedIdx := frand.Intn(len(trimmed))
-	actions = []proto4.WriteAction{
+	actions = []proto4.ModifyAction{
 		{Type: proto4.ActionUpdate, Root: trimmed[trimmedIdx], A: uint64(updateIdx)},
 	}
 
@@ -1169,14 +1280,11 @@ func TestRPCSectorRoots(t *testing.T) {
 		}
 		roots = append(roots, writeResult.Root)
 
-		modifyResult, err := rhp4.RPCModifySectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, []proto4.WriteAction{
-			{Type: proto4.ActionAppend, Root: writeResult.Root},
-		})
+		appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, []types.Hash256{writeResult.Root})
 		if err != nil {
 			t.Fatal(err)
 		}
-		revision.Revision = modifyResult.Revision
-
+		revision.Revision = appendResult.Revision
 		checkRoots(t, roots)
 	}
 }
@@ -1444,39 +1552,38 @@ func BenchmarkContractUpload(b *testing.B) {
 	token.Signature = renterKey.SignHash(token.SigHash())
 
 	var sectors [][proto4.SectorSize]byte
+	roots := make([]types.Hash256, 0, b.N)
 	for i := 0; i < b.N; i++ {
 		var sector [proto4.SectorSize]byte
 		frand.Read(sector[:256])
 		sectors = append(sectors, sector)
+		roots = append(roots, proto4.SectorRoot(&sector))
 	}
 
 	b.ResetTimer()
 	b.ReportAllocs()
 	b.SetBytes(proto4.SectorSize)
 
 	var wg sync.WaitGroup
-	actions := make([]proto4.WriteAction, b.N)
 	for i := 0; i < b.N; i++ {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
 			writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, NewReaderLen(sectors[i][:]), 5)
 			if err != nil {
 				b.Error(err)
-			}
-			actions[i] = proto4.WriteAction{
-				Type: proto4.ActionAppend,
-				Root: writeResult.Root,
+			} else if writeResult.Root != roots[i] {
+				b.Errorf("expected %v, got %v", roots[i], writeResult.Root)
 			}
 		}(i)
 	}
 
 	wg.Wait()
 
-	modifyResult, err := rhp4.RPCModifySectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, actions)
+	appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, roots)
 	if err != nil {
 		b.Fatal(err)
-	} else if modifyResult.Revision.Filesize != uint64(b.N)*proto4.SectorSize {
-		b.Fatalf("expected %v sectors, got %v", b.N, modifyResult.Revision.Filesize/proto4.SectorSize)
+	} else if appendResult.Revision.Filesize != uint64(b.N)*proto4.SectorSize {
+		b.Fatalf("expected %v sectors, got %v", b.N, appendResult.Revision.Filesize/proto4.SectorSize)
 	}
 }