Skip to content

Commit

Permalink
Merge branch 'dev' into pj/contract-usability
Browse files Browse the repository at this point in the history
  • Loading branch information
peterjan authored Nov 12, 2024
2 parents 7f2d604 + 4b0f42a commit 8c9f8a7
Show file tree
Hide file tree
Showing 12 changed files with 339 additions and 22 deletions.
5 changes: 5 additions & 0 deletions api/host.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@ type (
Subnets []string `json:"subnets"`
}

HostInfo struct {
PublicKey types.PublicKey `json:"publicKey"`
SiamuxAddr string `json:"siamuxAddr"`
}

HostInteractions struct {
TotalScans uint64 `json:"totalScans"`
LastScan time.Time `json:"lastScan"`
Expand Down
2 changes: 2 additions & 0 deletions bus/bus.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ type (
UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error
UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error
UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error
UsableHosts(ctx context.Context) ([]sql.HostInfo, error)
}

// A MetadataStore stores information about contracts and objects.
Expand Down Expand Up @@ -438,6 +439,7 @@ func (b *Bus) Handler() http.Handler {
"GET /contract/:id/roots": b.contractIDRootsHandlerGET,
"GET /contract/:id/size": b.contractSizeHandlerGET,

"GET /hosts": b.hostsHandlerGET,
"POST /hosts": b.hostsHandlerPOST,
"GET /hosts/allowlist": b.hostsAllowlistHandlerGET,
"PUT /hosts/allowlist": b.hostsAllowlistHandlerPUT,
Expand Down
8 changes: 8 additions & 0 deletions bus/client/hosts.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,11 @@ func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKe
err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck)
return
}

// UsableHosts returns a list of hosts that are ready to be used. That means
// they are deemed usable by the autopilot, they are not gouging, not blocked,
// not offline, etc.
func (c *Client) UsableHosts(ctx context.Context) (hosts []api.HostInfo, err error) {
err = c.c.WithContext(ctx).GET("/hosts", &hosts)
return
}
21 changes: 21 additions & 0 deletions bus/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,27 @@ func (b *Bus) walletPendingHandler(jc jape.Context) {
jc.Encode(events)
}

func (b *Bus) hostsHandlerGET(jc jape.Context) {
hosts, err := b.store.UsableHosts(jc.Request.Context())
if jc.Check("couldn't fetch hosts", err) != nil {
return
}

gp, err := b.gougingParams(jc.Request.Context())
if jc.Check("could not get gouging parameters", err) != nil {
return
}
gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, nil, nil)

var infos []api.HostInfo
for _, h := range hosts {
if !gc.Check(&h.HS, &h.PT).Gouging() {
infos = append(infos, h.HostInfo)
}
}
jc.Encode(infos)
}

func (b *Bus) hostsHandlerPOST(jc jape.Context) {
var req api.HostsRequest
if jc.Decode(&req) != nil {
Expand Down
43 changes: 27 additions & 16 deletions internal/test/e2e/gouging_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"time"

rhpv2 "go.sia.tech/core/rhp/v2"
"go.sia.tech/core/types"
"go.sia.tech/renterd/api"
"go.sia.tech/renterd/internal/test"
"lukechampine.com/frand"
Expand All @@ -30,9 +29,17 @@ func TestGouging(t *testing.T) {
cluster.MineBlocks(cfg.Period + 1)

// add hosts
tt.OKAll(cluster.AddHostsBlocking(int(test.AutopilotConfig.Contracts.Amount)))
n := int(test.AutopilotConfig.Contracts.Amount)
tt.OKAll(cluster.AddHostsBlocking(n))
cluster.WaitForAccounts()

// assert all hosts are usable
h, err := b.UsableHosts(context.Background())
tt.OK(err)
if len(h) != n {
t.Fatal("unexpected number of hosts")
}

// assert that the current period is greater than the period
tt.Retry(10, time.Second, func() error {
if ap, _ := b.Autopilot(context.Background(), api.DefaultAutopilotID); ap.CurrentPeriod <= cfg.Period {
Expand Down Expand Up @@ -62,35 +69,39 @@ func TestGouging(t *testing.T) {
t.Fatal("unexpected data")
}

// update the gouging settings to limit the max storage price to 100H
// fetch current host settings
settings := cluster.hosts[0].settings.Settings()

// update host settings
updated := settings
updated.StoragePrice = updated.StoragePrice.Mul64(2)
tt.OK(cluster.hosts[0].UpdateSettings(updated))

// update gouging settings
gs := test.GougingSettings
gs.MaxStoragePrice = types.NewCurrency64(100)
gs.MaxStoragePrice = settings.StoragePrice
if err := b.UpdateGougingSettings(context.Background(), gs); err != nil {
t.Fatal(err)
}

// fetch current contract set
contracts, err := b.Contracts(context.Background(), api.ContractsOpts{ContractSet: cfg.Set})
tt.OK(err)

// update one host's settings so it's gouging
hk := contracts[0].HostKey
host := hostsMap[hk.String()]
settings := host.settings.Settings()
settings.StoragePrice = types.NewCurrency64(101) // gouging
tt.OK(host.UpdateSettings(settings))

// make sure the price table expires so the worker is forced to fetch it
// again, this is necessary for the host to be considered price gouging
time.Sleep(defaultHostSettings.PriceTableValidity)

// assert all but one host are usable
h, err = b.UsableHosts(context.Background())
tt.OK(err)
if len(h) != n-1 {
t.Fatal("unexpected number of hosts", len(h))
}

// upload some data - should fail
tt.FailAll(w.UploadObject(context.Background(), bytes.NewReader(data), testBucket, path, api.UploadObjectOptions{}))

// update all host settings so they're gouging
for _, h := range cluster.hosts {
settings := h.settings.Settings()
settings.StoragePrice = types.NewCurrency64(101)
settings.StoragePrice = settings.StoragePrice.Mul64(2)
if err := h.UpdateSettings(settings); err != nil {
t.Fatal(err)
}
Expand Down
15 changes: 9 additions & 6 deletions internal/test/host.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,15 @@ func NewHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSet

func NewHostSettings() rhpv2.HostSettings {
return rhpv2.HostSettings{
AcceptingContracts: true,
Collateral: types.Siacoins(1).Div64(1 << 40),
MaxCollateral: types.Siacoins(10000),
MaxDuration: 144 * 7 * 12, // 12w
Version: "1.5.10",
RemainingStorage: 1 << 42, // 4 TiB
AcceptingContracts: true,
Collateral: types.Siacoins(1).Div64(1 << 40),
EphemeralAccountExpiry: time.Hour * 24,
MaxCollateral: types.Siacoins(10000),
MaxDuration: 144 * 7 * 12, // 12w
MaxEphemeralAccountBalance: types.Siacoins(1000),
Version: "1.5.10",
RemainingStorage: 1 << 42, // 4 TiB
SiaMuxPort: "9983",
}
}

Expand Down
8 changes: 8 additions & 0 deletions stores/hostdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,3 +116,11 @@ func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api
return tx.RecordPriceTables(ctx, priceTableUpdate)
})
}

func (s *SQLStore) UsableHosts(ctx context.Context) (hosts []sql.HostInfo, err error) {
err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
hosts, err = tx.UsableHosts(ctx)
return err
})
return
}
151 changes: 151 additions & 0 deletions stores/hostdb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"go.sia.tech/core/types"
"go.sia.tech/coreutils/chain"
"go.sia.tech/renterd/api"
"go.sia.tech/renterd/internal/gouging"
"go.sia.tech/renterd/internal/test"
sql "go.sia.tech/renterd/stores/sql"
)

Expand Down Expand Up @@ -446,6 +448,155 @@ func TestHosts(t *testing.T) {
}
}

func TestUsableHosts(t *testing.T) {
ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
defer ss.Close()
ctx := context.Background()

// add autopilot
err := ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: api.DefaultAutopilotID})
if err != nil {
t.Fatal(err)
}

// prepare hosts & contracts
//
// h1: usable
// h2: not usable - blocked
// h3: not usable - no host check
// h4: not usable - no contract
var hks []types.PublicKey
for i := 1; i <= 4; i++ {
// add host
hk := types.PublicKey{byte(i)}
addr := fmt.Sprintf("foo.com:100%d", i)
err := ss.addCustomTestHost(hk, addr)
if err != nil {
t.Fatal(err)
}
hks = append(hks, hk)

// add host scan
hs := test.NewHostSettings()
pt := test.NewHostPriceTable()
s1 := newTestScan(hk, time.Now(), hs, pt, true, nil, nil)
if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {
t.Fatal(err)
}

// add host check
if i != 3 {
hc := newTestHostCheck()
err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, hk, hc)
if err != nil {
t.Fatal(err)
}
}

// add contract
if i != 4 {
_, err = ss.addTestContract(types.FileContractID{byte(i)}, hk)
if err != nil {
t.Fatal(err)
}
}

// block host
if i == 2 {
err = ss.UpdateHostBlocklistEntries(context.Background(), []string{addr}, nil, false)
if err != nil {
t.Fatal(err)
}
}
}

// assert h1 is usable
hosts, err := ss.UsableHosts(ctx)
if err != nil {
t.Fatal(err)
} else if len(hosts) != 1 {
t.Fatal("unexpected", len(hosts))
} else if hosts[0].PublicKey != hks[0] {
t.Fatal("unexpected", hosts)
} else if hosts[0].SiamuxAddr != "foo.com:9983" {
t.Fatal("unexpected", hosts)
}

// create gouging checker
gs := test.GougingSettings
cs := api.ConsensusState{Synced: true}
gc := gouging.NewChecker(gs, cs, nil, nil)

// assert h1 is not gouging
h1 := hosts[0]
if gc.Check(&h1.HS, &h1.PT).Gouging() {
t.Fatal("unexpected")
}

// record a scan for h1 to make it gouging
hs := test.NewHostSettings()
pt := test.NewHostPriceTable()
pt.UploadBandwidthCost = gs.MaxUploadPrice
s1 := newTestScan(h1.PublicKey, time.Now(), hs, pt, true, nil, nil)
if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {
t.Fatal(err)
}

// fetch it again
hosts, err = ss.UsableHosts(ctx)
if err != nil {
t.Fatal(err)
} else if len(hosts) != 1 {
t.Fatal("unexpected", len(hosts))
}

// assert h1 is now gouging
h1 = hosts[0]
if !gc.Check(&h1.HS, &h1.PT).Gouging() {
t.Fatal("unexpected")
}

// create helper to assert number of usable hosts
assertNumUsableHosts := func(n int) {
t.Helper()
hosts, err = ss.UsableHosts(ctx)
if err != nil {
t.Fatal(err)
} else if len(hosts) != n {
t.Fatal("unexpected", len(hosts))
}
}

// unblock h2
if err := ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true); err != nil {
t.Fatal(err)
}

assertNumUsableHosts(2)

// add host check for h3
hc := newTestHostCheck()
err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, types.PublicKey{3}, hc)
if err != nil {
t.Fatal(err)
}

assertNumUsableHosts(3)

// add contract for h4
_, err = ss.addTestContract(types.FileContractID{byte(4)}, types.PublicKey{4})
if err != nil {
t.Fatal(err)
}

assertNumUsableHosts(4)

// add an allowlist
ss.UpdateHostAllowlistEntries(context.Background(), []types.PublicKey{{9}}, nil, false)

assertNumUsableHosts(0)
}

// TestRecordScan is a test for recording scans.
func TestRecordScan(t *testing.T) {
ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
Expand Down
5 changes: 5 additions & 0 deletions stores/sql/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,11 @@ type (
// present in the database.
UpsertContractSectors(ctx context.Context, contractSectors []ContractSector) error

// UsableHosts returns a list of hosts that are ready to be used. That
// means they are deemed usable by the autopilot, they are not gouging,
// not blocked, not offline, etc.
UsableHosts(ctx context.Context) ([]HostInfo, error)

// WalletEvents returns all wallet events in the database.
WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error)

Expand Down
Loading

0 comments on commit 8c9f8a7

Please sign in to comment.