From 7a4061513e1ef620599369f7e23688df617a737a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Sat, 14 Jun 2025 01:31:54 +0200 Subject: [PATCH 1/4] firewalldb: rename kv stores migration tests As the firewalldb migration will include more than just the migration of the kvstore data, we rename the migration tests that only migrate the kvstore data to make it clearer which tests only focus on migrating kv entries. --- firewalldb/sql_migration_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index c2442a819..09822f658 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -234,23 +234,23 @@ func TestFirewallDBMigration(t *testing.T) { }, }, { - name: "global entries", + name: "global kv entries", populateDB: globalEntries, }, { - name: "session specific entries", + name: "session specific kv entries", populateDB: sessionSpecificEntries, }, { - name: "feature specific entries", + name: "feature specific kv entries", populateDB: featureSpecificEntries, }, { - name: "all entry combinations", + name: "all kv entry combinations", populateDB: allEntryCombinations, }, { - name: "random entries", + name: "random kv entries", populateDB: randomKVEntries, }, } From 4d55bac2db752a761708832203fb74bebcaaaa0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Tue, 17 Jun 2025 17:34:31 +0200 Subject: [PATCH 2/4] firewalldb: prepare migration tests for more stores Currently, the migration tests for firewalldb only migrates the kv stores. In future commits, we will also migrate the privacy mapper and the actions in the firewalldb,. Before this commit, the expected results of the migrations tests could only be kv records, which will not be the case when we also migrate the privacy mapper and the actions. Therefore, we prepare the migration tests to expect more than just kv records. This commit introduces a new type of `expectedResult` type which the prep of the migration tests will use, which can specify more than just one type of expected result. --- firewalldb/sql_migration_test.go | 65 +++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index 09822f658..d48c0de11 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -34,6 +34,11 @@ var ( testEntryValue = []byte{1, 2, 3} ) +// expectedResult represents the expected result of a migration test. +type expectedResult struct { + kvEntries fn.Option[[]*kvEntry] +} + // TestFirewallDBMigration tests the migration of firewalldb from a bolt // backend to a SQL database. Note that this test does not attempt to be a // complete migration test. @@ -72,10 +77,10 @@ func TestFirewallDBMigration(t *testing.T) { return store, genericExecutor } - // The assertMigrationResults function will currently assert that + // The assertKvStoreMigrationResults function will currently assert that // the migrated kv stores entries in the SQLDB match the original kv // stores entries in the BoltDB. - assertMigrationResults := func(t *testing.T, sqlStore *SQLDB, + assertKvStoreMigrationResults := func(t *testing.T, sqlStore *SQLDB, kvEntries []*kvEntry) { var ( @@ -213,6 +218,20 @@ func TestFirewallDBMigration(t *testing.T) { } } + // The assertMigrationResults function will currently assert that + // the migrated kv stores records in the SQLDB match the original kv + // stores records in the BoltDB. + assertMigrationResults := func(t *testing.T, sqlStore *SQLDB, + expRes *expectedResult) { + + // If the expected result contains kv records, then we + // assert that the kv store migration results match + // the expected results. + expRes.kvEntries.WhenSome(func(kvEntries []*kvEntry) { + assertKvStoreMigrationResults(t, sqlStore, kvEntries) + }) + } + // The tests slice contains all the tests that we will run for the // migration of the firewalldb from a BoltDB to a SQLDB. // Note that the tests currently only test the migration of the KV @@ -221,16 +240,22 @@ func TestFirewallDBMigration(t *testing.T) { tests := []struct { name string populateDB func(t *testing.T, ctx context.Context, - boltDB *BoltDB, sessionStore session.Store) []*kvEntry + boltDB *BoltDB, sessionStore session.Store) *expectedResult }{ { name: "empty", populateDB: func(t *testing.T, ctx context.Context, boltDB *BoltDB, - sessionStore session.Store) []*kvEntry { + sessionStore session.Store) *expectedResult { + + // Don't populate the DB, and return empty kv + // records and privacy pairs. - // Don't populate the DB. - return make([]*kvEntry, 0) + return &expectedResult{ + kvEntries: fn.Some( + []*kvEntry{}, + ), + } }, }, { @@ -314,7 +339,7 @@ func TestFirewallDBMigration(t *testing.T) { // globalEntries populates the kv store with one global entry for the temp // store, and one for the perm store. func globalEntries(t *testing.T, ctx context.Context, boltDB *BoltDB, - _ session.Store) []*kvEntry { + _ session.Store) *expectedResult { return insertTempAndPermEntry( t, ctx, boltDB, testRuleName, fn.None[[]byte](), @@ -326,7 +351,7 @@ func globalEntries(t *testing.T, ctx context.Context, boltDB *BoltDB, // entry for the local temp store, and one session specific entry for the perm // local store. func sessionSpecificEntries(t *testing.T, ctx context.Context, boltDB *BoltDB, - sessionStore session.Store) []*kvEntry { + sessionStore session.Store) *expectedResult { groupAlias := getNewSessionAlias(t, ctx, sessionStore) @@ -340,7 +365,7 @@ func sessionSpecificEntries(t *testing.T, ctx context.Context, boltDB *BoltDB, // entry for the local temp store, and one feature specific entry for the perm // local store. func featureSpecificEntries(t *testing.T, ctx context.Context, boltDB *BoltDB, - sessionStore session.Store) []*kvEntry { + sessionStore session.Store) *expectedResult { groupAlias := getNewSessionAlias(t, ctx, sessionStore) @@ -358,11 +383,11 @@ func featureSpecificEntries(t *testing.T, ctx context.Context, boltDB *BoltDB, // any entries when the entry set is more complex than just a single entry at // each level. func allEntryCombinations(t *testing.T, ctx context.Context, boltDB *BoltDB, - sessionStore session.Store) []*kvEntry { + sessionStore session.Store) *expectedResult { var result []*kvEntry - add := func(entry []*kvEntry) { - result = append(result, entry...) + add := func(entry *expectedResult) { + result = append(result, entry.kvEntries.UnwrapOrFail(t)...) } // First lets create standard entries at all levels, which represents @@ -446,7 +471,9 @@ func allEntryCombinations(t *testing.T, ctx context.Context, boltDB *BoltDB, fn.Some(testFeatureName), testEntryKey4, emptyValue, )) - return result + return &expectedResult{ + kvEntries: fn.Some(result), + } } func getNewSessionAlias(t *testing.T, ctx context.Context, @@ -467,7 +494,7 @@ func getNewSessionAlias(t *testing.T, ctx context.Context, func insertTempAndPermEntry(t *testing.T, ctx context.Context, boltDB *BoltDB, ruleName string, groupAlias fn.Option[[]byte], featureNameOpt fn.Option[string], entryKey string, - entryValue []byte) []*kvEntry { + entryValue []byte) *expectedResult { tempKvEntry := &kvEntry{ ruleName: ruleName, @@ -491,7 +518,9 @@ func insertTempAndPermEntry(t *testing.T, ctx context.Context, insertKvEntry(t, ctx, boltDB, permKvEntry) - return []*kvEntry{tempKvEntry, permKvEntry} + return &expectedResult{ + kvEntries: fn.Some([]*kvEntry{tempKvEntry, permKvEntry}), + } } // insertKvEntry populates the kv store with passed entry, and asserts that the @@ -540,7 +569,7 @@ func insertKvEntry(t *testing.T, ctx context.Context, // across all possible combinations of different levels of entries in the kv // store. All values and different bucket names are randomly generated. func randomKVEntries(t *testing.T, ctx context.Context, - boltDB *BoltDB, sessionStore session.Store) []*kvEntry { + boltDB *BoltDB, sessionStore session.Store) *expectedResult { var ( // We set the number of entries to insert to 1000, as that @@ -651,7 +680,9 @@ func randomKVEntries(t *testing.T, ctx context.Context, insertedEntries = append(insertedEntries, entry) } - return insertedEntries + return &expectedResult{ + kvEntries: fn.Some(insertedEntries), + } } // randomString generates a random string of the passed length n. From 4d2a4233c872443d68c826374936660bb4ab88e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Wed, 23 Jul 2025 15:02:30 +0200 Subject: [PATCH 3/4] firewalldb: remove speculative doc comments This commit removes speculative comments from the firewalldb migration docs that forecast future implementations. Such comments can create confusion for developers looking at the current code base without knowing our plans how the migration will be further developed. --- firewalldb/sql_migration.go | 3 --- firewalldb/sql_migration_test.go | 12 +++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/firewalldb/sql_migration.go b/firewalldb/sql_migration.go index 1e114c12c..057a15d52 100644 --- a/firewalldb/sql_migration.go +++ b/firewalldb/sql_migration.go @@ -72,9 +72,6 @@ func (e *kvEntry) namespacedKey() string { // transaction to ensure that all rows in the stores are migrated or none at // all. // -// Note that this migration currently only migrates the kvstores, but will be -// extended in the future to also migrate the privacy mapper and action stores. -// // NOTE: As sessions may contain linked sessions and accounts, the sessions and // accounts sql migration MUST be run prior to this migration. func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB, diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index d48c0de11..eca8b7cb5 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -42,9 +42,6 @@ type expectedResult struct { // TestFirewallDBMigration tests the migration of firewalldb from a bolt // backend to a SQL database. Note that this test does not attempt to be a // complete migration test. -// This test only tests the migration of the KV stores currently, but will -// be extended in the future to also test the migration of the privacy mapper -// and the actions store in the future. func TestFirewallDBMigration(t *testing.T) { t.Parallel() @@ -218,9 +215,9 @@ func TestFirewallDBMigration(t *testing.T) { } } - // The assertMigrationResults function will currently assert that - // the migrated kv stores records in the SQLDB match the original kv - // stores records in the BoltDB. + // The assertMigrationResults asserts that the migrated entries in the + // firewall SQLDB match the expected results which should represent the + // original entries in the BoltDB. assertMigrationResults := func(t *testing.T, sqlStore *SQLDB, expRes *expectedResult) { @@ -234,9 +231,6 @@ func TestFirewallDBMigration(t *testing.T) { // The tests slice contains all the tests that we will run for the // migration of the firewalldb from a BoltDB to a SQLDB. - // Note that the tests currently only test the migration of the KV - // stores, but will be extended in the future to also test the migration - // of the privacy mapper and the actions store. tests := []struct { name string populateDB func(t *testing.T, ctx context.Context, From a4fc12eb62c8b687b0d143aa4e50c60b2ef4f3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Tue, 17 Jun 2025 17:35:21 +0200 Subject: [PATCH 4/4] firewalldb: add privacy mapper SQL migration This commit introduces the migration logic for transitioning the privacy mapper store from kvdb to SQL. Note that as of this commit, the migration is not yet triggered by any production code, i.e. only tests execute the migration logic. --- firewalldb/sql_migration.go | 282 ++++++++++++++++++++++++++++++- firewalldb/sql_migration_test.go | 206 +++++++++++++++++++++- 2 files changed, 485 insertions(+), 3 deletions(-) diff --git a/firewalldb/sql_migration.go b/firewalldb/sql_migration.go index 057a15d52..3add610f9 100644 --- a/firewalldb/sql_migration.go +++ b/firewalldb/sql_migration.go @@ -67,6 +67,11 @@ func (e *kvEntry) namespacedKey() string { return ns } +// privacyPairs is a type alias for a map that holds the privacy pairs, where +// the outer key is the group ID, and the value is a map of real to pseudo +// values. +type privacyPairs = map[int64]map[string]string + // MigrateFirewallDBToSQL runs the migration of the firwalldb stores from the // bbolt database to a SQL database. The migration is done in a single // transaction to ensure that all rows in the stores are migrated or none at @@ -84,10 +89,14 @@ func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB, return err } + err = migratePrivacyMapperDBToSQL(ctx, kvStore, sqlTx) + if err != nil { + return err + } + log.Infof("The rules DB has been migrated from KV to SQL.") - // TODO(viktor): Add migration for the privacy mapper and the action - // stores. + // TODO(viktor): Add migration for the action stores. return nil } @@ -487,3 +496,272 @@ func verifyBktKeys(bkt *bbolt.Bucket, errorOnKeyValues bool, return fmt.Errorf("unexpected key found: %s", key) }) } + +func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB, + sqlTx SQLQueries) error { + + log.Infof("Starting migration of the privacy mapper store to SQL") + + // 1) Collect all privacy pairs from the KV store. + privPairs, err := collectPrivacyPairs(ctx, kvStore, sqlTx) + if err != nil { + return fmt.Errorf("error migrating privacy mapper store: %w", + err) + } + + // 2) Insert all collected privacy pairs into the SQL database. + err = insertPrivacyPairs(ctx, sqlTx, privPairs) + if err != nil { + return fmt.Errorf("insertion of privacy pairs failed: %w", err) + } + + // 3) Validate that all inserted privacy pairs match the original values + // in the KV store. Note that this is done after all values have been + // inserted, to ensure that the migration doesn't overwrite any values + // after they were inserted. + err = validatePrivacyPairsMigration(ctx, sqlTx, privPairs) + if err != nil { + return fmt.Errorf("migration validation of privacy pairs "+ + "failed: %w", err) + } + + log.Infof("Migration of the privacy mapper stores to SQL completed. "+ + "Total number of rows migrated: %d", len(privPairs)) + return nil +} + +// collectPrivacyPairs collects all privacy pairs from the KV store. +func collectPrivacyPairs(ctx context.Context, kvStore *bbolt.DB, + sqlTx SQLQueries) (privacyPairs, error) { + + groupPairs := make(privacyPairs) + + return groupPairs, kvStore.View(func(kvTx *bbolt.Tx) error { + bkt := kvTx.Bucket(privacyBucketKey) + if bkt == nil { + // If we haven't generated any privacy bucket yet, + // we can skip the migration, as there are no privacy + // pairs to migrate. + return nil + } + + return bkt.ForEach(func(groupId, v []byte) error { + if v != nil { + return fmt.Errorf("expected only buckets "+ + "under %s bkt, but found value %s", + privacyBucketKey, v) + } + + gBkt := bkt.Bucket(groupId) + if gBkt == nil { + return fmt.Errorf("group bkt for group id "+ + "%s not found", groupId) + } + + groupSqlId, err := sqlTx.GetSessionIDByAlias( + ctx, groupId, + ) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("session with group id %x "+ + "not found in sql db", groupId) + } else if err != nil { + return err + } + + groupRealToPseudoPairs, err := collectGroupPairs(gBkt) + if err != nil { + return fmt.Errorf("processing group bkt "+ + "for group id %s (sqlID %d) failed: %w", + groupId, groupSqlId, err) + } + + groupPairs[groupSqlId] = groupRealToPseudoPairs + + return nil + }) + }) +} + +// collectGroupPairs collects all privacy pairs for a specific session group, +// i.e. the group buckets under the privacy mapper bucket in the KV store. +// The function returns them as a map, where the key is the real value, and +// the value for the key is the pseudo values. +// It also checks that the pairs are consistent, i.e. that for each real value +// there is a corresponding pseudo value, and vice versa. If the pairs are +// inconsistent, it returns an error indicating the mismatch. +func collectGroupPairs(bkt *bbolt.Bucket) (map[string]string, error) { + var ( + realToPseudoRes map[string]string + pseudoToRealRes map[string]string + err error + missMatchErr = errors.New("privacy mapper pairs mismatch") + ) + + if realBkt := bkt.Bucket(realToPseudoKey); realBkt != nil { + realToPseudoRes, err = collectPairs(realBkt) + if err != nil { + return nil, fmt.Errorf("fetching real to pseudo pairs "+ + "failed: %w", err) + } + } else { + return nil, fmt.Errorf("%s bucket not found", realToPseudoKey) + } + + if pseudoBkt := bkt.Bucket(pseudoToRealKey); pseudoBkt != nil { + pseudoToRealRes, err = collectPairs(pseudoBkt) + if err != nil { + return nil, fmt.Errorf("fetching pseudo to real pairs "+ + "failed: %w", err) + } + } else { + return nil, fmt.Errorf("%s bucket not found", pseudoToRealKey) + } + + if len(realToPseudoRes) != len(pseudoToRealRes) { + return nil, missMatchErr + } + + for realVal, pseudoVal := range realToPseudoRes { + if rv, ok := pseudoToRealRes[pseudoVal]; !ok || rv != realVal { + return nil, missMatchErr + } + } + + return realToPseudoRes, nil +} + +// collectPairs collects all privacy pairs from a specific realToPseudoKey or +// pseudoToRealKey bucket in the KV store. It returns a map where the key is +// the real value or pseudo value, and the value is the corresponding pseudo +// value or real value, respectively (depending on if the realToPseudo or +// pseudoToReal bucket is passed to the function). +func collectPairs(pairsBucket *bbolt.Bucket) (map[string]string, error) { + pairsRes := make(map[string]string) + + return pairsRes, pairsBucket.ForEach(func(k, v []byte) error { + if v == nil { + return fmt.Errorf("expected only key-values under "+ + "pairs bucket, but found bucket %s", k) + } + + if len(v) == 0 { + return fmt.Errorf("empty value stored for privacy "+ + "pairs key %s", k) + } + + pairsRes[string(k)] = string(v) + + return nil + }) +} + +// insertPrivacyPairs inserts the collected privacy pairs into the SQL database. +func insertPrivacyPairs(ctx context.Context, sqlTx SQLQueries, + pairs privacyPairs) error { + + for groupId, groupPairs := range pairs { + err := insertGroupPairs(ctx, sqlTx, groupPairs, groupId) + if err != nil { + return fmt.Errorf("inserting group pairs for group "+ + "id %d failed: %w", groupId, err) + } + } + + return nil +} + +// insertGroupPairs inserts the privacy pairs for a specific group into +// the SQL database. It checks for duplicates before inserting, and returns +// an error if a duplicate pair is found. The function takes a map of real +// to pseudo values, where the key is the real value and the value is the +// corresponding pseudo value. +func insertGroupPairs(ctx context.Context, sqlTx SQLQueries, + pairs map[string]string, groupID int64) error { + + for realVal, pseudoVal := range pairs { + err := sqlTx.InsertPrivacyPair( + ctx, sqlc.InsertPrivacyPairParams{ + GroupID: groupID, + RealVal: realVal, + PseudoVal: pseudoVal, + }, + ) + if err != nil { + return fmt.Errorf("inserting privacy pair %s:%s "+ + "failed: %w", realVal, pseudoVal, err) + } + } + + return nil +} + +// validatePrivacyPairsMigration validates that the migrated privacy pairs +// match the original values in the KV store. +func validatePrivacyPairsMigration(ctx context.Context, sqlTx SQLQueries, + pairs privacyPairs) error { + + for groupId, groupPairs := range pairs { + err := validateGroupPairsMigration( + ctx, sqlTx, groupPairs, groupId, + ) + if err != nil { + return fmt.Errorf("migration validation of privacy "+ + "pairs for group %d failed: %w", groupId, err) + } + } + + return nil +} + +// validateGroupPairsMigration validates that the migrated privacy pairs for +// a specific group match the original values in the KV store. It checks that +// for each real value, the pseudo value in the SQL database matches the +// original pseudo value, and vice versa. If any mismatch is found, it returns +// an error indicating the mismatch. +func validateGroupPairsMigration(ctx context.Context, sqlTx SQLQueries, + pairs map[string]string, groupID int64) error { + + for realVal, pseudoVal := range pairs { + resPseudoVal, err := sqlTx.GetPseudoForReal( + ctx, sqlc.GetPseudoForRealParams{ + GroupID: groupID, + RealVal: realVal, + }, + ) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("migrated privacy pair %s:%s not "+ + "found for real value", realVal, pseudoVal) + } + if err != nil { + return err + } + + if resPseudoVal != pseudoVal { + return fmt.Errorf("pseudo value in db %s, does not "+ + "match original value %s, for real value %s", + resPseudoVal, pseudoVal, realVal) + } + + resRealVal, err := sqlTx.GetRealForPseudo( + ctx, sqlc.GetRealForPseudoParams{ + GroupID: groupID, + PseudoVal: pseudoVal, + }, + ) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("migrated privacy pair %s:%s not "+ + "found for pseudo value", realVal, pseudoVal) + } + if err != nil { + return err + } + + if resRealVal != realVal { + return fmt.Errorf("real value in db %s, does not "+ + "match original value %s, for pseudo value %s", + resRealVal, realVal, pseudoVal) + } + } + + return nil +} diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index eca8b7cb5..c36bf339c 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -37,6 +37,7 @@ var ( // expectedResult represents the expected result of a migration test. type expectedResult struct { kvEntries fn.Option[[]*kvEntry] + privPairs fn.Option[privacyPairs] } // TestFirewallDBMigration tests the migration of firewalldb from a bolt @@ -215,6 +216,29 @@ func TestFirewallDBMigration(t *testing.T) { } } + assertPrivacyMapperMigrationResults := func(t *testing.T, + sqlStore *SQLDB, privPairs privacyPairs) { + + for groupID, groupPairs := range privPairs { + storePairs, err := sqlStore.GetAllPrivacyPairs( + ctx, groupID, + ) + require.NoError(t, err) + require.Len(t, storePairs, len(groupPairs)) + + for _, storePair := range storePairs { + // Assert that the store pair is in the + // original pairs. + pseudo, ok := groupPairs[storePair.RealVal] + require.True(t, ok) + + // Assert that the pseudo value matches + // the one in the store. + require.Equal(t, pseudo, storePair.PseudoVal) + } + } + } + // The assertMigrationResults asserts that the migrated entries in the // firewall SQLDB match the expected results which should represent the // original entries in the BoltDB. @@ -227,6 +251,13 @@ func TestFirewallDBMigration(t *testing.T) { expRes.kvEntries.WhenSome(func(kvEntries []*kvEntry) { assertKvStoreMigrationResults(t, sqlStore, kvEntries) }) + + // If the expected result contains privacy pairs, then we + // assert that the privacy mapper migration results match + // the expected results. + expRes.privPairs.WhenSome(func(pairs privacyPairs) { + assertPrivacyMapperMigrationResults(t, sqlStore, pairs) + }) } // The tests slice contains all the tests that we will run for the @@ -244,11 +275,11 @@ func TestFirewallDBMigration(t *testing.T) { // Don't populate the DB, and return empty kv // records and privacy pairs. - return &expectedResult{ kvEntries: fn.Some( []*kvEntry{}, ), + privPairs: fn.Some(make(privacyPairs)), } }, }, @@ -272,6 +303,26 @@ func TestFirewallDBMigration(t *testing.T) { name: "random kv entries", populateDB: randomKVEntries, }, + { + name: "one session and privacy pair", + populateDB: oneSessionAndPrivPair, + }, + { + name: "multiple sessions with one privacy pair", + populateDB: multiSessionsOnePrivPairs, + }, + { + name: "multiple privacy pairs", + populateDB: multipleSessionsAndPrivacyPairs, + }, + { + name: "random privacy pairs", + populateDB: randomPrivacyPairs, + }, + { + name: "random firewalldb entries", + populateDB: randomFirewallDBEntries, + }, } for _, test := range tests { @@ -467,6 +518,7 @@ func allEntryCombinations(t *testing.T, ctx context.Context, boltDB *BoltDB, return &expectedResult{ kvEntries: fn.Some(result), + privPairs: fn.None[privacyPairs](), } } @@ -514,6 +566,8 @@ func insertTempAndPermEntry(t *testing.T, ctx context.Context, return &expectedResult{ kvEntries: fn.Some([]*kvEntry{tempKvEntry, permKvEntry}), + // No privacy pairs are inserted in this test. + privPairs: fn.None[privacyPairs](), } } @@ -676,6 +730,156 @@ func randomKVEntries(t *testing.T, ctx context.Context, return &expectedResult{ kvEntries: fn.Some(insertedEntries), + // No privacy pairs are inserted in this test. + privPairs: fn.None[privacyPairs](), + } +} + +// multiSessionsOnePrivPairs inserts 1 session with 1 privacy pair into the +// boltDB. +func oneSessionAndPrivPair(t *testing.T, ctx context.Context, + boltDB *BoltDB, sessionStore session.Store) *expectedResult { + + return createPrivacyPairs(t, ctx, boltDB, sessionStore, 1, 1) +} + +// multiSessionsOnePrivPairs inserts 1 session with 10 privacy pairs into the +// boltDB. +func multiSessionsOnePrivPairs(t *testing.T, ctx context.Context, + boltDB *BoltDB, sessionStore session.Store) *expectedResult { + + return createPrivacyPairs(t, ctx, boltDB, sessionStore, 1, 10) +} + +// multipleSessionsAndPrivacyPairs inserts 5 sessions with 10 privacy pairs +// per session into the boltDB. +func multipleSessionsAndPrivacyPairs(t *testing.T, ctx context.Context, + boltDB *BoltDB, sessionStore session.Store) *expectedResult { + + return createPrivacyPairs(t, ctx, boltDB, sessionStore, 5, 10) +} + +// createPrivacyPairs is a helper function that creates a number of sessions +// with a number of privacy pairs per session. It returns an expectedResult +// struct that contains the expected privacy pairs and no kv records. +func createPrivacyPairs(t *testing.T, ctx context.Context, + boltDB *BoltDB, sessionStore session.Store, numSessions int, + numPairsPerSession int) *expectedResult { + + pairs := make(privacyPairs) + + sessSQLStore, ok := sessionStore.(*session.SQLStore) + require.True(t, ok) + + for i := 0; i < numSessions; i++ { + sess, err := sessionStore.NewSession( + ctx, fmt.Sprintf("session-%d", i), + session.Type(uint8(rand.Intn(5))), + time.Unix(1000, 0), randomString(rand.Intn(10)+1), + ) + require.NoError(t, err) + + groupID := sess.GroupID + sqlGroupID, err := sessSQLStore.GetSessionIDByAlias( + ctx, groupID[:], + ) + require.NoError(t, err) + + groupPairs := make(map[string]string) + + for j := 0; j < numPairsPerSession; j++ { + // Note that the real values will be the same across the + // sessions, as with real world data, the real value + // will often be the same across sessions. + realKey := fmt.Sprintf("real-%d", j) + pseudoKey := fmt.Sprintf("pseudo-%d-%d", i, j) + + f := func(ctx context.Context, tx PrivacyMapTx) error { + return tx.NewPair(ctx, realKey, pseudoKey) + } + + err := boltDB.PrivacyDB(groupID).Update(ctx, f) + require.NoError(t, err) + + groupPairs[realKey] = pseudoKey + } + + pairs[sqlGroupID] = groupPairs + } + + return &expectedResult{ + kvEntries: fn.None[[]*kvEntry](), + privPairs: fn.Some(pairs), + } +} + +// randomPrivacyPairs creates a random number of privacy pairs to 10 sessions. +func randomPrivacyPairs(t *testing.T, ctx context.Context, + boltDB *BoltDB, sessionStore session.Store) *expectedResult { + + numSessions := 10 + maxPairsPerSession := 20 + pairs := make(privacyPairs) + + sessSQLStore, ok := sessionStore.(*session.SQLStore) + require.True(t, ok) + + for i := 0; i < numSessions; i++ { + sess, err := sessionStore.NewSession( + ctx, fmt.Sprintf("rand-session-%d", i), + session.Type(uint8(rand.Intn(5))), + time.Unix(1000, 0), randomString(rand.Intn(10)+1), + ) + require.NoError(t, err) + + groupID := sess.GroupID + sqlGroupID, err := sessSQLStore.GetSessionIDByAlias( + ctx, groupID[:], + ) + require.NoError(t, err) + + numPairs := rand.Intn(maxPairsPerSession) + 1 + groupPairs := make(map[string]string) + + for j := 0; j < numPairs; j++ { + realKey := fmt.Sprintf("real-%s", + randomString(rand.Intn(10)+5)) + pseudoKey := fmt.Sprintf("pseudo-%s", + randomString(rand.Intn(10)+5)) + + f := func(ctx context.Context, tx PrivacyMapTx) error { + return tx.NewPair(ctx, realKey, pseudoKey) + } + + err := boltDB.PrivacyDB(groupID).Update(ctx, f) + require.NoError(t, err) + + groupPairs[realKey] = pseudoKey + } + + pairs[sqlGroupID] = groupPairs + } + + return &expectedResult{ + kvEntries: fn.None[[]*kvEntry](), + privPairs: fn.Some(pairs), + } +} + +// randomFirewallDBEntries populates the firewalldb with random entries for all +// types entries that are currently supported in the firewalldb. +// +// TODO(viktor): Extend this function to also populate it with random action +// entries, once the actions migration has been implemented. +func randomFirewallDBEntries(t *testing.T, ctx context.Context, + boltDB *BoltDB, sessionStore session.Store) *expectedResult { + + kvEntries := randomKVEntries(t, ctx, boltDB, sessionStore) + privPairs := randomPrivacyPairs(t, ctx, boltDB, sessionStore) + + return &expectedResult{ + kvEntries: kvEntries.kvEntries, + privPairs: privPairs.privPairs, } }