From 6a32ca823fcbe0f334a4a3c68175ff4371598c03 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 09:24:34 +0530 Subject: [PATCH 01/11] drop policies table on cleanup Signed-off-by: Pranav Singh --- models/meshmodel/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/meshmodel/registry.go b/models/meshmodel/registry.go index fd121486..8ef91a2c 100644 --- a/models/meshmodel/registry.go +++ b/models/meshmodel/registry.go @@ -89,6 +89,7 @@ func (rm *RegistryManager) Cleanup() { &v1alpha1.ModelDB{}, &v1alpha1.CategoryDB{}, &v1alpha1.RelationshipDefinitionDB{}, + &v1alpha1.PolicyDefinitionDB{}, ) } func (rm *RegistryManager) RegisterEntity(h Host, en Entity) error { From cd03771f26e9c0b2fcc7bc7ae5f1c0b00226a604 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 12:28:55 +0530 Subject: [PATCH 02/11] rm comment Signed-off-by: Pranav Singh --- models/meshmodel/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/meshmodel/registry.go b/models/meshmodel/registry.go index 8ef91a2c..d8697d10 100644 --- a/models/meshmodel/registry.go +++ b/models/meshmodel/registry.go @@ -133,7 +133,6 @@ func (rm *RegistryManager) RegisterEntity(h Host, en Entity) error { UpdatedAt: time.Now(), } return rm.db.Create(&entry).Error - //Add logic for Policies and other entities below case v1alpha1.PolicyDefinition: policyID, err := v1alpha1.CreatePolicy(rm.db, entity) if err != nil { From 2c4f855c7fb0d799c5de83665e54a8e19ff077f9 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 12:29:28 +0530 Subject: [PATCH 03/11] update policy structs attribute types and create func Signed-off-by: Pranav Singh --- models/meshmodel/core/v1alpha1/policy.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index f616cb4e..d923330e 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -15,7 +15,8 @@ type PolicyDefinition struct { TypeMeta Model Model `json:"model"` SubType string `json:"subType" yaml:"subType"` - Expression map[string]interface{} `json:"expression" yaml:"expression"` + Expression string `json:"expression" yaml:"expression"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` } @@ -25,7 +26,8 @@ type PolicyDefinitionDB struct { ModelID uuid.UUID `json:"-" gorm:"modelID"` TypeMeta SubType string `json:"subType" yaml:"subType"` - Expression []byte `json:"expression" yaml:"expression"` + Expression string `json:"expression" yaml:"expression"` + Metadata []byte `json:"metadata" yaml:"metadata"` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` } @@ -83,10 +85,11 @@ func (pdb *PolicyDefinitionDB) GetPolicyDefinition(m Model) (p PolicyDefinition) p.TypeMeta = pdb.TypeMeta p.Model = m p.SubType = pdb.SubType - if p.Expression == nil { - p.Expression = make(map[string]interface{}) + p.Expression = pdb.Expression + if p.Metadata == nil { + p.Metadata = make(map[string]interface{}) } - _ = json.Unmarshal(pdb.Expression, &p.Expression) + _ = json.Unmarshal(pdb.Metadata, &p.Metadata) return } @@ -111,6 +114,7 @@ func (p *PolicyDefinition) GetPolicyDefinitionDB() (pdb PolicyDefinitionDB) { pdb.TypeMeta = p.TypeMeta pdb.SubType = p.SubType pdb.ModelID = p.Model.ID - pdb.Expression, _ = json.Marshal(p.Expression) + pdb.Expression = p.Expression + pdb.Metadata, _ = json.Marshal(p.Metadata) return } From 96f7d565d4a951d03485de1ee5a902ef13ce9a99 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 15:25:22 +0530 Subject: [PATCH 04/11] add support counting policies Signed-off-by: Pranav Singh --- models/meshmodel/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/meshmodel/registry.go b/models/meshmodel/registry.go index d8697d10..d0ffb763 100644 --- a/models/meshmodel/registry.go +++ b/models/meshmodel/registry.go @@ -175,11 +175,11 @@ func (rm *RegistryManager) GetEntities(f types.Filter) ([]Entity, *int64) { return en, &count case *v1alpha1.PolicyFilter: en := make([]Entity, 0) - policies := v1alpha1.GetMeshModelPolicy(rm.db, *filter) + policies, count := v1alpha1.GetMeshModelPolicy(rm.db, *filter) for _, pol := range policies { en = append(en, pol) } - return en, nil + return en, &count default: return nil, nil } From 8b87912f5e9811f5251c126fc82d4ef9820a1720 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 15:26:05 +0530 Subject: [PATCH 05/11] fix gorm bugs; add filters support for querying DB Signed-off-by: Pranav Singh --- models/meshmodel/core/v1alpha1/policy.go | 47 ++++++++++++++++++++---- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index d923330e..9f54933d 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -8,6 +8,8 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/types" + "github.com/sirupsen/logrus" + "gorm.io/gorm/clause" ) type PolicyDefinition struct { @@ -35,7 +37,14 @@ type PolicyDefinitionDB struct { type PolicyFilter struct { Kind string SubType string - ModelName string + Name string + APIVersion string + ModelName string + Version string // future use for versioning + Sort string //asc or desc. Default behavior is asc + OrderOn string //Name of the field on which sorting is to be done + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int } func (pf *PolicyFilter) Create(m map[string]interface{}) { @@ -52,15 +61,18 @@ func (p PolicyDefinition) Type() types.CapabilityType { return types.PolicyDefinition } -func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefinition) { +func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefinition, count int64) { type componentDefinitionWithModel struct { PolicyDefinitionDB - Model + ModelDB + CategoryDB } + logrus.Debug("filter: ", f) var componentDefinitionsWithModel []componentDefinitionWithModel finder := db.Model(&PolicyDefinitionDB{}). - Select("policy_definition_dbs.*, models.*"). - Joins("JOIN model_dbs ON model_dbs.id = policy_definition_dbs.model_id") + Select("policy_definition_dbs.*, model_dbs.*"). + Joins("JOIN model_dbs ON model_dbs.id = policy_definition_dbs.model_id"). + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") if f.Kind != "" { finder = finder.Where("policy_definition_dbs.kind = ?", f.Kind) } @@ -70,14 +82,35 @@ func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefini if f.ModelName != "" { finder = finder.Where("model_dbs.name = ?", f.ModelName) } + if f.APIVersion != "" { + finder = finder.Where("policy_definition_dbs.api_version = ?", f.APIVersion) + } + if f.Name != "" { + finder = finder.Where("policy_definition_dbs.metadata ->> 'name'", f.Name) + } + if f.OrderOn != "" { + if f.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: f.OrderOn}, Desc: true}) + } else { + finder = finder.Order(f.OrderOn + " asc") + } + } + + finder.Count(&count) + + finder = finder.Offset(f.Offset) + if f.Limit != 0 { + finder = finder.Limit(f.Limit) + } + err := finder.Scan(&componentDefinitionsWithModel).Error if err != nil { fmt.Println(err.Error()) } for _, cm := range componentDefinitionsWithModel { - pl = append(pl, cm.PolicyDefinitionDB.GetPolicyDefinition(cm.Model)) + pl = append(pl, cm.PolicyDefinitionDB.GetPolicyDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db)))) } - return pl + return pl, count } func (pdb *PolicyDefinitionDB) GetPolicyDefinition(m Model) (p PolicyDefinition) { From 46ac7fce2bfc9ce5d5881bde3c3c834a3175e031 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 15:26:50 +0530 Subject: [PATCH 06/11] make RegoPolicyHandler dynamic to ingest any expression Signed-off-by: Pranav Singh --- models/meshmodel/core/policies/rego_policy_relationship.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index d884fd18..35aa4f17 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -10,12 +10,12 @@ import ( ) // RegoPolicyHandler takes the required inputs and run the query against all the policy files provided -func RegoPolicyHandler(ctx context.Context, policyDir []string, regoQueryString string, designFile []byte) (map[string]interface{}, error) { - regoPolicyLoader := rego.Load(policyDir, nil) +func RegoPolicyHandler(ctx context.Context, expression string, regoQueryString string, designFile []byte) (map[string]interface{}, error) { + // regoPolicyLoader := rego.Load(policyDir, nil) regoEngine, err := rego.New( rego.Query(regoQueryString), - regoPolicyLoader, + rego.Module("expression.rego", expression), ).PrepareForEval(ctx) if err != nil { logrus.Error("error preparing for evaluation", err) From e56309ddd90fa6ee6152b6c5fda8295a4208fea1 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 19:28:28 +0530 Subject: [PATCH 07/11] rm debug logs Signed-off-by: Pranav Singh --- models/meshmodel/core/policies/rego_policy_relationship.go | 1 - models/meshmodel/core/v1alpha1/policy.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 35aa4f17..df31f54c 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -11,7 +11,6 @@ import ( // RegoPolicyHandler takes the required inputs and run the query against all the policy files provided func RegoPolicyHandler(ctx context.Context, expression string, regoQueryString string, designFile []byte) (map[string]interface{}, error) { - // regoPolicyLoader := rego.Load(policyDir, nil) regoEngine, err := rego.New( rego.Query(regoQueryString), diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index 9f54933d..8cd867ce 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -67,7 +67,7 @@ func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefini ModelDB CategoryDB } - logrus.Debug("filter: ", f) + var componentDefinitionsWithModel []componentDefinitionWithModel finder := db.Model(&PolicyDefinitionDB{}). Select("policy_definition_dbs.*, model_dbs.*"). From 2a2d8276a38a0227787974edebc37de79310a8a3 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Thu, 25 May 2023 19:31:08 +0530 Subject: [PATCH 08/11] fix lint check Signed-off-by: Pranav Singh --- models/meshmodel/core/v1alpha1/policy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index 8cd867ce..b46b5ebe 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/types" - "github.com/sirupsen/logrus" "gorm.io/gorm/clause" ) From c2c0c0de67a3b18f8b056f5f8802ba28c025d9a8 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Fri, 26 May 2023 13:03:38 +0530 Subject: [PATCH 09/11] missing imports Signed-off-by: Pranav Singh --- models/meshmodel/core/v1alpha1/policy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index b46b5ebe..8cd867ce 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/types" + "github.com/sirupsen/logrus" "gorm.io/gorm/clause" ) From 91df29c15fdd0a8fba9ccaeb4348d39e37a58912 Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Fri, 26 May 2023 13:06:35 +0530 Subject: [PATCH 10/11] rm unwanted imports Signed-off-by: Pranav Singh --- models/meshmodel/core/v1alpha1/policy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index 8cd867ce..b46b5ebe 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/types" - "github.com/sirupsen/logrus" "gorm.io/gorm/clause" ) From 080f39b08d92cbaeddb6ed5e27e598f1ca2efd3a Mon Sep 17 00:00:00 2001 From: Pranav Singh Date: Fri, 26 May 2023 13:14:16 +0530 Subject: [PATCH 11/11] fix lint Signed-off-by: Pranav Singh --- models/meshmodel/core/v1alpha1/policy.go | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index b46b5ebe..70b1b1ec 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -16,8 +16,8 @@ type PolicyDefinition struct { TypeMeta Model Model `json:"model"` SubType string `json:"subType" yaml:"subType"` - Expression string `json:"expression" yaml:"expression"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + Expression string `json:"expression" yaml:"expression"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` } @@ -28,22 +28,22 @@ type PolicyDefinitionDB struct { TypeMeta SubType string `json:"subType" yaml:"subType"` Expression string `json:"expression" yaml:"expression"` - Metadata []byte `json:"metadata" yaml:"metadata"` + Metadata []byte `json:"metadata" yaml:"metadata"` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` } type PolicyFilter struct { - Kind string - SubType string - Name string - APIVersion string - ModelName string - Version string // future use for versioning - Sort string //asc or desc. Default behavior is asc - OrderOn string //Name of the field on which sorting is to be done - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int + Kind string + SubType string + Name string + APIVersion string + ModelName string + Version string // future use for versioning + Sort string //asc or desc. Default behavior is asc + OrderOn string //Name of the field on which sorting is to be done + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int } func (pf *PolicyFilter) Create(m map[string]interface{}) { @@ -71,7 +71,7 @@ func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefini finder := db.Model(&PolicyDefinitionDB{}). Select("policy_definition_dbs.*, model_dbs.*"). Joins("JOIN model_dbs ON model_dbs.id = policy_definition_dbs.model_id"). - Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") if f.Kind != "" { finder = finder.Where("policy_definition_dbs.kind = ?", f.Kind) }