From 96d3cd88c87addd10b6eea1509e68a66b009aeaa Mon Sep 17 00:00:00 2001
From: pieterlukasse <pieterlukasse@gmail.com>
Date: Thu, 23 Jan 2025 20:46:40 +0100
Subject: [PATCH] fix: fix scenario with multiple filters

---
 models/helper.go                  |  2 ++
 tests/models_tests/models_test.go | 32 +++++++++++++++++++++++++++++++
 utils/db.go                       | 11 +----------
 3 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/models/helper.go b/models/helper.go
index 054a177..f5de764 100644
--- a/models/helper.go
+++ b/models/helper.go
@@ -68,6 +68,8 @@ func QueryFilterByConceptDefHelper(query *gorm.DB, sourceId int, filterConceptDe
 	if filterConceptDef.Transformation != "" {
 		// simple filterConceptDef with just the concept id
 		simpleFilterConceptDef := utils.CustomConceptVariableDef{ConceptId: filterConceptDef.ConceptId}
+		query.Select(fmt.Sprintf("%s.person_id, %s.observation_concept_id, %s.value_as_number ",
+			observationTableAlias+"_a", observationTableAlias+"_a", observationTableAlias+"_a"))
 		query := QueryFilterByConceptDefHelper2(query, sourceId, simpleFilterConceptDef,
 			omopDataSource, "", personIdFieldForObservationJoin, "observation_continuous", observationTableAlias+"_a")
 		tmpTransformedTable, err := TransformDataIntoTempTable(omopDataSource, query, filterConceptDef)
diff --git a/tests/models_tests/models_test.go b/tests/models_tests/models_test.go
index 0951d11..716d81d 100644
--- a/tests/models_tests/models_test.go
+++ b/tests/models_tests/models_test.go
@@ -882,6 +882,38 @@ func TestRetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs
 	if len(data) != 2 {
 		t.Errorf("expected 2 histogram data but got %d", len(data))
 	}
+
+	// now filter repeated twice:
+	filterConceptDefsPlusCohortPairs = []interface{}{
+		utils.CustomConceptVariableDef{
+			ConceptId: histogramConceptId,
+			Filters: []utils.Filter{
+				{
+					Type:  ">=",
+					Value: utils.Float64Ptr(1.0),
+				},
+			},
+			Transformation: "z_score",
+		}, // ^ this filter will narrow down to 2 records
+		utils.CustomConceptVariableDef{
+			ConceptId: histogramConceptId,
+			Filters: []utils.Filter{
+				{
+					Type:  ">=",
+					Value: utils.Float64Ptr(0.7),
+				},
+			},
+			Transformation: "z_score", // > a subsequent transformation of z_score on the remaining 2 records, will result
+			// in different values compared to the first filter above, as the scores are now
+			// calculated over two values only
+		},
+	}
+	data, _ = cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(testSourceId, largestCohort.Id, histogramConceptId, filterConceptDefsPlusCohortPairs)
+	// make sure the filter worked on transformed values:
+	if len(data) != 1 {
+		t.Errorf("expected 1 histogram data but got %d", len(data))
+	}
+
 }
 
 func TestRetrieveHistogramDataBySourceIdAndConceptId(t *testing.T) {
diff --git a/utils/db.go b/utils/db.go
index b7e4122..1494e43 100644
--- a/utils/db.go
+++ b/utils/db.go
@@ -81,7 +81,7 @@ func (h DbAndSchema) GetViewDirective() string {
 		return ""
 	}
 }
-func ToSQL2(query *gorm.DB) string {
+func ToSQL(query *gorm.DB) string {
 	// Use db.ToSQL to generate the SQL string for the existing query
 	sql := query.ToSQL(func(tx *gorm.DB) *gorm.DB {
 		return tx.Session(&gorm.Session{DryRun: true}).Find([]interface{}{})
@@ -90,15 +90,6 @@ func ToSQL2(query *gorm.DB) string {
 	return sql
 }
 
-func ToSQL(query *gorm.DB) string {
-	// Clone the query object to avoid altering the original
-	//tempQuery := query.Session(&gorm.Session{DryRun: true})
-	sql := query.ToSQL(func(tx *gorm.DB) *gorm.DB {
-		return tx.Find([]interface{}{})
-	})
-	return sql
-}
-
 func TableExists(tx *gorm.DB, tableName string) bool {
 	query := fmt.Sprintf("SELECT 1 FROM %s WHERE 1 = 2", tableName)
 	err := tx.Exec(query).Error