diff --git a/controllers/cohortdata.go b/controllers/cohortdata.go index b780d328..620db7b1 100644 --- a/controllers/cohortdata.go +++ b/controllers/cohortdata.go @@ -40,7 +40,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co return } - filterConceptIds, cohortPairs, err := utils.ParseConceptIdsAndDichotomousDefs(c) + filterConceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids", "error": err.Error()}) c.Abort() @@ -59,7 +59,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co return } - cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIds, cohortPairs) + cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIdsAndValues, cohortPairs) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()}) c.Abort() @@ -76,6 +76,49 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co c.JSON(http.StatusOK, gin.H{"bins": histogramData}) } +func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Context) { + sourceIdStr := c.Param("sourceid") + log.Printf("Querying source: %s", sourceIdStr) + cohortIdStr := c.Param("cohortid") + log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr) + conceptIdStr := c.Param("conceptid") + if sourceIdStr == "" || cohortIdStr == "" || conceptIdStr == "" { + c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"}) + c.Abort() + return + } + + filterConceptIdsAndValues, cohortPairs, _ := utils.ParseConceptDefsAndDichotomousDefs(c) + + sourceId, _ := strconv.Atoi(sourceIdStr) + cohortId, _ := strconv.Atoi(cohortIdStr) + conceptId, _ := strconv.ParseInt(conceptIdStr, 10, 64) + + validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs) + if !validAccessRequest { + log.Printf("Error: invalid request") + c.JSON(http.StatusForbidden, gin.H{"message": "access denied"}) + c.Abort() + return + } + + cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()}) + c.Abort() + return + } + + conceptValues := []float64{} + for _, personData := range cohortData { + conceptValues = append(conceptValues, float64(*personData.ConceptValueAsNumber)) + } + + statsData := utils.GenerateStatsData(cohortId, conceptId, conceptValues) + + c.JSON(http.StatusOK, gin.H{"statsData": statsData}) +} + func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *gin.Context) { // TODO - add some validation to ensure that only calls from Argo are allowed through since it outputs FULL data? @@ -90,7 +133,9 @@ func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *g return } - conceptIds, cohortPairs, err := utils.ParseConceptIdsAndDichotomousDefs(c) + conceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c) + conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues) + if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids and dichotomous Ids", "error": err.Error()}) c.Abort() @@ -250,12 +295,13 @@ func populateConceptValue(row []string, cohortItem models.PersonConceptAndValue, func (u CohortDataController) RetrieveCohortOverlapStats(c *gin.Context) { errors := make([]error, 4) var sourceId, caseCohortId, controlCohortId int - var conceptIds []int64 + var conceptIdsAndValues []utils.CustomConceptVariableDef var cohortPairs []utils.CustomDichotomousVariableDef sourceId, errors[0] = utils.ParseNumericArg(c, "sourceid") caseCohortId, errors[1] = utils.ParseNumericArg(c, "casecohortid") controlCohortId, errors[2] = utils.ParseNumericArg(c, "controlcohortid") - conceptIds, cohortPairs, errors[3] = utils.ParseConceptIdsAndDichotomousDefs(c) + conceptIdsAndValues, cohortPairs, errors[3] = utils.ParseConceptDefsAndDichotomousDefs(c) + conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues) validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{caseCohortId, controlCohortId}, cohortPairs) if !validAccessRequest { diff --git a/controllers/concept.go b/controllers/concept.go index c32e6034..8b23520a 100644 --- a/controllers/concept.go +++ b/controllers/concept.go @@ -197,7 +197,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) { c.Abort() return } - _, cohortPairs := utils.GetConceptIdsAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs) + _, cohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs) validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs) if !validAccessRequest { log.Printf("Error: invalid request") @@ -266,7 +266,8 @@ func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId i } func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) { - filterConceptIds, filterCohortPairs := utils.GetConceptIdsAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs) + filterConceptIdsAndValues, filterCohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs) + filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptIdsAndValues) breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId) if err != nil { return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error()) @@ -274,8 +275,8 @@ func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, conceptValuesToPeopleCount := getConceptValueToPeopleCount(breakdownStats) variableName := "" switch convertedItem := conceptIdOrCohortPair.(type) { - case int64: - conceptInformation, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptId(sourceId, convertedItem) + case utils.CustomConceptVariableDef: + conceptInformation, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptId(sourceId, convertedItem.ConceptId) if err != nil { return nil, fmt.Errorf("could not retrieve concept details for %v due to error: %s", convertedItem, err.Error()) } diff --git a/models/cohortdata.go b/models/cohortdata.go index e6ade3e8..baa4c642 100644 --- a/models/cohortdata.go +++ b/models/cohortdata.go @@ -11,7 +11,7 @@ type CohortDataI interface { RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int64) ([]*PersonConceptAndValue, error) RetrieveCohortOverlapStats(sourceId int, caseCohortId int, controlCohortId int, otherFilterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) (CohortOverlapStats, error) RetrieveDataByOriginalCohortAndNewCohort(sourceId int, originalCohortDefinitionId int, cohortDefinitionId int) ([]*PersonIdAndCohort, error) - RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) + RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) RetrieveBarGraphDataBySourceIdAndCohortIdAndConceptIds(sourceId int, conceptId int64) ([]*NominalGroupData, error) RetrieveHistogramDataBySourceIdAndConceptId(sourceId int, histogramConceptId int64) ([]*PersonConceptAndValue, error) } @@ -97,7 +97,7 @@ func (h CohortData) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPerso return cohortData, meta_result.Error } -func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) { +func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) { var dataSourceModel = new(Source) omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop) resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results) @@ -110,7 +110,7 @@ func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCo Where("observation.observation_concept_id = ?", histogramConceptId). Where("observation.value_as_number is not null") - query = QueryFilterByConceptIdsHelper(query, sourceId, filterConceptIds, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id") + query = QueryFilterByConceptIdsAndValuesHelper(query, sourceId, filterConceptIdsAndValues, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id") query, cancel := utils.AddTimeoutToQuery(query) defer cancel() meta_result := query.Scan(&cohortData) diff --git a/models/concept.go b/models/concept.go index 246d430b..33f870d8 100644 --- a/models/concept.go +++ b/models/concept.go @@ -122,8 +122,9 @@ func (h Concept) RetrieveInfoBySourceIdAndConceptTypes(sourceId int, conceptType // how many persons in the cohort have that value in their observation records. // E.g. if we have a cohort of size N and a concept that can have values "A" or "B", // then it will return something like: -// {ConceptValue: "A", NPersonsInCohortWithValue: M}, -// {ConceptValue: "B", NPersonsInCohortWithValue: N-M}, +// +// {ConceptValue: "A", NPersonsInCohortWithValue: M}, +// {ConceptValue: "B", NPersonsInCohortWithValue: N-M}, func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId int, cohortDefinitionId int, breakdownConceptId int64) ([]*ConceptBreakdown, error) { // this is identical to the result of the function below if called with empty filterConceptIds[] and empty filterCohortPairs... so call that: filterConceptIds := []int64{} @@ -134,8 +135,10 @@ func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId int, cohor // Basically same goal as described in function above, but only count persons that have a non-null value for each // of the ids in the given filterConceptIds. So, using the example documented in the function above, it will // return something like: -// {ConceptValue: "A", NPersonsInCohortWithValue: M-X}, -// {ConceptValue: "B", NPersonsInCohortWithValue: N-M-X}, +// +// {ConceptValue: "A", NPersonsInCohortWithValue: M-X}, +// {ConceptValue: "B", NPersonsInCohortWithValue: N-M-X}, +// // where X is the number of persons that have NO value or just a "null" value for one or more of the ids in the given filterConceptIds. func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef, breakdownConceptId int64) ([]*ConceptBreakdown, error) { diff --git a/models/helper.go b/models/helper.go index 987454fe..ac48b53c 100644 --- a/models/helper.go +++ b/models/helper.go @@ -25,6 +25,27 @@ func QueryFilterByConceptIdsHelper(query *gorm.DB, sourceId int, filterConceptId return query } +// Same as Query Filter above but adds additional value filter as well +func QueryFilterByConceptIdsAndValuesHelper(query *gorm.DB, sourceId int, filterConceptIdsAndValues []utils.CustomConceptVariableDef, + omopDataSource *utils.DbAndSchema, resultSchemaName string, personIdFieldForObservationJoin string) *gorm.DB { + // iterate over the filterConceptIds, adding a new INNER JOIN and filters for each, so that the resulting set is the + // set of persons that have a non-null value for each and every one of the concepts: + for i, filterConceptIdAndValue := range filterConceptIdsAndValues { + observationTableAlias := fmt.Sprintf("observation_filter_%d", i) + log.Printf("Adding extra INNER JOIN with alias %s", observationTableAlias) + query = query.Joins("INNER JOIN "+omopDataSource.Schema+".observation_continuous as "+observationTableAlias+omopDataSource.GetViewDirective()+" ON "+observationTableAlias+".person_id = "+personIdFieldForObservationJoin). + Where(observationTableAlias+".observation_concept_id = ?", filterConceptIdAndValue.ConceptId) + + //If filter by value, add the value filtering clauses to the query + if len(filterConceptIdAndValue.ConceptValues) > 0 { + query = query.Where(observationTableAlias+".value_as_concept_id in ?", filterConceptIdAndValue.ConceptValues) + } else { + query = query.Where(GetConceptValueNotNullCheckBasedOnConceptType(observationTableAlias, sourceId, filterConceptIdAndValue.ConceptId)) + } + } + return query +} + // Helper function that adds extra filter clauses to the query, for the given filterCohortPairs, intersecting on the // right set of tables, excluding data where necessary, etc. // It basically iterates over the list of filterCohortPairs, adding relevant INTERSECT and EXCEPT clauses, so that the resulting set is the diff --git a/server/router.go b/server/router.go index 75a633dd..7997d0e2 100644 --- a/server/router.go +++ b/server/router.go @@ -53,6 +53,9 @@ func NewRouter() *gin.Engine { // full data endpoints: authorized.POST("/cohort-data/by-source-id/:sourceid/by-cohort-definition-id/:cohortid", cohortData.RetrieveDataBySourceIdAndCohortIdAndVariables) + // cohort data statistics + authorized.POST("/cohort-stats/by-source-id/:sourceid/by-cohort-definition-id/:cohortid/by-concept-id/:conceptid", cohortData.RetrieveStatsForCohortIdAndConceptId) + // histogram endpoint authorized.POST("/histogram/by-source-id/:sourceid/by-cohort-definition-id/:cohortid/by-histogram-concept-id/:histogramid", cohortData.RetrieveHistogramForCohortIdAndConceptId) diff --git a/tests/controllers_tests/controllers_test.go b/tests/controllers_tests/controllers_test.go index df8e187a..d49355e6 100644 --- a/tests/controllers_tests/controllers_test.go +++ b/tests/controllers_tests/controllers_test.go @@ -75,7 +75,7 @@ func (h dummyCohortDataModel) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrde return cohortData, nil } -func (h dummyCohortDataModel) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*models.PersonConceptAndValue, error) { +func (h dummyCohortDataModel) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*models.PersonConceptAndValue, error) { cohortData := []*models.PersonConceptAndValue{} return cohortData, nil @@ -907,13 +907,13 @@ func TestGetAttritionRowForConceptIdsAndCohortPairs(t *testing.T) { // mix of concept ids and CustomDichotomousVariableDef items: conceptIdsAndCohortPairs := []interface{}{ - int64(1234), - int64(5678), + utils.CustomConceptVariableDef{ConceptId: int64(1234), ConceptValues: []int64{}}, + utils.CustomConceptVariableDef{ConceptId: int64(5678), ConceptValues: []int64{}}, utils.CustomDichotomousVariableDef{ CohortDefinitionId1: 1, CohortDefinitionId2: 2, ProvidedName: "testA12"}, - int64(2090006880), + utils.CustomConceptVariableDef{ConceptId: int64(2090006880), ConceptValues: []int64{}}, utils.CustomDichotomousVariableDef{ CohortDefinitionId1: 3, CohortDefinitionId2: 4, @@ -1200,3 +1200,53 @@ func TestGenerateDataDictionary(t *testing.T) { } } + +func TestRetrieveStatsForCohortIdAndConceptIdWithWrongParams(t *testing.T) { + setUp(t) + requestContext := new(gin.Context) + requestContext.Params = append(requestContext.Params, gin.Param{Key: "sourceid", Value: strconv.Itoa(tests.GetTestSourceId())}) + requestContext.Params = append(requestContext.Params, gin.Param{Key: "cohortid", Value: "4"}) + requestContext.Writer = new(tests.CustomResponseWriter) + requestContext.Request = new(http.Request) + requestBody := "{\"variables\":[{\"variable_type\": \"custom_dichotomous\", \"cohort_ids\": [1, 3]}]}" + requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody)) + //requestContext.Writer = new(tests.CustomResponseWriter) + cohortDataController.RetrieveStatsForCohortIdAndConceptId(requestContext) + // Params above are wrong, so request should abort: + if !requestContext.IsAborted() { + t.Errorf("should have aborted") + } +} + +func TestRetrieveStatsForCohortIdAndConceptIdWithCorrectParams(t *testing.T) { + setUp(t) + requestContext := new(gin.Context) + requestContext.Params = append(requestContext.Params, gin.Param{Key: "sourceid", Value: strconv.Itoa(tests.GetTestSourceId())}) + requestContext.Params = append(requestContext.Params, gin.Param{Key: "cohortid", Value: "4"}) + requestContext.Params = append(requestContext.Params, gin.Param{Key: "conceptid", Value: "2000006885"}) + requestContext.Writer = new(tests.CustomResponseWriter) + requestContext.Request = new(http.Request) + requestBody := "{\"variables\":[{\"variable_type\": \"concept\", \"concept_id\": 2000000324},{\"variable_type\": \"custom_dichotomous\", \"cohort_ids\": [1, 3]}]}" + requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody)) + cohortDataController.RetrieveStatsForCohortIdAndConceptId(requestContext) + // Params above are correct, so request should NOT abort: + if requestContext.IsAborted() { + t.Errorf("Did not expect this request to abort") + } + result := requestContext.Writer.(*tests.CustomResponseWriter) + if !strings.Contains(result.CustomResponseWriterOut, "statsData") { + t.Errorf("Expected output starting with 'statsData,...'") + } + + // the same request should fail if the teamProject authorization fails: + requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody)) + cohortDataControllerWithFailingTeamProjectAuthz.RetrieveStatsForCohortIdAndConceptId(requestContext) + result = requestContext.Writer.(*tests.CustomResponseWriter) + // expect error: + if !strings.Contains(result.CustomResponseWriterOut, "access denied") { + t.Errorf("Expected 'access denied' as result") + } + if !requestContext.IsAborted() { + t.Errorf("Expected request to be aborted") + } +} diff --git a/tests/models_tests/models_test.go b/tests/models_tests/models_test.go index a566edff..a3f146e9 100644 --- a/tests/models_tests/models_test.go +++ b/tests/models_tests/models_test.go @@ -758,9 +758,9 @@ func TestGetCohortDefinitionByName(t *testing.T) { func TestRetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(t *testing.T) { setUp(t) - filterConceptIds := []int64{} + filterConceptIdsAndValues := []utils.CustomConceptVariableDef{} filterCohortPairs := []utils.CustomDichotomousVariableDef{} - data, _ := cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(testSourceId, largestCohort.Id, histogramConceptId, filterConceptIds, filterCohortPairs) + data, _ := cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(testSourceId, largestCohort.Id, histogramConceptId, filterConceptIdsAndValues, filterCohortPairs) // everyone in the largestCohort has the histogramConceptId, but one person has NULL in the value_as_number: if len(data) != largestCohort.CohortSize-1 { t.Errorf("expected %d histogram data but got %d", largestCohort.CohortSize, len(data)) @@ -774,7 +774,7 @@ func TestRetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(t ProvidedName: "test"}, } // then we expect histogram data for the overlapping population only (which is 5 for extendedCopyOfSecondLargestCohort and largestCohort): - data, _ = cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(testSourceId, largestCohort.Id, histogramConceptId, filterConceptIds, filterCohortPairs) + data, _ = cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(testSourceId, largestCohort.Id, histogramConceptId, filterConceptIdsAndValues, filterCohortPairs) if len(data) != 5 { t.Errorf("expected 5 histogram data but got %d", len(data)) } @@ -820,6 +820,51 @@ func TestQueryFilterByConceptIdsHelper(t *testing.T) { } } +func TestQueryFilterByConceptIdsAndValuesHelper(t *testing.T) { + // This test checks whether the query succeeds when the mainObservationTableAlias + // argument passed to QueryFilterByConceptIdsHelper (last argument) + // matches the alias used in the main query, and whether it fails otherwise. + + setUp(t) + omopDataSource := tests.GetOmopDataSource() + filterConceptIdsAndValues := []utils.CustomConceptVariableDef{{ConceptId: allConceptIds[0], ConceptValues: []int64{}}, {ConceptId: allConceptIds[1], ConceptValues: []int64{}}, {ConceptId: allConceptIds[2], ConceptValues: []int64{}}} + var personIds []struct { + PersonId int64 + } + + // Subtest1: correct alias "observation": + query := omopDataSource.Db.Table(omopDataSource.Schema + ".observation_continuous as observation" + omopDataSource.GetViewDirective()). + Select("observation.person_id") + query = models.QueryFilterByConceptIdsAndValuesHelper(query, testSourceId, filterConceptIdsAndValues, omopDataSource, "", "observation.person_id") + meta_result := query.Scan(&personIds) + if meta_result.Error != nil { + t.Errorf("Did NOT expect an error") + } + // Subtest2: incorrect alias "observation"...should fail: + query = omopDataSource.Db.Table(omopDataSource.Schema + ".observation_continuous as observationWRONG"). + Select("*") + query = models.QueryFilterByConceptIdsAndValuesHelper(query, testSourceId, filterConceptIdsAndValues, omopDataSource, "", "observation.person_id") + meta_result = query.Scan(&personIds) + if meta_result.Error == nil { + t.Errorf("Expected an error") + } + // Subtest3: limit result set by concept value: + filterConceptIdsAndValues = []utils.CustomConceptVariableDef{{ConceptId: 2000007027, ConceptValues: []int64{2000007028}}} + + query = omopDataSource.Db.Table(omopDataSource.Schema + ".observation_continuous as observation"). + Select("*") + query = models.QueryFilterByConceptIdsAndValuesHelper(query, testSourceId, filterConceptIdsAndValues, omopDataSource, "", "observation.person_id") + meta_result = query.Scan(&personIds) + if meta_result.Error != nil { + t.Errorf("Should have succeeded") + } + for _, id := range personIds { + if id.PersonId != 1 && id.PersonId != 7 { + t.Errorf("Filter did not work successfully") + } + } +} + func TestRetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(t *testing.T) { setUp(t) cohortDefinitions, _ := cohortDefinitionModel.GetAllCohortDefinitionsAndStatsOrderBySizeDesc(testSourceId, defaultTeamProject) diff --git a/tests/utils_tests/utils_test.go b/tests/utils_tests/utils_test.go index 67215469..72ea77b5 100644 --- a/tests/utils_tests/utils_test.go +++ b/tests/utils_tests/utils_test.go @@ -51,18 +51,18 @@ func TestParsePrefixedConceptIdsAndDichotomousIds(t *testing.T) { requestContext.Writer = new(tests.CustomResponseWriter) requestContext.Request = new(http.Request) requestBody := "{\"variables\":[{\"variable_type\": \"concept\", \"concept_id\": 2000000324}," + - "{\"variable_type\": \"concept\", \"concept_id\": 2000000123}," + + "{\"variable_type\": \"concept\", \"concept_id\": 2000000123, \"values\": [2000000237, 2000000238]}," + "{\"variable_type\": \"custom_dichotomous\", \"provided_name\": \"test\", \"cohort_ids\": [1, 3]}]}" requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody)) - conceptIds, cohortPairs, _ := utils.ParseConceptIdsAndDichotomousDefs(requestContext) + conceptDefs, cohortPairs, _ := utils.ParseConceptDefsAndDichotomousDefs(requestContext) if requestContext.IsAborted() { t.Errorf("Did not expect this request to abort") } - expectedPrefixedConceptIds := []int64{2000000324, 2000000123} - if !reflect.DeepEqual(conceptIds, expectedPrefixedConceptIds) { - t.Errorf("Expected %d but found %d", expectedPrefixedConceptIds, conceptIds) + expectedPrefixedConceptDefs := []utils.CustomConceptVariableDef{{ConceptId: 2000000324, ConceptValues: []int64{}}, {ConceptId: 2000000123, ConceptValues: []int64{2000000237, 2000000238}}} + if !reflect.DeepEqual(conceptDefs, expectedPrefixedConceptDefs) { + t.Errorf("Expected %d but found %d", expectedPrefixedConceptDefs, conceptDefs) } expectedCohortPairs := []utils.CustomDichotomousVariableDef{ @@ -316,3 +316,42 @@ func TestSubtract(t *testing.T) { t.Errorf("Expected [] but found %v", result) } } + +func TestGenerateStatsData(t *testing.T) { + setUp(t) + + var emptyData = []float64{} + result := utils.GenerateStatsData(1, 1, emptyData) + if result != nil { + t.Errorf("Expected a nil result for an empty data set") + } + + var expectedResult = &utils.ConceptStats{CohortId: 1, ConceptId: 1, NumberOfPeople: 11, Min: 6.0, Max: 49.0, Avg: 33.18181818181818, Sd: 15.134657288477642} + result = utils.GenerateStatsData(1, 1, testData) + if !reflect.DeepEqual(expectedResult, result) { + t.Errorf("Expected %v but found %v", expectedResult, result) + } +} + +func TestConvertConceptIdToCustomConceptVariablesDef(t *testing.T) { + setUp(t) + + expectedResult := []utils.CustomConceptVariableDef{{ConceptId: 1234, ConceptValues: []int64{}}, {ConceptId: 5678, ConceptValues: []int64{}}} + conceptIds := []int64{1234, 5678} + result := utils.ConvertConceptIdToCustomConceptVariablesDef(conceptIds) + + if !reflect.DeepEqual(expectedResult, result) { + t.Errorf("Expected %v but found %v", expectedResult, result) + } +} + +func TestExtractConceptIdsFromCustomConceptVariablesDef(t *testing.T) { + setUp(t) + + var testData = []utils.CustomConceptVariableDef{{ConceptId: 1234, ConceptValues: []int64{7890}}, {ConceptId: 5678, ConceptValues: []int64{}}} + expectedResult := []int64{1234, 5678} + result := utils.ExtractConceptIdsFromCustomConceptVariablesDef(testData) + if !reflect.DeepEqual(expectedResult, result) { + t.Errorf("Expected %v but found %v", expectedResult, result) + } +} diff --git a/utils/parsing.go b/utils/parsing.go index a7bb6029..c6d376ec 100644 --- a/utils/parsing.go +++ b/utils/parsing.go @@ -108,17 +108,24 @@ type CustomDichotomousVariableDef struct { ProvidedName string } +type CustomConceptVariableDef struct { + ConceptId int64 + ConceptValues []int64 +} + func GetCohortPairKey(firstCohortDefinitionId int, secondCohortDefinitionId int) string { return fmt.Sprintf("ID_%v_%v", firstCohortDefinitionId, secondCohortDefinitionId) } // This method expects a request body with a payload similar to the following example: // {"variables": [ -// {variable_type: "concept", concept_id: 2000000324}, -// {variable_type: "concept", concept_id: 2000006885}, -// {variable_type: "custom_dichotomous", provided_name: "name1", cohort_ids: [cohortX_id, cohortY_id]}, -// {variable_type: "custom_dichotomous", provided_name: "name2", cohort_ids: [cohortM_id, cohortN_id]}, -// ... +// +// {variable_type: "concept", concept_id: 2000000324}, +// {variable_type: "concept", concept_id: 2000006885}, +// {variable_type: "custom_dichotomous", provided_name: "name1", cohort_ids: [cohortX_id, cohortY_id]}, +// {variable_type: "custom_dichotomous", provided_name: "name2", cohort_ids: [cohortM_id, cohortN_id]}, +// ... +// // ]} // It returns the list with all concept_id values and custom dichotomous variable definitions. func ParseConceptIdsAndDichotomousDefsAsSingleList(c *gin.Context) ([]interface{}, error) { @@ -140,7 +147,19 @@ func ParseConceptIdsAndDichotomousDefsAsSingleList(c *gin.Context) ([]interface{ // accessing them...needs to be fixed to throw better errors: for _, variable := range variables { if variable["variable_type"] == "concept" { - conceptIdsAndCohortPairs = append(conceptIdsAndCohortPairs, int64(variable["concept_id"].(float64))) + convertedConceptValues := []int64{} + values, ok := variable["values"].([]interface{}) + // If the values are passed as parameter, add to list + if ok { + for _, val := range values { + convertedConceptValues = append(convertedConceptValues, int64(val.(float64))) + } + } + conceptVariableDef := CustomConceptVariableDef{ + ConceptId: int64(variable["concept_id"].(float64)), + ConceptValues: convertedConceptValues, + } + conceptIdsAndCohortPairs = append(conceptIdsAndCohortPairs, conceptVariableDef) } if variable["variable_type"] == "custom_dichotomous" { cohortPair := []int{} @@ -164,13 +183,13 @@ func ParseConceptIdsAndDichotomousDefsAsSingleList(c *gin.Context) ([]interface{ } // deprecated: for backwards compatibility -func ParseConceptIdsAndDichotomousDefs(c *gin.Context) ([]int64, []CustomDichotomousVariableDef, error) { +func ParseConceptDefsAndDichotomousDefs(c *gin.Context) ([]CustomConceptVariableDef, []CustomDichotomousVariableDef, error) { conceptIdsAndCohortPairs, err := ParseConceptIdsAndDichotomousDefsAsSingleList(c) if err != nil { log.Printf("Error: %s", err) return nil, nil, err } - conceptIds, cohortPairs := GetConceptIdsAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs) + conceptIds, cohortPairs := GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs) return conceptIds, cohortPairs, nil } @@ -247,18 +266,18 @@ func ParseSourceAndCohortId(c *gin.Context) (int, int, error) { } // separates a conceptIdsAndCohortPairs into a conceptIds list and a cohortPairs list -func GetConceptIdsAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs []interface{}) ([]int64, []CustomDichotomousVariableDef) { - conceptIds := []int64{} +func GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs []interface{}) ([]CustomConceptVariableDef, []CustomDichotomousVariableDef) { + conceptIdsAndValues := []CustomConceptVariableDef{} cohortPairs := []CustomDichotomousVariableDef{} for _, item := range conceptIdsAndCohortPairs { switch convertedItem := item.(type) { - case int64: - conceptIds = append(conceptIds, convertedItem) + case CustomConceptVariableDef: + conceptIdsAndValues = append(conceptIdsAndValues, convertedItem) case CustomDichotomousVariableDef: cohortPairs = append(cohortPairs, convertedItem) } } - return conceptIds, cohortPairs + return conceptIdsAndValues, cohortPairs } // deprecated: returns the conceptIds and cohortPairs as separate lists (for backwards compatibility) @@ -267,7 +286,8 @@ func ParseSourceIdAndCohortIdAndVariablesList(c *gin.Context) (int, int, []int64 if err != nil { return -1, -1, nil, nil, err } - conceptIds, cohortPairs := GetConceptIdsAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs) + conceptIdsAndValues, cohortPairs := GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs) + conceptIds := ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues) return sourceId, cohortId, conceptIds, cohortPairs, nil } @@ -348,3 +368,22 @@ func Subtract(list1 []int, list2 []int) []int { } return result } + +func ConvertConceptIdToCustomConceptVariablesDef(conceptIds []int64) []CustomConceptVariableDef { + result := []CustomConceptVariableDef{} + for _, val := range conceptIds { + variable := CustomConceptVariableDef{ConceptId: val, ConceptValues: []int64{}} + result = append(result, variable) + } + + return result +} + +func ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues []CustomConceptVariableDef) []int64 { + result := []int64{} + for _, val := range conceptIdsAndValues { + result = append(result, val.ConceptId) + } + + return result +} diff --git a/utils/stats.go b/utils/stats.go new file mode 100644 index 00000000..cafe9326 --- /dev/null +++ b/utils/stats.go @@ -0,0 +1,44 @@ +package utils + +import ( + "log" + + "github.com/montanaflynn/stats" +) + +type ConceptStats struct { + CohortId int `json:"cohortId"` + ConceptId int64 `json:"conceptId"` + NumberOfPeople int `json:"personCount"` + Min float64 `json:"min"` + Max float64 `json:"max"` + Avg float64 `json:"avg"` + Sd float64 `json:"sd"` +} + +func GenerateStatsData(cohortId int, conceptId int64, conceptValues []float64) *ConceptStats { + + if len(conceptValues) == 0 { + log.Printf("Data size is zero. Returning nil.") + return nil + } + + result := new(ConceptStats) + result.CohortId = cohortId + result.ConceptId = conceptId + result.NumberOfPeople = len(conceptValues) + + minValue, _ := stats.Min(conceptValues) + result.Min = minValue + + maxValue, _ := stats.Max(conceptValues) + result.Max = maxValue + + meanValue, _ := stats.Mean(conceptValues) + result.Avg = meanValue + + sdValue, _ := stats.StandardDeviation(conceptValues) + result.Sd = sdValue + + return result +}