Add new stats endpoint (#124)

* add new stats endpoitn
* add concept value filter capability

controllers/cohortdata.go

@@ -40,7 +40,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
 		return
 	}
 
-	filterConceptIds, cohortPairs, err := utils.ParseConceptIdsAndDichotomousDefs(c)
+	filterConceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids", "error": err.Error()})
 		c.Abort()
@@ -59,7 +59,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
 		return
 	}
 
-	cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIds, cohortPairs)
+	cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIdsAndValues, cohortPairs)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
 		c.Abort()
@@ -76,6 +76,49 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
 	c.JSON(http.StatusOK, gin.H{"bins": histogramData})
 }
 
+func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Context) {
+	sourceIdStr := c.Param("sourceid")
+	log.Printf("Querying source: %s", sourceIdStr)
+	cohortIdStr := c.Param("cohortid")
+	log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
+	conceptIdStr := c.Param("conceptid")
+	if sourceIdStr == "" || cohortIdStr == "" || conceptIdStr == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
+		c.Abort()
+		return
+	}
+
+	filterConceptIdsAndValues, cohortPairs, _ := utils.ParseConceptDefsAndDichotomousDefs(c)
+
+	sourceId, _ := strconv.Atoi(sourceIdStr)
+	cohortId, _ := strconv.Atoi(cohortIdStr)
+	conceptId, _ := strconv.ParseInt(conceptIdStr, 10, 64)
+
+	validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
+	if !validAccessRequest {
+		log.Printf("Error: invalid request")
+		c.JSON(http.StatusForbidden, gin.H{"message": "access denied"})
+		c.Abort()
+		return
+	}
+
+	cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
+		c.Abort()
+		return
+	}
+
+	conceptValues := []float64{}
+	for _, personData := range cohortData {
+		conceptValues = append(conceptValues, float64(*personData.ConceptValueAsNumber))
+	}
+
+	statsData := utils.GenerateStatsData(cohortId, conceptId, conceptValues)
+
+	c.JSON(http.StatusOK, gin.H{"statsData": statsData})
+}
+
 func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *gin.Context) {
 	// TODO - add some validation to ensure that only calls from Argo are allowed through since it outputs FULL data?
 
@@ -90,7 +133,9 @@ func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *g
 		return
 	}
 
-	conceptIds, cohortPairs, err := utils.ParseConceptIdsAndDichotomousDefs(c)
+	conceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
+	conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)
+
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids and dichotomous Ids", "error": err.Error()})
 		c.Abort()
@@ -250,12 +295,13 @@ func populateConceptValue(row []string, cohortItem models.PersonConceptAndValue,
 func (u CohortDataController) RetrieveCohortOverlapStats(c *gin.Context) {
 	errors := make([]error, 4)
 	var sourceId, caseCohortId, controlCohortId int
-	var conceptIds []int64
+	var conceptIdsAndValues []utils.CustomConceptVariableDef
 	var cohortPairs []utils.CustomDichotomousVariableDef
 	sourceId, errors[0] = utils.ParseNumericArg(c, "sourceid")
 	caseCohortId, errors[1] = utils.ParseNumericArg(c, "casecohortid")
 	controlCohortId, errors[2] = utils.ParseNumericArg(c, "controlcohortid")
-	conceptIds, cohortPairs, errors[3] = utils.ParseConceptIdsAndDichotomousDefs(c)
+	conceptIdsAndValues, cohortPairs, errors[3] = utils.ParseConceptDefsAndDichotomousDefs(c)
+	conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)
 
 	validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{caseCohortId, controlCohortId}, cohortPairs)
 	if !validAccessRequest {

controllers/concept.go

@@ -197,7 +197,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
 		c.Abort()
 		return
 	}
-	_, cohortPairs := utils.GetConceptIdsAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
+	_, cohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
 	validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
 	if !validAccessRequest {
 		log.Printf("Error: invalid request")
@@ -266,16 +266,17 @@ func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId i
 }
 
 func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
-	filterConceptIds, filterCohortPairs := utils.GetConceptIdsAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
+	filterConceptIdsAndValues, filterCohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
+	filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptIdsAndValues)
 	breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
 	if err != nil {
 		return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error())
 	}
 	conceptValuesToPeopleCount := getConceptValueToPeopleCount(breakdownStats)
 	variableName := ""
 	switch convertedItem := conceptIdOrCohortPair.(type) {
-	case int64:
-		conceptInformation, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptId(sourceId, convertedItem)
+	case utils.CustomConceptVariableDef:
+		conceptInformation, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptId(sourceId, convertedItem.ConceptId)
 		if err != nil {
 			return nil, fmt.Errorf("could not retrieve concept details for %v due to error: %s", convertedItem, err.Error())
 		}

models/cohortdata.go

@@ -11,7 +11,7 @@ type CohortDataI interface {
 	RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int64) ([]*PersonConceptAndValue, error)
 	RetrieveCohortOverlapStats(sourceId int, caseCohortId int, controlCohortId int, otherFilterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) (CohortOverlapStats, error)
 	RetrieveDataByOriginalCohortAndNewCohort(sourceId int, originalCohortDefinitionId int, cohortDefinitionId int) ([]*PersonIdAndCohort, error)
-	RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
+	RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
 	RetrieveBarGraphDataBySourceIdAndCohortIdAndConceptIds(sourceId int, conceptId int64) ([]*NominalGroupData, error)
 	RetrieveHistogramDataBySourceIdAndConceptId(sourceId int, histogramConceptId int64) ([]*PersonConceptAndValue, error)
 }
@@ -97,7 +97,7 @@ func (h CohortData) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPerso
 	return cohortData, meta_result.Error
 }
 
-func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
+func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
 	var dataSourceModel = new(Source)
 	omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
 	resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results)
@@ -110,7 +110,7 @@ func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCo
 		Where("observation.observation_concept_id = ?", histogramConceptId).
 		Where("observation.value_as_number is not null")
 
-	query = QueryFilterByConceptIdsHelper(query, sourceId, filterConceptIds, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id")
+	query = QueryFilterByConceptIdsAndValuesHelper(query, sourceId, filterConceptIdsAndValues, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id")
 	query, cancel := utils.AddTimeoutToQuery(query)
 	defer cancel()
 	meta_result := query.Scan(&cohortData)

models/concept.go

@@ -122,8 +122,9 @@ func (h Concept) RetrieveInfoBySourceIdAndConceptTypes(sourceId int, conceptType
 // how many persons in the cohort have that value in their observation records.
 // E.g. if we have a cohort of size N and a concept that can have values "A" or "B",
 // then it will return something like:
-//  {ConceptValue: "A", NPersonsInCohortWithValue: M},
-//  {ConceptValue: "B", NPersonsInCohortWithValue: N-M},
+//
+//	{ConceptValue: "A", NPersonsInCohortWithValue: M},
+//	{ConceptValue: "B", NPersonsInCohortWithValue: N-M},
 func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId int, cohortDefinitionId int, breakdownConceptId int64) ([]*ConceptBreakdown, error) {
 	// this is identical to the result of the function below if called with empty filterConceptIds[] and empty filterCohortPairs... so call that:
 	filterConceptIds := []int64{}
@@ -134,8 +135,10 @@ func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId int, cohor
 // Basically same goal as described in function above, but only count persons that have a non-null value for each
 // of the ids in the given filterConceptIds. So, using the example documented in the function above, it will
 // return something like:
-//  {ConceptValue: "A", NPersonsInCohortWithValue: M-X},
-//  {ConceptValue: "B", NPersonsInCohortWithValue: N-M-X},
+//
+//	{ConceptValue: "A", NPersonsInCohortWithValue: M-X},
+//	{ConceptValue: "B", NPersonsInCohortWithValue: N-M-X},
+//
 // where X is the number of persons that have NO value or just a "null" value for one or more of the ids in the given filterConceptIds.
 func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef, breakdownConceptId int64) ([]*ConceptBreakdown, error) {
 

models/helper.go

@@ -25,6 +25,27 @@ func QueryFilterByConceptIdsHelper(query *gorm.DB, sourceId int, filterConceptId
 	return query
 }
 
+// Same as Query Filter above but adds additional value filter as well
+func QueryFilterByConceptIdsAndValuesHelper(query *gorm.DB, sourceId int, filterConceptIdsAndValues []utils.CustomConceptVariableDef,
+	omopDataSource *utils.DbAndSchema, resultSchemaName string, personIdFieldForObservationJoin string) *gorm.DB {
+	// iterate over the filterConceptIds, adding a new INNER JOIN and filters for each, so that the resulting set is the
+	// set of persons that have a non-null value for each and every one of the concepts:
+	for i, filterConceptIdAndValue := range filterConceptIdsAndValues {
+		observationTableAlias := fmt.Sprintf("observation_filter_%d", i)
+		log.Printf("Adding extra INNER JOIN with alias %s", observationTableAlias)
+		query = query.Joins("INNER JOIN "+omopDataSource.Schema+".observation_continuous as "+observationTableAlias+omopDataSource.GetViewDirective()+" ON "+observationTableAlias+".person_id = "+personIdFieldForObservationJoin).
+			Where(observationTableAlias+".observation_concept_id = ?", filterConceptIdAndValue.ConceptId)
+
+		//If filter by value, add the value filtering clauses to the query
+		if len(filterConceptIdAndValue.ConceptValues) > 0 {
+			query = query.Where(observationTableAlias+".value_as_concept_id in ?", filterConceptIdAndValue.ConceptValues)
+		} else {
+			query = query.Where(GetConceptValueNotNullCheckBasedOnConceptType(observationTableAlias, sourceId, filterConceptIdAndValue.ConceptId))
+		}
+	}
+	return query
+}
+
 // Helper function that adds extra filter clauses to the query, for the given filterCohortPairs, intersecting on the
 // right set of tables, excluding data where necessary, etc.
 // It basically iterates over the list of filterCohortPairs, adding relevant INTERSECT and EXCEPT clauses, so that the resulting set is the

server/router.go

@@ -53,6 +53,9 @@ func NewRouter() *gin.Engine {
 		// full data endpoints:
 		authorized.POST("/cohort-data/by-source-id/:sourceid/by-cohort-definition-id/:cohortid", cohortData.RetrieveDataBySourceIdAndCohortIdAndVariables)
 
+		// cohort data statistics
+		authorized.POST("/cohort-stats/by-source-id/:sourceid/by-cohort-definition-id/:cohortid/by-concept-id/:conceptid", cohortData.RetrieveStatsForCohortIdAndConceptId)
+
 		// histogram endpoint
 		authorized.POST("/histogram/by-source-id/:sourceid/by-cohort-definition-id/:cohortid/by-histogram-concept-id/:histogramid", cohortData.RetrieveHistogramForCohortIdAndConceptId)
 

tests/controllers_tests/controllers_test.go

@@ -75,7 +75,7 @@ func (h dummyCohortDataModel) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrde
 	return cohortData, nil
 }
 
-func (h dummyCohortDataModel) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*models.PersonConceptAndValue, error) {
+func (h dummyCohortDataModel) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*models.PersonConceptAndValue, error) {
 
 	cohortData := []*models.PersonConceptAndValue{}
 	return cohortData, nil
@@ -907,13 +907,13 @@ func TestGetAttritionRowForConceptIdsAndCohortPairs(t *testing.T) {
 
 	// mix of concept ids and CustomDichotomousVariableDef items:
 	conceptIdsAndCohortPairs := []interface{}{
-		int64(1234),
-		int64(5678),
+		utils.CustomConceptVariableDef{ConceptId: int64(1234), ConceptValues: []int64{}},
+		utils.CustomConceptVariableDef{ConceptId: int64(5678), ConceptValues: []int64{}},
 		utils.CustomDichotomousVariableDef{
 			CohortDefinitionId1: 1,
 			CohortDefinitionId2: 2,
 			ProvidedName:        "testA12"},
-		int64(2090006880),
+		utils.CustomConceptVariableDef{ConceptId: int64(2090006880), ConceptValues: []int64{}},
 		utils.CustomDichotomousVariableDef{
 			CohortDefinitionId1: 3,
 			CohortDefinitionId2: 4,
@@ -1200,3 +1200,53 @@ func TestGenerateDataDictionary(t *testing.T) {
 	}
 
 }
+
+func TestRetrieveStatsForCohortIdAndConceptIdWithWrongParams(t *testing.T) {
+	setUp(t)
+	requestContext := new(gin.Context)
+	requestContext.Params = append(requestContext.Params, gin.Param{Key: "sourceid", Value: strconv.Itoa(tests.GetTestSourceId())})
+	requestContext.Params = append(requestContext.Params, gin.Param{Key: "cohortid", Value: "4"})
+	requestContext.Writer = new(tests.CustomResponseWriter)
+	requestContext.Request = new(http.Request)
+	requestBody := "{\"variables\":[{\"variable_type\": \"custom_dichotomous\", \"cohort_ids\": [1, 3]}]}"
+	requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody))
+	//requestContext.Writer = new(tests.CustomResponseWriter)
+	cohortDataController.RetrieveStatsForCohortIdAndConceptId(requestContext)
+	// Params above are wrong, so request should abort:
+	if !requestContext.IsAborted() {
+		t.Errorf("should have aborted")
+	}
+}
+
+func TestRetrieveStatsForCohortIdAndConceptIdWithCorrectParams(t *testing.T) {
+	setUp(t)
+	requestContext := new(gin.Context)
+	requestContext.Params = append(requestContext.Params, gin.Param{Key: "sourceid", Value: strconv.Itoa(tests.GetTestSourceId())})
+	requestContext.Params = append(requestContext.Params, gin.Param{Key: "cohortid", Value: "4"})
+	requestContext.Params = append(requestContext.Params, gin.Param{Key: "conceptid", Value: "2000006885"})
+	requestContext.Writer = new(tests.CustomResponseWriter)
+	requestContext.Request = new(http.Request)
+	requestBody := "{\"variables\":[{\"variable_type\": \"concept\", \"concept_id\": 2000000324},{\"variable_type\": \"custom_dichotomous\", \"cohort_ids\": [1, 3]}]}"
+	requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody))
+	cohortDataController.RetrieveStatsForCohortIdAndConceptId(requestContext)
+	// Params above are correct, so request should NOT abort:
+	if requestContext.IsAborted() {
+		t.Errorf("Did not expect this request to abort")
+	}
+	result := requestContext.Writer.(*tests.CustomResponseWriter)
+	if !strings.Contains(result.CustomResponseWriterOut, "statsData") {
+		t.Errorf("Expected output starting with 'statsData,...'")
+	}
+
+	// the same request should fail if the teamProject authorization fails:
+	requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody))
+	cohortDataControllerWithFailingTeamProjectAuthz.RetrieveStatsForCohortIdAndConceptId(requestContext)
+	result = requestContext.Writer.(*tests.CustomResponseWriter)
+	// expect error:
+	if !strings.Contains(result.CustomResponseWriterOut, "access denied") {
+		t.Errorf("Expected 'access denied' as result")
+	}
+	if !requestContext.IsAborted() {
+		t.Errorf("Expected request to be aborted")
+	}
+}