Merge pull request #10 from uc-cdis/fix_performance_issue_cohort_data

Fix performance issue in cohort-data

controllers/cohortdata.go

@@ -50,7 +50,7 @@ func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndConceptIds(c *
 	conceptIds := getConceptIdsFromPrefixedConceptIds(prefixedConceptIds.PrefixedConceptIds)
 
 	// call model method:
-	cohortData, err := cohortDataModel.RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId, cohortId, conceptIds)
+	cohortData, err := cohortDataModel.RetrieveDataLargeBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId, cohortId, conceptIds)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err})
 		c.Abort()

models/cohortdata.go

@@ -14,6 +14,7 @@ type PersonConceptAndValue struct {
 	ConceptValueAsNumber float32
 }
 
+// DEPRECATED - too slow for larger cohorts
 func (h CohortData) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int) ([]*PersonConceptAndValue, error) {
 	var dataSourceModel = new(Source)
 	omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
@@ -38,3 +39,27 @@ func (h CohortData) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPerso
 
 	return cohortData, nil
 }
+
+// Same as above, but for LARGE cohorts/dbs
+// Assumption is that both OMOP and RESULTS schemas
+// are on same DB
+func (h CohortData) RetrieveDataLargeBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int) ([]*PersonConceptAndValue, error) {
+	log.Printf(">> Using inner join impl. for large cohorts")
+	var dataSourceModel = new(Source)
+	omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
+
+	resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results)
+
+	// get the observations for the subjects and the concepts, to build up the data rows to return:
+	var cohortData []*PersonConceptAndValue
+	omopDataSource.Db.Model(&Observation{}).
+		Select("observation.person_id, concept.concept_id, concept.concept_name, observation.value_as_string as concept_value_as_string, observation.value_as_number as concept_value_as_number").
+		Joins("INNER JOIN "+omopDataSource.Schema+".concept as concept ON observation.observation_concept_id = concept.concept_id").
+		Joins("INNER JOIN "+resultsDataSource.Schema+".cohort as cohort ON cohort.subject_id = observation.person_id").
+		Where("cohort.cohort_definition_id = ?", cohortDefinitionId).
+		Where("concept.concept_id in (?)", conceptIds).
+		Order("observation.person_id asc"). // this order is important!
+		Scan(&cohortData)
+
+	return cohortData, nil
+}