Revert "Feat: support filters and transformation (#125)" (#127)

This reverts commit 3cb8424.

Author: pieterlukasse

.secrets.baseline

@@ -115,9 +115,9 @@
         "filename": "utils/dsn.go",
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
         "is_verified": false,
-        "line_number": 26
+        "line_number": 23
       }
     ]
   },
-  "generated_at": "2025-02-06T13:49:06Z"
+  "generated_at": "2022-04-19T19:51:52Z"
 }

README.md

@@ -138,7 +138,7 @@ curl -d '{"variables":[{"variable_type": "concept", "concept_id": 2000000324},{"
 
 Histogram endpoint:
 ```bash
-curl -d '{"variables":[{"variable_type": "custom_dichotomous", "cohort_ids": [1, 4]}, {"variable_type": "concept", "concept_id": 2000006885}]}' -H "Content-Type: application/json" -X POST http://localhost:8080/histogram/by-source-id/1/by-cohort-definition-id/4
+curl -d '{"variables":[{"variable_type": "custom_dichotomous", "cohort_ids": [1, 4]}]}' -H "Content-Type: application/json" -X POST http://localhost:8080/histogram/by-source-id/1/by-cohort-definition-id/4/by-histogram-concept-id/2000006885
 ```
 
 # Deployment steps

controllers/cohortdata.go

@@ -29,20 +29,28 @@ func NewCohortDataController(cohortDataModel models.CohortDataI, dataDictionaryM
 }
 
 func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Context) {
-	sourceId, cohortId, conceptIdsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
+	sourceIdStr := c.Param("sourceid")
+	log.Printf("Querying source: %s", sourceIdStr)
+	cohortIdStr := c.Param("cohortid")
+	log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
+	histogramIdStr := c.Param("histogramid")
+	if sourceIdStr == "" || cohortIdStr == "" || histogramIdStr == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
 		c.Abort()
 		return
 	}
 
-	// parse cohortPairs separately as well, so we can validate permissions
-	_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
+	filterConceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids", "error": err.Error()})
 		c.Abort()
 		return
 	}
+
+	sourceId, _ := strconv.Atoi(sourceIdStr)
+	cohortId, _ := strconv.Atoi(cohortIdStr)
+	histogramConceptId, _ := strconv.ParseInt(histogramIdStr, 10, 64)
+
 	validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
 	if !validAccessRequest {
 		log.Printf("Error: invalid request")
@@ -51,9 +59,9 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
 		return
 	}
 
-	cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs)
+	cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIdsAndValues, cohortPairs)
 	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving histogram data", "error": err.Error()})
+		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
 		c.Abort()
 		return
 	}
@@ -94,7 +102,7 @@ func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Contex
 		return
 	}
 
-	cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
+	cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
 		c.Abort()

controllers/concept.go

@@ -197,7 +197,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
 		c.Abort()
 		return
 	}
-	_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
+	_, cohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
 	validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
 	if !validAccessRequest {
 		log.Printf("Error: invalid request")
@@ -266,8 +266,8 @@ func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId i
 }
 
 func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
-	filterConceptDefsAndValues, filterCohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
-	filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptDefsAndValues)
+	filterConceptIdsAndValues, filterCohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
+	filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptIdsAndValues)
 	breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
 	if err != nil {
 		return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error())

models/cohortdata.go

@@ -5,15 +5,13 @@ import (
 	"log"
 
 	"github.com/uc-cdis/cohort-middleware/utils"
-	"gorm.io/gorm"
 )
 
 type CohortDataI interface {
 	RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int64) ([]*PersonConceptAndValue, error)
 	RetrieveCohortOverlapStats(sourceId int, caseCohortId int, controlCohortId int, otherFilterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) (CohortOverlapStats, error)
 	RetrieveDataByOriginalCohortAndNewCohort(sourceId int, originalCohortDefinitionId int, cohortDefinitionId int) ([]*PersonIdAndCohort, error)
-	RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
-	RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId int, cohortDefinitionId int, filterConceptDefsAndCohortPairs []interface{}) ([]*PersonConceptAndValue, error)
+	RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
 	RetrieveBarGraphDataBySourceIdAndCohortIdAndConceptIds(sourceId int, conceptId int64) ([]*NominalGroupData, error)
 	RetrieveHistogramDataBySourceIdAndConceptId(sourceId int, histogramConceptId int64) ([]*PersonConceptAndValue, error)
 }
@@ -99,44 +97,7 @@ func (h CohortData) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPerso
 	return cohortData, meta_result.Error
 }
 
-func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId int, cohortDefinitionId int, filterConceptDefsAndCohortPairs []interface{}) ([]*PersonConceptAndValue, error) {
-	var dataSourceModel = new(Source)
-	omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
-	resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results)
-	finalCohortAlias := "final_cohort_alias"
-	histogramConcept, err := utils.CheckAndGetLastCustomConceptVariableDef(filterConceptDefsAndCohortPairs)
-	if err != nil {
-		log.Fatalf("failed: %v", err)
-		return nil, err
-	}
-	// get the observations for the subjects and the concepts, to build up the data rows to return:
-	var cohortData []*PersonConceptAndValue
-	session := resultsDataSource.Db.Session(&gorm.Session{})
-	err = session.Transaction(func(query *gorm.DB) error { // TODO - rename query?
-		query, tmpTableName := QueryFilterByConceptDefsPlusCohortPairsHelper(query, sourceId, cohortDefinitionId, filterConceptDefsAndCohortPairs, omopDataSource, resultsDataSource, finalCohortAlias)
-		if tmpTableName != "" {
-			query = query.Select("distinct(" + tmpTableName + ".person_id), " + tmpTableName + ".observation_concept_id as concept_id, " + tmpTableName + ".value_as_number as concept_value_as_number")
-		} else {
-			query = query.Select("distinct(observation.person_id), observation.observation_concept_id as concept_id, observation.value_as_number as concept_value_as_number").
-				Joins("INNER JOIN "+omopDataSource.Schema+".observation_continuous as observation"+omopDataSource.GetViewDirective()+" ON "+finalCohortAlias+".subject_id = observation.person_id").
-				Where("observation.observation_concept_id = ?", histogramConcept.ConceptId).
-				Where("observation.value_as_number is not null")
-		}
-
-		query, cancel := utils.AddTimeoutToQuery(query)
-		defer cancel()
-		meta_result := query.Scan(&cohortData)
-		return meta_result.Error
-	})
-	if err != nil {
-		log.Fatalf("Transaction failed: %v", err)
-		return nil, err
-	}
-	return cohortData, err
-}
-
-// DEPRECATED - USE RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs instead.
-func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptDefs []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
+func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
 	var dataSourceModel = new(Source)
 	omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
 	resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results)
@@ -149,7 +110,7 @@ func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndC
 		Where("observation.observation_concept_id = ?", histogramConceptId).
 		Where("observation.value_as_number is not null")
 
-	query = QueryFilterByConceptDefsHelper(query, sourceId, filterConceptDefs, omopDataSource, resultsDataSource.Schema, "unionAndIntersect")
+	query = QueryFilterByConceptIdsAndValuesHelper(query, sourceId, filterConceptIdsAndValues, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id")
 	query, cancel := utils.AddTimeoutToQuery(query)
 	defer cancel()
 	meta_result := query.Scan(&cohortData)