Skip to content

Commit 3cb8424

Browse files
pieterlukassepieterlukasse
authored
Feat: support filters and transformation (#125)
feat: add new support for filters and transformation arguments in histogram endpoint * fix: fix the naming for concept lists ...from ids to definitions (defs) and add some parsing for the new filter entries in request body * feat: working histogram endpoint POC for using tmp table for transformations * feat: added first log transformation * feat: added z-score transformation * feat: add tempttablecache to include cleanup of oldest / least accessed item * tmp: temporarily disable strict validation until frontend is fixed * feat: add new ToSQL wrapper method * feat: remove histogramConceptId from histogram endpoint ...and expect it as part of the variables instead * fix: make sure only postgresql and sqlserver are accepted for now * fix: naming improvements and extra unit tests * fix: improve coverage of parsing.go * feat: improve coverage of TempTableCache * feat: improve coverage of TableExists
1 parent 69cfe0c commit 3cb8424

File tree

15 files changed

+1173
-173
lines changed

15 files changed

+1173
-173
lines changed

‎.secrets.baseline

+2-2
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,9 @@
115115
"filename": "utils/dsn.go",
116116
"hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
117117
"is_verified": false,
118-
"line_number": 23
118+
"line_number": 26
119119
}
120120
]
121121
},
122-
"generated_at": "2022-04-19T19:51:52Z"
122+
"generated_at": "2025-02-06T13:49:06Z"
123123
}

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ curl -d '{"variables":[{"variable_type": "concept", "concept_id": 2000000324},{"
138138

139139
Histogram endpoint:
140140
```bash
141-
curl -d '{"variables":[{"variable_type": "custom_dichotomous", "cohort_ids": [1, 4]}]}' -H "Content-Type: application/json" -X POST http://localhost:8080/histogram/by-source-id/1/by-cohort-definition-id/4/by-histogram-concept-id/2000006885
141+
curl -d '{"variables":[{"variable_type": "custom_dichotomous", "cohort_ids": [1, 4]}, {"variable_type": "concept", "concept_id": 2000006885}]}' -H "Content-Type: application/json" -X POST http://localhost:8080/histogram/by-source-id/1/by-cohort-definition-id/4
142142
```
143143

144144
# Deployment steps

‎controllers/cohortdata.go

+8-16
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,20 @@ func NewCohortDataController(cohortDataModel models.CohortDataI, dataDictionaryM
2929
}
3030

3131
func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Context) {
32-
sourceIdStr := c.Param("sourceid")
33-
log.Printf("Querying source: %s", sourceIdStr)
34-
cohortIdStr := c.Param("cohortid")
35-
log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
36-
histogramIdStr := c.Param("histogramid")
37-
if sourceIdStr == "" || cohortIdStr == "" || histogramIdStr == "" {
38-
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
32+
sourceId, cohortId, conceptIdsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
33+
if err != nil {
34+
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
3935
c.Abort()
4036
return
4137
}
4238

43-
filterConceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
39+
// parse cohortPairs separately as well, so we can validate permissions
40+
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
4441
if err != nil {
4542
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids", "error": err.Error()})
4643
c.Abort()
4744
return
4845
}
49-
50-
sourceId, _ := strconv.Atoi(sourceIdStr)
51-
cohortId, _ := strconv.Atoi(cohortIdStr)
52-
histogramConceptId, _ := strconv.ParseInt(histogramIdStr, 10, 64)
53-
5446
validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
5547
if !validAccessRequest {
5648
log.Printf("Error: invalid request")
@@ -59,9 +51,9 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
5951
return
6052
}
6153

62-
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIdsAndValues, cohortPairs)
54+
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs)
6355
if err != nil {
64-
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
56+
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving histogram data", "error": err.Error()})
6557
c.Abort()
6658
return
6759
}
@@ -102,7 +94,7 @@ func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Contex
10294
return
10395
}
10496

105-
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
97+
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
10698
if err != nil {
10799
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
108100
c.Abort()

‎controllers/concept.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
197197
c.Abort()
198198
return
199199
}
200-
_, cohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
200+
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
201201
validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
202202
if !validAccessRequest {
203203
log.Printf("Error: invalid request")
@@ -266,8 +266,8 @@ func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId i
266266
}
267267

268268
func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
269-
filterConceptIdsAndValues, filterCohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
270-
filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptIdsAndValues)
269+
filterConceptDefsAndValues, filterCohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
270+
filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptDefsAndValues)
271271
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
272272
if err != nil {
273273
return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error())

‎models/cohortdata.go

+42-3
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@ import (
55
"log"
66

77
"github.com/uc-cdis/cohort-middleware/utils"
8+
"gorm.io/gorm"
89
)
910

1011
type CohortDataI interface {
1112
RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int64) ([]*PersonConceptAndValue, error)
1213
RetrieveCohortOverlapStats(sourceId int, caseCohortId int, controlCohortId int, otherFilterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) (CohortOverlapStats, error)
1314
RetrieveDataByOriginalCohortAndNewCohort(sourceId int, originalCohortDefinitionId int, cohortDefinitionId int) ([]*PersonIdAndCohort, error)
14-
RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
15+
RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
16+
RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId int, cohortDefinitionId int, filterConceptDefsAndCohortPairs []interface{}) ([]*PersonConceptAndValue, error)
1517
RetrieveBarGraphDataBySourceIdAndCohortIdAndConceptIds(sourceId int, conceptId int64) ([]*NominalGroupData, error)
1618
RetrieveHistogramDataBySourceIdAndConceptId(sourceId int, histogramConceptId int64) ([]*PersonConceptAndValue, error)
1719
}
@@ -97,7 +99,44 @@ func (h CohortData) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPerso
9799
return cohortData, meta_result.Error
98100
}
99101

100-
func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
102+
func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId int, cohortDefinitionId int, filterConceptDefsAndCohortPairs []interface{}) ([]*PersonConceptAndValue, error) {
103+
var dataSourceModel = new(Source)
104+
omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
105+
resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results)
106+
finalCohortAlias := "final_cohort_alias"
107+
histogramConcept, err := utils.CheckAndGetLastCustomConceptVariableDef(filterConceptDefsAndCohortPairs)
108+
if err != nil {
109+
log.Fatalf("failed: %v", err)
110+
return nil, err
111+
}
112+
// get the observations for the subjects and the concepts, to build up the data rows to return:
113+
var cohortData []*PersonConceptAndValue
114+
session := resultsDataSource.Db.Session(&gorm.Session{})
115+
err = session.Transaction(func(query *gorm.DB) error { // TODO - rename query?
116+
query, tmpTableName := QueryFilterByConceptDefsPlusCohortPairsHelper(query, sourceId, cohortDefinitionId, filterConceptDefsAndCohortPairs, omopDataSource, resultsDataSource, finalCohortAlias)
117+
if tmpTableName != "" {
118+
query = query.Select("distinct(" + tmpTableName + ".person_id), " + tmpTableName + ".observation_concept_id as concept_id, " + tmpTableName + ".value_as_number as concept_value_as_number")
119+
} else {
120+
query = query.Select("distinct(observation.person_id), observation.observation_concept_id as concept_id, observation.value_as_number as concept_value_as_number").
121+
Joins("INNER JOIN "+omopDataSource.Schema+".observation_continuous as observation"+omopDataSource.GetViewDirective()+" ON "+finalCohortAlias+".subject_id = observation.person_id").
122+
Where("observation.observation_concept_id = ?", histogramConcept.ConceptId).
123+
Where("observation.value_as_number is not null")
124+
}
125+
126+
query, cancel := utils.AddTimeoutToQuery(query)
127+
defer cancel()
128+
meta_result := query.Scan(&cohortData)
129+
return meta_result.Error
130+
})
131+
if err != nil {
132+
log.Fatalf("Transaction failed: %v", err)
133+
return nil, err
134+
}
135+
return cohortData, err
136+
}
137+
138+
// DEPRECATED - USE RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs instead.
139+
func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptDefs []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
101140
var dataSourceModel = new(Source)
102141
omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
103142
resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results)
@@ -110,7 +149,7 @@ func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCo
110149
Where("observation.observation_concept_id = ?", histogramConceptId).
111150
Where("observation.value_as_number is not null")
112151

113-
query = QueryFilterByConceptIdsAndValuesHelper(query, sourceId, filterConceptIdsAndValues, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id")
152+
query = QueryFilterByConceptDefsHelper(query, sourceId, filterConceptDefs, omopDataSource, resultsDataSource.Schema, "unionAndIntersect")
114153
query, cancel := utils.AddTimeoutToQuery(query)
115154
defer cancel()
116155
meta_result := query.Scan(&cohortData)

0 commit comments

Comments
 (0)