Skip to content

Commit d027039

Browse files
tianj7tianj7
authored
Add new stats endpoint (#124)
* add new stats endpoitn * add concept value filter capability
1 parent c821cbb commit d027039

File tree

11 files changed

+333
-42
lines changed

11 files changed

+333
-42
lines changed

‎controllers/cohortdata.go

+51-5
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
4040
return
4141
}
4242

43-
filterConceptIds, cohortPairs, err := utils.ParseConceptIdsAndDichotomousDefs(c)
43+
filterConceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
4444
if err != nil {
4545
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids", "error": err.Error()})
4646
c.Abort()
@@ -59,7 +59,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
5959
return
6060
}
6161

62-
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIds, cohortPairs)
62+
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, histogramConceptId, filterConceptIdsAndValues, cohortPairs)
6363
if err != nil {
6464
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
6565
c.Abort()
@@ -76,6 +76,49 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
7676
c.JSON(http.StatusOK, gin.H{"bins": histogramData})
7777
}
7878

79+
func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Context) {
80+
sourceIdStr := c.Param("sourceid")
81+
log.Printf("Querying source: %s", sourceIdStr)
82+
cohortIdStr := c.Param("cohortid")
83+
log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
84+
conceptIdStr := c.Param("conceptid")
85+
if sourceIdStr == "" || cohortIdStr == "" || conceptIdStr == "" {
86+
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
87+
c.Abort()
88+
return
89+
}
90+
91+
filterConceptIdsAndValues, cohortPairs, _ := utils.ParseConceptDefsAndDichotomousDefs(c)
92+
93+
sourceId, _ := strconv.Atoi(sourceIdStr)
94+
cohortId, _ := strconv.Atoi(cohortIdStr)
95+
conceptId, _ := strconv.ParseInt(conceptIdStr, 10, 64)
96+
97+
validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
98+
if !validAccessRequest {
99+
log.Printf("Error: invalid request")
100+
c.JSON(http.StatusForbidden, gin.H{"message": "access denied"})
101+
c.Abort()
102+
return
103+
}
104+
105+
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
106+
if err != nil {
107+
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
108+
c.Abort()
109+
return
110+
}
111+
112+
conceptValues := []float64{}
113+
for _, personData := range cohortData {
114+
conceptValues = append(conceptValues, float64(*personData.ConceptValueAsNumber))
115+
}
116+
117+
statsData := utils.GenerateStatsData(cohortId, conceptId, conceptValues)
118+
119+
c.JSON(http.StatusOK, gin.H{"statsData": statsData})
120+
}
121+
79122
func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *gin.Context) {
80123
// TODO - add some validation to ensure that only calls from Argo are allowed through since it outputs FULL data?
81124

@@ -90,7 +133,9 @@ func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *g
90133
return
91134
}
92135

93-
conceptIds, cohortPairs, err := utils.ParseConceptIdsAndDichotomousDefs(c)
136+
conceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
137+
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)
138+
94139
if err != nil {
95140
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids and dichotomous Ids", "error": err.Error()})
96141
c.Abort()
@@ -250,12 +295,13 @@ func populateConceptValue(row []string, cohortItem models.PersonConceptAndValue,
250295
func (u CohortDataController) RetrieveCohortOverlapStats(c *gin.Context) {
251296
errors := make([]error, 4)
252297
var sourceId, caseCohortId, controlCohortId int
253-
var conceptIds []int64
298+
var conceptIdsAndValues []utils.CustomConceptVariableDef
254299
var cohortPairs []utils.CustomDichotomousVariableDef
255300
sourceId, errors[0] = utils.ParseNumericArg(c, "sourceid")
256301
caseCohortId, errors[1] = utils.ParseNumericArg(c, "casecohortid")
257302
controlCohortId, errors[2] = utils.ParseNumericArg(c, "controlcohortid")
258-
conceptIds, cohortPairs, errors[3] = utils.ParseConceptIdsAndDichotomousDefs(c)
303+
conceptIdsAndValues, cohortPairs, errors[3] = utils.ParseConceptDefsAndDichotomousDefs(c)
304+
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)
259305

260306
validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{caseCohortId, controlCohortId}, cohortPairs)
261307
if !validAccessRequest {

‎controllers/concept.go

+5-4
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
197197
c.Abort()
198198
return
199199
}
200-
_, cohortPairs := utils.GetConceptIdsAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
200+
_, cohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
201201
validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
202202
if !validAccessRequest {
203203
log.Printf("Error: invalid request")
@@ -266,16 +266,17 @@ func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId i
266266
}
267267

268268
func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
269-
filterConceptIds, filterCohortPairs := utils.GetConceptIdsAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
269+
filterConceptIdsAndValues, filterCohortPairs := utils.GetConceptIdsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
270+
filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptIdsAndValues)
270271
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
271272
if err != nil {
272273
return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error())
273274
}
274275
conceptValuesToPeopleCount := getConceptValueToPeopleCount(breakdownStats)
275276
variableName := ""
276277
switch convertedItem := conceptIdOrCohortPair.(type) {
277-
case int64:
278-
conceptInformation, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptId(sourceId, convertedItem)
278+
case utils.CustomConceptVariableDef:
279+
conceptInformation, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptId(sourceId, convertedItem.ConceptId)
279280
if err != nil {
280281
return nil, fmt.Errorf("could not retrieve concept details for %v due to error: %s", convertedItem, err.Error())
281282
}

‎models/cohortdata.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ type CohortDataI interface {
1111
RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int64) ([]*PersonConceptAndValue, error)
1212
RetrieveCohortOverlapStats(sourceId int, caseCohortId int, controlCohortId int, otherFilterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) (CohortOverlapStats, error)
1313
RetrieveDataByOriginalCohortAndNewCohort(sourceId int, originalCohortDefinitionId int, cohortDefinitionId int) ([]*PersonIdAndCohort, error)
14-
RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
14+
RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error)
1515
RetrieveBarGraphDataBySourceIdAndCohortIdAndConceptIds(sourceId int, conceptId int64) ([]*NominalGroupData, error)
1616
RetrieveHistogramDataBySourceIdAndConceptId(sourceId int, histogramConceptId int64) ([]*PersonConceptAndValue, error)
1717
}
@@ -97,7 +97,7 @@ func (h CohortData) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPerso
9797
return cohortData, meta_result.Error
9898
}
9999

100-
func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
100+
func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIdsAndValues []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*PersonConceptAndValue, error) {
101101
var dataSourceModel = new(Source)
102102
omopDataSource := dataSourceModel.GetDataSource(sourceId, Omop)
103103
resultsDataSource := dataSourceModel.GetDataSource(sourceId, Results)
@@ -110,7 +110,7 @@ func (h CohortData) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCo
110110
Where("observation.observation_concept_id = ?", histogramConceptId).
111111
Where("observation.value_as_number is not null")
112112

113-
query = QueryFilterByConceptIdsHelper(query, sourceId, filterConceptIds, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id")
113+
query = QueryFilterByConceptIdsAndValuesHelper(query, sourceId, filterConceptIdsAndValues, omopDataSource, resultsDataSource.Schema, "unionAndIntersect.subject_id")
114114
query, cancel := utils.AddTimeoutToQuery(query)
115115
defer cancel()
116116
meta_result := query.Scan(&cohortData)

‎models/concept.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,9 @@ func (h Concept) RetrieveInfoBySourceIdAndConceptTypes(sourceId int, conceptType
122122
// how many persons in the cohort have that value in their observation records.
123123
// E.g. if we have a cohort of size N and a concept that can have values "A" or "B",
124124
// then it will return something like:
125-
// {ConceptValue: "A", NPersonsInCohortWithValue: M},
126-
// {ConceptValue: "B", NPersonsInCohortWithValue: N-M},
125+
//
126+
// {ConceptValue: "A", NPersonsInCohortWithValue: M},
127+
// {ConceptValue: "B", NPersonsInCohortWithValue: N-M},
127128
func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId int, cohortDefinitionId int, breakdownConceptId int64) ([]*ConceptBreakdown, error) {
128129
// this is identical to the result of the function below if called with empty filterConceptIds[] and empty filterCohortPairs... so call that:
129130
filterConceptIds := []int64{}
@@ -134,8 +135,10 @@ func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId int, cohor
134135
// Basically same goal as described in function above, but only count persons that have a non-null value for each
135136
// of the ids in the given filterConceptIds. So, using the example documented in the function above, it will
136137
// return something like:
137-
// {ConceptValue: "A", NPersonsInCohortWithValue: M-X},
138-
// {ConceptValue: "B", NPersonsInCohortWithValue: N-M-X},
138+
//
139+
// {ConceptValue: "A", NPersonsInCohortWithValue: M-X},
140+
// {ConceptValue: "B", NPersonsInCohortWithValue: N-M-X},
141+
//
139142
// where X is the number of persons that have NO value or just a "null" value for one or more of the ids in the given filterConceptIds.
140143
func (h Concept) RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef, breakdownConceptId int64) ([]*ConceptBreakdown, error) {
141144

‎models/helper.go

+21
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,27 @@ func QueryFilterByConceptIdsHelper(query *gorm.DB, sourceId int, filterConceptId
2525
return query
2626
}
2727

28+
// Same as Query Filter above but adds additional value filter as well
29+
func QueryFilterByConceptIdsAndValuesHelper(query *gorm.DB, sourceId int, filterConceptIdsAndValues []utils.CustomConceptVariableDef,
30+
omopDataSource *utils.DbAndSchema, resultSchemaName string, personIdFieldForObservationJoin string) *gorm.DB {
31+
// iterate over the filterConceptIds, adding a new INNER JOIN and filters for each, so that the resulting set is the
32+
// set of persons that have a non-null value for each and every one of the concepts:
33+
for i, filterConceptIdAndValue := range filterConceptIdsAndValues {
34+
observationTableAlias := fmt.Sprintf("observation_filter_%d", i)
35+
log.Printf("Adding extra INNER JOIN with alias %s", observationTableAlias)
36+
query = query.Joins("INNER JOIN "+omopDataSource.Schema+".observation_continuous as "+observationTableAlias+omopDataSource.GetViewDirective()+" ON "+observationTableAlias+".person_id = "+personIdFieldForObservationJoin).
37+
Where(observationTableAlias+".observation_concept_id = ?", filterConceptIdAndValue.ConceptId)
38+
39+
//If filter by value, add the value filtering clauses to the query
40+
if len(filterConceptIdAndValue.ConceptValues) > 0 {
41+
query = query.Where(observationTableAlias+".value_as_concept_id in ?", filterConceptIdAndValue.ConceptValues)
42+
} else {
43+
query = query.Where(GetConceptValueNotNullCheckBasedOnConceptType(observationTableAlias, sourceId, filterConceptIdAndValue.ConceptId))
44+
}
45+
}
46+
return query
47+
}
48+
2849
// Helper function that adds extra filter clauses to the query, for the given filterCohortPairs, intersecting on the
2950
// right set of tables, excluding data where necessary, etc.
3051
// It basically iterates over the list of filterCohortPairs, adding relevant INTERSECT and EXCEPT clauses, so that the resulting set is the

‎server/router.go

+3
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ func NewRouter() *gin.Engine {
5353
// full data endpoints:
5454
authorized.POST("/cohort-data/by-source-id/:sourceid/by-cohort-definition-id/:cohortid", cohortData.RetrieveDataBySourceIdAndCohortIdAndVariables)
5555

56+
// cohort data statistics
57+
authorized.POST("/cohort-stats/by-source-id/:sourceid/by-cohort-definition-id/:cohortid/by-concept-id/:conceptid", cohortData.RetrieveStatsForCohortIdAndConceptId)
58+
5659
// histogram endpoint
5760
authorized.POST("/histogram/by-source-id/:sourceid/by-cohort-definition-id/:cohortid/by-histogram-concept-id/:histogramid", cohortData.RetrieveHistogramForCohortIdAndConceptId)
5861

‎tests/controllers_tests/controllers_test.go

+54-4
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ func (h dummyCohortDataModel) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrde
7575
return cohortData, nil
7676
}
7777

78-
func (h dummyCohortDataModel) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*models.PersonConceptAndValue, error) {
78+
func (h dummyCohortDataModel) RetrieveHistogramDataBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, histogramConceptId int64, filterConceptIds []utils.CustomConceptVariableDef, filterCohortPairs []utils.CustomDichotomousVariableDef) ([]*models.PersonConceptAndValue, error) {
7979

8080
cohortData := []*models.PersonConceptAndValue{}
8181
return cohortData, nil
@@ -907,13 +907,13 @@ func TestGetAttritionRowForConceptIdsAndCohortPairs(t *testing.T) {
907907

908908
// mix of concept ids and CustomDichotomousVariableDef items:
909909
conceptIdsAndCohortPairs := []interface{}{
910-
int64(1234),
911-
int64(5678),
910+
utils.CustomConceptVariableDef{ConceptId: int64(1234), ConceptValues: []int64{}},
911+
utils.CustomConceptVariableDef{ConceptId: int64(5678), ConceptValues: []int64{}},
912912
utils.CustomDichotomousVariableDef{
913913
CohortDefinitionId1: 1,
914914
CohortDefinitionId2: 2,
915915
ProvidedName: "testA12"},
916-
int64(2090006880),
916+
utils.CustomConceptVariableDef{ConceptId: int64(2090006880), ConceptValues: []int64{}},
917917
utils.CustomDichotomousVariableDef{
918918
CohortDefinitionId1: 3,
919919
CohortDefinitionId2: 4,
@@ -1200,3 +1200,53 @@ func TestGenerateDataDictionary(t *testing.T) {
12001200
}
12011201

12021202
}
1203+
1204+
func TestRetrieveStatsForCohortIdAndConceptIdWithWrongParams(t *testing.T) {
1205+
setUp(t)
1206+
requestContext := new(gin.Context)
1207+
requestContext.Params = append(requestContext.Params, gin.Param{Key: "sourceid", Value: strconv.Itoa(tests.GetTestSourceId())})
1208+
requestContext.Params = append(requestContext.Params, gin.Param{Key: "cohortid", Value: "4"})
1209+
requestContext.Writer = new(tests.CustomResponseWriter)
1210+
requestContext.Request = new(http.Request)
1211+
requestBody := "{\"variables\":[{\"variable_type\": \"custom_dichotomous\", \"cohort_ids\": [1, 3]}]}"
1212+
requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody))
1213+
//requestContext.Writer = new(tests.CustomResponseWriter)
1214+
cohortDataController.RetrieveStatsForCohortIdAndConceptId(requestContext)
1215+
// Params above are wrong, so request should abort:
1216+
if !requestContext.IsAborted() {
1217+
t.Errorf("should have aborted")
1218+
}
1219+
}
1220+
1221+
func TestRetrieveStatsForCohortIdAndConceptIdWithCorrectParams(t *testing.T) {
1222+
setUp(t)
1223+
requestContext := new(gin.Context)
1224+
requestContext.Params = append(requestContext.Params, gin.Param{Key: "sourceid", Value: strconv.Itoa(tests.GetTestSourceId())})
1225+
requestContext.Params = append(requestContext.Params, gin.Param{Key: "cohortid", Value: "4"})
1226+
requestContext.Params = append(requestContext.Params, gin.Param{Key: "conceptid", Value: "2000006885"})
1227+
requestContext.Writer = new(tests.CustomResponseWriter)
1228+
requestContext.Request = new(http.Request)
1229+
requestBody := "{\"variables\":[{\"variable_type\": \"concept\", \"concept_id\": 2000000324},{\"variable_type\": \"custom_dichotomous\", \"cohort_ids\": [1, 3]}]}"
1230+
requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody))
1231+
cohortDataController.RetrieveStatsForCohortIdAndConceptId(requestContext)
1232+
// Params above are correct, so request should NOT abort:
1233+
if requestContext.IsAborted() {
1234+
t.Errorf("Did not expect this request to abort")
1235+
}
1236+
result := requestContext.Writer.(*tests.CustomResponseWriter)
1237+
if !strings.Contains(result.CustomResponseWriterOut, "statsData") {
1238+
t.Errorf("Expected output starting with 'statsData,...'")
1239+
}
1240+
1241+
// the same request should fail if the teamProject authorization fails:
1242+
requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody))
1243+
cohortDataControllerWithFailingTeamProjectAuthz.RetrieveStatsForCohortIdAndConceptId(requestContext)
1244+
result = requestContext.Writer.(*tests.CustomResponseWriter)
1245+
// expect error:
1246+
if !strings.Contains(result.CustomResponseWriterOut, "access denied") {
1247+
t.Errorf("Expected 'access denied' as result")
1248+
}
1249+
if !requestContext.IsAborted() {
1250+
t.Errorf("Expected request to be aborted")
1251+
}
1252+
}

0 commit comments

Comments
 (0)