Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: add filter and transformation support to breakdown / attrition endpoint #126

Merged
merged 6 commits into from
Mar 4, 2025
Merged

Feat: add filter and transformation support to breakdown / attrition endpoint #126

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2c9901b
feat: add filter and transformation support to breakdown / attrition …
pieterlukasse Jan 28, 2025
1ee8ee6
fix: restore variable type validation
pieterlukasse Feb 11, 2025
a1e9790
feat: remove QueryFilterByConceptIdsHelper deprecated method
pieterlukasse Feb 11, 2025
af0320d
fix: fix breakdown query so it checks on the breakdown concept values
pieterlukasse Feb 12, 2025
0d75f2b
fix: fix breakdown query
pieterlukasse Feb 12, 2025
04194da
fix: better naming and more tests covering filter/transformation
pieterlukasse Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 46 additions & 50 deletions controllers/cohortdata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co

// parse cohortPairs separately as well, so we can validate permissions
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids", "error": err.Error()})
c.Abort()
return
}

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
log.Printf("Error: invalid request")
Expand All @@ -69,22 +65,15 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
}

func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Context) {
sourceIdStr := c.Param("sourceid")
log.Printf("Querying source: %s", sourceIdStr)
cohortIdStr := c.Param("cohortid")
log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
conceptIdStr := c.Param("conceptid")
if sourceIdStr == "" || cohortIdStr == "" || conceptIdStr == "" {
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
sourceId, cohortId, conceptIdsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}

filterConceptIdsAndValues, cohortPairs, _ := utils.ParseConceptDefsAndDichotomousDefs(c)

sourceId, _ := strconv.Atoi(sourceIdStr)
cohortId, _ := strconv.Atoi(cohortIdStr)
conceptId, _ := strconv.ParseInt(conceptIdStr, 10, 64)
// parse cohortPairs separately as well, so we can validate permissions
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
Expand All @@ -94,7 +83,7 @@ func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Contex
return
}

cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
c.Abort()
Expand All @@ -105,37 +94,30 @@ func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Contex
for _, personData := range cohortData {
conceptValues = append(conceptValues, float64(*personData.ConceptValueAsNumber))
}
conceptToStat, errGetLast := utils.CheckAndGetLastCustomConceptVariableDef(conceptIdsAndCohortPairs)
if errGetLast != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error: last variable should be of numeric type", "error": errGetLast.Error()})
c.Abort()
return
}

statsData := utils.GenerateStatsData(cohortId, conceptId, conceptValues)
statsData := utils.GenerateStatsData(cohortId, conceptToStat.ConceptId, conceptValues)

c.JSON(http.StatusOK, gin.H{"statsData": statsData})
}

func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *gin.Context) {
// TODO - add some validation to ensure that only calls from Argo are allowed through since it outputs FULL data?

// parse and validate all parameters:
sourceIdStr := c.Param("sourceid")
log.Printf("Querying source: %s", sourceIdStr)
cohortIdStr := c.Param("cohortid")
log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
if sourceIdStr == "" || cohortIdStr == "" {
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
c.Abort()
return
}

conceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)

// -> this concern is considered to be addressed by https://github.com/uc-cdis/cloud-automation/pull/1884
sourceId, cohortId, conceptDefsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids and dichotomous Ids", "error": err.Error()})
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}

sourceId, _ := strconv.Atoi(sourceIdStr)
cohortId, _ := strconv.Atoi(cohortIdStr)
// parse cohortPairs separately as well, so we can validate permissions
conceptDefs, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
Expand All @@ -145,17 +127,32 @@ func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *g
return
}

// call model method:
cohortData, err := u.cohortDataModel.RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId, cohortId, conceptIds)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
c.Abort()
return
// Iterate over conceptDefsAndCohortPairs and collect the concept values for each person:
// {PersonId:1, ConceptId:1, ConceptValue: "A value with, comma!"},
// {PersonId:1, ConceptId:2, ConceptValue: B},
// {PersonId:2, ConceptId:1, ConceptValue: C},
var variablesToQuery []interface{}
var finalConceptDataset []*models.PersonConceptAndValue
for _, item := range conceptDefsAndCohortPairs {
variablesToQuery = append(variablesToQuery, item)
// if item is of type CustomConceptVariableDef, get the data:
if _, ok := item.(utils.CustomConceptVariableDef); ok {
// use variablesToQuery to query an increasingly tight set (simulating the attrition table that generated this query)
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, variablesToQuery)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
c.Abort()
return
}
// add to final concept data set:
finalConceptDataset = append(finalConceptDataset, cohortData...)
}
}

partialCSV := GeneratePartialCSV(sourceId, cohortData, conceptIds)
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptDefs)
partialCSV := GeneratePartialCSV(sourceId, finalConceptDataset, conceptIds) // use conceptdefs to improve column description? nah...no person is reading this table....just needs to be unique

personIdToCSVValues, err := u.RetrievePeopleIdAndCohort(sourceId, cohortId, cohortPairs, cohortData)
personIdToCSVValues, err := u.RetrievePeopleIdAndCohort(sourceId, cohortId, cohortPairs, finalConceptDataset)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving people ID to csv value map", "error": err.Error()})
c.Abort()
Expand Down Expand Up @@ -287,13 +284,12 @@ func populateConceptValue(row []string, cohortItem models.PersonConceptAndValue,
func (u CohortDataController) RetrieveCohortOverlapStats(c *gin.Context) {
errors := make([]error, 4)
var sourceId, caseCohortId, controlCohortId int
var conceptIdsAndValues []utils.CustomConceptVariableDef
var cohortPairs []utils.CustomDichotomousVariableDef
var conceptDefsAndCohortPairs []interface{}
sourceId, errors[0] = utils.ParseNumericArg(c, "sourceid")
caseCohortId, errors[1] = utils.ParseNumericArg(c, "casecohortid")
controlCohortId, errors[2] = utils.ParseNumericArg(c, "controlcohortid")
conceptIdsAndValues, cohortPairs, errors[3] = utils.ParseConceptDefsAndDichotomousDefs(c)
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)
conceptDefsAndCohortPairs, errors[3] = utils.ParseConceptDefsAndDichotomousDefsAsSingleList(c)
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{caseCohortId, controlCohortId}, cohortPairs)
if !validAccessRequest {
Expand All @@ -309,7 +305,7 @@ func (u CohortDataController) RetrieveCohortOverlapStats(c *gin.Context) {
return
}
overlapStats, err := u.cohortDataModel.RetrieveCohortOverlapStats(sourceId, caseCohortId,
controlCohortId, conceptIds, cohortPairs)
controlCohortId, conceptDefsAndCohortPairs)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving stats", "error": err.Error()})
c.Abort()
Expand Down
31 changes: 17 additions & 14 deletions controllers/concept.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,13 +125,16 @@ func (u ConceptController) RetrieveBreakdownStatsBySourceIdAndCohortId(c *gin.Co
}

func (u ConceptController) RetrieveBreakdownStatsBySourceIdAndCohortIdAndVariables(c *gin.Context) {
sourceId, cohortId, conceptIds, cohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesList(c)
sourceId, cohortId, conceptDefsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}

// parse cohortPairs separately as well, so we can validate permissions
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
log.Printf("Error: invalid request")
Expand All @@ -147,7 +150,7 @@ func (u ConceptController) RetrieveBreakdownStatsBySourceIdAndCohortIdAndVariabl
c.Abort()
return
}
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, conceptIds, cohortPairs, breakdownConceptId)
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, conceptDefsAndCohortPairs, breakdownConceptId)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving stats", "error": err.Error()})
Expand Down Expand Up @@ -190,14 +193,14 @@ func generateRowForVariable(variableName string, breakdownConceptValuesToPeopleC
}

func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
sourceId, cohortId, conceptIdsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
sourceId, cohortId, conceptDefsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)
validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
log.Printf("Error: invalid request")
Expand Down Expand Up @@ -238,7 +241,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
c.Abort()
return
}
otherAttritionRows, err := u.GetAttritionRowForConceptIdsAndCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs, breakdownConceptId, sortedConceptValues)
otherAttritionRows, err := u.GetAttritionRowForConceptDefsAndCohortPairs(sourceId, cohortId, conceptDefsAndCohortPairs, breakdownConceptId, sortedConceptValues)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown rows for filter conceptIds and cohortPairs", "error": err.Error()})
Expand All @@ -249,13 +252,13 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
c.String(http.StatusOK, b.String())
}

func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId int, cohortId int, conceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([][]string, error) {
func (u ConceptController) GetAttritionRowForConceptDefsAndCohortPairs(sourceId int, cohortId int, conceptDefsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([][]string, error) {
var otherAttritionRows [][]string
for idx, conceptIdOrCohortPair := range conceptIdsAndCohortPairs {
// attrition filter: run each query with an increasingly longer list of filterConceptIdsAndCohortPairs, until the last query is run with them all:
filterConceptIdsAndCohortPairs := conceptIdsAndCohortPairs[0 : idx+1]
for idx, conceptIdOrCohortPair := range conceptDefsAndCohortPairs {
// attrition filter: run each query with an increasingly longer list of filterConceptDefsAndCohortPairs, until the last query is run with them all:
filterConceptDefsAndCohortPairs := conceptDefsAndCohortPairs[0 : idx+1]

attritionRow, err := u.GetAttritionRowForConceptIdOrCohortPair(sourceId, cohortId, conceptIdOrCohortPair, filterConceptIdsAndCohortPairs, breakdownConceptId, sortedConceptValues)
attritionRow, err := u.GetAttritionRowForConceptDefOrCohortPair(sourceId, cohortId, conceptIdOrCohortPair, filterConceptDefsAndCohortPairs, breakdownConceptId, sortedConceptValues)
if err != nil {
log.Printf("Error: %s", err.Error())
return nil, err
Expand All @@ -265,10 +268,10 @@ func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId i
return otherAttritionRows, nil
}

func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
filterConceptDefsAndValues, filterCohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
func (u ConceptController) GetAttritionRowForConceptDefOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptDefsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
filterConceptDefsAndValues, filterCohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(filterConceptDefsAndCohortPairs)
filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptDefsAndValues)
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, filterConceptDefsAndCohortPairs, breakdownConceptId)
if err != nil {
return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error())
}
Expand Down
Loading
Loading