Skip to content

Commit 6390ae4

Browse files
nepruneneprune
committed
[Alloy] Allow using a join to check if defects are linked.
Rather than separate per-item parent calls which are hitting the rate limit. mysociety/societyworks#4682
1 parent 7aadc5d commit 6390ae4

File tree

4 files changed

+132
-65
lines changed

4 files changed

+132
-65
lines changed

‎perllib/Open311/Endpoint/Integration/AlloyV2.pm

+20
Original file line numberDiff line numberDiff line change
@@ -765,6 +765,10 @@ It fetches the item's parents to find any that have a C<rfs_design> designCode,
765765
assuming that's the 'inspection' associated with the 'defect' (or indeed, the
766766
defect associated with the job).
767767
768+
If C<use_joins_rather_than_parent_calls> is set, C<inspection_to_defect_attribute_link>
769+
and C<inspection_specific_attribute> will be used in a join to get C<rfs_design> parents
770+
in the query for updated resources, rather than querying the parents of each defect separately.
771+
768772
It uses the C<defect_attribute_mapping> status entry to find the attribute
769773
containing the status, and then uses C<defect_status_mapping> to map that
770774
Alloy value to a status.
@@ -842,6 +846,10 @@ fetch any new reports in those designs. As with updates, it fetches any
842846
resources updated in the time window. It ignores any in C<ignored_defect_types>
843847
or any with a C<rfs_design> parent.
844848
849+
If C<use_joins_rather_than_parent_calls> is set, C<inspection_to_defect_attribute_link>
850+
and C<inspection_specific_attribute> will be used in a join to get C<rfs_design> parents
851+
in the query for updated resources, rather than querying the parents of each defect separately.
852+
845853
It uses C<defect_sourcetype_category_mapping> on the designCode to fetch a
846854
default and a possible types mapping, which it uses to set a category if it
847855
finds an attribute with a particular fixed list of matches. It also might
@@ -978,6 +986,12 @@ sub fetch_updated_resources {
978986
}]
979987
};
980988

989+
if ($self->config->{use_joins_rather_than_parent_calls}) {
990+
my $inspection_to_defect_attribute_link = $self->config->{inspection_to_defect_attribute_link};
991+
my $inspection_specific_attribute = $self->config->{inspection_specific_attribute};
992+
$body_base->{joinAttributes} = ["root^$inspection_to_defect_attribute_link.$inspection_specific_attribute"]
993+
}
994+
981995
return $self->alloy->search( $body_base );
982996
}
983997

@@ -1187,6 +1201,12 @@ sub _get_defect_inspection {
11871201
sub _get_defect_inspection_parents {
11881202
my ($self, $defect) = @_;
11891203

1204+
if ($self->config->{use_joins_rather_than_parent_calls}) {
1205+
# We should have already fetched the parents by joining in the initial query
1206+
# so we can return these IDs.
1207+
return @{ $defect->{joinedItemIDs} // [] };
1208+
}
1209+
11901210
my $parents = $self->alloy->api_call(call => "item/$defect->{itemId}/parents")->{results};
11911211
my @linked_defects;
11921212
foreach (@$parents) {

‎t/open311/endpoint/alloyv2.t

+68-65
Original file line numberDiff line numberDiff line change
@@ -492,73 +492,76 @@ subtest "create problem with no resource_id" => sub {
492492
restore_time;
493493
};
494494

495-
subtest "check fetch updates" => sub {
496-
my $res = $endpoint->run_test_request(
497-
GET => '/servicerequestupdates.json?jurisdiction_id=dummy&start_date=2019-01-01T00:00:00Z&end_date=2019-03-01T02:00:00Z',
498-
);
495+
for my $use_joins_rather_than_parent_calls (1, 0) {
496+
subtest "check fetch updates (using join: $use_joins_rather_than_parent_calls)" => sub {
497+
$endpoint->config->{use_joins_rather_than_parent_calls} = $use_joins_rather_than_parent_calls;
498+
my $res = $endpoint->run_test_request(
499+
GET => '/servicerequestupdates.json?jurisdiction_id=dummy&start_date=2019-01-01T00:00:00Z&end_date=2019-03-01T02:00:00Z',
500+
);
499501

500-
my $sent = pop @sent;
501-
ok $res->is_success, 'valid request'
502-
or diag $res->content;
502+
my $sent = pop @sent;
503+
ok $res->is_success, 'valid request'
504+
or diag $res->content;
503505

504-
is_deeply decode_json($res->content),
505-
[ {
506-
status => 'investigating',
507-
service_request_id => '3027029',
508-
description => 'This is an updated customer response',
509-
updated_datetime => '2019-01-01T00:32:40Z',
510-
update_id => '3027029_20190101003240',
511-
media_url => '',
512-
extras => { latest_data_only => 1 },
513-
},
514-
{
515-
status => 'investigating',
516-
service_request_id => '3027030',
517-
description => '',
518-
updated_datetime => '2019-01-01T01:42:40Z',
519-
update_id => '3027030_20190101014240',
520-
media_url => '',
521-
extras => { latest_data_only => 1 },
522-
},
523-
{
524-
status => 'not_councils_responsibility',
525-
service_request_id => '3027031',
526-
description => '',
527-
updated_datetime => '2019-01-01T01:43:40Z',
528-
update_id => '3027031_20190101014340',
529-
media_url => '',
530-
external_status_code => '01b51bb5c0de101a004154b5',
531-
extras => { latest_data_only => 1 },
532-
},
533-
{
534-
status => 'action_scheduled',
535-
service_request_id => '3027032',
536-
description => '',
537-
updated_datetime => '2019-01-01T01:48:13Z',
538-
update_id => '4947501_20190101014813',
539-
media_url => '',
540-
extras => { latest_data_only => 1 },
541-
},
542-
{
543-
status => 'investigating',
544-
service_request_id => '3027034',
545-
description => '',
546-
updated_datetime => '2019-01-01T01:49:13Z',
547-
update_id => '3027034_20190101014913',
548-
media_url => '',
549-
extras => { latest_data_only => 1 },
550-
},
551-
{
552-
status => 'open',
553-
service_request_id => '4947502',
554-
description => '',
555-
updated_datetime => '2019-01-01T01:51:08Z',
556-
update_id => '4947502_20190101015108',
557-
media_url => '',
558-
extras => { latest_data_only => 1 },
559-
}
560-
], 'correct json returned';
561-
};
506+
is_deeply decode_json($res->content),
507+
[ {
508+
status => 'investigating',
509+
service_request_id => '3027029',
510+
description => 'This is an updated customer response',
511+
updated_datetime => '2019-01-01T00:32:40Z',
512+
update_id => '3027029_20190101003240',
513+
media_url => '',
514+
extras => { latest_data_only => 1 },
515+
},
516+
{
517+
status => 'investigating',
518+
service_request_id => '3027030',
519+
description => '',
520+
updated_datetime => '2019-01-01T01:42:40Z',
521+
update_id => '3027030_20190101014240',
522+
media_url => '',
523+
extras => { latest_data_only => 1 },
524+
},
525+
{
526+
status => 'not_councils_responsibility',
527+
service_request_id => '3027031',
528+
description => '',
529+
updated_datetime => '2019-01-01T01:43:40Z',
530+
update_id => '3027031_20190101014340',
531+
media_url => '',
532+
external_status_code => '01b51bb5c0de101a004154b5',
533+
extras => { latest_data_only => 1 },
534+
},
535+
{
536+
status => 'action_scheduled',
537+
service_request_id => '3027032',
538+
description => '',
539+
updated_datetime => '2019-01-01T01:48:13Z',
540+
update_id => '4947501_20190101014813',
541+
media_url => '',
542+
extras => { latest_data_only => 1 },
543+
},
544+
{
545+
status => 'investigating',
546+
service_request_id => '3027034',
547+
description => '',
548+
updated_datetime => '2019-01-01T01:49:13Z',
549+
update_id => '3027034_20190101014913',
550+
media_url => '',
551+
extras => { latest_data_only => 1 },
552+
},
553+
{
554+
status => 'open',
555+
service_request_id => '4947502',
556+
description => '',
557+
updated_datetime => '2019-01-01T01:51:08Z',
558+
update_id => '4947502_20190101015108',
559+
media_url => '',
560+
extras => { latest_data_only => 1 },
561+
}
562+
], 'correct json returned';
563+
};
564+
}
562565

563566
subtest "check fetch updates with cobrand skipping update where job has unchanged parent defect" => sub {
564567
my $res = $oxfordshire_endpoint->run_test_request(

‎t/open311/endpoint/alloyv2.yml

+4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
"irg_config_code": "ASSET_INSPECTIONS",
88
"resource_attachment_attribute_id": "attributes_filesAttachableAttachments",
99

10+
"use_joins_rather_than_parent_calls": 0,
11+
"inspection_to_defect_attribute_link": "defect_to_inspection_link_attribute",
12+
"inspection_specific_attribute": "inspection_only_attribute",
13+
1014
"category_list_code": "designs_listFixMyStreetCategories1001257_5d3210e1fe2ad806f8df98c1",
1115
"category_title_attribute": "title",
1216
# List of categories/subcategories and parent attribute id

‎t/open311/endpoint/json/alloyv2/defect_search.json

+40
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,46 @@
11
{
22
"page": 1,
33
"pageSize": 20,
4+
"joinResults": [
5+
{
6+
"itemId": "4947501",
7+
"joinQueries": [
8+
{
9+
"item": {
10+
"itemId": "3027032"
11+
},
12+
"attributes": [
13+
{
14+
"attributeCode": "inspection_only_attribute",
15+
"value": "doesn't matter"
16+
}
17+
]
18+
}
19+
],
20+
"joinAttributes": [
21+
"root.defect_to_inspection_link_attribute.inspection_only_attribute"
22+
]
23+
},
24+
{
25+
"itemId": "0203824fc0de101a008fbb4f",
26+
"joinQueries": [
27+
{
28+
"item": {
29+
"itemId": "irrelevant"
30+
},
31+
"attributes": [
32+
{
33+
"attributeCode": "inspection_only_attribute",
34+
"value": "doesn't matter"
35+
}
36+
]
37+
}
38+
],
39+
"joinAttributes": [
40+
"root.defect_to_inspection_link_attribute.inspection_only_attribute"
41+
]
42+
}
43+
],
444
"results": [
545
{
646
"itemId": "4947501",

0 commit comments

Comments
 (0)