Skip to content

Commit 42a80de

Browse files
struanstruan
committed
initial version of an auto points script for one row per council
Currently very focused on what's needed to import some automatic points data from some FOI responses but should be the basis of a more generic auto points script at some stage
1 parent 7b6114e commit 42a80de

File tree

1 file changed

+265
-0
lines changed

1 file changed

+265
-0
lines changed

‎crowdsourcer/management/commands/add_automatic_points_from_council_list.py

+265
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
from collections import defaultdict
2+
3+
from django.contrib.auth.models import User
4+
5+
import pandas as pd
6+
7+
from crowdsourcer.import_utils import BaseImporter
8+
from crowdsourcer.models import Option, Question, Response
9+
10+
11+
class Command(BaseImporter):
12+
help = "Apply a list of responses to councils"
13+
14+
def add_arguments(self, parser):
15+
parser.add_argument(
16+
"-q", "--quiet", action="store_true", help="Silence debug text."
17+
)
18+
19+
parser.add_argument(
20+
"--session",
21+
action="store",
22+
required=True,
23+
help="Marking session to use questions with",
24+
)
25+
26+
parser.add_argument(
27+
"--response_list",
28+
action="store",
29+
required=True,
30+
help="CSV file containing the councils and responses",
31+
)
32+
33+
parser.add_argument(
34+
"--question_list",
35+
action="store",
36+
required=True,
37+
help="CSV file containing the questions",
38+
)
39+
40+
parser.add_argument(
41+
"--option_map",
42+
action="store",
43+
help="CSV file containing option mapping from previous answers",
44+
)
45+
parser.add_argument(
46+
"--stage",
47+
action="store",
48+
required=True,
49+
help="marking stage to add responses to",
50+
)
51+
52+
parser.add_argument(
53+
"--authority_map",
54+
action="store",
55+
help="CSV file containing bad_name, good_name columns to map from bad councils names",
56+
)
57+
58+
parser.add_argument(
59+
"--update_existing_responses",
60+
action="store_true",
61+
help="Always update existings responses",
62+
)
63+
64+
parser.add_argument(
65+
"--commit", action="store_true", help="Commits changes to DB"
66+
)
67+
68+
def get_df(self, file):
69+
df = pd.read_csv(file)
70+
71+
return df
72+
73+
def get_option_map(self, file):
74+
if file == "" or file is None:
75+
return {}
76+
77+
df = pd.read_csv(file)
78+
df.question = df.question.astype(str)
79+
80+
option_map = defaultdict(dict)
81+
for _, option in df.iterrows():
82+
if option_map[option["section"]].get(option["question"]) is None:
83+
option_map[option["section"]][option["question"]] = {}
84+
85+
option_map[option["section"]][option["question"]][option["prev_option"]] = (
86+
option["new_option"]
87+
)
88+
89+
return option_map
90+
91+
def get_mapped_answer(self, answer, q, answer_map):
92+
if (
93+
answer_map.get(q.section.title) is not None
94+
and answer_map[q.section.title].get(q.number_and_part) is not None
95+
and answer_map[q.section.title][q.number_and_part].get(answer) is not None
96+
):
97+
return answer_map[q.section.title][q.number_and_part][answer]
98+
99+
return answer
100+
101+
def handle(
102+
self,
103+
quiet: bool = False,
104+
commit: bool = False,
105+
response_list: str = "",
106+
question_list: str = "",
107+
stage: str = "",
108+
session: str = "",
109+
option_map: str = "",
110+
authority_map: str = "",
111+
update_existing_responses: bool = False,
112+
*args,
113+
**kwargs,
114+
):
115+
self.quiet = quiet
116+
117+
u, _ = User.objects.get_or_create(
118+
username="Auto_point_script",
119+
)
120+
121+
rt, ms = self.get_stage_and_session(stage, session)
122+
123+
responses = self.get_df(response_list)
124+
questions = self.get_df(question_list)
125+
answer_map = self.get_option_map(option_map)
126+
self.council_lookup = self.get_council_lookup()
127+
self.set_authority_map(authority_map)
128+
129+
page_number = 0
130+
evidence = "Council owns less than 100 homes or no homes at all"
131+
132+
responses_added = 0
133+
responses_overidden = 0
134+
existing_responses = 0
135+
with self.get_atomic_context(commit):
136+
for _, q in questions.iterrows():
137+
q_args = {"number": q["question_number"]}
138+
if (
139+
not pd.isna(q["question_part"])
140+
and q.get("question_part", None) is not None
141+
):
142+
q_args["number_part"] = q["question_part"].strip()
143+
144+
try:
145+
question = Question.objects.get(
146+
section__marking_session=ms,
147+
section__title=q["section"],
148+
**q_args,
149+
)
150+
except Question.DoesNotExist:
151+
self.print_error(
152+
f"no matching question for {q['section']}, {q_args}"
153+
)
154+
continue
155+
156+
for _, r in responses.iterrows():
157+
158+
council_name = r["public_body"]
159+
council = self.get_authority(council_name, ms)
160+
if council is None:
161+
self.print_error(f"{council_name} not found")
162+
continue
163+
164+
answer = self.get_mapped_answer(
165+
r["Staff Review"].strip(), question, answer_map
166+
)
167+
try:
168+
option = Option.objects.get(
169+
question=question, description=answer
170+
)
171+
except Option.DoesNotExist:
172+
self.print_error(
173+
f"no matching option for {question.number_and_part}, {q['section']} - '{answer}'"
174+
)
175+
continue
176+
177+
add_response = False
178+
179+
options = None
180+
try:
181+
response = Response.objects.get(
182+
question=question, authority=council, response_type=rt
183+
)
184+
if question.question_type == "multiple_choice":
185+
if response.multi_option is not None:
186+
options = [x.id for x in response.multi_option.all()]
187+
if option.id not in options:
188+
self.print_info(
189+
f"existing response does not contain expected response for {question.number_and_part}, {q['section']}, {council.name}"
190+
)
191+
else:
192+
if response.option != option:
193+
self.print_info(
194+
f"different existing response for {question.number_and_part}, {q['section']}, {council.name}"
195+
)
196+
self.print_debug(
197+
f"response exists for {council.name} for {question.number_and_part}, {q['section']}"
198+
)
199+
existing_responses += 1
200+
except Response.DoesNotExist:
201+
add_response = True
202+
203+
response_opts = {
204+
"user": u,
205+
"question": question,
206+
"authority": council,
207+
"response_type": rt,
208+
"private_notes": "Automatically assigned mark",
209+
"page_number": page_number,
210+
"evidence": evidence,
211+
"public_notes": r["request_url"],
212+
}
213+
214+
if add_response:
215+
responses_added += 1
216+
self.print_debug(
217+
f"creating response for {council.name} for {question.number_and_part}, {q['section']}"
218+
)
219+
220+
if question.question_type == "multiple_choice":
221+
r = Response.objects.create(**response_opts)
222+
if options is not None:
223+
for o in options:
224+
r.multi_option.add(o)
225+
else:
226+
r.multi_option.add(option.id)
227+
else:
228+
response_opts["option"] = option
229+
r = Response.objects.create(**response_opts)
230+
231+
elif update_existing_responses:
232+
responses_overidden += 1
233+
self.print_info(
234+
f"overriding response for {council.name} for {question.number_and_part}, {q['section']}"
235+
)
236+
237+
if question.question_type != "multiple_choice":
238+
response.option = option
239+
240+
response.private_notes = (
241+
response_opts["private_notes"]
242+
+ "\n"
243+
+ "Overridden by automatic assignment"
244+
)
245+
246+
response.public_notes = response_opts["public_notes"]
247+
response.evidence = response_opts["evidence"]
248+
response.page_number = response_opts["page_number"]
249+
250+
response.save()
251+
if question.question_type == "multiple_choice":
252+
response.multi_option.clear()
253+
if options is not None:
254+
for o in options:
255+
response.multi_option.add(o)
256+
else:
257+
response.multi_option.add(option.id)
258+
259+
self.print_success(
260+
f"Added {responses_added} responses for {q['section']} {question.number_and_part}, {existing_responses} existing responses, {responses_overidden} responses overridden"
261+
)
262+
if not commit:
263+
self.print_info(
264+
"call with --commit to commit changed to database",
265+
)

0 commit comments

Comments
 (0)