Skip to content

Commit 3814ffd

Browse files
artczartcz
committed
first draft of downloading and storing the pretalx data
1 parent 91c1bbe commit 3814ffd

File tree

4 files changed

+266
-0
lines changed

4 files changed

+266
-0
lines changed

‎intbot/core/integrations/pretalx.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
from typing import Any
2+
3+
import httpx
4+
from core.models import PretalxData
5+
from django.conf import settings
6+
7+
PRETALX_EVENT = "ep2025"
8+
base_url = f"https://pretalx.com/api/events/{PRETALX_EVENT}/"
9+
10+
RESOURCES = {
11+
# Questions need to be passed to include answers in the same endpoint,
12+
# saving us later time with joining the answers.
13+
PretalxData.PretalxEndpoints.submissions: "submissions?questions=all",
14+
PretalxData.PretalxEndpoints.speakers: "speakers?questions=all",
15+
}
16+
17+
18+
JsonType = dict[str, Any]
19+
20+
21+
def fetch_pretalx_data(resource) -> list[JsonType]:
22+
headers = {
23+
"Authorization": f"Token {settings.PRETALX_API_TOKEN}",
24+
"Content-Type": "application/json",
25+
}
26+
27+
endpoint = RESOURCES[resource]
28+
url = base_url + f"{endpoint}"
29+
30+
# Pretalx paginates the output, so we will need to do multiple requests and
31+
# then merge mutliple pages to one big dictionary
32+
res0 = []
33+
data = {"next": url}
34+
n = 0
35+
while url := data["next"]:
36+
n += 1
37+
response = httpx.get(url, headers=headers)
38+
39+
if response.status_code != 200:
40+
raise Exception(f"Error {response.status_code}: {response.text}")
41+
42+
data = response.json()
43+
res0 += data["results"]
44+
45+
return res0
46+
47+
48+
def download_latest_submissions() -> PretalxData:
49+
data = fetch_pretalx_data(PretalxData.PretalxEndpoints.submissions)
50+
51+
pretalx_data = PretalxData.objects.create(
52+
endpoint=PretalxData.PretalxEndpoints.submissions,
53+
content=data,
54+
)
55+
56+
return pretalx_data
57+
58+
59+
def download_latest_speakers() -> PretalxData:
60+
data = fetch_pretalx_data(PretalxData.PretalxEndpoints.speakers)
61+
62+
pretalx_data = PretalxData.objects.create(
63+
endpoint=PretalxData.PretalxEndpoints.speakers,
64+
content=data,
65+
)
66+
67+
return pretalx_data

‎intbot/core/models.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,32 @@ def summary(self) -> str:
8181

8282
def __str__(self):
8383
return f"{self.uuid} {self.author}: {self.content[:30]}"
84+
85+
86+
class PretalxData(models.Model):
87+
"""
88+
Table to store raw data download from pretalx for later parsing.
89+
90+
We first download data from pretalx to this table, and then fire a separate
91+
background task that pulls data from this table and stores in separate
92+
"business" tables, like "Proposal" or "Speaker".
93+
"""
94+
95+
class PretalxEndpoints(models.TextChoices):
96+
submissions = "submissions", "Submissions"
97+
speakers = "speakers", "Speakers"
98+
schedule = "schedule", "Schedule"
99+
100+
uuid = models.UUIDField(default=uuid.uuid4)
101+
endpoint = models.CharField(
102+
max_length=255,
103+
choices=PretalxEndpoints.choices,
104+
)
105+
content = models.JSONField()
106+
107+
created_at = models.DateTimeField(auto_now_add=True)
108+
modified_at = models.DateTimeField(auto_now=True)
109+
processed_at = models.DateTimeField(blank=True, null=True)
110+
111+
def __str__(self):
112+
return f"{self.uuid}"

‎intbot/intbot/settings.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,9 @@ def get(name) -> str:
197197
ZAMMAD_GROUP_SPONSORS = get("ZAMMAD_GROUP_SPONSORS")
198198
ZAMMAD_GROUP_GRANTS = get("ZAMMAD_GROUP_GRANTS")
199199

200+
# Pretalx
201+
PRETALX_API_TOKEN = get("PRETALX_API_TOKEN")
202+
200203

201204
if DJANGO_ENV == "dev":
202205
DEBUG = True
@@ -282,6 +285,8 @@ def get(name) -> str:
282285
ZAMMAD_GROUP_HELPDESK = "TestZammad Helpdesk"
283286
ZAMMAD_GROUP_BILLING = "TestZammad Billing"
284287

288+
PRETALX_API_TOKEN = "Test-Pretalx-API-token"
289+
285290

286291
elif DJANGO_ENV == "local_container":
287292
DEBUG = False

‎intbot/tests/test_integrations/test_pretalx.py

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import respx
2+
import pytest
3+
from core.integrations import pretalx
4+
from core.models import PretalxData
5+
from httpx import Response
6+
7+
8+
@respx.mock
9+
def test_fetch_submissions_from_pretalx():
10+
endpoint = pretalx.RESOURCES[PretalxData.PretalxEndpoints.submissions]
11+
url = pretalx.base_url + endpoint
12+
respx.get(url).mock(
13+
return_value=Response(
14+
200,
15+
json={
16+
"results": [
17+
{"hello": "world"},
18+
],
19+
"next": f"{url}&page=2",
20+
},
21+
)
22+
)
23+
respx.get(url + "&page=2").mock(
24+
return_value=Response(
25+
200,
26+
json={
27+
"results": [
28+
{"foo": "bar"},
29+
],
30+
# It's important to make it last page in tests.
31+
# Otherwise it will be infinite loop :)
32+
"next": None,
33+
},
34+
)
35+
)
36+
37+
submissions = pretalx.fetch_pretalx_data(
38+
PretalxData.PretalxEndpoints.submissions,
39+
)
40+
41+
assert submissions == [
42+
{"hello": "world"},
43+
{"foo": "bar"},
44+
]
45+
46+
47+
@respx.mock
48+
def test_fetch_speakers_from_pretalx():
49+
endpoint = pretalx.RESOURCES[PretalxData.PretalxEndpoints.speakers]
50+
url = pretalx.base_url + endpoint
51+
respx.get(url).mock(
52+
return_value=Response(
53+
200,
54+
json={
55+
"results": [
56+
{"hello": "world"},
57+
],
58+
"next": f"{url}&page=2",
59+
},
60+
)
61+
)
62+
respx.get(url + "&page=2").mock(
63+
return_value=Response(
64+
200,
65+
json={
66+
"results": [
67+
{"foo": "bar"},
68+
],
69+
# It's important to make it last page in tests.
70+
# Otherwise it will be infinite loop :)
71+
"next": None,
72+
},
73+
)
74+
)
75+
76+
submissions = pretalx.fetch_pretalx_data(
77+
PretalxData.PretalxEndpoints.speakers,
78+
)
79+
80+
assert submissions == [
81+
{"hello": "world"},
82+
{"foo": "bar"},
83+
]
84+
85+
86+
@respx.mock
87+
@pytest.mark.django_db
88+
def test_download_latest_submissions():
89+
endpoint = pretalx.RESOURCES[PretalxData.PretalxEndpoints.submissions]
90+
url = pretalx.base_url + endpoint
91+
respx.get(url).mock(
92+
return_value=Response(
93+
200,
94+
json={
95+
"results": [
96+
{"hello": "world"},
97+
],
98+
"next": f"{url}&page=2",
99+
},
100+
)
101+
)
102+
respx.get(url + "&page=2").mock(
103+
return_value=Response(
104+
200,
105+
json={
106+
"results": [
107+
{"foo": "bar"},
108+
],
109+
# It's important to make it last page in tests.
110+
# Otherwise it will be infinite loop :)
111+
"next": None,
112+
},
113+
)
114+
)
115+
116+
pretalx.download_latest_submissions()
117+
118+
pd = PretalxData.objects.get(endpoint=PretalxData.PretalxEndpoints.submissions)
119+
120+
assert pd.endpoint == "submissions"
121+
assert pd.content == [
122+
{"hello": "world"},
123+
{"foo": "bar"},
124+
]
125+
126+
@respx.mock
127+
@pytest.mark.django_db
128+
def test_download_latest_speakers():
129+
endpoint = pretalx.RESOURCES[PretalxData.PretalxEndpoints.speakers]
130+
url = pretalx.base_url + endpoint
131+
respx.get(url).mock(
132+
return_value=Response(
133+
200,
134+
json={
135+
"results": [
136+
{"hello": "world"},
137+
],
138+
"next": f"{url}&page=2",
139+
},
140+
)
141+
)
142+
respx.get(url + "&page=2").mock(
143+
return_value=Response(
144+
200,
145+
json={
146+
"results": [
147+
{"foo": "bar"},
148+
],
149+
# It's important to make it last page in tests.
150+
# Otherwise it will be infinite loop :)
151+
"next": None,
152+
},
153+
)
154+
)
155+
156+
pretalx.download_latest_speakers()
157+
158+
pd = PretalxData.objects.get(endpoint=PretalxData.PretalxEndpoints.speakers)
159+
160+
assert pd.endpoint == "speakers"
161+
assert pd.content == [
162+
{"hello": "world"},
163+
{"foo": "bar"},
164+
]
165+

0 commit comments

Comments
 (0)