download pretix data

deploy/playbooks/04_cron.yml

@@ -4,7 +4,13 @@
   tasks:
     - name: "Download pretalx data every hour"
       ansible.builtin.cron:
-        name: "Download pretalx data every hour"
+        name: "Download pretalx data once a day at 05:05am"
         minute: "5"
-        hour: "8"     # run once a day at 08:05 am
+        hour: "5"     # run once a day at 08:05 am
         job: "make prod/cron/pretalx"
+
+      ansible.builtin.cron:
+        name: "Download pretix data once a day at 06:05am"
+        minute: "5"
+        hour: "6"
+        job: "make prod/cron/pretix"

deploy/templates/app/Makefile.app.j2

@@ -18,5 +18,9 @@ prod/manage:
 prod/cron/pretalx:
 	$(MAKE_APP) in-container/manage ARG="download_pretalx_data --event=europython-2025"
 
+
+prod/cron/pretix:
+	$(MAKE_APP) in-container/manage ARG="download_pretix_data --event=ep2025"
+
 logs:
 	docker compose logs -f

intbot/core/admin.py

@@ -1,6 +1,6 @@
 import json
 
-from core.models import DiscordMessage, PretalxData, Webhook
+from core.models import DiscordMessage, PretalxData, PretixData, Webhook
 from django.contrib import admin
 from django.utils.html import format_html
 
@@ -93,6 +93,33 @@ def pretty_content(self, obj: PretalxData):
     pretty_content.short_description = "Content"
 
 
+class PretixDataAdmin(admin.ModelAdmin):
+    list_display = [
+        "uuid",
+        "resource",
+        "created_at",
+        "modified_at",
+    ]
+    list_filter = [
+        "created_at",
+        "resource",
+    ]
+    readonly_fields = fields = [
+        "uuid",
+        "resource",
+        "pretty_content",
+        "created_at",
+        "modified_at",
+        "processed_at",
+    ]
+
+    def pretty_content(self, obj: PretixData):
+        return format_html("<pre>{}</pre>", json.dumps(obj.content, indent=4))
+
+    pretty_content.short_description = "Content"
+
+
 admin.site.register(Webhook, WebhookAdmin)
 admin.site.register(DiscordMessage, DiscordMessageAdmin)
 admin.site.register(PretalxData, PretalxDataAdmin)
+admin.site.register(PretixData, PretixDataAdmin)

intbot/core/integrations/pretix.py

@@ -0,0 +1,99 @@
+import logging
+from typing import Any
+
+import httpx
+from core.models import PretixData
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+PRETALX_EVENTS = [
+    "2022",
+    "ep2023",
+    "ep2024",
+    "ep2025",
+]
+
+ENDPOINTS = {
+    PretixData.PretixResources.orders: "orders/",
+    PretixData.PretixResources.products: "items/",
+    PretixData.PretixResources.vouchers: "vouchers/",
+}
+
+
+JsonType = dict[str, Any]
+
+
+def get_event_url(event):
+    assert event in PRETALX_EVENTS
+
+    pretix_url = "https://tickets.europython.eu"
+    url = f"{pretix_url}/api/v1/organizers/europython/events/{event}/"
+    return url
+
+
+def fetch_pretix_data(
+    event: str, resource: PretixData.PretixResources
+) -> list[JsonType]:
+    headers = {
+        "Authorization": f"Token {settings.PRETIX_API_TOKEN}",
+        "Content-Type": "application/json",
+    }
+
+    base_url = get_event_url(event)
+    endpoint = ENDPOINTS[resource]
+    url = f"{base_url}{endpoint}"
+
+    # Pretix paginates the output, so we will need to do multiple requests and
+    # then merge multiple pages to one big dictionary
+    results = []
+    page = 0
+
+    # This takes advantage of the fact that url will contain a url to the
+    # next page, until there is more data to fetch. If this is the last page,
+    # then the url will be None (falsy), and thus stop the while loop.
+    while url:
+        page += 1
+        response = httpx.get(url, headers=headers)
+
+        if response.status_code != 200:
+            raise Exception(f"Error {response.status_code}: {response.text}")
+
+        logger.info("Fetching data from %s, page %s", url, page)
+
+        data = response.json()
+        results += data["results"]
+        url = data["next"]
+
+    return results
+
+
+def download_latest_orders(event: str) -> PretixData:
+    data = fetch_pretix_data(event, PretixData.PretixResources.orders)
+
+    pretix_data = PretixData.objects.create(
+        resource=PretixData.PretixResources.orders,
+        content=data,
+    )
+
+    return pretix_data
+
+def download_latest_products(event: str) -> PretixData:
+    data = fetch_pretix_data(event, PretixData.PretixResources.products)
+
+    pretix_data = PretixData.objects.create(
+        resource=PretixData.PretixResources.products,
+        content=data,
+    )
+
+    return pretix_data
+
+def download_latest_vouchers(event: str) -> PretixData:
+    data = fetch_pretix_data(event, PretixData.PretixResources.vouchers)
+
+    pretix_data = PretixData.objects.create(
+        resource=PretixData.PretixResources.vouchers,
+        content=data,
+    )
+
+    return pretix_data

intbot/core/management/commands/download_pretix_data.py

@@ -0,0 +1,32 @@
+from core.integrations.pretalx import (
+    PRETIX_EVENTS,
+    download_latest_orders,
+    download_latest_products,
+    download_latest_vouchers,
+)
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Downloads latest pretix data"
+
+    def add_arguments(self, parser):
+        # Add keyword argument event
+        parser.add_argument(
+            "--event",
+            choices=PRETIX_EVENTS,
+            help="slug of the event (for example `ep2025`)",
+            required=True,
+        )
+
+    def handle(self, **kwargs):
+        event = kwargs["event"]
+
+        self.stdout.write(f"Downloading latest products from pretix... {event}")
+        download_latest_products(event)
+
+        self.stdout.write(f"Downloading latest vouchers from pretix... {event}")
+        download_latest_vouchers(event)
+
+        self.stdout.write(f"Downloading latest orders from pretix... {event}")
+        download_latest_orders(event)

intbot/core/migrations/0006_add_pretix_data_model.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.1.4 on 2025-04-24 22:08
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0005_add_pretalx_data_model'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PretixData',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4)),
+                ('resource', models.CharField(choices=[('orders', 'Orders'), ('products', 'Products'), ('vouchers', 'Vouchers')], max_length=255)),
+                ('content', models.JSONField()),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('modified_at', models.DateTimeField(auto_now=True)),
+                ('processed_at', models.DateTimeField(blank=True, null=True)),
+            ],
+        ),
+    ]

intbot/core/models.py

@@ -87,9 +87,7 @@ class PretalxData(models.Model):
     """
     Table to store raw data download from pretalx for later parsing.
 
-    We first download data from pretalx to this table, and then fire a separate
-    background task that pulls data from this table and stores in separate
-    "business" tables, like "Proposal" or "Speaker".
+    We first download data from pretix, then we later parse the latest jsons.
     """
 
     class PretalxResources(models.TextChoices):
@@ -110,3 +108,30 @@ class PretalxResources(models.TextChoices):
 
     def __str__(self):
         return f"{self.uuid}"
+
+
+class PretixData(models.Model):
+    """
+    Table to store raw data download from pretix for later parsing.
+
+    We first download data from pretix, then we later parse the latest jsons.
+    """
+
+    class PretixResources(models.TextChoices):
+        orders = "orders", "Orders"
+        products = "products", "Products"
+        vouchers = "vouchers", "Vouchers"
+
+    uuid = models.UUIDField(default=uuid.uuid4)
+    resource = models.CharField(
+        max_length=255,
+        choices=PretixResources.choices,
+    )
+    content = models.JSONField()
+
+    created_at = models.DateTimeField(auto_now_add=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    processed_at = models.DateTimeField(blank=True, null=True)
+
+    def __str__(self):
+        return f"{self.uuid}"

intbot/intbot/settings.py

@@ -200,6 +200,9 @@ def get(name) -> str:
 # Pretalx
 PRETALX_API_TOKEN = get("PRETALX_API_TOKEN")
 
+# Pretix
+PRETIX_API_TOKEN = get("PRETIX_API_TOKEN")
+
 
 if DJANGO_ENV == "dev":
     DEBUG = True

intbot/tests/test_admin.py

@@ -2,7 +2,7 @@
 Sanity checks (mostly) if the admin resources are available
 """
 
-from core.models import DiscordMessage, PretalxData, Webhook
+from core.models import DiscordMessage, PretalxData, PretixData, Webhook
 
 
 def test_admin_for_webhooks_sanity_check(admin_client):
@@ -34,33 +34,69 @@ def test_admin_for_discordmessages_sanity_check(admin_client):
     assert dm.channel_name.encode() in response.content
 
 
-def test_admin_list_for_pretalx_data(admin_client):
-    """Simple sanity check if the page loads correctly"""
-    url = "/admin/core/pretalxdata/"
-    pd = PretalxData.objects.create(
-        resource=PretalxData.PretalxResources.speakers,
-        content={},
-    )
-    assert pd.uuid
-
-    response = admin_client.get(url)
-
-    assert response.status_code == 200
-    assert str(pd.uuid).encode() in response.content
-    assert pd.get_resource_display().encode() in response.content
-
-
-def test_admin_change_for_pretalx_data(admin_client):
-    """Simple sanity check if the page loads correctly"""
-    url = "/admin/core/pretalxdata/"
-    pd = PretalxData.objects.create(
-        resource=PretalxData.PretalxResources.speakers,
-        content={},
-    )
-    assert pd.uuid
-
-    response = admin_client.get(f"{url}{pd.pk}/change/")
-
-    assert response.status_code == 200
-    assert str(pd.uuid).encode() in response.content
-    assert pd.get_resource_display().encode() in response.content
+class TestPretalxDataAdmin:
+    """This class exists only for namespacing purposes"""
+
+    def test_admin_list_for_pretalx_data(self, admin_client):
+        """Simple sanity check if the page loads correctly"""
+        url = "/admin/core/pretalxdata/"
+        pd = PretalxData.objects.create(
+            resource=PretalxData.PretalxResources.speakers,
+            content={},
+        )
+        assert pd.uuid
+
+        response = admin_client.get(url)
+
+        assert response.status_code == 200
+        assert str(pd.uuid).encode() in response.content
+        assert pd.get_resource_display().encode() in response.content
+
+    def test_admin_change_for_pretalx_data(self, admin_client):
+        """Simple sanity check if the page loads correctly"""
+        url = "/admin/core/pretalxdata/"
+        pd = PretalxData.objects.create(
+            resource=PretalxData.PretalxResources.speakers,
+            content={},
+        )
+        assert pd.uuid
+
+        response = admin_client.get(f"{url}{pd.pk}/change/")
+
+        assert response.status_code == 200
+        assert str(pd.uuid).encode() in response.content
+        assert pd.get_resource_display().encode() in response.content
+
+
+class TestPretixDataAdmin:
+    """This class exists only for namespacing purposes"""
+
+    def test_admin_list_for_pretix_data(self, admin_client):
+        """Simple sanity check if the page loads correctly"""
+        url = "/admin/core/pretixdata/"
+        pd = PretixData.objects.create(
+            resource=PretixData.PretixResources.orders,
+            content={},
+        )
+        assert pd.uuid
+
+        response = admin_client.get(url)
+
+        assert response.status_code == 200
+        assert str(pd.uuid).encode() in response.content
+        assert pd.get_resource_display().encode() in response.content
+
+    def test_admin_change_for_pretix_data(self, admin_client):
+        """Simple sanity check if the page loads correctly"""
+        url = "/admin/core/pretixdata/"
+        pd = PretixData.objects.create(
+            resource=PretixData.PretixResources.orders,
+            content={},
+        )
+        assert pd.uuid
+
+        response = admin_client.get(f"{url}{pd.pk}/change/")
+
+        assert response.status_code == 200
+        assert str(pd.uuid).encode() in response.content
+        assert pd.get_resource_display().encode() in response.content