Email attachment feature (#7)

omerfarukabaci · gokselcoban · taylanpince · web-flow · commit f2e95311a232 · 2024-11-05T19:14:12.000+01:00
* Implement email attachment feature * Bump version to 1.0.0 * Refactor email attachments feature * Fix typo * Email monitoring and tracking (#8) * add email monitoring and tracking fields * add bounce type fields * add migration * add events extra requirements * add monitoring fields to SentEmailAdmin * send_email method returns sent_email * add ordering to SentEmailAdmin * if the email is opened, clicked or complained, it is also delivered * message_id may not be exists in extra_headers for testing * task required to return a serializable * Update version --------- Co-authored-by: Goksel Coban <goksel326@gmail.com> Co-authored-by: Taylan Pince <taylanpince@gmail.com>
diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,5 @@ coverage.xml
 .coverage
 __pycache__
 .DS_Store
-*.sqlite*
+*.sqlite*
+.vscode
diff --git a/django_ses_plus/admin.py b/django_ses_plus/admin.py
@@ -7,10 +7,14 @@ class SentEmailAdmin(admin.ModelAdmin):
     fieldsets = (
         ("Details", {"fields": ("id", "subject", "from_email", "to_email", "creation_datetime",)}),
         ("Content", {"fields": ("html",)}),
+        ("Monitoring", {"fields": ("message_id", "status", "bounce_type", "bounce_sub_type", "is_opened", "is_clicked", "is_complained")}),
     )
-    readonly_fields = ("id", "subject", "from_email", "to_email", "creation_datetime", "html")
-    list_display = ("id", "to_email", "subject", "creation_datetime")
+    readonly_fields = ("id", "subject", "from_email", "to_email", "creation_datetime", "html", "message_id", "status", "bounce_type", "bounce_sub_type", "is_opened", "is_clicked", "is_complained")
+    list_display = ("id", "to_email", "subject", "status", "creation_datetime")
+    list_filter = ("status", "is_opened", "is_clicked", "is_complained")
     search_fields = ("to_email", "subject", )
+    ordering = ("-creation_datetime",)
+    date_hierarchy = "creation_datetime"
 
     def get_actions(self, request):
         actions = super().get_actions(request)
diff --git a/django_ses_plus/migrations/0002_add_sent_email_attachment_model.py b/django_ses_plus/migrations/0002_add_sent_email_attachment_model.py
@@ -0,0 +1,32 @@
+# Generated by Django 3.1.1 on 2021-01-26 14:20
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_ses_plus.utils
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_ses_plus', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SentEmailAttachment',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('filename', models.CharField(max_length=255)),
+                ('content', models.FileField(upload_to=django_ses_plus.utils.sent_email_attachment_upload_path)),
+                ('mimetype', models.CharField(help_text='e.g. text/html, application/pdf, image/png...', max_length=255)),
+                ('creation_datetime', models.DateTimeField(auto_now_add=True)),
+                ('update_datetime', models.DateTimeField(auto_now=True)),
+                ('sent_email', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='django_ses_plus.sentemail')),
+            ],
+            options={
+                'verbose_name': 'Sent Email Attachment',
+                'verbose_name_plural': 'Sent Email Attachments',
+            },
+        ),
+    ]
diff --git a/django_ses_plus/migrations/0003_add_monitoring_and_tracking_fields.py b/django_ses_plus/migrations/0003_add_monitoring_and_tracking_fields.py
@@ -0,0 +1,48 @@
+# Generated by Django 3.1.3 on 2021-01-27 16:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_ses_plus', '0002_add_sent_email_attachment_model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sentemail',
+            name='bounce_sub_type',
+            field=models.CharField(blank=True, max_length=50),
+        ),
+        migrations.AddField(
+            model_name='sentemail',
+            name='bounce_type',
+            field=models.CharField(blank=True, max_length=50),
+        ),
+        migrations.AddField(
+            model_name='sentemail',
+            name='is_clicked',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='sentemail',
+            name='is_complained',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='sentemail',
+            name='is_opened',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='sentemail',
+            name='message_id',
+            field=models.CharField(blank=True, max_length=100),
+        ),
+        migrations.AddField(
+            model_name='sentemail',
+            name='status',
+            field=models.CharField(choices=[('unknown', 'Unknown'), ('sent', 'Sent'), ('delivered', 'Delivered'), ('bounced', 'Bounced')], default='unknown', max_length=50),
+        ),
+    ]
diff --git a/django_ses_plus/models.py b/django_ses_plus/models.py
@@ -1,17 +1,28 @@
+import base64
 import uuid
 
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import models
+from django.dispatch import receiver
 from django.template.loader import render_to_string
 from django.utils import translation
+from django_ses import signals
 from django.utils.translation import gettext_lazy as _
 
 from django_ses_plus.settings import DJANGO_SES_PLUS_SETTINGS
-from .utils import sent_email_upload_path
+from .utils import sent_email_upload_path, sent_email_attachment_upload_path
 
 
 class SentEmail(models.Model):
+    class Status(models.TextChoices):
+        UNKNOWN = 'unknown', 'Unknown'
+        SENT = 'sent', 'Sent'
+        DELIVERED = 'delivered', 'Delivered'
+        BOUNCED = 'bounced', 'Bounced'
+        # Don't support rejects and rendering failures for now.
+        # REJECTED = 'rejected', 'Rejected'
+        # RENDERING_FAILURE = 'rendering-failure', 'Rendering Failure'
 
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
 
@@ -21,6 +32,15 @@ class SentEmail(models.Model):
     subject = models.TextField()
     html = models.FileField(upload_to=sent_email_upload_path)
 
+    # tracking & monitoring
+    message_id = models.CharField(max_length=100, blank=True)
+    status = models.CharField(choices=Status.choices, default=Status.UNKNOWN, max_length=50)
+    bounce_type = models.CharField(max_length=50, blank=True)
+    bounce_sub_type = models.CharField(max_length=50, blank=True)
+    is_opened = models.BooleanField(default=False)
+    is_clicked = models.BooleanField(default=False)
+    is_complained = models.BooleanField(default=False)
+
     creation_datetime = models.DateTimeField(auto_now_add=True)
     update_datetime = models.DateTimeField(auto_now=True)
 
@@ -31,13 +51,107 @@ class Meta:
     def __str__(self):
         return f"{self.subject} to {self.to_email}"
 
+    @staticmethod
+    @receiver(signals.send_received)
+    def send_handler(sender, mail_obj, send_obj, **kwargs):
+        message_id = mail_obj['messageId']
+        SentEmail.objects.filter(
+            message_id=message_id,
+            status=SentEmail.Status.UNKNOWN,
+        ).update(
+            status=SentEmail.Status.SENT
+        )
+
+    @staticmethod
+    @receiver(signals.delivery_received)
+    def delivery_handler(sender, mail_obj, delivery_obj, **kwargs):
+        message_id = mail_obj['messageId']
+        recipients = delivery_obj['recipients']
+        SentEmail.objects.filter(
+            message_id=message_id,
+            to_email__in=recipients,
+        ).update(
+            status=SentEmail.Status.DELIVERED
+        )
+
+    @staticmethod
+    @receiver(signals.bounce_received)
+    def bounce_handler(sender, mail_obj, bounce_obj, **kwargs):
+        message_id = mail_obj['messageId']
+        recipients = [r['emailAddress'] for r in bounce_obj['bouncedRecipients']]
+        SentEmail.objects.filter(
+            message_id=message_id,
+            to_email__in=recipients,
+        ).update(
+            status=SentEmail.Status.BOUNCED,
+            bounce_type=bounce_obj['bounceType'],
+            bounce_sub_type=bounce_obj['bounceSubType'],
+        )
+
+    @staticmethod
+    @receiver(signals.open_received)
+    def open_handler(sender, mail_obj, open_obj, **kwargs):
+        # open_obj does not provide recipient emails.
+        message_id = mail_obj['messageId']
+        SentEmail.objects.filter(
+            message_id=message_id,
+        ).update(
+            status=SentEmail.Status.DELIVERED,
+            is_opened=True
+        )
+
+    @staticmethod
+    @receiver(signals.click_received)
+    def click_handler(sender, mail_obj, click_obj, **kwargs):
+        # click_obj does not provide recipient emails.
+        message_id = mail_obj['messageId']
+        SentEmail.objects.filter(
+            message_id=message_id,
+        ).update(
+            status=SentEmail.Status.DELIVERED,
+            is_clicked=True
+        )
+
+    @staticmethod
+    @receiver(signals.complaint_received)
+    def compliant_handler(sender, mail_obj, complaint_obj, **kwargs):
+        message_id = mail_obj['messageId']
+        recipients = complaint_obj['complainedRecipients']
+        SentEmail.objects.filter(
+            message_id=message_id,
+            to_email__in=recipients,
+        ).update(
+            status=SentEmail.Status.DELIVERED,
+            is_complained=True
+        )
+
+
+class SentEmailAttachment(models.Model):
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+
+    sent_email = models.ForeignKey(to="django_ses_plus.SentEmail", related_name="attachments", on_delete=models.CASCADE)
+    filename = models.CharField(max_length=255)
+    content = models.FileField(upload_to=sent_email_attachment_upload_path)
+    mimetype = models.CharField(help_text="e.g. text/html, application/pdf, image/png...", max_length=255)
+
+    creation_datetime = models.DateTimeField(auto_now_add=True)
+    update_datetime = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        verbose_name = _("Sent Email Attachment")
+        verbose_name_plural = _("Sent Email Attachments")
+
+    def __str__(self):
+        return f"{self.filename} ({self.mimetype})"
+
 
 class SendEmailMixin(object):
 
     def get_to_email(self):
         return self.email
 
-    def send_email(self, subject, template_path, context, from_email=None, language=None):
+    def send_email(self, subject, template_path, context, from_email=None, language=None, attachments=None):
         from .tasks import send_email
         if not DJANGO_SES_PLUS_SETTINGS["SEND_EMAIL"]:
             return _("Email cannot be sent due to SEND_EMAIL flag in project settings.")
@@ -50,11 +164,25 @@ def send_email(self, subject, template_path, context, from_email=None, language=
         if language:
             translation.activate(language)
 
+        if attachments is not None:
+            assert isinstance(attachments, list), "Attachments should be a `list` of `dict` objects."
+
+            for attachment in attachments:
+                assert all([key in attachment for key in ["filename", "content", "mimetype"]]), "Attachments should contain `filename`, `content` and `mimetype`."
+
+                if isinstance(attachment["content"], bytes):
+                    # Since celery only accepts JSON serializable types and `bytes` is not JSON serializable,
+                    # Base64 encoding is used to be able to pass attachment content to the celery task,
+                    attachment["content"] = base64.b64encode(attachment["content"]).decode("utf-8")
+                else:
+                    assert False, f"Attachment contents should be `bytes`, not {type(attachment['content'])}."
+
         html_message = render_to_string(template_path, context)
         send_email.delay(
             subject=subject,
             to_email=self.get_to_email(),
             html_message=html_message,
+            attachments=attachments,
             from_email=from_email,
             recipient_id=recipient_id
         )
diff --git a/django_ses_plus/tasks.py b/django_ses_plus/tasks.py
@@ -1,41 +1,55 @@
+import base64
 from uuid import uuid4
 
 from celery import shared_task
 
 from django_ses_plus import logger
-from .settings import DJANGO_SES_PLUS_SETTINGS
 from django.core.files.base import ContentFile
 from django.core.mail import send_mail
 from django.utils.translation import gettext_lazy as _
 
-from .models import SentEmail
+from .models import SentEmail, SentEmailAttachment
+from .settings import DJANGO_SES_PLUS_SETTINGS
+from .utils import send_mail
 
 
 @shared_task(retry_kwargs=DJANGO_SES_PLUS_SETTINGS["CELERY_TASK_RETRY_KWARGS"])
-def send_email(subject, to_email, html_message, from_email=None, message=None, recipient_id=None):
+def send_email(subject, to_email, html_message, from_email=None, message=None, recipient_id=None, attachments=None):
     if not DJANGO_SES_PLUS_SETTINGS["SEND_EMAIL"]:
         return _("Email cannot be sent due to SEND_EMAIL flag in project settings.")
 
     if not from_email:
         from_email = DJANGO_SES_PLUS_SETTINGS["DEFAULT_FROM_EMAIL"]
 
-    send_mail(
+    num_sent, mail = send_mail(
         subject=subject,
         message=message,
         html_message=html_message,
+        attachments=attachments,
         from_email=from_email,
         recipient_list=[to_email],
         fail_silently=False,
     )
 
     try:
-        SentEmail.objects.create(
+        sent_email = SentEmail.objects.create(
+            message_id=mail.extra_headers.get('message_id', ''),
             recipient_id=recipient_id,
             subject=subject,
             html=ContentFile(content=bytes(html_message, encoding="utf8"), name="{}.html".format(uuid4())),
             from_email=from_email,
             to_email=to_email,
         )
+        if attachments:
+            for attachment in attachments:
+                SentEmailAttachment.objects.create(
+                    sent_email=sent_email,
+                    filename=attachment["filename"],
+                    content=ContentFile(content=base64.b64decode(attachment["content"]), name=attachment["filename"]),
+                    mimetype=attachment["mimetype"]
+                )
     except Exception as e:
         # Do not retry if object creation fails.
         logger.error(str(e), exc_info=e, extra={'trace': True})
+    else:
+        return sent_email.id
diff --git a/django_ses_plus/utils.py b/django_ses_plus/utils.py
@@ -1,6 +1,33 @@
+import base64
+from os import path
 from time import strftime
+from uuid import uuid4
+
+from django.core.mail import get_connection
+from django.core.mail.message import EmailMultiAlternatives
 
 
 def sent_email_upload_path(sent_email, filename):
     filename = filename.split(".")[0]
     return strftime("emails/%Y/%m/%d/%Y-%m-%d-%H-%M-%S-{}.html".format(filename))
+
+
+def sent_email_attachment_upload_path(sent_email_attachment, filename):
+    extension = path.splitext(filename)[1]
+    return strftime(f"email-attachments/%Y/%m/%d/{uuid4().hex}{extension}")
+
+
+def send_mail(subject, message, from_email, recipient_list, fail_silently=False, html_message=None, attachments=None):
+    connection = get_connection(fail_silently=fail_silently)
+
+    mail = EmailMultiAlternatives(subject, message, from_email, recipient_list, connection=connection)
+
+    if html_message:
+        mail.attach_alternative(html_message, 'text/html')
+
+    if attachments:
+        for attachment in attachments:
+            mail.attach(attachment["filename"], base64.b64decode(attachment["content"]), attachment["mimetype"])
+
+    num_sent = mail.send()
+    return num_sent, mail
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
 Django >= 3.2
 celery >= 4
-django-ses >= 1.0.0
+django-ses >= 2.1.1
diff --git a/setup.py b/setup.py
