Track notification answers

todo/mail/consumers/tracker.py

@@ -1,8 +1,9 @@
+import re
 import logging
 
+from email.charset import Charset as EMailCharset
 from django.db import transaction
 from django.db.models import Count
-from email.charset import Charset as EMailCharset
 from html2text import html2text
 from todo.models import Comment, Task, TaskList
 
@@ -23,6 +24,7 @@ def message_find_mime(message, mime_type):
     for submessage in message.walk():
         if submessage.get_content_type() == mime_type:
             return submessage
+    return None
 
 
 def message_text(message):
@@ -45,6 +47,34 @@ def format_task_title(format_string, message):
     )
 
 
+DJANGO_TODO_THREAD = re.compile(r'<thread-(\d+)@django-todo>')
+
+
+def parse_references(task_list, references):
+    related_messages = []
+    answer_thread = None
+    for related_message in references.split():
+        logger.info("checking reference: %r", related_message)
+        match = re.match(DJANGO_TODO_THREAD, related_message)
+        if match is None:
+            related_messages.append(related_message)
+            continue
+
+        thread_id = int(match.group(1))
+        new_answer_thread = Task.objects.filter(
+            task_list=task_list,
+            pk=thread_id
+        ).first()
+        if new_answer_thread is not None:
+            answer_thread = new_answer_thread
+
+    if answer_thread is None:
+        logger.info("no answer thread found in references")
+    else:
+        logger.info("found an answer thread: %d", answer_thread)
+    return related_messages, answer_thread
+
+
 def insert_message(task_list, message, priority, task_title_format):
     if "message-id" not in message:
         logger.warning("missing message id, ignoring message")
@@ -62,6 +92,7 @@ def insert_message(task_list, message, priority, task_title_format):
         "received message:\t"
         f"[Subject: {message['subject']}]\t"
         f"[Message-ID: {message['message-id']}]\t"
+        f"[References: {message['references']}]\t"
         f"[To: {message['to']}]\t"
         f"[From: {message['from']}]"
     )
@@ -70,7 +101,8 @@ def insert_message(task_list, message, priority, task_title_format):
     message_from = message["from"]
     text = message_text(message)
 
-    related_messages = message.get("references", "").split()
+    related_messages, answer_thread = \
+        parse_references(task_list, message.get("references", ""))
 
     # find the most relevant task to add a comment on.
     # among tasks in the selected task list, find the task having the
@@ -85,6 +117,11 @@ def insert_message(task_list, message, priority, task_title_format):
         .first()
     )
 
+    # if no related comment is found but a thread message-id
+    # (generated by django-todo) could be found, use it
+    if best_task is None and answer_thread is not None:
+        best_task = answer_thread
+
     with transaction.atomic():
         if best_task is None:
             best_task = Task.objects.create(
@@ -92,14 +129,14 @@ def insert_message(task_list, message, priority, task_title_format):
                 title=format_task_title(task_title_format, message),
                 task_list=task_list
             )
-        logger.info(f"using task: {repr(best_task)}")
+        logger.info("using task: %r", best_task)
 
         comment, comment_created = Comment.objects.get_or_create(
             task=best_task,
             email_message_id=message_id,
             defaults={"email_from": message_from, "body": text},
         )
-        logger.info(f"created comment: {repr(comment)}")
+        logger.info("created comment: %r", comment)
 
 
 def tracker_consumer(producer, group=None, task_list_slug=None,
@@ -107,7 +144,7 @@ def tracker_consumer(producer, group=None, task_list_slug=None,
     task_list = TaskList.objects.get(group__name=group, slug=task_list_slug)
     for message in producer:
         try:
-            insert_message(task_list, message, priority, title_format)
+            insert_message(task_list, message, priority, task_title_format)
         except Exception:
             # ignore exceptions during insertion, in order to avoid
             logger.exception("got exception while inserting message")

todo/mail/delivery.py

@@ -0,0 +1,20 @@
+import importlib
+
+def _declare_backend(backend_path):
+    backend_path = backend_path.split('.')
+    backend_module_name = '.'.join(backend_path[:-1])
+    class_name = backend_path[-1]
+
+    def backend(*args, headers={}, from_address=None, **kwargs):
+        def _backend():
+            backend_module = importlib.import_module(backend_module_name)
+            backend = getattr(backend_module, class_name)
+            return backend(*args, **kwargs)
+        _backend.from_address = from_address
+        _backend.headers = headers
+        return _backend
+    return backend
+
+
+smtp_backend = _declare_backend('django.core.mail.backends.smtp.EmailBackend')
+console_backend = _declare_backend('django.core.mail.backends.console.EmailBackend')

todo/mail/producers/imap.py

@@ -16,9 +16,9 @@ def imap_check(command_tuple):
 
 
 @contextmanager
-def imap_connect(host, port, user, password):
+def imap_connect(host, port, username, password):
     conn = imaplib.IMAP4_SSL(host=host, port=port)
-    conn.login(user, password)
+    conn.login(username, password)
     imap_check(conn.list())
     try:
         yield conn
@@ -49,7 +49,7 @@ def imap_producer(
     preserve=False,
     host=None,
     port=993,
-    user=None,
+    username=None,
     password=None,
     nap_duration=1,
     input_folder="INBOX",
@@ -60,7 +60,7 @@ def imap_producer(
     def process_batch():
         logger.debug("starting to process batch")
         # reconnect each time to avoid repeated failures due to a lost connection
-        with imap_connect(host, port, user, password) as conn:
+        with imap_connect(host, port, username, password) as conn:
             # select the requested folder
             imap_check(conn.select(input_folder, readonly=False))
 

todo/utils.py

@@ -1,5 +1,10 @@
+import email.utils
+import functools
+import time
 from django.conf import settings
 from django.contrib.sites.models import Site
+from django.core import mail
+from django.core.exceptions import PermissionDenied
 from django.core.mail import send_mail
 from django.template.loader import render_to_string
 from todo.models import Comment, Task
@@ -18,45 +23,136 @@ def staff_check(user):
         return True
 
 
+def todo_get_backend(task):
+    '''returns a mail backend for some task'''
+    mail_backends = getattr(settings, "TODO_MAIL_BACKENDS", None)
+    if mail_backends is None:
+        return None
+
+    task_backend = mail_backends[task.task_list.slug]
+    if task_backend is None:
+        return None
+
+    return task_backend
+
+
+def todo_get_mailer(user, task):
+    """a mailer is a (from_address, backend) pair"""
+    task_backend = todo_get_backend(task)
+    if task_backend is None:
+        return (None, mail.get_connection)
+
+    from_address = getattr(task_backend, "from_address", None)
+    if from_address is None:
+        # worst fallback ever
+        from_address = user.email
+
+    from_address = email.utils.formataddr((user.username, from_address))
+    return (from_address, task_backend)
+
+
+def todo_send_mail(user, task, subject, body, recip_list):
+    '''Send an email attached to task, triggered by user'''
+    references = Comment.objects.filter(task=task).only('email_message_id')
+    references = (ref.email_message_id for ref in references)
+    references = ' '.join(filter(bool, references))
+
+    from_address, backend = todo_get_mailer(user, task)
+    message_hash = hash((
+        subject,
+        body,
+        from_address,
+        frozenset(recip_list),
+        references,
+    ))
+
+    message_id = (
+        # the task_id enables attaching back notification answers
+        "<notif-{task_id}."
+        # the message hash / epoch pair enables deduplication
+        "{message_hash:x}."
+        "{epoch}@django-todo>"
+    ).format(
+        task_id=task.pk,
+        # avoid the -hexstring case (hashes can be negative)
+        message_hash=abs(message_hash),
+        epoch=int(time.time())
+    )
+
+    import sys
+    print("sending mail", file=sys.stderr)
+    # the thread message id is used as a common denominator between all
+    # notifications for some task. This message doesn't actually exist,
+    # it's just there to make threading possible
+    thread_message_id = "<thread-{}@django-todo>".format(task.pk)
+    references = '{} {}'.format(references, thread_message_id)
+
+    with backend() as connection:
+        message = mail.EmailMessage(
+            subject,
+            body,
+            from_address,
+            recip_list,
+            [], # Bcc
+            headers={
+                **getattr(backend, 'headers', {}),
+                'Message-ID': message_id,
+                'References': references,
+                'In-reply-to': thread_message_id,
+            },
+            connection=connection,
+        )
+        # import pdb; pdb.set_trace();
+        message.send()
+
+
 def send_notify_mail(new_task):
-    # Send email to assignee if task is assigned to someone other than submittor.
+    '''
-    # Unassigned tasks should not try to notify.
+    Send email to assignee if task is assigned to someone other than submittor.
+    Unassigned tasks should not try to notify.
+    '''
 
-    if not new_task.assigned_to == new_task.created_by:
+    if new_task.assigned_to == new_task.created_by:
-        current_site = Site.objects.get_current()
+        return
-        email_subject = render_to_string("todo/email/assigned_subject.txt", {"task": new_task})
-        email_body = render_to_string(
-            "todo/email/assigned_body.txt", {"task": new_task, "site": current_site}
-        )
 
-        send_mail(
+    current_site = Site.objects.get_current()
-            email_subject,
+    subject = render_to_string("todo/email/assigned_subject.txt", {"task": new_task})
-            email_body,
+    body = render_to_string(
-            new_task.created_by.email,
+        "todo/email/assigned_body.txt", {"task": new_task, "site": current_site}
-            [new_task.assigned_to.email],
+    )
-            fail_silently=False,
+
-        )
+    recip_list = [new_task.assigned_to.email]
+    todo_send_mail(new_task.created_by, new_task, subject, body, recip_list)
 
 
 def send_email_to_thread_participants(task, msg_body, user, subject=None):
-    # Notify all previous commentors on a Task about a new comment.
+    '''Notify all previous commentors on a Task about a new comment.'''
 
     current_site = Site.objects.get_current()
-    email_subject = (
+    email_subject = subject
-        subject if subject else render_to_string("todo/email/assigned_subject.txt", {"task": task})
+    if not subject:
-    )
+        subject = render_to_string(
+            "todo/email/assigned_subject.txt",
+            {"task": task}
+        )
+
     email_body = render_to_string(
         "todo/email/newcomment_body.txt",
         {"task": task, "body": msg_body, "site": current_site, "user": user},
     )
 
-    # Get list of all thread participants - everyone who has commented, plus task creator.
+    # Get all thread participants
     commenters = Comment.objects.filter(task=task)
-    recip_list = [ca.author.email for ca in commenters]
+    recip_list = set(
-    recip_list.append(task.created_by.email)
+        ca.author.email
-    recip_list = list(set(recip_list))  # Eliminate duplicates
+        for ca in commenters
+        if ca.author is not None
+    )
+    for user_email in (task.created_by.email, task.assigned_to.email):
+        recip_list.add(user_email)
+    recip_list = list(m for m in recip_list if m)
 
-    send_mail(email_subject, email_body, task.created_by.email, recip_list, fail_silently=False)
+    todo_send_mail(user, task, email_subject, email_body, recip_list)
 
 
 def toggle_task_completed(task_id: int) -> bool: