Implement mail tracker system

* Implement mail tracking

Signed-off-by: Victor "multun" Collod <victor.collod@prologin.org>

* Implement task merging

* Add a mail tracker title format pattern

* Autocomplete task names

* Fix comment display

* Track notification answers

* Add a socket timeout for the mail worker

A mail worker is a long running application. And sometimes, the IMAP server
just hangs for hours for no apparent reason. imaplib doesn't enable setting
a timeout, and setting it globally seems fine.

* Only validate the merge form when submitted

* Redirect to the new form when merging

* Prettier task edit UI

* Make task merging optional

* Test mail tracking

* Update documentation for mail tracking

* Update dependencies

* Add the TODO_COMMENT_CLASSES setting

* Fix dependencies install order

* Remove debug leftovers, improve documentation

* Fail on missing from_address

todo/mail/consumers/__init__.py

@@ -0,0 +1,9 @@
+def tracker_consumer(**kwargs):
+    def tracker_factory(producer):
+        # the import needs to be delayed until call to enable
+        # using the wrapper in the django settings
+        from .tracker import tracker_consumer
+
+        return tracker_consumer(producer, **kwargs)
+
+    return tracker_factory

todo/mail/consumers/tracker.py

@@ -0,0 +1,150 @@
+import re
+import logging
+
+from email.charset import Charset as EMailCharset
+from django.db import transaction
+from django.db.models import Count
+from html2text import html2text
+from todo.models import Comment, Task, TaskList
+
+logger = logging.getLogger(__name__)
+
+
+def part_decode(message):
+    charset = ("ascii", "ignore")
+    email_charset = message.get_content_charset()
+    if email_charset:
+        charset = (EMailCharset(email_charset).input_charset,)
+
+    body = message.get_payload(decode=True)
+    return body.decode(*charset)
+
+
+def message_find_mime(message, mime_type):
+    for submessage in message.walk():
+        if submessage.get_content_type() == mime_type:
+            return submessage
+    return None
+
+
+def message_text(message):
+    text_part = message_find_mime(message, "text/plain")
+    if text_part is not None:
+        return part_decode(text_part)
+
+    html_part = message_find_mime(message, "text/html")
+    if html_part is not None:
+        return html2text(part_decode(html_part))
+
+    # TODO: find something smart to do when no text if found
+    return ""
+
+
+def format_task_title(format_string, message):
+    return format_string.format(
+        subject=message["subject"],
+        author=message["from"],
+    )
+
+
+DJANGO_TODO_THREAD = re.compile(r'<thread-(\d+)@django-todo>')
+
+
+def parse_references(task_list, references):
+    related_messages = []
+    answer_thread = None
+    for related_message in references.split():
+        logger.info("checking reference: %r", related_message)
+        match = re.match(DJANGO_TODO_THREAD, related_message)
+        if match is None:
+            related_messages.append(related_message)
+            continue
+
+        thread_id = int(match.group(1))
+        new_answer_thread = Task.objects.filter(
+            task_list=task_list,
+            pk=thread_id
+        ).first()
+        if new_answer_thread is not None:
+            answer_thread = new_answer_thread
+
+    if answer_thread is None:
+        logger.info("no answer thread found in references")
+    else:
+        logger.info("found an answer thread: %d", answer_thread)
+    return related_messages, answer_thread
+
+
+def insert_message(task_list, message, priority, task_title_format):
+    if "message-id" not in message:
+        logger.warning("missing message id, ignoring message")
+        return
+
+    if "from" not in message:
+        logger.warning('missing "From" header, ignoring message')
+        return
+
+    if "subject" not in message:
+        logger.warning('missing "Subject" header, ignoring message')
+        return
+
+    logger.info(
+        "received message:\t"
+        f"[Subject: {message['subject']}]\t"
+        f"[Message-ID: {message['message-id']}]\t"
+        f"[References: {message['references']}]\t"
+        f"[To: {message['to']}]\t"
+        f"[From: {message['from']}]"
+    )
+
+    message_id = message["message-id"]
+    message_from = message["from"]
+    text = message_text(message)
+
+    related_messages, answer_thread = \
+        parse_references(task_list, message.get("references", ""))
+
+    # find the most relevant task to add a comment on.
+    # among tasks in the selected task list, find the task having the
+    # most email comments the current message references
+    best_task = (
+        Task.objects.filter(
+            task_list=task_list, comment__email_message_id__in=related_messages
+        )
+        .annotate(num_comments=Count("comment"))
+        .order_by("-num_comments")
+        .only("id")
+        .first()
+    )
+
+    # if no related comment is found but a thread message-id
+    # (generated by django-todo) could be found, use it
+    if best_task is None and answer_thread is not None:
+        best_task = answer_thread
+
+    with transaction.atomic():
+        if best_task is None:
+            best_task = Task.objects.create(
+                priority=priority,
+                title=format_task_title(task_title_format, message),
+                task_list=task_list
+            )
+        logger.info("using task: %r", best_task)
+
+        comment, comment_created = Comment.objects.get_or_create(
+            task=best_task,
+            email_message_id=message_id,
+            defaults={"email_from": message_from, "body": text},
+        )
+        logger.info("created comment: %r", comment)
+
+
+def tracker_consumer(producer, group=None, task_list_slug=None,
+                     priority=1, task_title_format="[MAIL] {subject}"):
+    task_list = TaskList.objects.get(group__name=group, slug=task_list_slug)
+    for message in producer:
+        try:
+            insert_message(task_list, message, priority, task_title_format)
+        except Exception:
+            # ignore exceptions during insertion, in order to avoid
+            logger.exception("got exception while inserting message")

todo/mail/delivery.py

@@ -0,0 +1,25 @@
+import importlib
+
+def _declare_backend(backend_path):
+    backend_path = backend_path.split('.')
+    backend_module_name = '.'.join(backend_path[:-1])
+    class_name = backend_path[-1]
+
+    def backend(*args, headers={}, from_address=None, **kwargs):
+        def _backend():
+            backend_module = importlib.import_module(backend_module_name)
+            backend = getattr(backend_module, class_name)
+            return backend(*args, **kwargs)
+
+        if from_address is None:
+            raise ValueError("missing from_address")
+
+        _backend.from_address = from_address
+        _backend.headers = headers
+        return _backend
+    return backend
+
+
+smtp_backend = _declare_backend('django.core.mail.backends.smtp.EmailBackend')
+console_backend = _declare_backend('django.core.mail.backends.console.EmailBackend')
+locmem_backend = _declare_backend('django.core.mail.backends.locmem.EmailBackend')

todo/mail/producers/__init__.py

@@ -0,0 +1,9 @@
+def imap_producer(**kwargs):
+    def imap_producer_factory():
+        # the import needs to be delayed until call to enable
+        # using the wrapper in the django settings
+        from .imap import imap_producer
+
+        return imap_producer(**kwargs)
+
+    return imap_producer_factory

todo/mail/producers/imap.py

@@ -0,0 +1,100 @@
+import email
+import email.parser
+import imaplib
+import logging
+import time
+
+from email.policy import default
+from contextlib import contextmanager
+
+logger = logging.getLogger(__name__)
+
+
+def imap_check(command_tuple):
+    status, ids = command_tuple
+    assert status == "OK", ids
+
+
+@contextmanager
+def imap_connect(host, port, username, password):
+    conn = imaplib.IMAP4_SSL(host=host, port=port)
+    conn.login(username, password)
+    imap_check(conn.list())
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+def parse_message(message):
+    for response_part in message:
+        if not isinstance(response_part, tuple):
+            continue
+
+        message_metadata, message_content = response_part
+        email_parser = email.parser.BytesFeedParser(policy=default)
+        email_parser.feed(message_content)
+        return email_parser.close()
+
+
+def search_message(conn, *filters):
+    status, message_ids = conn.search(None, *filters)
+    for message_id in message_ids[0].split():
+        status, message = conn.fetch(message_id, "(RFC822)")
+        yield message_id, parse_message(message)
+
+
+def imap_producer(
+    process_all=False,
+    preserve=False,
+    host=None,
+    port=993,
+    username=None,
+    password=None,
+    nap_duration=1,
+    input_folder="INBOX",
+):
+    logger.debug("starting IMAP worker")
+    imap_filter = "(ALL)" if process_all else "(UNSEEN)"
+
+    def process_batch():
+        logger.debug("starting to process batch")
+        # reconnect each time to avoid repeated failures due to a lost connection
+        with imap_connect(host, port, username, password) as conn:
+            # select the requested folder
+            imap_check(conn.select(input_folder, readonly=False))
+
+            try:
+                for message_uid, message in search_message(conn, imap_filter):
+                    logger.info(f"received message {message_uid}")
+                    try:
+                        yield message
+                    except Exception:
+                        logger.exception(
+                            f"something went wrong while processing {message_uid}"
+                        )
+                        raise
+
+                    if not preserve:
+                        # tag the message for deletion
+                        conn.uid("STORE", message_uid, "+FLAGS", "(\\Deleted)")
+                else:
+                    logger.debug("did not receive any message")
+            finally:
+                if not preserve:
+                    # flush deleted messages
+                    conn.expunge()
+
+    while True:
+        try:
+            yield from process_batch()
+        except (GeneratorExit, KeyboardInterrupt):
+            # the generator was closed, due to the consumer
+            # breaking out of the loop, or an exception occuring
+            raise
+        except Exception:
+            logger.exception("mail fetching went wrong, retrying")
+
+        # sleep to avoid using too much resources
+        # TODO: get notified when a new message arrives
+        time.sleep(nap_duration)