Validate dates and TaskList; convert errors to list of dictionaries

todo/data/import_example.csv

@@ -1,6 +1,6 @@
 Title,Group,Task List,Created Date,Due Date,Completed,Created By,Assigned To,Note,Priority
-Make dinner,Scuba Divers,Example List,2012-03-14,,No,shacker,shacker,This is as good as it gets,3
+Make dinner,Scuba Divers,Groovy,2012-03-12,2012-03-14,No,shacker,shacker,This is as good as it gets,3
 Bake bread,Scuba Divers,Example List,2012-03-14,2012-03-14,,nonexistentusername,,,
-Eat food,Coyotes,Example List,,2015-06-24,Yes,user3,user2,Every generation throws a hero up the pop charts,77
+Eat food,Scuba Divers,Groovy,,2015-06-24,Yes,user1,user1,Every generation throws a hero up the pop charts,77
 Be glad,Scuba Divers,Example List,2019-03-07,,,user3,user2,,1
 Dog food,Scuba Divers,Example List,2019-03-07,,,,user2,,1

todo/operations/csv_importer.py

@@ -2,6 +2,7 @@ import csv
 import logging
 import sys
 from pathlib import Path
+import datetime
 
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Group
@@ -21,7 +22,8 @@ class CSVImporter:
     make sense to create new groups from here. In other words, the ingested CSV must accurately represent the current
     database. Non-conforming rows are skipped and logged. Unlike manual task creation, we won't assume that the person
     running this ingestion is the task creator - the creator must be specified, and a blank cell is an error. We also
-    do not create new lists - they must already exist.
+    do not create new lists - they must already exist (because if we did create new lists we'd also have to add the user to it,
+    etc.)
 
     Supplies a detailed log of what was and was not imported at the end."""
 
@@ -49,9 +51,13 @@ class CSVImporter:
                     ic(newrow)
                     print("\n")
 
-            # Report
+            # Report. Stored errors has the form:
-            for msg in self.errors:
+            # self.errors = [{3: ["Incorrect foo", "Non-existent bar"]}, {7: [...]}]
-                print(msg)
+            for error_dict in self.errors:
+                for k, error_list in error_dict.items():
+                    print(f"Skipped row {k}:")
+                    for msg in error_list:
+                        print(f"\t{msg}")
 
             print(f"\nProcessed {self.line_count} rows")
             print(f"Inserted xxx rows")
@@ -60,19 +66,20 @@ class CSVImporter:
         """Perform data integrity checks and set default values. Returns a valid object for insertion, or False.
         Errors are stored for later display."""
 
+        row_errors = []
+
         # Task creator must exist
         if not row.get("Created By"):
-            msg = f"Skipped row {self.line_count}: Missing required task creator."
+            msg = f"Missing required task creator."
-            self.errors.append(msg)
+            row_errors.append(msg)
-            return False
 
         created_by = get_user_model().objects.filter(username=row.get("Created By"))
         if created_by.exists():
             creator = created_by.first()
         else:
-            msg = f"Skipped row {self.line_count}: Invalid task creator {row.get('Created By')}"
+            creator = None
-            self.errors.append(msg)
+            msg = f"Invalid task creator {row.get('Created By')}"
-            return False
+            row_errors.append(msg)
 
         # If specified, Assignee must exist
         if row.get("Assigned To"):
@@ -80,9 +87,8 @@ class CSVImporter:
             if assigned.exists():
                 assignee = assigned.first()
             else:
-                msg = f"Skipped row {self.line_count}: Missing or invalid task assignee {row.get('Assigned To')}"
+                msg = f"Missing or invalid task assignee {row.get('Assigned To')}"
-                self.errors.append(msg)
+                row_errors.append(msg)
-                return False
         else:
             assignee = None  # Perfectly valid
 
@@ -90,21 +96,18 @@ class CSVImporter:
         try:
             target_group = Group.objects.get(name=row.get("Group"))
         except Group.DoesNotExist:
-            msg = f"Skipped row {self.line_count}: Could not find group {row.get('Group')}."
+            msg = f"Could not find group {row.get('Group')}."
-            self.errors.append(msg)
+            row_errors.append(msg)
-            return False
 
         # Task creator must be in the target group
-        if target_group not in creator.groups.all():
+        if creator and target_group not in creator.groups.all():
-            msg = f"Skipped row {self.line_count}: {creator} is not in group {target_group}"
+            msg = f"{creator} is not in group {target_group}"
-            self.errors.append(msg)
+            row_errors.append(msg)
-            return False
 
         # Assignee must be in the target group
         if assignee and target_group not in assignee.groups.all():
-            msg = f"Skipped row {self.line_count}: {assignee} is not in group {target_group}"
+            msg = f"{assignee} is not in group {target_group}"
-            self.errors.append(msg)
+            row_errors.append(msg)
-            return False
 
         # Group membership checks have passed
         row["Created By"] = creator
@@ -112,7 +115,40 @@ class CSVImporter:
         if assignee:
             row["Assigned To"] = assignee
 
+        # Task list must exist in the target group
+        try:
+            tasklist = TaskList.objects.get(name=row.get("Task List"), group=target_group)
+            row["Task List"] = tasklist
+        except TaskList.DoesNotExist:
+            msg = (
+                f"Task list {row.get('Task List')} in group {target_group} does not exist"
+            )
+            row_errors.append(msg)
+
+        # Validate Due Date
+        dd = row.get("Due Date")
+        if dd:
+            try:
+                row["Due Date"] = datetime.datetime.strptime(dd, '%Y-%m-%d')
+            except ValueError:
+                msg = f"Could not convert Due Date {dd} to python date"
+                row_errors.append(msg)
+
+        # Validate Created Date
+        cd = row.get("Created Date")
+        if cd:
+            try:
+                row["Created Date"] = datetime.datetime.strptime(cd, '%Y-%m-%d')
+            except ValueError:
+                msg = f"Could not convert Created Date {cd} to python date"
+                row_errors.append(msg)
+
         # Set Completed default
         row["Completed"] = True if row.get("Completed") == "Yes" else False
 
+        if row_errors:
+            self.errors.append({self.line_count: row_errors})
+            return False
+
+        # No errors:
         return row