diff --git a/todo/data/import_example.csv b/todo/data/import_example.csv index bc33c15..8a312dd 100644 --- a/todo/data/import_example.csv +++ b/todo/data/import_example.csv @@ -1,6 +1,6 @@ Title,Group,Task List,Created Date,Due Date,Completed,Created By,Assigned To,Note,Priority -Make dinner,Scuba Divers,Example List,2012-03-14,,No,shacker,shacker,This is as good as it gets,3 +Make dinner,Scuba Divers,Groovy,2012-03-12,2012-03-14,No,shacker,shacker,This is as good as it gets,3 Bake bread,Scuba Divers,Example List,2012-03-14,2012-03-14,,nonexistentusername,,, -Eat food,Coyotes,Example List,,2015-06-24,Yes,user3,user2,Every generation throws a hero up the pop charts,77 +Eat food,Scuba Divers,Groovy,,2015-06-24,Yes,user1,user1,Every generation throws a hero up the pop charts,77 Be glad,Scuba Divers,Example List,2019-03-07,,,user3,user2,,1 Dog food,Scuba Divers,Example List,2019-03-07,,,,user2,,1 \ No newline at end of file diff --git a/todo/operations/csv_importer.py b/todo/operations/csv_importer.py index 78916b5..7e1b7fc 100644 --- a/todo/operations/csv_importer.py +++ b/todo/operations/csv_importer.py @@ -2,6 +2,7 @@ import csv import logging import sys from pathlib import Path +import datetime from django.contrib.auth import get_user_model from django.contrib.auth.models import Group @@ -21,7 +22,8 @@ class CSVImporter: make sense to create new groups from here. In other words, the ingested CSV must accurately represent the current database. Non-conforming rows are skipped and logged. Unlike manual task creation, we won't assume that the person running this ingestion is the task creator - the creator must be specified, and a blank cell is an error. We also - do not create new lists - they must already exist. + do not create new lists - they must already exist (because if we did create new lists we'd also have to add the user to it, + etc.) Supplies a detailed log of what was and was not imported at the end.""" @@ -49,9 +51,13 @@ class CSVImporter: ic(newrow) print("\n") - # Report - for msg in self.errors: - print(msg) + # Report. Stored errors has the form: + # self.errors = [{3: ["Incorrect foo", "Non-existent bar"]}, {7: [...]}] + for error_dict in self.errors: + for k, error_list in error_dict.items(): + print(f"Skipped row {k}:") + for msg in error_list: + print(f"\t{msg}") print(f"\nProcessed {self.line_count} rows") print(f"Inserted xxx rows") @@ -60,19 +66,20 @@ class CSVImporter: """Perform data integrity checks and set default values. Returns a valid object for insertion, or False. Errors are stored for later display.""" + row_errors = [] + # Task creator must exist if not row.get("Created By"): - msg = f"Skipped row {self.line_count}: Missing required task creator." - self.errors.append(msg) - return False + msg = f"Missing required task creator." + row_errors.append(msg) created_by = get_user_model().objects.filter(username=row.get("Created By")) if created_by.exists(): creator = created_by.first() else: - msg = f"Skipped row {self.line_count}: Invalid task creator {row.get('Created By')}" - self.errors.append(msg) - return False + creator = None + msg = f"Invalid task creator {row.get('Created By')}" + row_errors.append(msg) # If specified, Assignee must exist if row.get("Assigned To"): @@ -80,9 +87,8 @@ class CSVImporter: if assigned.exists(): assignee = assigned.first() else: - msg = f"Skipped row {self.line_count}: Missing or invalid task assignee {row.get('Assigned To')}" - self.errors.append(msg) - return False + msg = f"Missing or invalid task assignee {row.get('Assigned To')}" + row_errors.append(msg) else: assignee = None # Perfectly valid @@ -90,21 +96,18 @@ class CSVImporter: try: target_group = Group.objects.get(name=row.get("Group")) except Group.DoesNotExist: - msg = f"Skipped row {self.line_count}: Could not find group {row.get('Group')}." - self.errors.append(msg) - return False + msg = f"Could not find group {row.get('Group')}." + row_errors.append(msg) # Task creator must be in the target group - if target_group not in creator.groups.all(): - msg = f"Skipped row {self.line_count}: {creator} is not in group {target_group}" - self.errors.append(msg) - return False + if creator and target_group not in creator.groups.all(): + msg = f"{creator} is not in group {target_group}" + row_errors.append(msg) # Assignee must be in the target group if assignee and target_group not in assignee.groups.all(): - msg = f"Skipped row {self.line_count}: {assignee} is not in group {target_group}" - self.errors.append(msg) - return False + msg = f"{assignee} is not in group {target_group}" + row_errors.append(msg) # Group membership checks have passed row["Created By"] = creator @@ -112,7 +115,40 @@ class CSVImporter: if assignee: row["Assigned To"] = assignee + # Task list must exist in the target group + try: + tasklist = TaskList.objects.get(name=row.get("Task List"), group=target_group) + row["Task List"] = tasklist + except TaskList.DoesNotExist: + msg = ( + f"Task list {row.get('Task List')} in group {target_group} does not exist" + ) + row_errors.append(msg) + + # Validate Due Date + dd = row.get("Due Date") + if dd: + try: + row["Due Date"] = datetime.datetime.strptime(dd, '%Y-%m-%d') + except ValueError: + msg = f"Could not convert Due Date {dd} to python date" + row_errors.append(msg) + + # Validate Created Date + cd = row.get("Created Date") + if cd: + try: + row["Created Date"] = datetime.datetime.strptime(cd, '%Y-%m-%d') + except ValueError: + msg = f"Could not convert Created Date {cd} to python date" + row_errors.append(msg) + # Set Completed default row["Completed"] = True if row.get("Completed") == "Yes" else False + if row_errors: + self.errors.append({self.line_count: row_errors}) + return False + + # No errors: return row