From 57b99d4d4388bb9e0bb8d72f1fce3fa0bf03c232 Mon Sep 17 00:00:00 2001 From: Scot Hacker Date: Sat, 9 Mar 2019 23:22:49 -0800 Subject: [PATCH] Print msgs from the mgmt command, not the operations module --- README.md | 12 +++++---- todo/management/commands/import_csv.py | 19 +++++++++++++- todo/operations/csv_importer.py | 36 ++++++++++++-------------- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index dcc5631..ad14b8e 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ The previous `tox` system was removed with the v2 release, since we no longer ai # Importing Tasks via CSV -django-todo has the ability to batch-import ("upsert") tasks from a specifically formatted CSV spreadsheet. This ability is provided through both a management command or the web interface. +django-todo has the ability to batch-import ("upsert") tasks from a specifically formatted CSV spreadsheet. This ability is provided through both a management command and a web interface. ## Management Command @@ -176,13 +176,13 @@ django-todo has the ability to batch-import ("upsert") tasks from a specifically ## Web Importer -Link from your navigation to `{url todo:import_csv}` +Link from your navigation to `{url "todo:import_csv"}` ## Import Logic -Because data entered via CSV is not going through the same view permissions enforced in the rest of django-todo, and to simplify the logic of when to update vs create a record, etc., the importer will *not* create new users, groups, or task lists. All users, groups, and task lists referenced i your CSV must already exist, and memberships must be correct (if you have a row specifying a user in an incorrect group, the importer will skip that row). +Because data entered via CSV is not going through the same view permissions enforced in the rest of django-todo, and to simplify data dependency logic, and to pre-empt disagreements between django-todo users, the importer will *not* create new users, groups, or task lists. All users, groups, and task lists referenced in your CSV must already exist, and group memberships must be correct (if you have a row specifying a user in an incorrect group, the importer will skip that row). -Any validation error (e.g. unparse-able dates) results in that row being skipped. +Any validation error (e.g. unparse-able dates) will result in that row being skipped. A report of rows upserted and rows skipped (with line numbers and reasons) is provided at the end of the run. @@ -190,12 +190,14 @@ A report of rows upserted and rows skipped (with line numbers and reasons) is pr Copy `todo/data/import_example.csv` to another location on your system and edit in a spreadsheet or directly. +**Do not edit the header row!** + The "Created By", "Task List" and "Group" columns are required -- all others are optional and should work pretty much exactly like manual task entry via the web UI. Note: Internally, Tasks are keyed to TaskLists, not to Groups (TaskLists are in Gruops). However, we request the Group in the CSV because it's possible to have multiple TaskLists with the same name in different groups; i.e. we need it for namespacing and permissions. -## Upsert Logic: +## Upsert Logic For each valid row, we need to decide whether to create a new task or update an existing one. django-todo matches on the unique combination of Task List, Task Title, and Created By. If we find a task that matches those three, we *update* the rest of the columns. In other words, if you import a CSV once, then edit the Assigned To for a task and import it again, the original task will be updated with a new assignee (and same for the other columns). diff --git a/todo/management/commands/import_csv.py b/todo/management/commands/import_csv.py index f170350..e6c593d 100644 --- a/todo/management/commands/import_csv.py +++ b/todo/management/commands/import_csv.py @@ -28,4 +28,21 @@ class Command(BaseCommand): filepath = str(options.get("file")) importer = CSVImporter() - importer.upsert(filepath) + results = importer.upsert(filepath) + + # Report successes, failures and summaries + print() + for upsert_msg in results.get("upserts"): + print(upsert_msg) + + # Stored errors has the form: + # self.errors = [{3: ["Incorrect foo", "Non-existent bar"]}, {7: [...]}] + for error_dict in results.get("errors"): + for k, error_list in error_dict.items(): + print(f"\nSkipped CSV row {k}:") + for msg in error_list: + print(f"- {msg}") + + print() + for summary_msg in results.get("summaries"): + print(summary_msg) diff --git a/todo/operations/csv_importer.py b/todo/operations/csv_importer.py index 13034ee..ca2493c 100644 --- a/todo/operations/csv_importer.py +++ b/todo/operations/csv_importer.py @@ -1,12 +1,11 @@ import csv +import datetime import logging import sys from pathlib import Path -import datetime from django.contrib.auth import get_user_model from django.contrib.auth.models import Group -from icecream import ic from todo.models import Task, TaskList @@ -19,7 +18,9 @@ class CSVImporter: """ def __init__(self): - self.errors = [] + self.error_msgs = [] + self.upsert_msgs = [] + self.summary_msgs = [] self.line_count = 0 self.upsert_count = 0 @@ -34,7 +35,6 @@ class CSVImporter: # Header row is: # Title, Group, Task List, Created Date, Due Date, Completed, Created By, Assigned To, Note, Priority - print("\n") csv_reader = csv.DictReader(csv_file) for row in csv_reader: self.line_count += 1 @@ -57,22 +57,21 @@ class CSVImporter: }, ) self.upsert_count += 1 - print( - f"Upserted task {obj.id}: \"{obj.title}\"" - f"in list \"{obj.task_list}\" (group \"{obj.task_list.group}\")" + msg = ( + f'Upserted task {obj.id}: "{obj.title}"' + f' in list "{obj.task_list}" (group "{obj.task_list.group}")' ) + self.upsert_msgs.append(msg) - # Report. Stored errors has the form: - # self.errors = [{3: ["Incorrect foo", "Non-existent bar"]}, {7: [...]}] - print("\n") - for error_dict in self.errors: - for k, error_list in error_dict.items(): - print(f"Skipped CSV row {k}:") - for msg in error_list: - print(f"\t{msg}") + self.summary_msgs.append(f"\nProcessed {self.line_count} CSV rows") + self.summary_msgs.append(f"Upserted {self.upsert_count} rows") - print(f"\nProcessed {self.line_count} CSV rows") - print(f"Upserted {self.upsert_count} rows") + _res = { + "errors": self.error_msgs, + "upserts": self.upsert_msgs, + "summaries": self.summary_msgs, + } + return _res def validate_row(self, row): """Perform data integrity checks and set default values. Returns a valid object for insertion, or False. @@ -147,7 +146,6 @@ class CSVImporter: else: row["Created Date"] = None # Override default empty string '' value - # ####################### # Validate Created Date cd = row.get("Created Date") @@ -172,7 +170,7 @@ class CSVImporter: # ####################### if row_errors: - self.errors.append({self.line_count: row_errors}) + self.error_msgs.append({self.line_count: row_errors}) return False # No errors: