Add support for appending rows

README.md

@@ -13,7 +13,7 @@ Datasette plugin for uploading CSV files and converting them to database tables
 
 ## Usage
 
-The plugin adds an interface at `/-/upload-csvs` for uploading a CSV file and using it to create a new database table.
+The plugin adds an interface at `/-/upload-csvs` for uploading a CSV file and using it to create a new database table. The interface provides an option to append rows to an existent table in the database.
 
 By default only [the root actor](https://datasette.readthedocs.io/en/stable/authentication.html#using-the-root-actor) can access the page - so you'll need to run Datasette with the `--root` option and click on the link shown in the terminal to sign in and access the page.
 

datasette_upload_csvs/__init__.py

@@ -120,12 +120,13 @@ async def upload_csvs(scope, receive, datasette, request):
         if table_name.endswith(".csv"):
             table_name = table_name[:-4]
 
-    # If the table already exists, add a suffix
-    suffix = 2
-    base_table_name = table_name
-    while await db.table_exists(table_name):
-        table_name = "{}_{}".format(base_table_name, suffix)
-        suffix += 1
+    if not formdata.get('append') == 'true':
+        # If the table already exists, add a suffix
+        suffix = 2
+        base_table_name = table_name
+        while await db.table_exists(table_name):
+            table_name = "{}_{}".format(base_table_name, suffix)
+            suffix += 1
 
     total_size = get_temporary_file_size(csv.file)
     task_id = str(uuid.uuid4())

datasette_upload_csvs/templates/upload_csv.html

@@ -71,6 +71,10 @@ <h1>Upload CSV</h1>
     <label for="id_table_name">Table name</label>&nbsp; &nbsp;
     <input required id="id_table_name" type="text" name="table_name">
 </p>
+<p style="margin-top: 1em">
+    <label for="id_table_append">Append rows <br/> (if the table exists)</label>&nbsp; &nbsp;
+    <input id="id_table_append" type="checkbox" name="table_append">
+</p>
 <p><input type="submit" value="Upload file" class="button"></p>
 </form>
 </div>
@@ -83,6 +87,7 @@ <h1>Upload CSV</h1>
 var progressLabel = document.getElementById("progress-label");
 var label = dropArea.getElementsByTagName("label")[0];
 var tableName = document.getElementById("id_table_name");
+var tableAppend = document.getElementById("id_table_append");
 var databaseName = document.getElementById("id_database");
 
 // State that holds the most-recent uploaded File, from a FileList
@@ -211,6 +216,9 @@ <h1>Upload CSV</h1>
   formData.append("csrftoken", "{{ csrftoken() }}");
   formData.append("csv", currentFile);
   formData.append("table", tableName.value);
+  if (tableAppend.checked) {
+    formData.append("append", tableAppend.checked);
+  }
   if (databaseName) {
     formData.append("database", databaseName.value);
   }

tests/test_datasette_upload_csvs.py

@@ -61,6 +61,7 @@ async def test_menu(tmpdir, auth, has_database):
 
 
 SIMPLE = b"name,age\nCleo,5\nPancakes,4"
+SIMPLE_EXISTED = [{"name": "Sam", "age": 6}]
 SIMPLE_EXPECTED = [{"name": "Cleo", "age": 5}, {"name": "Pancakes", "age": 4}]
 NOT_UTF8 = (
     b"IncidentNumber,DateTimeOfCall,CalYear,FinYear,TypeOfIncident,PumpCount,PumpHoursTotal,HourlyNotionalCost(\xa3),IncidentNotionalCost(\xa3)\r\n"
@@ -98,25 +99,33 @@ async def test_menu(tmpdir, auth, has_database):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    "filename,content,expected_table,expected_rows",
+    "filename,content,expected_table,expected_rows,append",
     (
-        ("dogs.csv", SIMPLE, "dogs", SIMPLE_EXPECTED),
+        ("dogs.csv", SIMPLE, "dogs", SIMPLE_EXPECTED, False),
         (
             "weird ~ filename here.csv.csv",
             SIMPLE,
             "weird ~ filename here.csv",
             SIMPLE_EXPECTED,
+            False
         ),
-        ("not-utf8.csv", NOT_UTF8, "not-utf8", NOT_UTF8_EXPECTED),
-        ("latin1-after-x.csv", "LATIN1_AFTER_FIRST_2KB", "latin1-after-x", ANY),
+        ("not-utf8.csv", NOT_UTF8, "not-utf8", NOT_UTF8_EXPECTED, False),
+        ("latin1-after-x.csv", "LATIN1_AFTER_FIRST_2KB", "latin1-after-x", ANY, False),
         # This table already exists
-        ("already_exists.csv", SIMPLE, "already_exists_2", SIMPLE_EXPECTED),
+        ("already_exists.csv", SIMPLE, "already_exists_2", SIMPLE_EXPECTED, False),
+        (
+            "already_exists.csv",
+            SIMPLE,
+            "already_exists",
+            SIMPLE_EXISTED + SIMPLE_EXPECTED,
+            True
+        ),
     ),
 )
 @pytest.mark.parametrize("use_xhr", (True, False))
 @pytest.mark.parametrize("database", ("data", "data2"))
 async def test_upload(
-    tmpdir, filename, content, expected_table, expected_rows, use_xhr, database
+    tmpdir, filename, content, expected_table, expected_rows, append, use_xhr, database
 ):
     expected_url = "/{}/{}".format(database, tilde_encode(expected_table))
     path = str(tmpdir / "data.db")
@@ -127,7 +136,7 @@ async def test_upload(
         dbs_by_name[pathlib.Path(p).stem] = db
         db.vacuum()
         db.enable_wal()
-        db["already_exists"].insert({"id": 1})
+        db["already_exists"].insert_all(SIMPLE_EXISTED)
         binary_content = content
         # Trick to avoid a 12MB string being part of the pytest rendered test name:
         if content == "LATIN1_AFTER_FIRST_2KB":
@@ -164,6 +173,7 @@ async def test_upload(
                 "csrftoken": csrftoken,
                 "xhr": "1" if use_xhr else "",
                 "database": database,
+                "append": append,
             },
             files=files,
         )