From 21a20e3d025cc3b87a499f3809a49743b8740259 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Mar 2024 08:44:57 +0000 Subject: [PATCH 1/2] Added support for appending rows (#1) * Added append flag * Pass append to formdata * Don't add a suffix name to the table if appending * Prepared tests for append * Fixed the flag * Updated the readme * Enabled append test --- README.md | 2 +- datasette_upload_csvs/__init__.py | 13 +++++----- .../templates/upload_csv.html | 8 +++++++ tests/test_datasette_upload_csvs.py | 24 +++++++++++++------ 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 978fc8e..8dda7d8 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Datasette plugin for uploading CSV files and converting them to database tables ## Usage -The plugin adds an interface at `/-/upload-csvs` for uploading a CSV file and using it to create a new database table. +The plugin adds an interface at `/-/upload-csvs` for uploading a CSV file and using it to create a new database table. An interface provide an option to append rows to an existent table in the database. By default only [the root actor](https://datasette.readthedocs.io/en/stable/authentication.html#using-the-root-actor) can access the page - so you'll need to run Datasette with the `--root` option and click on the link shown in the terminal to sign in and access the page. diff --git a/datasette_upload_csvs/__init__.py b/datasette_upload_csvs/__init__.py index 5d85ec0..fa60eec 100644 --- a/datasette_upload_csvs/__init__.py +++ b/datasette_upload_csvs/__init__.py @@ -120,12 +120,13 @@ async def upload_csvs(scope, receive, datasette, request): if table_name.endswith(".csv"): table_name = table_name[:-4] - # If the table already exists, add a suffix - suffix = 2 - base_table_name = table_name - while await db.table_exists(table_name): - table_name = "{}_{}".format(base_table_name, suffix) - suffix += 1 + if not formdata.get('append') == 'true': + # If the table already exists, add a suffix + suffix = 2 + base_table_name = table_name + while await db.table_exists(table_name): + table_name = "{}_{}".format(base_table_name, suffix) + suffix += 1 total_size = get_temporary_file_size(csv.file) task_id = str(uuid.uuid4()) diff --git a/datasette_upload_csvs/templates/upload_csv.html b/datasette_upload_csvs/templates/upload_csv.html index 661abc0..a9dd0f2 100644 --- a/datasette_upload_csvs/templates/upload_csv.html +++ b/datasette_upload_csvs/templates/upload_csv.html @@ -71,6 +71,10 @@

Upload CSV

   

+

+     + +

@@ -83,6 +87,7 @@

Upload CSV

var progressLabel = document.getElementById("progress-label"); var label = dropArea.getElementsByTagName("label")[0]; var tableName = document.getElementById("id_table_name"); +var tableAppend = document.getElementById("id_table_append"); var databaseName = document.getElementById("id_database"); // State that holds the most-recent uploaded File, from a FileList @@ -211,6 +216,9 @@

Upload CSV

formData.append("csrftoken", "{{ csrftoken() }}"); formData.append("csv", currentFile); formData.append("table", tableName.value); + if (tableAppend.checked) { + formData.append("append", tableAppend.checked); + } if (databaseName) { formData.append("database", databaseName.value); } diff --git a/tests/test_datasette_upload_csvs.py b/tests/test_datasette_upload_csvs.py index 7f1aad0..5494a6f 100644 --- a/tests/test_datasette_upload_csvs.py +++ b/tests/test_datasette_upload_csvs.py @@ -61,6 +61,7 @@ async def test_menu(tmpdir, auth, has_database): SIMPLE = b"name,age\nCleo,5\nPancakes,4" +SIMPLE_EXISTED = [{"name": "Sam", "age": 6}] SIMPLE_EXPECTED = [{"name": "Cleo", "age": 5}, {"name": "Pancakes", "age": 4}] NOT_UTF8 = ( b"IncidentNumber,DateTimeOfCall,CalYear,FinYear,TypeOfIncident,PumpCount,PumpHoursTotal,HourlyNotionalCost(\xa3),IncidentNotionalCost(\xa3)\r\n" @@ -98,25 +99,33 @@ async def test_menu(tmpdir, auth, has_database): @pytest.mark.asyncio @pytest.mark.parametrize( - "filename,content,expected_table,expected_rows", + "filename,content,expected_table,expected_rows,append", ( - ("dogs.csv", SIMPLE, "dogs", SIMPLE_EXPECTED), + ("dogs.csv", SIMPLE, "dogs", SIMPLE_EXPECTED, False), ( "weird ~ filename here.csv.csv", SIMPLE, "weird ~ filename here.csv", SIMPLE_EXPECTED, + False ), - ("not-utf8.csv", NOT_UTF8, "not-utf8", NOT_UTF8_EXPECTED), - ("latin1-after-x.csv", "LATIN1_AFTER_FIRST_2KB", "latin1-after-x", ANY), + ("not-utf8.csv", NOT_UTF8, "not-utf8", NOT_UTF8_EXPECTED, False), + ("latin1-after-x.csv", "LATIN1_AFTER_FIRST_2KB", "latin1-after-x", ANY, False), # This table already exists - ("already_exists.csv", SIMPLE, "already_exists_2", SIMPLE_EXPECTED), + ("already_exists.csv", SIMPLE, "already_exists_2", SIMPLE_EXPECTED, False), + ( + "already_exists.csv", + SIMPLE, + "already_exists", + SIMPLE_EXISTED + SIMPLE_EXPECTED, + True + ), ), ) @pytest.mark.parametrize("use_xhr", (True, False)) @pytest.mark.parametrize("database", ("data", "data2")) async def test_upload( - tmpdir, filename, content, expected_table, expected_rows, use_xhr, database + tmpdir, filename, content, expected_table, expected_rows, append, use_xhr, database ): expected_url = "/{}/{}".format(database, tilde_encode(expected_table)) path = str(tmpdir / "data.db") @@ -127,7 +136,7 @@ async def test_upload( dbs_by_name[pathlib.Path(p).stem] = db db.vacuum() db.enable_wal() - db["already_exists"].insert({"id": 1}) + db["already_exists"].insert_all(SIMPLE_EXISTED) binary_content = content # Trick to avoid a 12MB string being part of the pytest rendered test name: if content == "LATIN1_AFTER_FIRST_2KB": @@ -164,6 +173,7 @@ async def test_upload( "csrftoken": csrftoken, "xhr": "1" if use_xhr else "", "database": database, + "append": append, }, files=files, ) From b1a9d831a346855a66ea55a967f7502e5e8f3f50 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Mar 2024 09:01:49 +0000 Subject: [PATCH 2/2] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8dda7d8..38c72aa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Datasette plugin for uploading CSV files and converting them to database tables ## Usage -The plugin adds an interface at `/-/upload-csvs` for uploading a CSV file and using it to create a new database table. An interface provide an option to append rows to an existent table in the database. +The plugin adds an interface at `/-/upload-csvs` for uploading a CSV file and using it to create a new database table. The interface provides an option to append rows to an existent table in the database. By default only [the root actor](https://datasette.readthedocs.io/en/stable/authentication.html#using-the-root-actor) can access the page - so you'll need to run Datasette with the `--root` option and click on the link shown in the terminal to sign in and access the page.