From 80083572730ac98c696702ba15a7a80c03901302 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Mon, 22 Feb 2021 12:56:12 -0700 Subject: [PATCH 01/21] Add ability to import Gmail Takeout mbox --- README.md | 6 ++ google_takeout_to_sqlite/cli.py | 21 +++++ google_takeout_to_sqlite/utils.py | 122 ++++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 150 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3168098..9a7e710 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,12 @@ Your location history records latitude, longitude and timestame for where Google $ google-takeout-to-sqlite location-history takeout.db ~/Downloads/takeout-20190530.zip +## Email History + +You can import your emails from your Gmail mbox using this command: + + $ google-takeout-to-sqlite mbox takeout.db ~/Downloads/gmail.mbox + ## Browsing your data with Datasette Once you have imported Google data into a SQLite database file you can browse your data using [Datasette](https://github.com/simonw/datasette). Install Datasette like so: diff --git a/google_takeout_to_sqlite/cli.py b/google_takeout_to_sqlite/cli.py index a40cc4f..48572e6 100644 --- a/google_takeout_to_sqlite/cli.py +++ b/google_takeout_to_sqlite/cli.py @@ -47,3 +47,24 @@ def my_activity(db_path, zip_path): db = sqlite_utils.Database(db_path) zf = zipfile.ZipFile(zip_path) utils.save_location_history(db, zf) + + +@cli.command(name="mbox") +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument( + "mbox_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +def my_mbox(db_path, mbox_path): + """ + Import all emails from Gmail mbox to SQLite + + Usage: google-takeout-to-sqlite mbox mygmail.db /path/to/gmail.mbox + """ + db = sqlite_utils.Database(db_path) + utils.save_emails(db, mbox_path) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 428843c..0fa8010 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -1,6 +1,11 @@ import json import hashlib import datetime +import email +import mailbox +import traceback +from rich.progress import track +from email.utils import parsedate_tz, mktime_tz def save_my_activity(db, zf): @@ -53,3 +58,120 @@ def id_for_location_history(row): datetime.datetime.utcfromtimestamp(int(row["timestampMs"]) / 1000).isoformat(), first_six, ) + + +def get_mbox(mbox_file): + num_errors = 0 + print('Preparing to process emails...') + mbox = mailbox.mbox(mbox_file) + print('Processing {} emails'.format(len(mbox))) + + # These are all the Gmail email fields available + # ['X-GM-THRID', 'X-Gmail-Labels', 'Delivered-To', 'Received', 'Received', + # 'Return-Path', 'Received', 'Received-SPF', 'Authentication-Results', + # 'Received', 'Mailing-List', 'Precedence', 'List-Post', 'List-Help', + # 'List-Unsubscribe', 'List-Subscribe', 'Delivered-To', 'Received', + # 'Message-ID', 'Date', 'From', 'To', 'MIME-Version', 'Content-Type', + # 'Content-Transfer-Encoding', 'X-Nabble-From', 'X-pstn-neptune', + # 'X-pstn-levels', 'X-pstn-settings', 'X-pstn-addresses', 'Subject'] + + for email in track(mbox): + try: + message = {} + message['Message-Id'] = email['Message-Id'] + message['X-GM-THRID'] = email['X-GM-THRID'] + message['X-Gmail-Labels'] = email['X-Gmail-Labels'] + + # These following try/excepts are here because for some reason + # these items returned from the mbox module are sometimes strings + # and sometimes headers and sometimes None. + + try: + email['From'].decode('utf-8') + except AttributeError: + message['From'] = str(email['From']) + try: + email['To'].decode('utf-8') + except AttributeError: + message["To"] = str(email['To']) + + try: + email['Subject'].decode('utf-8') + except AttributeError: + message["Subject"] = str(email['Subject']) + + + message["date"] = get_message_date(email.get('Date'), + email.get_from()) + message["body"] = get_email_body(email) + + yield message + except (TypeError, ValueError, AttributeError, LookupError) as e: + # How does this project want to handle logging? For now we're just + # printing out variables + num_errors = num_errors + 1 + print('Errors: {}'.format(num_errors)) + print(traceback.format_exc()) + continue + + +def save_emails(db, mbox_file): + """ + Import Gmail mbox from google takeout + """ + db["mbox_emails"].upsert_all( + ( + { + "id": message['Message-Id'], + 'X-GM-THRID': message['X-GM-THRID'], + 'X-Gmail-Labels': message['X-Gmail-Labels'], + "From": message['From'], + "To": message['To'], + "Subject": message['Subject'], + "when": message['date'], + "body": message['body'], + } + + for message in get_mbox(mbox_file) + ), + pk="id", + alter=True, + ) + print('Finished loading emails into {}.'.format(mbox_file)) + print('Enabling full text search on "body" and "Subject" fields') + db["mbox_emails"].enable_fts(["body", "Subject"]) + print('Finished!') + + +def get_email_body(message): + ''' + return the email body contents + ''' + body = None + if message.is_multipart(): + for part in message.walk(): + if part.is_multipart(): + for subpart in part.walk(): + if subpart.get_content_type() == 'text/plain': + body = subpart.get_payload(decode=True) + elif part.get_content_type() == 'text/plain': + body = part.get_payload(decode=True) + elif message.get_content_type() == 'text/plain': + body = message.get_payload(decode=True) + return body + + +def get_message_date(get_date, get_from): + if get_date: + mail_date = get_date + else: + mail_date = get_from.strip()[-30:] + + datetime_tuple = email.utils.parsedate_tz(mail_date) + if datetime_tuple: + unix_time = email.utils.mktime_tz(datetime_tuple) + mail_date_iso8601 = datetime.datetime.utcfromtimestamp(unix_time).isoformat(' ') + else: + mail_date_iso8601 = '' + + return mail_date_iso8601 diff --git a/setup.py b/setup.py index 5dd8d1f..72f43da 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_long_description(): [console_scripts] google-takeout-to-sqlite=google_takeout_to_sqlite.cli:cli """, - install_requires=["sqlite-utils~=1.11"], + install_requires=["sqlite-utils~=1.11", "rich"], extras_require={"test": ["pytest"]}, tests_require=["google-takeout-to-sqlite[test]"], ) From 50e7e8dacb1e1dbd58742ce7f2f86d50672d5b61 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Mon, 22 Feb 2021 14:19:15 -0700 Subject: [PATCH 02/21] Add some tests --- tests/mbox_contents/small.gmail.mbox | 147 +++++++++++++++++++++++++++ tests/test_gmail_import.py | 47 +++++++++ 2 files changed, 194 insertions(+) create mode 100644 tests/mbox_contents/small.gmail.mbox create mode 100644 tests/test_gmail_import.py diff --git a/tests/mbox_contents/small.gmail.mbox b/tests/mbox_contents/small.gmail.mbox new file mode 100644 index 0000000..c885379 --- /dev/null +++ b/tests/mbox_contents/small.gmail.mbox @@ -0,0 +1,147 @@ +From 1277085061787347926@xxx Tue Aug 05 08:00:23 +0000 2008 +X-GM-THRID: 1277085061787347926 +X-Gmail-Labels: Unread +Delivered-To: asdfereasdf@gmail.com +Received: by 10.142.98.16 with SMTP id v16cs5204wfb; + Tue, 5 Aug 2008 01:00:23 -0700 (PDT) +Received: by 10.151.26.12 with SMTP id d12mr926013ybj.145.1217923223126; + Tue, 05 Aug 2008 01:00:23 -0700 (PDT) +Return-Path: +Received: from www.zend.com (lists.zend.com [67.15.86.102]) + by mx.google.com with SMTP id 6si1509903yxg.6.2008.08.05.01.00.22; + Tue, 05 Aug 2008 01:00:23 -0700 (PDT) +Received-SPF: pass (google.com: domain of fw-general-return-20503-test=gmail.com@lists.zend.com designates 67.15.86.102 as permitted sender) client-ip=67.15.86.102; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of fw-general-return-20503-test=gmail.com@lists.zend.com designates 67.15.86.102 as permitted sender) smtp.mail=fw-general-return-20503-test=gmail.com@lists.zend.com +Received: (qmail 28326 invoked by uid 505); 5 Aug 2008 08:00:15 -0000 +Mailing-List: contact fw-general-help@lists.zend.com; run by ezmlm +Precedence: bulk +List-Post: +List-Help: +List-Unsubscribe: +List-Subscribe: +Delivered-To: mailing list fw-general@lists.zend.com +Received: (qmail 28319 invoked from network); 5 Aug 2008 08:00:15 -0000 +Message-ID: <18826312.post@talk.nabble.com> +Date: Tue, 5 Aug 2008 01:00:12 -0700 (PDT) +From: =?UTF-8?Q?=C5=82_Zieli=C5=84ski?= +To: fw-general@lists.zend.com +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +X-Nabble-From: personasdflkj@gmail.com +X-pstn-neptune: 0/0/0.00/0 +X-pstn-levels: (S: 5.98501/99.90000 CV:99.0000 P:95.9108 M:88.1613 C:98.6951 ) +X-pstn-settings: 1 (0.1500:0.1500) cv gt3 gt2 gt1 p m c +X-pstn-addresses: from [638/31] +Subject: [fw-general] Zend_Form and generating fields + + +Unfortunately it is slow! For 10 products it takes 0.6 sec. to generate. +Is there a better (more efficient) method to build such forms via Zend_Form? + +The same I noticed when tried to create a select element which contained +many options (i.e. list of countries). Without ajax (autocomplete) it takes +ages to generate and seems to be useless in this case. Shame. + +I wonder if Zend_Form can be used when it comes to generate a lot of +inputs/options in select or I`m forced to create it by hand? + + + +From 1278212428183604564@xxx Sun Aug 17 18:39:23 +0000 2008 +X-GM-THRID: 1278204036336346264 +X-Gmail-Labels: Unread +Delivered-To: testasdfasdf@gmail.com +Received: by 10.142.98.16 with SMTP id v16cs546946wfb; + Sun, 17 Aug 2008 11:39:24 -0700 (PDT) +Received: by 10.90.100.17 with SMTP id x17mr545483agb.48.1218998363996; + Sun, 17 Aug 2008 11:39:23 -0700 (PDT) +Return-Path: +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) + by mx.google.com with ESMTP id c44si5785715hsc.16.2008.08.17.11.39.23; + Sun, 17 Aug 2008 11:39:23 -0700 (PDT) +Received-SPF: pass (google.com: domain of gnumed-devel-bounces+asdflkjer=gmail.com@gnu.org designates 199.232.76.165 as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of gnumed-devel-bounces+asdflkjelrkj=gmail.com@gnu.org designates 199.232.76.165 as permitted sender) smtp.mail=gnumed-devel-bounces+asdlkjwer=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:51303 helo=lists.gnu.org) + by lists.gnu.org with esmtp (Exim 4.43) + id 1KUn9v-0005Uo-D7 + for lkwelrkj@gmail.com; Sun, 17 Aug 2008 14:39:23 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) + id 1KUn9s-0005Sa-Ct + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:20 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) + id 1KUn9q-0005QU-TY + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:20 -0400 +Received: from [199.232.76.173] (port=33439 helo=monty-python.gnu.org) + by lists.gnu.org with esmtp (Exim 4.43) id 1KUn9q-0005Q4-No + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:18 -0400 +Received: from mail.gmx.net ([213.165.64.20]:56165) + by monty-python.gnu.org with smtp (Exim 4.60) + (envelope-from ) id 1KUn9q-0008Gg-44 + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:18 -0400 +Received: (qmail invoked by alias); 17 Aug 2008 18:39:16 -0000 +Received: from A7ee9.a.strato-dslnet.de (EHLO merkur.person.loc) + [89.62.126.233] + by mail.gmx.net (mp066) with SMTP; 17 Aug 2008 20:39:16 +0200 +X-Authenticated: #1433807 +X-Provags-ID: V01U2FsdGVkX1+dsZZuQ0OdkW7jgzdpHkRRth5+XhDeDDCDx7naLk + 3bct2wCsMl/clU +Received: from ncq by merkur.person.loc with local (Exim 4.69) + (envelope-from ) id 1KUn9n-0003eu-Ew + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 20:39:15 +0200 +Date: Sun, 17 Aug 2008 20:39:15 +0200 +From: Person Person +To: gnumed-devel@gnu.org +Subject: Re: [Gnumed-devel] Tree view formatting +Message-ID: <20080817183915.GM3992@merkur.person.loc> +Mail-Followup-To: gnumed-devel@gnu.org +References: <272a08710808170925y47484f3fmcfb26f8686727762@mail.gmail.com> + <20080817170549.GH3992@merkur.person.loc> + <272a08710808171034o239f6167q3f7039727802dd09@mail.gmail.com> + <20080817180622.GK3992@merkur.person.loc> + <272a08710808171109x4b6429a1g89f6e594408080db@mail.gmail.com> +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline +In-Reply-To: <272a08710808171109x4b6429a1g89f6e594408080db@mail.gmail.com> +User-Agent: Mutt/1.5.18 (2008-05-17) +X-Y-GMX-Trusted: 0 +X-FuHaFi: 0.6899999999999999 +X-detected-kernel: by monty-python.gnu.org: Genre and OS details not + recognized. +X-BeenThere: gnumed-devel@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: gnumed-devel.gnu.org +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: gnumed-devel-bounces+asdlfkj=gmail.com@gnu.org +Errors-To: gnumed-devel-bounces+asdflkj=gmail.com@gnu.org + +On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote: + +> when you say you have changed it ... can I assume it will make the 0.3.0 release +yes + +> or is the release READY +I hope it is "ready" so I can release it within the next few +days. I usually wait a few days to see whether any errors +show up. That's why we need you guys to test like mad. + +> and all our future discussions on this list +> will from now on to be implemented on the 0.3.1 ? + +Not quite yet. And, rather 0.3+. + +Person + + +_______________________________________________ +Gnumed-devel mailing list +Gnumed-devel@gnu.org +http://lists.gnu.org/mailman/listinfo/gnumed-devel diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py new file mode 100644 index 0000000..26e009a --- /dev/null +++ b/tests/test_gmail_import.py @@ -0,0 +1,47 @@ +from google_takeout_to_sqlite.utils import save_emails +import pathlib +import sqlite_utils + + +def test_import_gmails(): + path = pathlib.Path(__file__).parent / "mbox_contents/small.gmail.mbox" + db = sqlite_utils.Database(memory=True) + save_emails(db, path) + assert "mbox_emails" in set(db.table_names()) + mbox_emails = list(sorted(db["mbox_emails"].rows, key=lambda r: r["id"])) + assert [ + {'From': '=?UTF-8?Q?=C5=82_Zieli=C5=84ski?= ', + 'Subject': '[fw-general] Zend_Form and generating fields', + 'To': 'fw-general@lists.zend.com', + 'X-GM-THRID': '1277085061787347926', + 'X-Gmail-Labels': 'Unread', + 'body': b'\r\nUnfortunately it is slow! For 10 products it takes 0.6 sec. to' + b' generate.\r\nIs there a better (more efficient) method to build s' + b'uch forms via Zend_Form?\r\n\r\nThe same I noticed when tried to' + b' create a select element which contained\r\nmany options (i.e. lis' + b't of countries). Without ajax (autocomplete) it takes\r\nages to g' + b'enerate and seems to be useless in this case. Shame.\r\n\r\nI wo' + b'nder if Zend_Form can be used when it comes to generate a lot of' + b'\r\ninputs/options in select or I`m forced to create it by han' + b'd?\r\n\r\n\r\n\r\n', + 'id': '<18826312.post@talk.nabble.com>', + 'when': '2008-08-05 08:00:12'}, + {'From': 'Person Person ', + 'Subject': 'Re: [Gnumed-devel] Tree view formatting', + 'To': 'gnumed-devel@gnu.org', + 'X-GM-THRID': '1278204036336346264', + 'X-Gmail-Labels': 'Unread', + 'body': b'On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote:\r\n\r\n' + b'> when you say you have changed it ... can I assume it will make' + b' the 0.3.0 release\r\nyes\r\n\r\n> or is the release READY\r\nI ' + b'hope it is "ready" so I can release it within the next few\r\ndays' + b'. I usually wait a few days to see whether any errors\r\nshow up. ' + b"That's why we need you guys to test like mad.\r\n\r\n> and all o" + b'ur future discussions on this list\r\n> will from now on to be imp' + b'lemented on the 0.3.1 ?\r\n\r\nNot quite yet. And, rather 0.3+.\r' + b'\n\r\nPerson\r\n\r\n\r\n_________________________________________' + b'______\r\nGnumed-devel mailing list\r\nGnumed-devel@gnu.org\r\nhtt' + b'p://lists.gnu.org/mailman/listinfo/gnumed-devel\r\n', + 'id': '<20080817183915.GM3992@merkur.person.loc>', + 'when': '2008-08-17 18:39:15'} + ] == mbox_emails From a3de045eba0fae4b309da21aa3119102b0efc576 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Tue, 23 Feb 2021 17:34:16 -0700 Subject: [PATCH 03/21] Format with Black --- google_takeout_to_sqlite/utils.py | 65 +++++++++++++-------------- tests/test_gmail_import.py | 74 ++++++++++++++++--------------- 2 files changed, 70 insertions(+), 69 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 0fa8010..d9ac4e5 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -62,9 +62,9 @@ def id_for_location_history(row): def get_mbox(mbox_file): num_errors = 0 - print('Preparing to process emails...') + print("Preparing to process emails...") mbox = mailbox.mbox(mbox_file) - print('Processing {} emails'.format(len(mbox))) + print("Processing {} emails".format(len(mbox))) # These are all the Gmail email fields available # ['X-GM-THRID', 'X-Gmail-Labels', 'Delivered-To', 'Received', 'Received', @@ -78,31 +78,29 @@ def get_mbox(mbox_file): for email in track(mbox): try: message = {} - message['Message-Id'] = email['Message-Id'] - message['X-GM-THRID'] = email['X-GM-THRID'] - message['X-Gmail-Labels'] = email['X-Gmail-Labels'] + message["Message-Id"] = email["Message-Id"] + message["X-GM-THRID"] = email["X-GM-THRID"] + message["X-Gmail-Labels"] = email["X-Gmail-Labels"] # These following try/excepts are here because for some reason # these items returned from the mbox module are sometimes strings # and sometimes headers and sometimes None. try: - email['From'].decode('utf-8') + email["From"].decode("utf-8") except AttributeError: - message['From'] = str(email['From']) + message["From"] = str(email["From"]) try: - email['To'].decode('utf-8') + email["To"].decode("utf-8") except AttributeError: - message["To"] = str(email['To']) + message["To"] = str(email["To"]) try: - email['Subject'].decode('utf-8') + email["Subject"].decode("utf-8") except AttributeError: - message["Subject"] = str(email['Subject']) + message["Subject"] = str(email["Subject"]) - - message["date"] = get_message_date(email.get('Date'), - email.get_from()) + message["date"] = get_message_date(email.get("Date"), email.get_from()) message["body"] = get_email_body(email) yield message @@ -110,7 +108,7 @@ def get_mbox(mbox_file): # How does this project want to handle logging? For now we're just # printing out variables num_errors = num_errors + 1 - print('Errors: {}'.format(num_errors)) + print("Errors: {}".format(num_errors)) print(traceback.format_exc()) continue @@ -122,41 +120,40 @@ def save_emails(db, mbox_file): db["mbox_emails"].upsert_all( ( { - "id": message['Message-Id'], - 'X-GM-THRID': message['X-GM-THRID'], - 'X-Gmail-Labels': message['X-Gmail-Labels'], - "From": message['From'], - "To": message['To'], - "Subject": message['Subject'], - "when": message['date'], - "body": message['body'], + "id": message["Message-Id"], + "X-GM-THRID": message["X-GM-THRID"], + "X-Gmail-Labels": message["X-Gmail-Labels"], + "From": message["From"], + "To": message["To"], + "Subject": message["Subject"], + "when": message["date"], + "body": message["body"], } - - for message in get_mbox(mbox_file) + for message in get_mbox(mbox_file) ), pk="id", alter=True, ) - print('Finished loading emails into {}.'.format(mbox_file)) + print("Finished loading emails into {}.".format(mbox_file)) print('Enabling full text search on "body" and "Subject" fields') db["mbox_emails"].enable_fts(["body", "Subject"]) - print('Finished!') + print("Finished!") def get_email_body(message): - ''' + """ return the email body contents - ''' + """ body = None if message.is_multipart(): for part in message.walk(): if part.is_multipart(): for subpart in part.walk(): - if subpart.get_content_type() == 'text/plain': + if subpart.get_content_type() == "text/plain": body = subpart.get_payload(decode=True) - elif part.get_content_type() == 'text/plain': + elif part.get_content_type() == "text/plain": body = part.get_payload(decode=True) - elif message.get_content_type() == 'text/plain': + elif message.get_content_type() == "text/plain": body = message.get_payload(decode=True) return body @@ -170,8 +167,8 @@ def get_message_date(get_date, get_from): datetime_tuple = email.utils.parsedate_tz(mail_date) if datetime_tuple: unix_time = email.utils.mktime_tz(datetime_tuple) - mail_date_iso8601 = datetime.datetime.utcfromtimestamp(unix_time).isoformat(' ') + mail_date_iso8601 = datetime.datetime.utcfromtimestamp(unix_time).isoformat(" ") else: - mail_date_iso8601 = '' + mail_date_iso8601 = "" return mail_date_iso8601 diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py index 26e009a..bc12b7c 100644 --- a/tests/test_gmail_import.py +++ b/tests/test_gmail_import.py @@ -10,38 +10,42 @@ def test_import_gmails(): assert "mbox_emails" in set(db.table_names()) mbox_emails = list(sorted(db["mbox_emails"].rows, key=lambda r: r["id"])) assert [ - {'From': '=?UTF-8?Q?=C5=82_Zieli=C5=84ski?= ', - 'Subject': '[fw-general] Zend_Form and generating fields', - 'To': 'fw-general@lists.zend.com', - 'X-GM-THRID': '1277085061787347926', - 'X-Gmail-Labels': 'Unread', - 'body': b'\r\nUnfortunately it is slow! For 10 products it takes 0.6 sec. to' - b' generate.\r\nIs there a better (more efficient) method to build s' - b'uch forms via Zend_Form?\r\n\r\nThe same I noticed when tried to' - b' create a select element which contained\r\nmany options (i.e. lis' - b't of countries). Without ajax (autocomplete) it takes\r\nages to g' - b'enerate and seems to be useless in this case. Shame.\r\n\r\nI wo' - b'nder if Zend_Form can be used when it comes to generate a lot of' - b'\r\ninputs/options in select or I`m forced to create it by han' - b'd?\r\n\r\n\r\n\r\n', - 'id': '<18826312.post@talk.nabble.com>', - 'when': '2008-08-05 08:00:12'}, - {'From': 'Person Person ', - 'Subject': 'Re: [Gnumed-devel] Tree view formatting', - 'To': 'gnumed-devel@gnu.org', - 'X-GM-THRID': '1278204036336346264', - 'X-Gmail-Labels': 'Unread', - 'body': b'On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote:\r\n\r\n' - b'> when you say you have changed it ... can I assume it will make' - b' the 0.3.0 release\r\nyes\r\n\r\n> or is the release READY\r\nI ' - b'hope it is "ready" so I can release it within the next few\r\ndays' - b'. I usually wait a few days to see whether any errors\r\nshow up. ' - b"That's why we need you guys to test like mad.\r\n\r\n> and all o" - b'ur future discussions on this list\r\n> will from now on to be imp' - b'lemented on the 0.3.1 ?\r\n\r\nNot quite yet. And, rather 0.3+.\r' - b'\n\r\nPerson\r\n\r\n\r\n_________________________________________' - b'______\r\nGnumed-devel mailing list\r\nGnumed-devel@gnu.org\r\nhtt' - b'p://lists.gnu.org/mailman/listinfo/gnumed-devel\r\n', - 'id': '<20080817183915.GM3992@merkur.person.loc>', - 'when': '2008-08-17 18:39:15'} - ] == mbox_emails + { + "From": "=?UTF-8?Q?=C5=82_Zieli=C5=84ski?= ", + "Subject": "[fw-general] Zend_Form and generating fields", + "To": "fw-general@lists.zend.com", + "X-GM-THRID": "1277085061787347926", + "X-Gmail-Labels": "Unread", + "body": b"\r\nUnfortunately it is slow! For 10 products it takes 0.6 sec. to" + b" generate.\r\nIs there a better (more efficient) method to build s" + b"uch forms via Zend_Form?\r\n\r\nThe same I noticed when tried to" + b" create a select element which contained\r\nmany options (i.e. lis" + b"t of countries). Without ajax (autocomplete) it takes\r\nages to g" + b"enerate and seems to be useless in this case. Shame.\r\n\r\nI wo" + b"nder if Zend_Form can be used when it comes to generate a lot of" + b"\r\ninputs/options in select or I`m forced to create it by han" + b"d?\r\n\r\n\r\n\r\n", + "id": "<18826312.post@talk.nabble.com>", + "when": "2008-08-05 08:00:12", + }, + { + "From": "Person Person ", + "Subject": "Re: [Gnumed-devel] Tree view formatting", + "To": "gnumed-devel@gnu.org", + "X-GM-THRID": "1278204036336346264", + "X-Gmail-Labels": "Unread", + "body": b"On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote:\r\n\r\n" + b"> when you say you have changed it ... can I assume it will make" + b" the 0.3.0 release\r\nyes\r\n\r\n> or is the release READY\r\nI " + b'hope it is "ready" so I can release it within the next few\r\ndays' + b". I usually wait a few days to see whether any errors\r\nshow up. " + b"That's why we need you guys to test like mad.\r\n\r\n> and all o" + b"ur future discussions on this list\r\n> will from now on to be imp" + b"lemented on the 0.3.1 ?\r\n\r\nNot quite yet. And, rather 0.3+.\r" + b"\n\r\nPerson\r\n\r\n\r\n_________________________________________" + b"______\r\nGnumed-devel mailing list\r\nGnumed-devel@gnu.org\r\nhtt" + b"p://lists.gnu.org/mailman/listinfo/gnumed-devel\r\n", + "id": "<20080817183915.GM3992@merkur.person.loc>", + "when": "2008-08-17 18:39:15", + }, + ] == mbox_emails From 72802a83fee282eb5d02d388567731ba4301050d Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Wed, 21 Jul 2021 22:03:41 -0700 Subject: [PATCH 04/21] Manually parse mbox format Parsing the mbox file manually instead of using Python's built-in parser allows us to process large files without loading them into memory all at once. --- google_takeout_to_sqlite/utils.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index d9ac4e5..5cbd594 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -2,9 +2,7 @@ import hashlib import datetime import email -import mailbox import traceback -from rich.progress import track from email.utils import parsedate_tz, mktime_tz @@ -59,12 +57,26 @@ def id_for_location_history(row): first_six, ) +def parse_mbox(mbox_file): + with open(mbox_file, 'rb') as f: + lines = [] + while True: + line = f.readline() + + is_boundary = not line or line.startswith(b'From ') + if is_boundary: + if len(lines) > 0: + message = b''.join(lines) + yield email.message_from_bytes(message) + lines = [] + else: + lines.append(line) + + if not line: + break def get_mbox(mbox_file): num_errors = 0 - print("Preparing to process emails...") - mbox = mailbox.mbox(mbox_file) - print("Processing {} emails".format(len(mbox))) # These are all the Gmail email fields available # ['X-GM-THRID', 'X-Gmail-Labels', 'Delivered-To', 'Received', 'Received', @@ -75,7 +87,7 @@ def get_mbox(mbox_file): # 'Content-Transfer-Encoding', 'X-Nabble-From', 'X-pstn-neptune', # 'X-pstn-levels', 'X-pstn-settings', 'X-pstn-addresses', 'Subject'] - for email in track(mbox): + for email in parse_mbox(mbox_file): try: message = {} message["Message-Id"] = email["Message-Id"] @@ -100,7 +112,7 @@ def get_mbox(mbox_file): except AttributeError: message["Subject"] = str(email["Subject"]) - message["date"] = get_message_date(email.get("Date"), email.get_from()) + message["date"] = get_message_date(email.get("Date"), email.get("From")) message["body"] = get_email_body(email) yield message From 4bc70103582c10802c85a523ef1e99a8a2154aa9 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Thu, 22 Jul 2021 09:44:45 -0700 Subject: [PATCH 05/21] Fix import for messages that don't have a Date This fixes a regression introduced by the previous commit where messages no longer fetch the date from the mbox 'From ' line. For messages without a Date header this means we lose information about the delivery date. --- google_takeout_to_sqlite/utils.py | 46 ++++++++++++++++--------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 5cbd594..8d41aae 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -3,8 +3,6 @@ import datetime import email import traceback -from email.utils import parsedate_tz, mktime_tz - def save_my_activity(db, zf): my_activities = [ @@ -59,20 +57,26 @@ def id_for_location_history(row): def parse_mbox(mbox_file): with open(mbox_file, 'rb') as f: + delivery_date = '' lines = [] + while True: line = f.readline() - is_boundary = not line or line.startswith(b'From ') - if is_boundary: - if len(lines) > 0: - message = b''.join(lines) - yield email.message_from_bytes(message) - lines = [] + is_new_record = line.startswith(b'From ') + is_eof = len(line) == 0 + + if is_eof or is_new_record: + message = b''.join(lines) + if message: + yield delivery_date, email.message_from_bytes(message) else: lines.append(line) - if not line: + if is_new_record: + delivery_date = line.strip()[-30:] + lines = [] + elif is_eof: break def get_mbox(mbox_file): @@ -87,7 +91,7 @@ def get_mbox(mbox_file): # 'Content-Transfer-Encoding', 'X-Nabble-From', 'X-pstn-neptune', # 'X-pstn-levels', 'X-pstn-settings', 'X-pstn-addresses', 'Subject'] - for email in parse_mbox(mbox_file): + for delivery_date, email in parse_mbox(mbox_file): try: message = {} message["Message-Id"] = email["Message-Id"] @@ -112,7 +116,11 @@ def get_mbox(mbox_file): except AttributeError: message["Subject"] = str(email["Subject"]) - message["date"] = get_message_date(email.get("Date"), email.get("From")) + if "Date" in email: + message["date"] = parse_mail_date(email["Date"]) + else: + message["date"] = parse_mail_date(delivery_date) + message["body"] = get_email_body(email) yield message @@ -170,17 +178,11 @@ def get_email_body(message): return body -def get_message_date(get_date, get_from): - if get_date: - mail_date = get_date - else: - mail_date = get_from.strip()[-30:] - +def parse_mail_date(mail_date): datetime_tuple = email.utils.parsedate_tz(mail_date) - if datetime_tuple: - unix_time = email.utils.mktime_tz(datetime_tuple) - mail_date_iso8601 = datetime.datetime.utcfromtimestamp(unix_time).isoformat(" ") - else: - mail_date_iso8601 = "" + if not datetime_tuple: + return "" + unix_time = email.utils.mktime_tz(datetime_tuple) + mail_date_iso8601 = datetime.datetime.utcfromtimestamp(unix_time).isoformat(" ") return mail_date_iso8601 From 8ee555c2889a38ff42b95664ee074b4a01a82f06 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Tue, 27 Jul 2021 23:41:03 -0700 Subject: [PATCH 06/21] Use thread id as pkey if missing message id. Some messages (like gchat logs) don't have message ids and therefore don't save properly. This commit uses the gmail X-GM-THRID if the Message-Id is missing. --- google_takeout_to_sqlite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 8d41aae..befa510 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -140,7 +140,7 @@ def save_emails(db, mbox_file): db["mbox_emails"].upsert_all( ( { - "id": message["Message-Id"], + "id": message["Message-Id"] if "Message-Id" in message else message["X-GM-THRID"], "X-GM-THRID": message["X-GM-THRID"], "X-Gmail-Labels": message["X-Gmail-Labels"], "From": message["From"], From e1fdef7a565d708167f36203372fb96cc3c1492d Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Tue, 27 Jul 2021 23:47:24 -0700 Subject: [PATCH 07/21] Fix parse exception: convert delivery_date to a str The function email.utils.parsedate_tz expects a str, but we were passing bytes. Casting to str fixes an exception in messages where the Date header is missing and the delivery time must be inferred from the mbox header. --- google_takeout_to_sqlite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index befa510..4f3d942 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -74,7 +74,7 @@ def parse_mbox(mbox_file): lines.append(line) if is_new_record: - delivery_date = line.strip()[-30:] + delivery_date = str(line.strip()[-30:]) lines = [] elif is_eof: break From 8939f5be3a23c9964bb73d584efd983b8eb109da Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Fri, 6 Aug 2021 14:34:54 -0700 Subject: [PATCH 08/21] Use message id from mbox header if none exists in MIME. Some messages (like chats) don't have a Message-Id mime header, so the message is saved without a primary key. A previous commit used the thread id in this situation, but the same thread id can be used for multiple messages. This id, which is the message id used by the gmail api, should be unique across all messages. --- google_takeout_to_sqlite/utils.py | 10 +++++++--- tests/mbox_contents/small.gmail.mbox | 8 ++++++++ tests/test_gmail_import.py | 10 ++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 4f3d942..dc09976 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -3,6 +3,7 @@ import datetime import email import traceback +import re def save_my_activity(db, zf): my_activities = [ @@ -58,6 +59,7 @@ def id_for_location_history(row): def parse_mbox(mbox_file): with open(mbox_file, 'rb') as f: delivery_date = '' + message_id = '' lines = [] while True: @@ -69,12 +71,12 @@ def parse_mbox(mbox_file): if is_eof or is_new_record: message = b''.join(lines) if message: - yield delivery_date, email.message_from_bytes(message) + yield delivery_date, message_id, email.message_from_bytes(message) else: lines.append(line) if is_new_record: - delivery_date = str(line.strip()[-30:]) + (message_id, delivery_date) = re.match(r'^From (\w+)@xxx (.+)\r\n', line.decode('utf-8')).groups() lines = [] elif is_eof: break @@ -91,10 +93,12 @@ def get_mbox(mbox_file): # 'Content-Transfer-Encoding', 'X-Nabble-From', 'X-pstn-neptune', # 'X-pstn-levels', 'X-pstn-settings', 'X-pstn-addresses', 'Subject'] - for delivery_date, email in parse_mbox(mbox_file): + for delivery_date, gmail_message_id, email in parse_mbox(mbox_file): try: message = {} message["Message-Id"] = email["Message-Id"] + if message["Message-Id"] is None: + message["Message-Id"] = gmail_message_id message["X-GM-THRID"] = email["X-GM-THRID"] message["X-Gmail-Labels"] = email["X-Gmail-Labels"] diff --git a/tests/mbox_contents/small.gmail.mbox b/tests/mbox_contents/small.gmail.mbox index c885379..622bc2e 100644 --- a/tests/mbox_contents/small.gmail.mbox +++ b/tests/mbox_contents/small.gmail.mbox @@ -145,3 +145,11 @@ _______________________________________________ Gnumed-devel mailing list Gnumed-devel@gnu.org http://lists.gnu.org/mailman/listinfo/gnumed-devel +From 1529555944956622147@xxx Wed Mar 23 01:57:00 +0000 2016 +X-GM-THRID: 1529553825574740118 +X-Gmail-Labels: Chat +From: Person Person +MIME-Version: 1.0 +Content-Type: text/plain + +Nothing worse than being without good wifi diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py index bc12b7c..cc72903 100644 --- a/tests/test_gmail_import.py +++ b/tests/test_gmail_import.py @@ -10,6 +10,16 @@ def test_import_gmails(): assert "mbox_emails" in set(db.table_names()) mbox_emails = list(sorted(db["mbox_emails"].rows, key=lambda r: r["id"])) assert [ + { + "id": "1529555944956622147", + "when": "2016-03-23 01:57:00", + "From": "Person Person ", + "body": b'Nothing worse than being without good wifi\r\n', + "Subject": 'None', + "To": 'None', + "X-GM-THRID": "1529553825574740118", + "X-Gmail-Labels": "Chat", + }, { "From": "=?UTF-8?Q?=C5=82_Zieli=C5=84ski?= ", "Subject": "[fw-general] Zend_Form and generating fields", From 953e7eb0698d37e486103090e075dbdc492bb335 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Fri, 6 Aug 2021 14:53:53 -0700 Subject: [PATCH 09/21] Explicitly parse email with compat32 policy. The docs note: "The policy keyword should always be specified; The default will change to email.policy.default in a future version of Python." --- google_takeout_to_sqlite/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index dc09976..ab2e282 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -2,6 +2,7 @@ import hashlib import datetime import email +from email import policy import traceback import re @@ -71,7 +72,7 @@ def parse_mbox(mbox_file): if is_eof or is_new_record: message = b''.join(lines) if message: - yield delivery_date, message_id, email.message_from_bytes(message) + yield delivery_date, message_id, email.message_from_bytes(message, policy=policy.compat32) else: lines.append(line) From 50cc88332a26975944a129d435c4c2bbde4527b3 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Fri, 6 Aug 2021 15:36:25 -0700 Subject: [PATCH 10/21] Simplify handling of headers with binary data. This shouldn't happen in RFC-abiding messages, but raw unicode or other non-ascii content will cause the header parser to return a Header object rather than a str. Improve handling of this case and add a simple unit test. --- google_takeout_to_sqlite/utils.py | 28 +++++++++++----------------- tests/mbox_contents/small.gmail.mbox | 11 +++++++++++ tests/test_gmail_import.py | 16 +++++++++++++--- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index ab2e282..4afca1c 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -103,23 +103,9 @@ def get_mbox(mbox_file): message["X-GM-THRID"] = email["X-GM-THRID"] message["X-Gmail-Labels"] = email["X-Gmail-Labels"] - # These following try/excepts are here because for some reason - # these items returned from the mbox module are sometimes strings - # and sometimes headers and sometimes None. - - try: - email["From"].decode("utf-8") - except AttributeError: - message["From"] = str(email["From"]) - try: - email["To"].decode("utf-8") - except AttributeError: - message["To"] = str(email["To"]) - - try: - email["Subject"].decode("utf-8") - except AttributeError: - message["Subject"] = str(email["Subject"]) + message["From"] = get_email_header(email, "From") + message["To"] = get_email_header(email, "To") + message["Subject"] = get_email_header(email, "Subject") if "Date" in email: message["date"] = parse_mail_date(email["Date"]) @@ -164,6 +150,14 @@ def save_emails(db, mbox_file): db["mbox_emails"].enable_fts(["body", "Subject"]) print("Finished!") +def get_email_header(message, name, failobj=None): + # get will either return a str, email.header.Header, or None. + # This function converts the Header to a str if one is returned. + value = message.get(name) + if value is None: + return failobj + else: + return str(value) def get_email_body(message): """ diff --git a/tests/mbox_contents/small.gmail.mbox b/tests/mbox_contents/small.gmail.mbox index 622bc2e..71c628b 100644 --- a/tests/mbox_contents/small.gmail.mbox +++ b/tests/mbox_contents/small.gmail.mbox @@ -153,3 +153,14 @@ MIME-Version: 1.0 Content-Type: text/plain Nothing worse than being without good wifi + +From 1705761119401391280@xxx Tue Jul 20 00:22:49 +0000 2021 +From: 罗 <123@example.net> +To: Person Person +X-GM-THRID: 1705761119401391280 +X-Gmail-Labels: Chat +MIME-Version: 1.0 +Subject: test +Content-Type: text/plain; charset=utf-8 + +好好学习，天天向上 diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py index cc72903..069112f 100644 --- a/tests/test_gmail_import.py +++ b/tests/test_gmail_import.py @@ -14,12 +14,22 @@ def test_import_gmails(): "id": "1529555944956622147", "when": "2016-03-23 01:57:00", "From": "Person Person ", - "body": b'Nothing worse than being without good wifi\r\n', - "Subject": 'None', - "To": 'None', + "body": b'Nothing worse than being without good wifi\r\n\r\n', + "Subject": None, + "To": None, "X-GM-THRID": "1529553825574740118", "X-Gmail-Labels": "Chat", }, + { + "Subject": "test", + "From": "��� <123@example.net>", + "To": "Person Person ", + "id": "1705761119401391280", + "body": "好好学习，天天向上\r\n".encode("utf-8"), + "when": "2021-07-20 00:22:49", + "X-GM-THRID": "1705761119401391280", + "X-Gmail-Labels": "Chat", + }, { "From": "=?UTF-8?Q?=C5=82_Zieli=C5=84ski?= ", "Subject": "[fw-general] Zend_Form and generating fields", From 770bc0e01f8983b5dfd7202e6c507d6d786b48fb Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Fri, 6 Aug 2021 15:59:17 -0700 Subject: [PATCH 11/21] Add RFC 2047 parsing to deal with unicode headers. --- google_takeout_to_sqlite/utils.py | 14 ++++++++++++-- tests/mbox_contents/small.gmail.mbox | 2 +- tests/test_gmail_import.py | 4 ++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 4afca1c..cf7bf81 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -2,7 +2,7 @@ import hashlib import datetime import email -from email import policy +from email import policy, header import traceback import re @@ -157,7 +157,17 @@ def get_email_header(message, name, failobj=None): if value is None: return failobj else: - return str(value) + parts = [] + for part, enc in header.decode_header(value): + try: + part = part.decode(enc or 'utf-8') + except LookupError: # encoding not found + part = part.decode('utf-8') + except AttributeError: # part was already a str + pass + parts.append(part) + + return ''.join(parts) def get_email_body(message): """ diff --git a/tests/mbox_contents/small.gmail.mbox b/tests/mbox_contents/small.gmail.mbox index 71c628b..702d23d 100644 --- a/tests/mbox_contents/small.gmail.mbox +++ b/tests/mbox_contents/small.gmail.mbox @@ -23,7 +23,7 @@ Delivered-To: mailing list fw-general@lists.zend.com Received: (qmail 28319 invoked from network); 5 Aug 2008 08:00:15 -0000 Message-ID: <18826312.post@talk.nabble.com> Date: Tue, 5 Aug 2008 01:00:12 -0700 (PDT) -From: =?UTF-8?Q?=C5=82_Zieli=C5=84ski?= +From: =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= To: fw-general@lists.zend.com MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py index 069112f..2b6c335 100644 --- a/tests/test_gmail_import.py +++ b/tests/test_gmail_import.py @@ -22,7 +22,7 @@ def test_import_gmails(): }, { "Subject": "test", - "From": "��� <123@example.net>", + "From": "罗 <123@example.net>", "To": "Person Person ", "id": "1705761119401391280", "body": "好好学习，天天向上\r\n".encode("utf-8"), @@ -31,7 +31,7 @@ def test_import_gmails(): "X-Gmail-Labels": "Chat", }, { - "From": "=?UTF-8?Q?=C5=82_Zieli=C5=84ski?= ", + "From": "Keld Jørn Simonsen ", "Subject": "[fw-general] Zend_Form and generating fields", "To": "fw-general@lists.zend.com", "X-GM-THRID": "1277085061787347926", From 4f50ff4823c217f7e59e7c86ab550f102073f4bc Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Fri, 6 Aug 2021 16:55:44 -0700 Subject: [PATCH 12/21] Deal with invalid rfc 2047 strings. If the string is invalid, the undecoded string is returned instead. --- google_takeout_to_sqlite/utils.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index cf7bf81..757afc2 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -157,17 +157,26 @@ def get_email_header(message, name, failobj=None): if value is None: return failobj else: - parts = [] - for part, enc in header.decode_header(value): - try: - part = part.decode(enc or 'utf-8') - except LookupError: # encoding not found - part = part.decode('utf-8') - except AttributeError: # part was already a str - pass - parts.append(part) - - return ''.join(parts) + try: + return decode_rfc_2047_str(value) + except: + # If the value is invalid, return the un-decoded string. + return value + +def decode_rfc_2047_str(value): + parts = [] + + for part, enc in header.decode_header(value): + try: + part = part.decode(enc or 'utf-8') + except LookupError: # encoding not found + part = part.decode('utf-8') + except AttributeError: # part was already a str + pass + + parts.append(part) + + return ''.join(parts) def get_email_body(message): """ From abb4dfd5383588e0ba99422e0dbcecd9d83002bb Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Fri, 6 Aug 2021 17:54:42 -0700 Subject: [PATCH 13/21] Make [body] a TEXT column instead of a BLOB. --- google_takeout_to_sqlite/utils.py | 8 +++++- tests/test_gmail_import.py | 44 +++++++++++++++---------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 757afc2..b762019 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -112,7 +112,13 @@ def get_mbox(mbox_file): else: message["date"] = parse_mail_date(delivery_date) - message["body"] = get_email_body(email) + body = get_email_body(email) + try: + message["body"] = body.decode('utf-8') + except UnicodeDecodeError: + message["body"] = body + except AttributeError: + message["body"] = body yield message except (TypeError, ValueError, AttributeError, LookupError) as e: diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py index 2b6c335..fe963aa 100644 --- a/tests/test_gmail_import.py +++ b/tests/test_gmail_import.py @@ -14,7 +14,7 @@ def test_import_gmails(): "id": "1529555944956622147", "when": "2016-03-23 01:57:00", "From": "Person Person ", - "body": b'Nothing worse than being without good wifi\r\n\r\n', + "body": 'Nothing worse than being without good wifi\r\n\r\n', "Subject": None, "To": None, "X-GM-THRID": "1529553825574740118", @@ -25,7 +25,7 @@ def test_import_gmails(): "From": "罗 <123@example.net>", "To": "Person Person ", "id": "1705761119401391280", - "body": "好好学习，天天向上\r\n".encode("utf-8"), + "body": "好好学习，天天向上\r\n", "when": "2021-07-20 00:22:49", "X-GM-THRID": "1705761119401391280", "X-Gmail-Labels": "Chat", @@ -36,15 +36,15 @@ def test_import_gmails(): "To": "fw-general@lists.zend.com", "X-GM-THRID": "1277085061787347926", "X-Gmail-Labels": "Unread", - "body": b"\r\nUnfortunately it is slow! For 10 products it takes 0.6 sec. to" - b" generate.\r\nIs there a better (more efficient) method to build s" - b"uch forms via Zend_Form?\r\n\r\nThe same I noticed when tried to" - b" create a select element which contained\r\nmany options (i.e. lis" - b"t of countries). Without ajax (autocomplete) it takes\r\nages to g" - b"enerate and seems to be useless in this case. Shame.\r\n\r\nI wo" - b"nder if Zend_Form can be used when it comes to generate a lot of" - b"\r\ninputs/options in select or I`m forced to create it by han" - b"d?\r\n\r\n\r\n\r\n", + "body": "\r\nUnfortunately it is slow! For 10 products it takes 0.6 sec. to" + " generate.\r\nIs there a better (more efficient) method to build s" + "uch forms via Zend_Form?\r\n\r\nThe same I noticed when tried to" + " create a select element which contained\r\nmany options (i.e. lis" + "t of countries). Without ajax (autocomplete) it takes\r\nages to g" + "enerate and seems to be useless in this case. Shame.\r\n\r\nI wo" + "nder if Zend_Form can be used when it comes to generate a lot of" + "\r\ninputs/options in select or I`m forced to create it by han" + "d?\r\n\r\n\r\n\r\n", "id": "<18826312.post@talk.nabble.com>", "when": "2008-08-05 08:00:12", }, @@ -54,17 +54,17 @@ def test_import_gmails(): "To": "gnumed-devel@gnu.org", "X-GM-THRID": "1278204036336346264", "X-Gmail-Labels": "Unread", - "body": b"On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote:\r\n\r\n" - b"> when you say you have changed it ... can I assume it will make" - b" the 0.3.0 release\r\nyes\r\n\r\n> or is the release READY\r\nI " - b'hope it is "ready" so I can release it within the next few\r\ndays' - b". I usually wait a few days to see whether any errors\r\nshow up. " - b"That's why we need you guys to test like mad.\r\n\r\n> and all o" - b"ur future discussions on this list\r\n> will from now on to be imp" - b"lemented on the 0.3.1 ?\r\n\r\nNot quite yet. And, rather 0.3+.\r" - b"\n\r\nPerson\r\n\r\n\r\n_________________________________________" - b"______\r\nGnumed-devel mailing list\r\nGnumed-devel@gnu.org\r\nhtt" - b"p://lists.gnu.org/mailman/listinfo/gnumed-devel\r\n", + "body": "On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote:\r\n\r\n" + "> when you say you have changed it ... can I assume it will make" + " the 0.3.0 release\r\nyes\r\n\r\n> or is the release READY\r\nI " + 'hope it is "ready" so I can release it within the next few\r\ndays' + ". I usually wait a few days to see whether any errors\r\nshow up. " + "That's why we need you guys to test like mad.\r\n\r\n> and all o" + "ur future discussions on this list\r\n> will from now on to be imp" + "lemented on the 0.3.1 ?\r\n\r\nNot quite yet. And, rather 0.3+.\r" + "\n\r\nPerson\r\n\r\n\r\n_________________________________________" + "______\r\nGnumed-devel mailing list\r\nGnumed-devel@gnu.org\r\nhtt" + "p://lists.gnu.org/mailman/listinfo/gnumed-devel\r\n", "id": "<20080817183915.GM3992@merkur.person.loc>", "when": "2008-08-17 18:39:15", }, From 2a31dd4aff49069ff2cc18b3032dc58922f4357e Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Sun, 8 Aug 2021 13:48:14 -0700 Subject: [PATCH 14/21] Don't default pkey to thread ID if missing. --- google_takeout_to_sqlite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index b762019..a9f6e36 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -137,7 +137,7 @@ def save_emails(db, mbox_file): db["mbox_emails"].upsert_all( ( { - "id": message["Message-Id"] if "Message-Id" in message else message["X-GM-THRID"], + "id": message["Message-Id"], "X-GM-THRID": message["X-GM-THRID"], "X-Gmail-Labels": message["X-Gmail-Labels"], "From": message["From"], From d3cf088672d748961c04a4c92efcf40f1e57c56a Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Sun, 8 Aug 2021 13:48:57 -0700 Subject: [PATCH 15/21] Format with black --- google_takeout_to_sqlite/utils.py | 36 ++++++++++++++++++++----------- tests/test_gmail_import.py | 2 +- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index a9f6e36..3d81d0d 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -6,6 +6,7 @@ import traceback import re + def save_my_activity(db, zf): my_activities = [ f.filename for f in zf.filelist if f.filename.endswith("My Activity.json") @@ -57,31 +58,37 @@ def id_for_location_history(row): first_six, ) + def parse_mbox(mbox_file): - with open(mbox_file, 'rb') as f: - delivery_date = '' - message_id = '' + with open(mbox_file, "rb") as f: + delivery_date = "" + message_id = "" lines = [] while True: line = f.readline() - is_new_record = line.startswith(b'From ') + is_new_record = line.startswith(b"From ") is_eof = len(line) == 0 if is_eof or is_new_record: - message = b''.join(lines) + message = b"".join(lines) if message: - yield delivery_date, message_id, email.message_from_bytes(message, policy=policy.compat32) + yield delivery_date, message_id, email.message_from_bytes( + message, policy=policy.compat32 + ) else: lines.append(line) if is_new_record: - (message_id, delivery_date) = re.match(r'^From (\w+)@xxx (.+)\r\n', line.decode('utf-8')).groups() + (message_id, delivery_date) = re.match( + r"^From (\w+)@xxx (.+)\r\n", line.decode("utf-8") + ).groups() lines = [] elif is_eof: break + def get_mbox(mbox_file): num_errors = 0 @@ -114,7 +121,7 @@ def get_mbox(mbox_file): body = get_email_body(email) try: - message["body"] = body.decode('utf-8') + message["body"] = body.decode("utf-8") except UnicodeDecodeError: message["body"] = body except AttributeError: @@ -156,6 +163,7 @@ def save_emails(db, mbox_file): db["mbox_emails"].enable_fts(["body", "Subject"]) print("Finished!") + def get_email_header(message, name, failobj=None): # get will either return a str, email.header.Header, or None. # This function converts the Header to a str if one is returned. @@ -169,20 +177,22 @@ def get_email_header(message, name, failobj=None): # If the value is invalid, return the un-decoded string. return value + def decode_rfc_2047_str(value): parts = [] for part, enc in header.decode_header(value): try: - part = part.decode(enc or 'utf-8') - except LookupError: # encoding not found - part = part.decode('utf-8') - except AttributeError: # part was already a str + part = part.decode(enc or "utf-8") + except LookupError: # encoding not found + part = part.decode("utf-8") + except AttributeError: # part was already a str pass parts.append(part) - return ''.join(parts) + return "".join(parts) + def get_email_body(message): """ diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py index fe963aa..a81534d 100644 --- a/tests/test_gmail_import.py +++ b/tests/test_gmail_import.py @@ -14,7 +14,7 @@ def test_import_gmails(): "id": "1529555944956622147", "when": "2016-03-23 01:57:00", "From": "Person Person ", - "body": 'Nothing worse than being without good wifi\r\n\r\n', + "body": "Nothing worse than being without good wifi\r\n\r\n", "Subject": None, "To": None, "X-GM-THRID": "1529553825574740118", From 6a3832c833b60b0b0fc77d7d72f46fee2256313a Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Sun, 8 Aug 2021 13:49:51 -0700 Subject: [PATCH 16/21] Remove dependency on rich. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 72f43da..5dd8d1f 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_long_description(): [console_scripts] google-takeout-to-sqlite=google_takeout_to_sqlite.cli:cli """, - install_requires=["sqlite-utils~=1.11", "rich"], + install_requires=["sqlite-utils~=1.11"], extras_require={"test": ["pytest"]}, tests_require=["google-takeout-to-sqlite[test]"], ) From 25ee0a243bbfa86f6debd0a1da4b256f92eb65ca Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Sun, 8 Aug 2021 13:58:04 -0700 Subject: [PATCH 17/21] Create table before inserting, ensuring proper column types. In some instances tables would be created with the wrong column types if the initial records had unexpected types. This fixes the issue by explicitly creating the table and specifying types. --- google_takeout_to_sqlite/utils.py | 15 +++++++++++++++ setup.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 3d81d0d..00ed439 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -141,6 +141,21 @@ def save_emails(db, mbox_file): """ Import Gmail mbox from google takeout """ + if not db["mbox_emails"].exists(): + db["mbox_emails"].create( + { + "id": str, + "X-GM-THRID": str, + "X-Gmail-Labels": str, + "From": str, + "To": str, + "Subject": str, + "when": str, + "body": str, + }, + pk="id", + ) + db["mbox_emails"].upsert_all( ( { diff --git a/setup.py b/setup.py index 5dd8d1f..35cc452 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_long_description(): [console_scripts] google-takeout-to-sqlite=google_takeout_to_sqlite.cli:cli """, - install_requires=["sqlite-utils~=1.11"], + install_requires=["sqlite-utils~=2.0"], extras_require={"test": ["pytest"]}, tests_require=["google-takeout-to-sqlite[test]"], ) From 0bfe031a0d076342c237ffb0894321b1d50108b7 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Sun, 8 Aug 2021 14:18:00 -0700 Subject: [PATCH 18/21] Add back progress tracking with Rich. --- google_takeout_to_sqlite/utils.py | 71 ++++++++++++++++++++----------- setup.py | 2 +- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 00ed439..f156bc3 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -5,6 +5,8 @@ from email import policy, header import traceback import re +import os +from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn def save_my_activity(db, zf): @@ -61,32 +63,49 @@ def id_for_location_history(row): def parse_mbox(mbox_file): with open(mbox_file, "rb") as f: - delivery_date = "" - message_id = "" - lines = [] - - while True: - line = f.readline() - - is_new_record = line.startswith(b"From ") - is_eof = len(line) == 0 - - if is_eof or is_new_record: - message = b"".join(lines) - if message: - yield delivery_date, message_id, email.message_from_bytes( - message, policy=policy.compat32 - ) - else: - lines.append(line) - - if is_new_record: - (message_id, delivery_date) = re.match( - r"^From (\w+)@xxx (.+)\r\n", line.decode("utf-8") - ).groups() - lines = [] - elif is_eof: - break + f.seek(0, os.SEEK_END) + file_len = f.tell() + f.seek(0) + + progress = Progress( + TextColumn("[bold blue]{task.description}", justify="right"), + BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + DownloadColumn(), + ) + + with progress: + task = progress.add_task("[red]Processing...", total=file_len) + + delivery_date = "" + message_id = "" + lines = [] + + while True: + line = f.readline() + + progress.update(task, advance=len(line)) + + is_new_record = line.startswith(b"From ") + is_eof = len(line) == 0 + + if is_eof or is_new_record: + message = b"".join(lines) + if message: + yield delivery_date, message_id, email.message_from_bytes( + message, policy=policy.compat32 + ) + else: + lines.append(line) + + if is_new_record: + (message_id, delivery_date) = re.match( + r"^From (\w+)@xxx (.+)\r\n", line.decode("utf-8") + ).groups() + lines = [] + elif is_eof: + break def get_mbox(mbox_file): diff --git a/setup.py b/setup.py index 35cc452..d517123 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_long_description(): [console_scripts] google-takeout-to-sqlite=google_takeout_to_sqlite.cli:cli """, - install_requires=["sqlite-utils~=2.0"], + install_requires=["sqlite-utils~=2.0", "rich"], extras_require={"test": ["pytest"]}, tests_require=["google-takeout-to-sqlite[test]"], ) From 98d89bf1b03e3443575159a22639efec445462c9 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Tue, 10 Aug 2021 14:40:47 -0700 Subject: [PATCH 19/21] Use Python 3.6 email parser. Using this newer email parsing code enables parsing of attachments and easier parsing of html emails in the future. --- google_takeout_to_sqlite/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index f156bc3..b08eba2 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -2,11 +2,16 @@ import hashlib import datetime import email -from email import policy, header import traceback import re import os from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn +from email import policy, header, headerregistry + +# This policy is similar to policy.default but without strict header parsing. +# Many emails contain invalid headers that cannot be parsed according to spec. +header_factory = headerregistry.HeaderRegistry(use_default_map=False) +email_policy = policy.EmailPolicy(header_factory=header_factory) def save_my_activity(db, zf): @@ -94,7 +99,7 @@ def parse_mbox(mbox_file): message = b"".join(lines) if message: yield delivery_date, message_id, email.message_from_bytes( - message, policy=policy.compat32 + message, policy=email_policy ) else: lines.append(line) From c081ed30f6f321d3e7ee2a156d08b82eb3a315d2 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Tue, 10 Aug 2021 15:37:47 -0700 Subject: [PATCH 20/21] Use EmailMessage.get_body. This may be more robust than the tree-walking method we were using earlier, and will enable parsing of html email contents in a future commit. --- google_takeout_to_sqlite/utils.py | 37 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index b08eba2..b6e0629 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -11,6 +11,9 @@ # This policy is similar to policy.default but without strict header parsing. # Many emails contain invalid headers that cannot be parsed according to spec. header_factory = headerregistry.HeaderRegistry(use_default_map=False) +header_factory.map_to_type( + "content-disposition", headerregistry.ContentDispositionHeader +) email_policy = policy.EmailPolicy(header_factory=header_factory) @@ -143,13 +146,7 @@ def get_mbox(mbox_file): else: message["date"] = parse_mail_date(delivery_date) - body = get_email_body(email) - try: - message["body"] = body.decode("utf-8") - except UnicodeDecodeError: - message["body"] = body - except AttributeError: - message["body"] = body + message["body"] = get_email_body(email) yield message except (TypeError, ValueError, AttributeError, LookupError) as e: @@ -237,18 +234,20 @@ def get_email_body(message): """ return the email body contents """ - body = None - if message.is_multipart(): - for part in message.walk(): - if part.is_multipart(): - for subpart in part.walk(): - if subpart.get_content_type() == "text/plain": - body = subpart.get_payload(decode=True) - elif part.get_content_type() == "text/plain": - body = part.get_payload(decode=True) - elif message.get_content_type() == "text/plain": - body = message.get_payload(decode=True) - return body + try: + body = message.get_body(preferencelist=("plain",)) + except AttributeError: + return message.get_payload(decode=True) + + if not body: + return None + + try: + return body.get_content() + except: + # If headers are malformed, get_content may throw an exception. + # Fall back to get_payload method that doesn't use the ContentManager. + return body.get_payload(decode=True) def parse_mail_date(mail_date): From 8e6d487b697ce2e8ad885acf613a157bfba84c59 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Tue, 10 Aug 2021 16:00:41 -0700 Subject: [PATCH 21/21] Parse html emails to plaintext. (Only if no text/plain alternative exists) --- google_takeout_to_sqlite/utils.py | 14 +++++++--- setup.py | 2 +- tests/mbox_contents/small.gmail.mbox | 38 ++++++++++++++++++++++++++++ tests/test_gmail_import.py | 20 +++++++++++++++ 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index b6e0629..9038b6f 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -5,6 +5,7 @@ import traceback import re import os +from bs4 import BeautifulSoup from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn from email import policy, header, headerregistry @@ -235,19 +236,26 @@ def get_email_body(message): return the email body contents """ try: - body = message.get_body(preferencelist=("plain",)) + body = message.get_body(preferencelist=("plain", "html")) except AttributeError: + # Work around https://bugs.python.org/issue42892 return message.get_payload(decode=True) if not body: return None try: - return body.get_content() + content = body.get_content() except: # If headers are malformed, get_content may throw an exception. # Fall back to get_payload method that doesn't use the ContentManager. - return body.get_payload(decode=True) + content = body.get_payload(decode=True) + + if body.get_content_type() == "text/html": + doc = BeautifulSoup(content, features="lxml") + return doc.get_text(strip=True, separator=" ") + else: + return content def parse_mail_date(mail_date): diff --git a/setup.py b/setup.py index d517123..ffa0fcb 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_long_description(): [console_scripts] google-takeout-to-sqlite=google_takeout_to_sqlite.cli:cli """, - install_requires=["sqlite-utils~=2.0", "rich"], + install_requires=["sqlite-utils~=2.0", "rich", "beautifulsoup4"], extras_require={"test": ["pytest"]}, tests_require=["google-takeout-to-sqlite[test]"], ) diff --git a/tests/mbox_contents/small.gmail.mbox b/tests/mbox_contents/small.gmail.mbox index 702d23d..19729d3 100644 --- a/tests/mbox_contents/small.gmail.mbox +++ b/tests/mbox_contents/small.gmail.mbox @@ -164,3 +164,41 @@ Subject: test Content-Type: text/plain; charset=utf-8 好好学习，天天向上 +From 1318905722553808470@xxx Mon Mar 22 09:41:09 -0800 1993 +From: Nathaniel Borenstein +To: Ned Freed +Date: Mon, 22 Mar 1993 09:41:09 -0800 (PST) +Subject: Formatted text mail +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary=boundary42 + +--boundary42 +Content-Type: text/plain; charset=us-ascii + +plain text version of message + +--boundary42 +Content-Type: text/html + +html version of message + +--boundary42-- + +From 2086381499394883780@xxx Tue Jul 20 00:22:49 +0000 2021 +From: foo1@bar.net +To: foo2@bar.net +Subject: A simple example +Mime-Version: 1.0 +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: 8bit + + + + +

Acute accent

+The following two lines look have the same screen rendering:

+E with acute accent becomes É.
+E with acute accent becomes É.

+Try clicking +here.

+ diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py index a81534d..026a1c6 100644 --- a/tests/test_gmail_import.py +++ b/tests/test_gmail_import.py @@ -10,6 +10,16 @@ def test_import_gmails(): assert "mbox_emails" in set(db.table_names()) mbox_emails = list(sorted(db["mbox_emails"].rows, key=lambda r: r["id"])) assert [ + { + "From": "Nathaniel Borenstein ", + "Subject": "Formatted text mail", + "To": "Ned Freed ", + "X-GM-THRID": None, + "X-Gmail-Labels": None, + "body": "plain text version of message\r\n", + "id": "1318905722553808470", + "when": "1993-03-22 17:41:09", + }, { "id": "1529555944956622147", "when": "2016-03-23 01:57:00", @@ -30,6 +40,16 @@ def test_import_gmails(): "X-GM-THRID": "1705761119401391280", "X-Gmail-Labels": "Chat", }, + { + "From": "foo1@bar.net", + "Subject": "A simple example", + "To": "foo2@bar.net", + "X-GM-THRID": None, + "X-Gmail-Labels": None, + "body": "Acute accent The following two lines look have the same screen rendering: E with acute accent becomes É. E with acute accent becomes É. Try clicking here.", + "id": "2086381499394883780", + "when": "2021-07-20 00:22:49", + }, { "From": "Keld Jørn Simonsen ", "Subject": "[fw-general] Zend_Form and generating fields",