diff --git a/README.md b/README.md index 3168098..9a7e710 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,12 @@ Your location history records latitude, longitude and timestame for where Google $ google-takeout-to-sqlite location-history takeout.db ~/Downloads/takeout-20190530.zip +## Email History + +You can import your emails from your Gmail mbox using this command: + + $ google-takeout-to-sqlite mbox takeout.db ~/Downloads/gmail.mbox + ## Browsing your data with Datasette Once you have imported Google data into a SQLite database file you can browse your data using [Datasette](https://github.com/simonw/datasette). Install Datasette like so: diff --git a/google_takeout_to_sqlite/cli.py b/google_takeout_to_sqlite/cli.py index a40cc4f..48572e6 100644 --- a/google_takeout_to_sqlite/cli.py +++ b/google_takeout_to_sqlite/cli.py @@ -47,3 +47,24 @@ def my_activity(db_path, zip_path): db = sqlite_utils.Database(db_path) zf = zipfile.ZipFile(zip_path) utils.save_location_history(db, zf) + + +@cli.command(name="mbox") +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument( + "mbox_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +def my_mbox(db_path, mbox_path): + """ + Import all emails from Gmail mbox to SQLite + + Usage: google-takeout-to-sqlite mbox mygmail.db /path/to/gmail.mbox + """ + db = sqlite_utils.Database(db_path) + utils.save_emails(db, mbox_path) diff --git a/google_takeout_to_sqlite/utils.py b/google_takeout_to_sqlite/utils.py index 428843c..9038b6f 100644 --- a/google_takeout_to_sqlite/utils.py +++ b/google_takeout_to_sqlite/utils.py @@ -1,6 +1,21 @@ import json import hashlib import datetime +import email +import traceback +import re +import os +from bs4 import BeautifulSoup +from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn +from email import policy, header, headerregistry + +# This policy is similar to policy.default but without strict header parsing. +# Many emails contain invalid headers that cannot be parsed according to spec. +header_factory = headerregistry.HeaderRegistry(use_default_map=False) +header_factory.map_to_type( + "content-disposition", headerregistry.ContentDispositionHeader +) +email_policy = policy.EmailPolicy(header_factory=header_factory) def save_my_activity(db, zf): @@ -53,3 +68,201 @@ def id_for_location_history(row): datetime.datetime.utcfromtimestamp(int(row["timestampMs"]) / 1000).isoformat(), first_six, ) + + +def parse_mbox(mbox_file): + with open(mbox_file, "rb") as f: + f.seek(0, os.SEEK_END) + file_len = f.tell() + f.seek(0) + + progress = Progress( + TextColumn("[bold blue]{task.description}", justify="right"), + BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + DownloadColumn(), + ) + + with progress: + task = progress.add_task("[red]Processing...", total=file_len) + + delivery_date = "" + message_id = "" + lines = [] + + while True: + line = f.readline() + + progress.update(task, advance=len(line)) + + is_new_record = line.startswith(b"From ") + is_eof = len(line) == 0 + + if is_eof or is_new_record: + message = b"".join(lines) + if message: + yield delivery_date, message_id, email.message_from_bytes( + message, policy=email_policy + ) + else: + lines.append(line) + + if is_new_record: + (message_id, delivery_date) = re.match( + r"^From (\w+)@xxx (.+)\r\n", line.decode("utf-8") + ).groups() + lines = [] + elif is_eof: + break + + +def get_mbox(mbox_file): + num_errors = 0 + + # These are all the Gmail email fields available + # ['X-GM-THRID', 'X-Gmail-Labels', 'Delivered-To', 'Received', 'Received', + # 'Return-Path', 'Received', 'Received-SPF', 'Authentication-Results', + # 'Received', 'Mailing-List', 'Precedence', 'List-Post', 'List-Help', + # 'List-Unsubscribe', 'List-Subscribe', 'Delivered-To', 'Received', + # 'Message-ID', 'Date', 'From', 'To', 'MIME-Version', 'Content-Type', + # 'Content-Transfer-Encoding', 'X-Nabble-From', 'X-pstn-neptune', + # 'X-pstn-levels', 'X-pstn-settings', 'X-pstn-addresses', 'Subject'] + + for delivery_date, gmail_message_id, email in parse_mbox(mbox_file): + try: + message = {} + message["Message-Id"] = email["Message-Id"] + if message["Message-Id"] is None: + message["Message-Id"] = gmail_message_id + message["X-GM-THRID"] = email["X-GM-THRID"] + message["X-Gmail-Labels"] = email["X-Gmail-Labels"] + + message["From"] = get_email_header(email, "From") + message["To"] = get_email_header(email, "To") + message["Subject"] = get_email_header(email, "Subject") + + if "Date" in email: + message["date"] = parse_mail_date(email["Date"]) + else: + message["date"] = parse_mail_date(delivery_date) + + message["body"] = get_email_body(email) + + yield message + except (TypeError, ValueError, AttributeError, LookupError) as e: + # How does this project want to handle logging? For now we're just + # printing out variables + num_errors = num_errors + 1 + print("Errors: {}".format(num_errors)) + print(traceback.format_exc()) + continue + + +def save_emails(db, mbox_file): + """ + Import Gmail mbox from google takeout + """ + if not db["mbox_emails"].exists(): + db["mbox_emails"].create( + { + "id": str, + "X-GM-THRID": str, + "X-Gmail-Labels": str, + "From": str, + "To": str, + "Subject": str, + "when": str, + "body": str, + }, + pk="id", + ) + + db["mbox_emails"].upsert_all( + ( + { + "id": message["Message-Id"], + "X-GM-THRID": message["X-GM-THRID"], + "X-Gmail-Labels": message["X-Gmail-Labels"], + "From": message["From"], + "To": message["To"], + "Subject": message["Subject"], + "when": message["date"], + "body": message["body"], + } + for message in get_mbox(mbox_file) + ), + pk="id", + alter=True, + ) + print("Finished loading emails into {}.".format(mbox_file)) + print('Enabling full text search on "body" and "Subject" fields') + db["mbox_emails"].enable_fts(["body", "Subject"]) + print("Finished!") + + +def get_email_header(message, name, failobj=None): + # get will either return a str, email.header.Header, or None. + # This function converts the Header to a str if one is returned. + value = message.get(name) + if value is None: + return failobj + else: + try: + return decode_rfc_2047_str(value) + except: + # If the value is invalid, return the un-decoded string. + return value + + +def decode_rfc_2047_str(value): + parts = [] + + for part, enc in header.decode_header(value): + try: + part = part.decode(enc or "utf-8") + except LookupError: # encoding not found + part = part.decode("utf-8") + except AttributeError: # part was already a str + pass + + parts.append(part) + + return "".join(parts) + + +def get_email_body(message): + """ + return the email body contents + """ + try: + body = message.get_body(preferencelist=("plain", "html")) + except AttributeError: + # Work around https://bugs.python.org/issue42892 + return message.get_payload(decode=True) + + if not body: + return None + + try: + content = body.get_content() + except: + # If headers are malformed, get_content may throw an exception. + # Fall back to get_payload method that doesn't use the ContentManager. + content = body.get_payload(decode=True) + + if body.get_content_type() == "text/html": + doc = BeautifulSoup(content, features="lxml") + return doc.get_text(strip=True, separator=" ") + else: + return content + + +def parse_mail_date(mail_date): + datetime_tuple = email.utils.parsedate_tz(mail_date) + if not datetime_tuple: + return "" + + unix_time = email.utils.mktime_tz(datetime_tuple) + mail_date_iso8601 = datetime.datetime.utcfromtimestamp(unix_time).isoformat(" ") + return mail_date_iso8601 diff --git a/setup.py b/setup.py index 5dd8d1f..ffa0fcb 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_long_description(): [console_scripts] google-takeout-to-sqlite=google_takeout_to_sqlite.cli:cli """, - install_requires=["sqlite-utils~=1.11"], + install_requires=["sqlite-utils~=2.0", "rich", "beautifulsoup4"], extras_require={"test": ["pytest"]}, tests_require=["google-takeout-to-sqlite[test]"], ) diff --git a/tests/mbox_contents/small.gmail.mbox b/tests/mbox_contents/small.gmail.mbox new file mode 100644 index 0000000..19729d3 --- /dev/null +++ b/tests/mbox_contents/small.gmail.mbox @@ -0,0 +1,204 @@ +From 1277085061787347926@xxx Tue Aug 05 08:00:23 +0000 2008 +X-GM-THRID: 1277085061787347926 +X-Gmail-Labels: Unread +Delivered-To: asdfereasdf@gmail.com +Received: by 10.142.98.16 with SMTP id v16cs5204wfb; + Tue, 5 Aug 2008 01:00:23 -0700 (PDT) +Received: by 10.151.26.12 with SMTP id d12mr926013ybj.145.1217923223126; + Tue, 05 Aug 2008 01:00:23 -0700 (PDT) +Return-Path: +Received: from www.zend.com (lists.zend.com [67.15.86.102]) + by mx.google.com with SMTP id 6si1509903yxg.6.2008.08.05.01.00.22; + Tue, 05 Aug 2008 01:00:23 -0700 (PDT) +Received-SPF: pass (google.com: domain of fw-general-return-20503-test=gmail.com@lists.zend.com designates 67.15.86.102 as permitted sender) client-ip=67.15.86.102; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of fw-general-return-20503-test=gmail.com@lists.zend.com designates 67.15.86.102 as permitted sender) smtp.mail=fw-general-return-20503-test=gmail.com@lists.zend.com +Received: (qmail 28326 invoked by uid 505); 5 Aug 2008 08:00:15 -0000 +Mailing-List: contact fw-general-help@lists.zend.com; run by ezmlm +Precedence: bulk +List-Post: +List-Help: +List-Unsubscribe: +List-Subscribe: +Delivered-To: mailing list fw-general@lists.zend.com +Received: (qmail 28319 invoked from network); 5 Aug 2008 08:00:15 -0000 +Message-ID: <18826312.post@talk.nabble.com> +Date: Tue, 5 Aug 2008 01:00:12 -0700 (PDT) +From: =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= +To: fw-general@lists.zend.com +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +X-Nabble-From: personasdflkj@gmail.com +X-pstn-neptune: 0/0/0.00/0 +X-pstn-levels: (S: 5.98501/99.90000 CV:99.0000 P:95.9108 M:88.1613 C:98.6951 ) +X-pstn-settings: 1 (0.1500:0.1500) cv gt3 gt2 gt1 p m c +X-pstn-addresses: from [638/31] +Subject: [fw-general] Zend_Form and generating fields + + +Unfortunately it is slow! For 10 products it takes 0.6 sec. to generate. +Is there a better (more efficient) method to build such forms via Zend_Form? + +The same I noticed when tried to create a select element which contained +many options (i.e. list of countries). Without ajax (autocomplete) it takes +ages to generate and seems to be useless in this case. Shame. + +I wonder if Zend_Form can be used when it comes to generate a lot of +inputs/options in select or I`m forced to create it by hand? + + + +From 1278212428183604564@xxx Sun Aug 17 18:39:23 +0000 2008 +X-GM-THRID: 1278204036336346264 +X-Gmail-Labels: Unread +Delivered-To: testasdfasdf@gmail.com +Received: by 10.142.98.16 with SMTP id v16cs546946wfb; + Sun, 17 Aug 2008 11:39:24 -0700 (PDT) +Received: by 10.90.100.17 with SMTP id x17mr545483agb.48.1218998363996; + Sun, 17 Aug 2008 11:39:23 -0700 (PDT) +Return-Path: +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) + by mx.google.com with ESMTP id c44si5785715hsc.16.2008.08.17.11.39.23; + Sun, 17 Aug 2008 11:39:23 -0700 (PDT) +Received-SPF: pass (google.com: domain of gnumed-devel-bounces+asdflkjer=gmail.com@gnu.org designates 199.232.76.165 as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of gnumed-devel-bounces+asdflkjelrkj=gmail.com@gnu.org designates 199.232.76.165 as permitted sender) smtp.mail=gnumed-devel-bounces+asdlkjwer=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:51303 helo=lists.gnu.org) + by lists.gnu.org with esmtp (Exim 4.43) + id 1KUn9v-0005Uo-D7 + for lkwelrkj@gmail.com; Sun, 17 Aug 2008 14:39:23 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) + id 1KUn9s-0005Sa-Ct + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:20 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) + id 1KUn9q-0005QU-TY + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:20 -0400 +Received: from [199.232.76.173] (port=33439 helo=monty-python.gnu.org) + by lists.gnu.org with esmtp (Exim 4.43) id 1KUn9q-0005Q4-No + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:18 -0400 +Received: from mail.gmx.net ([213.165.64.20]:56165) + by monty-python.gnu.org with smtp (Exim 4.60) + (envelope-from ) id 1KUn9q-0008Gg-44 + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 14:39:18 -0400 +Received: (qmail invoked by alias); 17 Aug 2008 18:39:16 -0000 +Received: from A7ee9.a.strato-dslnet.de (EHLO merkur.person.loc) + [89.62.126.233] + by mail.gmx.net (mp066) with SMTP; 17 Aug 2008 20:39:16 +0200 +X-Authenticated: #1433807 +X-Provags-ID: V01U2FsdGVkX1+dsZZuQ0OdkW7jgzdpHkRRth5+XhDeDDCDx7naLk + 3bct2wCsMl/clU +Received: from ncq by merkur.person.loc with local (Exim 4.69) + (envelope-from ) id 1KUn9n-0003eu-Ew + for gnumed-devel@gnu.org; Sun, 17 Aug 2008 20:39:15 +0200 +Date: Sun, 17 Aug 2008 20:39:15 +0200 +From: Person Person +To: gnumed-devel@gnu.org +Subject: Re: [Gnumed-devel] Tree view formatting +Message-ID: <20080817183915.GM3992@merkur.person.loc> +Mail-Followup-To: gnumed-devel@gnu.org +References: <272a08710808170925y47484f3fmcfb26f8686727762@mail.gmail.com> + <20080817170549.GH3992@merkur.person.loc> + <272a08710808171034o239f6167q3f7039727802dd09@mail.gmail.com> + <20080817180622.GK3992@merkur.person.loc> + <272a08710808171109x4b6429a1g89f6e594408080db@mail.gmail.com> +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline +In-Reply-To: <272a08710808171109x4b6429a1g89f6e594408080db@mail.gmail.com> +User-Agent: Mutt/1.5.18 (2008-05-17) +X-Y-GMX-Trusted: 0 +X-FuHaFi: 0.6899999999999999 +X-detected-kernel: by monty-python.gnu.org: Genre and OS details not + recognized. +X-BeenThere: gnumed-devel@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: gnumed-devel.gnu.org +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: gnumed-devel-bounces+asdlfkj=gmail.com@gnu.org +Errors-To: gnumed-devel-bounces+asdflkj=gmail.com@gnu.org + +On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote: + +> when you say you have changed it ... can I assume it will make the 0.3.0 release +yes + +> or is the release READY +I hope it is "ready" so I can release it within the next few +days. I usually wait a few days to see whether any errors +show up. That's why we need you guys to test like mad. + +> and all our future discussions on this list +> will from now on to be implemented on the 0.3.1 ? + +Not quite yet. And, rather 0.3+. + +Person + + +_______________________________________________ +Gnumed-devel mailing list +Gnumed-devel@gnu.org +http://lists.gnu.org/mailman/listinfo/gnumed-devel +From 1529555944956622147@xxx Wed Mar 23 01:57:00 +0000 2016 +X-GM-THRID: 1529553825574740118 +X-Gmail-Labels: Chat +From: Person Person +MIME-Version: 1.0 +Content-Type: text/plain + +Nothing worse than being without good wifi + +From 1705761119401391280@xxx Tue Jul 20 00:22:49 +0000 2021 +From: 罗 <123@example.net> +To: Person Person +X-GM-THRID: 1705761119401391280 +X-Gmail-Labels: Chat +MIME-Version: 1.0 +Subject: test +Content-Type: text/plain; charset=utf-8 + +好好学习，天天向上 +From 1318905722553808470@xxx Mon Mar 22 09:41:09 -0800 1993 +From: Nathaniel Borenstein +To: Ned Freed +Date: Mon, 22 Mar 1993 09:41:09 -0800 (PST) +Subject: Formatted text mail +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary=boundary42 + +--boundary42 +Content-Type: text/plain; charset=us-ascii + +plain text version of message + +--boundary42 +Content-Type: text/html + +html version of message + +--boundary42-- + +From 2086381499394883780@xxx Tue Jul 20 00:22:49 +0000 2021 +From: foo1@bar.net +To: foo2@bar.net +Subject: A simple example +Mime-Version: 1.0 +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: 8bit + + + + +

Acute accent

+The following two lines look have the same screen rendering:

+E with acute accent becomes É.
+E with acute accent becomes É.

+Try clicking +here.

+ diff --git a/tests/test_gmail_import.py b/tests/test_gmail_import.py new file mode 100644 index 0000000..026a1c6 --- /dev/null +++ b/tests/test_gmail_import.py @@ -0,0 +1,91 @@ +from google_takeout_to_sqlite.utils import save_emails +import pathlib +import sqlite_utils + + +def test_import_gmails(): + path = pathlib.Path(__file__).parent / "mbox_contents/small.gmail.mbox" + db = sqlite_utils.Database(memory=True) + save_emails(db, path) + assert "mbox_emails" in set(db.table_names()) + mbox_emails = list(sorted(db["mbox_emails"].rows, key=lambda r: r["id"])) + assert [ + { + "From": "Nathaniel Borenstein ", + "Subject": "Formatted text mail", + "To": "Ned Freed ", + "X-GM-THRID": None, + "X-Gmail-Labels": None, + "body": "plain text version of message\r\n", + "id": "1318905722553808470", + "when": "1993-03-22 17:41:09", + }, + { + "id": "1529555944956622147", + "when": "2016-03-23 01:57:00", + "From": "Person Person ", + "body": "Nothing worse than being without good wifi\r\n\r\n", + "Subject": None, + "To": None, + "X-GM-THRID": "1529553825574740118", + "X-Gmail-Labels": "Chat", + }, + { + "Subject": "test", + "From": "罗 <123@example.net>", + "To": "Person Person ", + "id": "1705761119401391280", + "body": "好好学习，天天向上\r\n", + "when": "2021-07-20 00:22:49", + "X-GM-THRID": "1705761119401391280", + "X-Gmail-Labels": "Chat", + }, + { + "From": "foo1@bar.net", + "Subject": "A simple example", + "To": "foo2@bar.net", + "X-GM-THRID": None, + "X-Gmail-Labels": None, + "body": "Acute accent The following two lines look have the same screen rendering: E with acute accent becomes É. E with acute accent becomes É. Try clicking here.", + "id": "2086381499394883780", + "when": "2021-07-20 00:22:49", + }, + { + "From": "Keld Jørn Simonsen ", + "Subject": "[fw-general] Zend_Form and generating fields", + "To": "fw-general@lists.zend.com", + "X-GM-THRID": "1277085061787347926", + "X-Gmail-Labels": "Unread", + "body": "\r\nUnfortunately it is slow! For 10 products it takes 0.6 sec. to" + " generate.\r\nIs there a better (more efficient) method to build s" + "uch forms via Zend_Form?\r\n\r\nThe same I noticed when tried to" + " create a select element which contained\r\nmany options (i.e. lis" + "t of countries). Without ajax (autocomplete) it takes\r\nages to g" + "enerate and seems to be useless in this case. Shame.\r\n\r\nI wo" + "nder if Zend_Form can be used when it comes to generate a lot of" + "\r\ninputs/options in select or I`m forced to create it by han" + "d?\r\n\r\n\r\n\r\n", + "id": "<18826312.post@talk.nabble.com>", + "when": "2008-08-05 08:00:12", + }, + { + "From": "Person Person ", + "Subject": "Re: [Gnumed-devel] Tree view formatting", + "To": "gnumed-devel@gnu.org", + "X-GM-THRID": "1278204036336346264", + "X-Gmail-Labels": "Unread", + "body": "On Sun, Aug 17, 2008 at 03:09:55PM -0300, Bob Luz wrote:\r\n\r\n" + "> when you say you have changed it ... can I assume it will make" + " the 0.3.0 release\r\nyes\r\n\r\n> or is the release READY\r\nI " + 'hope it is "ready" so I can release it within the next few\r\ndays' + ". I usually wait a few days to see whether any errors\r\nshow up. " + "That's why we need you guys to test like mad.\r\n\r\n> and all o" + "ur future discussions on this list\r\n> will from now on to be imp" + "lemented on the 0.3.1 ?\r\n\r\nNot quite yet. And, rather 0.3+.\r" + "\n\r\nPerson\r\n\r\n\r\n_________________________________________" + "______\r\nGnumed-devel mailing list\r\nGnumed-devel@gnu.org\r\nhtt" + "p://lists.gnu.org/mailman/listinfo/gnumed-devel\r\n", + "id": "<20080817183915.GM3992@merkur.person.loc>", + "when": "2008-08-17 18:39:15", + }, + ] == mbox_emails