pypi v0.0.2

Author: onlyphantom

README.md

@@ -14,13 +14,14 @@ For the social scientists, creating social networks from your mailbox data and a
 * Study social actors (most emails from Marketing involve Peter and Andy) and their relative influence  
 * Identify the key social groups (Sales team hangs out a lot, but the IT / product division less so)
 * Key account managers of the company (Despite being with the company only recently, Margaretha is connected to more key clients than her peers)
+* Compare distributions and patterns of email behaviors and aggregated statistics between groups of employees  
 
 
 If you're a graph theorist and looking for something more statistical:
-* Support directed and undirected graphs (**implemented in version 0.0.2**, see below)
+* Support directed and undirected graphs (**already implemented in version 0.0.2**, see below)
 * Also output statistical measurements such as centrality distribution (**planned for version 0.0.3**)
 * Betweenness, closeness, hubness, distance histograms plotting (**planned for version 0.0.3**) 
-* Exports to `.graphml` format for use in other graphing software (**implemented in version 0.0.2**)
+* Exports to `.graphml` format for use in other graphing software (**already implemented in version 0.0.2**, see below)
 
 ## Dependencies
 * Python 3.7+
@@ -155,9 +156,11 @@ All tests are located in the `/tests/` directory.
 
 ## Authors and Copyright
 
-Samuel Chan, Supertype [Supertype](https://supertype.ai)
+Samuel Chan, [Supertype](https://supertype.ai)
+ - Github: [onlyphantom](https://github.com/onlyphantom)
 
-Vincentius Christopher Calvin, Supertype [https://supertype.ai](https://supertype.ai)
+Vincentius Christopher Calvin, [Supertype](https://supertype.ai)
+ - Github: [vccalvin33](https://github.com/vccalvin33)
 
 If you find the code useful in your project, please link to this repository in your citation.
 

build/lib/emailnetwork/emails.py

@@ -1,7 +1,6 @@
-
 from dataclasses import dataclass
 from datetime import datetime
-from .utils import parse_date
+from emailnetwork.utils import parse_date
 
 @dataclass
 class EmailMeta:
@@ -22,30 +21,15 @@ def __post_init__(self):
         self.origin_domain = self.sender.domain
         self.date = parse_date(self.date)
 
-    def __eq__(self, datestring):
-        try:
-            targetdate = datetime.fromisoformat(datestring)
-        except ValueError:
-            print(ValueError)
-            return "Please use the ISO format for comparison: YYYY-MM-DD"
+    def __eq__(self, targetdate):
         if isinstance(targetdate, datetime):
             return self.date.date() == targetdate.date()
 
-    def __ge__(self, datestring):
-        try:
-            targetdate = datetime.fromisoformat(datestring)
-        except ValueError:
-            print(ValueError)
-            return "Please use the ISO format for comparison: YYYY-MM-DD"
+    def __ge__(self, targetdate):
         if isinstance(targetdate, datetime):
             return self.date.date() >= targetdate.date()
 
-    def __le__(self, datestring):
-        try:
-            targetdate = datetime.fromisoformat(datestring)
-        except ValueError:
-            print(ValueError)
-            return "Please use the ISO format for comparison: YYYY-MM-DD"
+    def __le__(self, targetdate):
         if isinstance(targetdate, datetime):
             return self.date.date() <= targetdate.date()
 
@@ -65,3 +49,7 @@ def __getitem__(self):
     def domain(self):
         return self.email.split('@')[-1] or None
 
+@dataclass
+class EmailBody:
+    subject: str = None
+    body: str = None

build/lib/emailnetwork/extract.py

@@ -1,25 +1,38 @@
+from datetime import datetime
 from email.utils import getaddresses
 from mailbox import mbox
 
-from emailnetwork.emails import EmailAddress, EmailMeta
-# try:
-#     from .emails import EmailAddress, EmailMeta
-# except:
-#     from emailnetwork.emails import EmailAddress, EmailMeta
+from mailbox import mboxMessage
+
+from emailnetwork.utils import clean_subject, clean_body
+from emailnetwork.emails import EmailAddress, EmailMeta, EmailBody
+from emailnetwork.summary import DomainSummary
+
+from emailnetwork.header import HeaderCounter
+
 
 def extract_meta(email):
 
     recs = email.get_all('To', []) + email.get_all('Resent-To', [])
     ccs = email.get_all('Cc', []) + email.get_all('Resent-Cc', [])
-    
+
     return EmailMeta(
         sender=EmailAddress(getaddresses(email.get_all('From'))[0]),
         recipients=[EmailAddress(rec) for rec in getaddresses(recs)],
         cc=[EmailAddress(cc) for cc in getaddresses(ccs)],
-        subject=email.get('Subject', '').strip() or None,
+        subject=clean_subject(email['Subject']) or None,
         date=email['Date']
     )
 
+
+def extract_body(email):
+
+    return EmailBody(
+        subject=clean_subject(email['Subject']) or None,
+        body=clean_body(email)
+    )
+
+
 class MBoxReader(object):
     """ A class that extends python's `mailbox` module to provide additional 
     functionalities such as length, date filtering and parsing. A key component of 
@@ -31,7 +44,7 @@ class MBoxReader(object):
     Args:
         object ([type]): Instantiate this class by specifying a path to an `.mbox` object
     """
-    
+
     def __init__(self, path) -> None:
         super().__init__()
         self.path = path
@@ -49,7 +62,7 @@ def count(self):
         Count the number of emails in the mbox instance.
         Helper function to implement __len__ 
         """
-        return self.mbox.keys()[-1]+1 
+        return self.mbox.keys()[-1]+1
         # return len(self.mbox.keys())
 
     def extract(self):
@@ -66,38 +79,73 @@ def extract(self):
                 print(e)
                 continue
 
-    
-    def filter_by_date(self, operator:str, datestring:str):
-        if operator not in ['>=', '==', '<=']:
+    def filter_emails(self, emailaddress=None, datestring=None, dateoperator="=="):
+        if emailaddress != None:
+            if type(emailaddress) != str:
+                raise ValueError(
+                    "Please use a valid string representing an email address")
+
+        if dateoperator not in ['>=', '==', '<=']:
             raise ValueError("Please use one of ['>=', '==', '<=']")
-        
+
+        if datestring != None:
+            try:
+                targetdate = datetime.strptime(datestring, "%Y-%m-%d")
+            except ValueError:
+                print(ValueError)
+                return "Please use the ISO format for comparison: YYYY-MM-DD"
+
         val = []
-        for email in self.mbox:
-            emailmeta = extract_meta(email)
-            if operator == '>=':
-                if emailmeta >= datestring:
-                    val.append(emailmeta)
-            elif operator == '==':
-                if emailmeta == datestring:
-                    val.append(emailmeta)
-            elif operator == '<=':
-                if emailmeta <= datestring:
+        if emailaddress == None and datestring == None:
+            for email in self.mbox:
+                emailmeta = extract_meta(email)
+                val.append(emailmeta)
+        elif emailaddress != None and datestring == None:
+            for email in self.mbox:
+                emailmeta = extract_meta(email)
+                checkers = [emailmeta.sender.email] + [recipient.email for recipient in emailmeta.recipients]
+                if emailaddress in checkers:
                     val.append(emailmeta)
-        return val
+        elif emailaddress == None and datestring != None:
+            for email in self.mbox:
+                emailmeta = extract_meta(email)
+                if dateoperator == '>=':
+                    if emailmeta >= targetdate:
+                        val.append(emailmeta)
+                elif dateoperator == '==':
+                    if emailmeta == targetdate:
+                        val.append(emailmeta)
+                elif dateoperator == '<=':
+                    if emailmeta <= targetdate:
+                        val.append(emailmeta)
+        else:
+            for email in self.mbox:
+                emailmeta = extract_meta(email)
+                checkers = [emailmeta.sender.email] + [recipient.email for recipient in emailmeta.recipients]
+                if emailaddress in checkers:
+                    if dateoperator == '>=':
+                        if emailmeta >= targetdate:
+                            val.append(emailmeta)
+                    elif dateoperator == '==':
+                        if emailmeta == targetdate:
+                            val.append(emailmeta)
+                    elif dateoperator == '<=':
+                        if emailmeta <= targetdate:
+                            val.append(emailmeta)
 
+        return val
 
 
 if __name__ == '__main__':
-    # reader = MBoxReader('/Users/samuel/Footprints/samuel-supertype.mbox')
-    reader = MBoxReader('/Users/samuel/Footprints/emailnetwork/emailnetwork/tests/test.mbox')
-    print(f'{len(reader)} emails in the sample mbox.')
-    # email = reader.mbox[646]
-    email = reader.mbox[0]
-    emailmsg = extract_meta(email)
+    reader = MBoxReader('/Users/samuel/Footprints/samuel-supertype.mbox')
+    # reader = MBoxReader('/Users/vincentiuscalvin/Documents/Supertype/mbox-dataset/Ori_Sample_01.mbox')
+    headers = HeaderCounter(reader)
+    k = headers.keys()
+    spamheaders = list(filter(lambda v: "spam" in v.lower(), k))
+    
+    summary = DomainSummary(reader)
 
-    thisyearmails = reader.filter_by_date(">=", "2021-01-05")
-    # print(emailmsg.recipients)
-    # print(emailmsg.recipients[0].domain)
-    emails = reader.extract()
-    #[email.origin_domain for email in emails]
-    
+    email = reader.mbox[1]
+    emailmsg = extract_meta(email)
+    emailbody = extract_body(email)
+    mails = reader.filter_emails(datestring='2020-12-31', dateoperator="==")

build/lib/emailnetwork/graph.py

@@ -210,7 +210,8 @@ def plot_undirected(reader:MBoxReader, layout:str='shell', graphml:bool=False):
 if __name__ == '__main__':
     MBOX_PATH = f'{os.path.dirname(__file__)}/tests/test.mbox'
 
-    reader = MBoxReader('/Users/samuel/Footprints/emailnetwork/emailnetwork/tests/test.mbox')
+    # reader = MBoxReader('/Users/samuel/Footprints/emailnetwork/emailnetwork/tests/test.mbox')
+    reader = MBoxReader('/Users/vincentiuscalvin/Documents/Supertype/mbox-dataset/Ori_Sample_01.mbox')
     # reader = MBoxReader('/Users/samuel/Footprints/samuel-supertype.mbox')
     # plot_single_directed(reader,300)
     # plot_single_directed(reader, 1, True)

build/lib/emailnetwork/header.py

@@ -0,0 +1,57 @@
+from collections import Counter
+from email.header import decode_header
+from emailnetwork.utils import clean_subject
+
+
+class HeaderCounter(Counter):
+    """[summary]
+
+    Args:
+        Counter ([type]): [description]
+    """
+
+    def __init__(self, reader):
+        super().__init__()
+        self = self.build_from(reader)
+
+    def __str__(self):
+        return f'{self.most_common()}'
+
+    def build_from(self, reader):
+        for email in reader:
+            for k in email.keys():
+                self[k] += 1
+
+        return self
+
+    def histogram(self, n=25):
+        from matplotlib import pyplot as plt
+        plt.style.use('fivethirtyeight')
+        k, v = (list(self.keys())[:n], list(self.values())[:n])
+        fig = plt.figure(figsize=(7, 10))
+        ax = fig.add_subplot(111)
+        y_pos = [i for i in range(n)]
+        ax.barh(y_pos, v, color='plum')
+        ax.set_yticks(y_pos)
+        ax.set_yticklabels(k)
+        ax.invert_yaxis()
+        ax.set_xlabel('Frequency')
+        ax.set_title('Email Header Analysis')
+        plt.tight_layout()
+        plt.show()
+
+
+if __name__ == '__main__':
+    from emailnetwork.extract import MBoxReader
+    # reader = MBoxReader('/Users/samuel/Footprints/samuel-supertype.mbox')
+    reader = MBoxReader('/Users/vincentiuscalvin/Documents/Supertype/mbox-dataset/Ori_Sample_01.mbox')
+    headers = HeaderCounter(reader)
+
+    k = headers.keys()
+
+    containspam = list(filter(lambda v: "spam" in v.lower(), k))
+
+    for email in reader:
+        for key in email.keys():
+            if key in containspam:
+                print({key: decode_header(email[key])})