MediaSum Corpus + pkg_resources Issue Fix (#332)

Author: seanzhangkx8

convokit/coordination/coordination.py

@@ -2,7 +2,7 @@
 from typing import Callable, Tuple, List, Dict, Optional, Collection, Union
 import copy
 
-import pkg_resources
+from importlib import resources
 
 from convokit.model import Corpus, Speaker, Utterance
 from convokit.transformer import Transformer
@@ -376,9 +376,7 @@ def score_report(self, corpus: Corpus, scores: CoordinationScore):
 
     # helper functions
     def _compute_liwc_reverse_dict(self) -> None:
-        with open(
-            pkg_resources.resource_filename("convokit", "data/coord-liwc-patterns.txt"), "r"
-        ) as f:
+        with open(resources.files("convokit").joinpath("data/coord-liwc-patterns.txt"), "r") as f:
             all_words = []
             for line in f:
                 cat, pat = line.strip().split("\t")

convokit/politeness_collections/politeness_api/features/politeness_strategies.py

@@ -1,5 +1,4 @@
-import pkg_resources
-import os
+from importlib import resources
 
 #####
 # Word lists
@@ -107,17 +106,14 @@
 ]
 
 # Positive and negative words from Liu
-pos_filename = pkg_resources.resource_filename(
-    "convokit", os.path.join("data", "liu-positive-words.txt")
-)
-neg_filename = pkg_resources.resource_filename(
-    "convokit", os.path.join("data", "liu-negative-words.txt")
-)
+data_root = resources.files("convokit").joinpath("data")
+pos_filename = data_root.joinpath("liu-positive-words.txt")
+neg_filename = data_root.joinpath("liu-negative-words.txt")
 
 
-positive_words = set(map(lambda x: x.strip(), open(pos_filename).read().splitlines()))
+positive_words = set(map(lambda x: x.strip(), pos_filename.read_text().splitlines()))
 negative_words = set(
-    map(lambda x: x.strip(), open(neg_filename, encoding="ISO-8859-1").read().splitlines())
+    map(lambda x: x.strip(), neg_filename.read_text(encoding="ISO-8859-1").splitlines())
 )
 
 #####

convokit/politeness_collections/politeness_cscw_zh/strategy_extractor.py

@@ -1,8 +1,7 @@
 import json
-import os
 import re
 from itertools import chain
-import pkg_resources
+from importlib import resources
 from collections import defaultdict
 from typing import Dict, List, Tuple
 
@@ -14,17 +13,10 @@
 
 LEXICON_DIR = "politeness_collections/politeness_cscw_zh/lexicons"
 
-ngram_path = pkg_resources.resource_filename(
-    "convokit", os.path.join(LEXICON_DIR, "ngram_markers.json")
-)
-
-starter_path = pkg_resources.resource_filename(
-    "convokit", os.path.join(LEXICON_DIR, "starter_markers.json")
-)
-
-non_starter_path = pkg_resources.resource_filename(
-    "convokit", os.path.join(LEXICON_DIR, "non_starter_markers.json")
-)
+lexicon_root = resources.files("convokit").joinpath(LEXICON_DIR)
+ngram_path = lexicon_root.joinpath("ngram_markers.json")
+starter_path = lexicon_root.joinpath("starter_markers.json")
+non_starter_path = lexicon_root.joinpath("non_starter_markers.json")
 
 
 PLEASE_PATTERN = re.compile(r"([烦劳还]?\s?请)|([烦劳]您)")

convokit/politeness_collections/politeness_local/strategy_extractor.py

@@ -1,8 +1,7 @@
 import json
-import os
 import re
 from itertools import chain
-import pkg_resources
+from importlib import resources
 from collections import defaultdict
 from typing import Dict, List, Tuple, Set, Optional
 
@@ -14,17 +13,10 @@
 
 LEXICON_DIR = "politeness_collections/politeness_local/lexicons"
 
-ngram_path = pkg_resources.resource_filename(
-    "convokit", os.path.join(LEXICON_DIR, "ngram_markers.json")
-)
-
-starter_path = pkg_resources.resource_filename(
-    "convokit", os.path.join(LEXICON_DIR, "starter_markers.json")
-)
-
-non_starter_path = pkg_resources.resource_filename(
-    "convokit", os.path.join(LEXICON_DIR, "non_starter_markers.json")
-)
+lexicon_root = resources.files("convokit").joinpath(LEXICON_DIR)
+ngram_path = lexicon_root.joinpath("ngram_markers.json")
+starter_path = lexicon_root.joinpath("starter_markers.json")
+non_starter_path = lexicon_root.joinpath("non_starter_markers.json")
 
 # strategy functions
 

docs/source/mediasum.rst

@@ -92,9 +92,9 @@ To download directly with ConvoKit:
 For some quick stats:
 
 >>> corpus.print_summary_stats()
-Number of Speakers: 700
-Number of Utterances: 67373
-Number of Conversations: 3107
+Number of Speakers: 718483
+Number of Utterances: 13919244
+Number of Conversations: 463596
 
 Get all paraphrase pairs from the corpus metadata