Skip to content

Commit 434458a

Browse files
kylepjohnsonkylepjohnson
committed
add 1k greek corpus and conversion
1 parent ed7bb85 commit 434458a

2 files changed

Lines changed: 282 additions & 204 deletions

File tree

‎2 Import corpora.ipynb‎

Lines changed: 238 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@
3333
{
3434
"cell_type": "code",
3535
"execution_count": 2,
36-
"metadata": {},
36+
"metadata": {
37+
"collapsed": true
38+
},
3739
"outputs": [],
3840
"source": [
3941
"# See https://github.com/cltk for all official corpora\n",
@@ -105,7 +107,7 @@
105107
},
106108
{
107109
"cell_type": "code",
108-
"execution_count": 6,
110+
"execution_count": 2,
109111
"metadata": {},
110112
"outputs": [
111113
{
@@ -121,16 +123,17 @@
121123
" 'greek_lexica_perseus',\n",
122124
" 'greek_training_set_sentence_cltk',\n",
123125
" 'greek_word2vec_cltk',\n",
124-
" 'greek_text_lacus_curtius']"
126+
" 'greek_text_lacus_curtius',\n",
127+
" 'greek_text_first1kgreek']"
125128
]
126129
},
127-
"execution_count": 6,
130+
"execution_count": 2,
128131
"metadata": {},
129132
"output_type": "execute_result"
130133
}
131134
],
132135
"source": [
133-
"# Let's get a Greek corpus, too\n",
136+
"# Let's get some Greek corpora, too\n",
134137
"\n",
135138
"my_greek_downloader = CorpusImporter('greek')\n",
136139
"my_greek_downloader.list_corpora"
@@ -139,7 +142,9 @@
139142
{
140143
"cell_type": "code",
141144
"execution_count": 6,
142-
"metadata": {},
145+
"metadata": {
146+
"collapsed": true
147+
},
143148
"outputs": [],
144149
"source": [
145150
"my_greek_downloader.import_corpus('greek_text_lacus_curtius')"
@@ -152,6 +157,230 @@
152157
"Likewise, verify with `ls -l ~/cltk_data/greek/text/greek_text_lacus_curtius/plain/`"
153158
]
154159
},
160+
{
161+
"cell_type": "code",
162+
"execution_count": 3,
163+
"metadata": {
164+
"scrolled": true
165+
},
166+
"outputs": [
167+
{
168+
"name": "stdout",
169+
"output_type": "stream",
170+
"text": [
171+
"Downloaded 5% 2.22 MiB | 4.15 MiB/s \r",
172+
"Downloaded 6% 2.22 MiB | 4.15 MiB/s \r",
173+
"Downloaded 7% 2.22 MiB | 4.15 MiB/s \r",
174+
"Downloaded 8% 2.22 MiB | 4.15 MiB/s \r",
175+
"Downloaded 8% 2.22 MiB | 4.15 MiB/s \r",
176+
"Downloaded 9% 2.22 MiB | 4.15 MiB/s \r",
177+
"Downloaded 10% 5.62 MiB | 5.43 MiB/s \r",
178+
"Downloaded 11% 5.62 MiB | 5.43 MiB/s \r",
179+
"Downloaded 12% 5.62 MiB | 5.43 MiB/s \r",
180+
"Downloaded 13% 5.62 MiB | 5.43 MiB/s \r",
181+
"Downloaded 14% 5.62 MiB | 5.43 MiB/s \r",
182+
"Downloaded 15% 5.62 MiB | 5.43 MiB/s \r",
183+
"Downloaded 16% 5.62 MiB | 5.43 MiB/s \r",
184+
"Downloaded 17% 5.62 MiB | 5.43 MiB/s \r",
185+
"Downloaded 18% 5.62 MiB | 5.43 MiB/s \r",
186+
"Downloaded 19% 5.62 MiB | 5.43 MiB/s \r",
187+
"Downloaded 20% 5.62 MiB | 5.43 MiB/s \r",
188+
"Downloaded 21% 14.47 MiB | 9.42 MiB/s \r",
189+
"Downloaded 22% 14.47 MiB | 9.42 MiB/s \r",
190+
"Downloaded 23% 14.47 MiB | 9.42 MiB/s \r",
191+
"Downloaded 24% 14.47 MiB | 9.42 MiB/s \r",
192+
"Downloaded 24% 17.04 MiB | 8.32 MiB/s \r",
193+
"Downloaded 25% 17.04 MiB | 8.32 MiB/s \r",
194+
"Downloaded 26% 17.04 MiB | 8.32 MiB/s \r",
195+
"Downloaded 27% 17.04 MiB | 8.32 MiB/s \r",
196+
"Downloaded 28% 19.54 MiB | 7.33 MiB/s \r",
197+
"Downloaded 29% 19.54 MiB | 7.33 MiB/s \r",
198+
"Downloaded 30% 19.54 MiB | 7.33 MiB/s \r",
199+
"Downloaded 30% 19.54 MiB | 7.33 MiB/s \r",
200+
"Downloaded 31% 19.54 MiB | 7.33 MiB/s \r",
201+
"Downloaded 32% 19.54 MiB | 7.33 MiB/s \r",
202+
"Downloaded 33% 19.54 MiB | 7.33 MiB/s \r",
203+
"Downloaded 34% 19.54 MiB | 7.33 MiB/s \r",
204+
"Downloaded 35% 19.54 MiB | 7.33 MiB/s \r",
205+
"Downloaded 36% 19.54 MiB | 7.33 MiB/s \r",
206+
"Downloaded 37% 23.64 MiB | 7.33 MiB/s \r",
207+
"Downloaded 38% 24.18 MiB | 6.49 MiB/s \r",
208+
"Downloaded 38% 26.77 MiB | 6.18 MiB/s \r",
209+
"Downloaded 39% 26.77 MiB | 6.18 MiB/s \r",
210+
"Downloaded 40% 26.77 MiB | 6.18 MiB/s \r",
211+
"Downloaded 41% 26.77 MiB | 6.18 MiB/s \r",
212+
"Downloaded 42% 26.77 MiB | 6.18 MiB/s \r",
213+
"Downloaded 43% 26.77 MiB | 6.18 MiB/s \r",
214+
"Downloaded 44% 26.77 MiB | 6.18 MiB/s \r",
215+
"Downloaded 44% 29.04 MiB | 5.83 MiB/s \r",
216+
"Downloaded 45% 29.04 MiB | 5.83 MiB/s \r",
217+
"Downloaded 46% 32.96 MiB | 5.45 MiB/s \r",
218+
"Downloaded 46% 32.96 MiB | 5.45 MiB/s \r",
219+
"Downloaded 47% 34.78 MiB | 4.05 MiB/s \r",
220+
"Downloaded 48% 34.78 MiB | 4.05 MiB/s \r",
221+
"Downloaded 49% 34.78 MiB | 4.05 MiB/s \r",
222+
"Downloaded 49% 37.17 MiB | 4.02 MiB/s \r",
223+
"Downloaded 50% 37.17 MiB | 4.02 MiB/s \r",
224+
"Downloaded 51% 39.68 MiB | 4.07 MiB/s \r",
225+
"Downloaded 51% 39.68 MiB | 4.07 MiB/s \r",
226+
"Downloaded 52% 42.10 MiB | 3.68 MiB/s \r",
227+
"Downloaded 53% 42.10 MiB | 3.68 MiB/s \r",
228+
"Downloaded 54% 42.10 MiB | 3.68 MiB/s \r",
229+
"Downloaded 55% 42.10 MiB | 3.68 MiB/s \r",
230+
"Downloaded 56% 42.10 MiB | 3.68 MiB/s \r",
231+
"Downloaded 57% 42.10 MiB | 3.68 MiB/s \r",
232+
"Downloaded 58% 42.10 MiB | 3.68 MiB/s \r",
233+
"Downloaded 59% 42.10 MiB | 3.68 MiB/s \r",
234+
"Downloaded 60% 44.94 MiB | 4.14 MiB/s \r",
235+
"Downloaded 61% 44.94 MiB | 4.14 MiB/s \r",
236+
"Downloaded 62% 44.94 MiB | 4.14 MiB/s \r",
237+
"Downloaded 62% 44.94 MiB | 4.14 MiB/s \r",
238+
"Downloaded 63% 46.48 MiB | 3.99 MiB/s \r",
239+
"Downloaded 64% 46.48 MiB | 3.99 MiB/s \r",
240+
"Downloaded 64% 50.14 MiB | 4.04 MiB/s \r",
241+
"Downloaded 65% 50.14 MiB | 4.04 MiB/s \r",
242+
"Downloaded 65% 52.54 MiB | 4.17 MiB/s \r",
243+
"Downloaded 66% 55.90 MiB | 4.08 MiB/s \r",
244+
"Downloaded 66% 57.35 MiB | 3.87 MiB/s \r",
245+
"Downloaded 67% 59.61 MiB | 3.81 MiB/s \r",
246+
"Downloaded 67% 59.61 MiB | 3.81 MiB/s \r",
247+
"Downloaded 68% 63.14 MiB | 3.48 MiB/s \r",
248+
"Downloaded 68% 63.14 MiB | 3.48 MiB/s \r",
249+
"Downloaded 69% 65.30 MiB | 3.52 MiB/s \r",
250+
"Downloaded 70% 65.30 MiB | 3.52 MiB/s \r",
251+
"Downloaded 70% 69.03 MiB | 3.83 MiB/s \r",
252+
"Downloaded 71% 70.43 MiB | 3.62 MiB/s \r",
253+
"Downloaded 72% 70.43 MiB | 3.62 MiB/s \r",
254+
"Downloaded 72% 70.43 MiB | 3.62 MiB/s \r",
255+
"Downloaded 73% 72.05 MiB | 3.57 MiB/s \r",
256+
"Downloaded 74% 72.05 MiB | 3.57 MiB/s \r",
257+
"Downloaded 75% 74.07 MiB | 3.66 MiB/s \r",
258+
"Downloaded 76% 76.26 MiB | 3.80 MiB/s \r",
259+
"Downloaded 76% 76.26 MiB | 3.80 MiB/s \r",
260+
"Downloaded 77% 78.72 MiB | 3.79 MiB/s \r",
261+
"Downloaded 77% 78.72 MiB | 3.79 MiB/s \r",
262+
"Downloaded 78% 78.72 MiB | 3.79 MiB/s \r",
263+
"Downloaded 79% 78.72 MiB | 3.79 MiB/s \r",
264+
"Downloaded 80% 85.18 MiB | 4.70 MiB/s \r",
265+
"Downloaded 81% 85.18 MiB | 4.70 MiB/s \r",
266+
"Downloaded 82% 85.18 MiB | 4.70 MiB/s \r",
267+
"Downloaded 83% 85.18 MiB | 4.70 MiB/s \r",
268+
"Downloaded 84% 85.18 MiB | 4.70 MiB/s \r",
269+
"Downloaded 85% 85.18 MiB | 4.70 MiB/s \r",
270+
"Downloaded 86% 85.18 MiB | 4.70 MiB/s \r",
271+
"Downloaded 87% 92.76 MiB | 6.02 MiB/s \r",
272+
"Downloaded 88% 92.76 MiB | 6.02 MiB/s \r",
273+
"Downloaded 89% 92.76 MiB | 6.02 MiB/s \r",
274+
"Downloaded 89% 92.76 MiB | 6.02 MiB/s \r",
275+
"Downloaded 90% 92.76 MiB | 6.02 MiB/s \r",
276+
"Downloaded 91% 92.76 MiB | 6.02 MiB/s \r",
277+
"Downloaded 92% 92.76 MiB | 6.02 MiB/s \r",
278+
"Downloaded 93% 102.39 MiB | 7.76 MiB/s \r",
279+
"Downloaded 94% 102.39 MiB | 7.76 MiB/s \r",
280+
"Downloaded 95% 113.15 MiB | 9.19 MiB/s \r",
281+
"Downloaded 95% 113.15 MiB | 9.19 MiB/s \r",
282+
"Downloaded 96% 114.78 MiB | 9.33 MiB/s \r",
283+
"Downloaded 97% 114.78 MiB | 9.33 MiB/s \r",
284+
"Downloaded 98% 123.62 MiB | 10.40 MiB/s \r",
285+
"Downloaded 98% 125.96 MiB | 9.94 MiB/s \r",
286+
"Downloaded 98% 125.96 MiB | 9.94 MiB/s \r",
287+
"Downloaded 98% 136.67 MiB | 11.52 MiB/s \r",
288+
"Downloaded 99% 138.21 MiB | 10.27 MiB/s \r",
289+
"Downloaded 99% 146.75 MiB | 8.54 MiB/s \r",
290+
"Downloaded 99% 146.75 MiB | 8.54 MiB/s \r",
291+
"Downloaded 99% 153.79 MiB | 7.57 MiB/s \r",
292+
"Downloaded 100% 157.59 MiB | 6.26 MiB/s \r",
293+
"Downloaded 100% 160.82 MiB | 5.18 MiB/s \r",
294+
"Downloaded 100% 163.52 MiB | 5.21 MiB/s \r"
295+
]
296+
}
297+
],
298+
"source": [
299+
"my_greek_downloader.import_corpus('greek_text_first1kgreek')"
300+
]
301+
},
302+
{
303+
"cell_type": "code",
304+
"execution_count": 4,
305+
"metadata": {},
306+
"outputs": [
307+
{
308+
"name": "stdout",
309+
"output_type": "stream",
310+
"text": [
311+
"total 2176\r\n",
312+
"-rw-r--r-- 1 root root 126919 Jul 13 10:05 Committing Issues using GitHub.docx\r\n",
313+
"-rwxr-xr-x 1 root root 1889 Jul 13 10:05 cselstats.pl\r\n",
314+
"drwxr-xr-x 118 root root 4096 Jul 13 10:05 data\r\n",
315+
"-rwxr-xr-x 1 root root 1955024 Jul 13 10:05 #gelasius-kg.xml#\r\n",
316+
"-rwxr-xr-x 1 root root 2414 Jul 13 10:05 greek-justwork.txt\r\n",
317+
"-rwxr-xr-x 1 root root 3249 Jul 13 10:05 greek.txt\r\n",
318+
"-rwxr-xr-x 1 root root 19777 Jul 13 10:05 Greek-works.txt\r\n",
319+
"-rw-r--r-- 1 root root 19125 Jul 13 10:05 license.md\r\n",
320+
"-rw-r--r-- 1 root root 58346 Jul 13 10:05 new_edition_metadata.csv\r\n",
321+
"-rw-r--r-- 1 root root 697 Jul 13 10:05 pages.sh\r\n",
322+
"-rwxr-xr-x 1 root root 1901 Jul 13 10:05 pnumber.xsl\r\n",
323+
"-rw-r--r-- 1 root root 1658 Jul 13 10:05 README.md\r\n",
324+
"drwxr-xr-x 2 root root 4096 Jul 13 10:05 save\r\n",
325+
"drwxr-xr-x 48 root root 4096 Jul 13 10:05 split\r\n",
326+
"drwxr-xr-x 2 root root 4096 Jul 13 10:05 volume_xml\r\n"
327+
]
328+
}
329+
],
330+
"source": [
331+
"!ls -l ~/cltk_data/greek/text/greek_text_first1kgreek/"
332+
]
333+
},
334+
{
335+
"cell_type": "markdown",
336+
"metadata": {},
337+
"source": [
338+
"# Convert TEI XML corpus\n",
339+
"\n",
340+
"Here we'll convert the 1K Years' Greek corpus from TEI XML to plaintext"
341+
]
342+
},
343+
{
344+
"cell_type": "code",
345+
"execution_count": 3,
346+
"metadata": {
347+
"collapsed": true
348+
},
349+
"outputs": [],
350+
"source": [
351+
"from cltk.corpus.greek.tei import onekgreek_tei_xml_to_text"
352+
]
353+
},
354+
{
355+
"cell_type": "code",
356+
"execution_count": 4,
357+
"metadata": {},
358+
"outputs": [],
359+
"source": [
360+
"#! If you get the following error: 'Install `bs4` and `lxml` to parse these TEI files.'\n",
361+
"# then run: `pip install bs4 lxml`\n",
362+
"\n",
363+
"onekgreek_tei_xml_to_text()"
364+
]
365+
},
366+
{
367+
"cell_type": "code",
368+
"execution_count": 5,
369+
"metadata": {},
370+
"outputs": [
371+
{
372+
"name": "stdout",
373+
"output_type": "stream",
374+
"text": [
375+
"677\r\n"
376+
]
377+
}
378+
],
379+
"source": [
380+
"# count the converted plaintext files:\n",
381+
"!ls -l ~/cltk_data/greek/text/greek_text_first1kgreek_plaintext/ | wc -l"
382+
]
383+
},
155384
{
156385
"cell_type": "markdown",
157386
"metadata": {},
@@ -184,7 +413,9 @@
184413
{
185414
"cell_type": "code",
186415
"execution_count": 7,
187-
"metadata": {},
416+
"metadata": {
417+
"collapsed": true
418+
},
188419
"outputs": [],
189420
"source": [
190421
"my_greek_downloader.import_corpus('tlg', '~/cltk/corpora/TLG_E/')"

0 commit comments

Comments
 (0)