Skip to content

Commit 737b76e

Browse files
ChilippChilipp
authored
Merge pull request #83 from EMPD2/empd2_release
EMPD2 release
2 parents 5b20673 + fbdbf8e commit 737b76e

3,335 files changed

Lines changed: 58731 additions & 47514 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎external/tarasov.tsv‎

Lines changed: 799 additions & 0 deletions
Large diffs are not rendered by default.

‎meta.tsv‎

Lines changed: 8747 additions & 9430 deletions
Large diffs are not rendered by default.

‎postgres/EMPD2.sql‎

Lines changed: 4817 additions & 885 deletions
Large diffs are not rendered by default.

‎postgres/scripts/import_into_empd2.py‎

Lines changed: 35 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -333,27 +333,31 @@ def clean_doi(doi):
333333
list_of_errors.append(str(e))
334334
print('IntegrityError ', x, e)
335335
err += 1
336+
continue
336337
except psql.DataError as e:
337338
if str(e) not in list_of_errors:
338339
list_of_errors.append(str(e))
339340
print('DataError ', x, e)
340341
conn = psql.connect(db_url)
341342
cursor = conn.cursor()
342343
err += 1
344+
continue
343345
except AttributeError as e:
344346
if str(e) not in list_of_errors:
345347
list_of_errors.append(str(e))
346348
print('AttributeError ', x, e)
347349
conn = psql.connect(db_url)
348350
cursor = conn.cursor()
349351
err += 1
352+
continue
350353
except Exception as e:
351354
if str(e) not in list_of_errors:
352355
list_of_errors.append(str(e))
353356
print('Error ', x, e)
354357
conn = psql.connect(db_url)
355358
cursor = conn.cursor()
356359
err += 1
360+
continue
357361

358362
temperature = METADATA.iloc[x]['Temperature']
359363
precip = METADATA.iloc[x]['Precipitation']
@@ -455,32 +459,39 @@ def clean_doi(doi):
455459
conn.commit()
456460
else:
457461
VAR_ = res[0][0]
458-
samplename = row.samplename
459-
try:
460-
if row['count'] > 0:
461-
val = round(row['count'])
462-
try:
462+
try:
463+
if row['count'] > 0:
464+
val = round(row['count'])
465+
try:
466+
cursor.execute(
467+
"INSERT INTO p_counts (sampleName, var_, count) "
468+
"VALUES ('%s', %d, %d)" % (
469+
samplename, VAR_, val))
470+
conn.commit()
471+
except psql.IntegrityError as e:
472+
conn = psql.connect(db_url)
473+
cursor = conn.cursor()
474+
if 'duplicate key value violates unique constraint "p_counts_pkey"' in str(e):
475+
cursor.execute(
476+
"SELECT count FROM p_counts WHERE "
477+
"sampleName = '%s' AND var_ = %d" % (
478+
samplename, VAR_))
479+
new_val = cursor.fetchall()[0][0] + val
463480
cursor.execute(
464-
"INSERT INTO p_counts (sampleName, var_, count) "
465-
"VALUES ('%s', %d, %d)" % (
466-
samplename, VAR_, val))
481+
"UPDATE p_counts SET count=%d WHERE "
482+
"sampleName = '%s' AND var_ = %d" % (
483+
new_val, samplename, VAR_))
467484
conn.commit()
468-
except psql.IntegrityError as e:
469-
conn = psql.connect(db_url)
470-
cursor = conn.cursor()
471-
if 'duplicate key value violates unique constraint "p_counts_pkey"' in str(e):
472-
cursor.execute(
473-
"SELECT count FROM p_counts WHERE "
474-
"sampleName = '%s' AND var_ = %d" % (
475-
samplename, VAR_))
476-
new_val = cursor.fetchall()[0][0] + val
477-
cursor.execute(
478-
"UPDATE p_counts SET count=%d WHERE "
479-
"sampleName = '%s' AND var_ = %d" % (
480-
new_val, samplename, VAR_))
481-
conn.commit()
482-
except Exception:
483-
print(samplename, VAR_, "!" + str(row['count']) + "!")
485+
except Exception as e:
486+
error = (samplename, VAR_, "!" + str(row['count']) + ":" +
487+
str(e))
488+
if error not in list_of_errors:
489+
list_of_errors.append(error)
490+
print('Error ', error)
491+
conn = psql.connect(db_url)
492+
cursor = conn.cursor()
493+
err += 1
494+
continue
484495

485496

486497
assert err == 0, '\n'.join(list_of_errors)

‎postgres/scripts/tables/Country.tsv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,3 +84,4 @@ Yemen Yemen
8484
Black Sea
8585
Dead Sea
8686
Adriatic Sea
87+
Uzbekistan

‎samples/AlbaSanchez_a1.tsv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ samplename original_varname acc_varname groupid count percentage
22
AlbaSanchez_a1 Abies pinsapo Abies pinsapo TRSH 14 4.5016077
33
AlbaSanchez_a1 Alnus Alnus TRSH 1 0.32154341
44
AlbaSanchez_a1 Betula Betula TRSH 2 0.64308682
5-
AlbaSanchez_a1 Daphne gnidium t. Daphne DWAR 14 4.5016077
5+
AlbaSanchez_a1 Daphne gnidium t. Daphne TRSH 14 4.5016077
66
AlbaSanchez_a1 Erica scoparia t. Erica arborea-type TRSH 1 0.32154341
77
AlbaSanchez_a1 Olea Olea europaea TRSH 7 2.2508039
88
AlbaSanchez_a1 Pinus pinea/halepensis t. Pinus halepensis-type TRSH 38 12.21865

‎samples/AlbaSanchez_a11.tsv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
samplename original_varname acc_varname groupid count percentage
22
AlbaSanchez_a11 Abies pinsapo Abies pinsapo TRSH 4 1.1363636
3-
AlbaSanchez_a11 Daphne gnidium t. Daphne DWAR 1 0.28409091
3+
AlbaSanchez_a11 Daphne gnidium t. Daphne TRSH 1 0.28409091
44
AlbaSanchez_a11 Olea Olea europaea TRSH 8 2.2727273
55
AlbaSanchez_a11 Pinus pinea/halepensis t. Pinus halepensis-type TRSH 196 55.681818
66
AlbaSanchez_a11 Pinus pinaster Pinus pinaster TRSH 1 0.28409091

‎samples/AlbaSanchez_a14.tsv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
samplename original_varname acc_varname groupid count percentage
22
AlbaSanchez_a14 Abies pinsapo Abies pinsapo TRSH 47 15.210356
33
AlbaSanchez_a14 Corylus Corylus TRSH 1 0.3236246
4-
AlbaSanchez_a14 Daphne gnidium t. Daphne DWAR 1 0.3236246
4+
AlbaSanchez_a14 Daphne gnidium t. Daphne TRSH 1 0.3236246
55
AlbaSanchez_a14 Erica scoparia t. Erica arborea-type TRSH 3 0.97087379
66
AlbaSanchez_a14 Jasminum t. Jasminum TRSH 1 0.3236246
77
AlbaSanchez_a14 Juniperus t. Juniperus-type TRSH 3 0.97087379

‎samples/AlbaSanchez_a23.tsv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ samplename original_varname acc_varname groupid count percentage
22
AlbaSanchez_a23 Abies pinsapo Abies pinsapo TRSH 4 1.4388489
33
AlbaSanchez_a23 Alnus Alnus TRSH 2 0.71942446
44
AlbaSanchez_a23 Betula Betula TRSH 1 0.35971223
5-
AlbaSanchez_a23 Daphne gnidium t. Daphne DWAR 1 0.35971223
5+
AlbaSanchez_a23 Daphne gnidium t. Daphne TRSH 1 0.35971223
66
AlbaSanchez_a23 Erica scoparia t. Erica arborea-type TRSH 5 1.7985612
77
AlbaSanchez_a23 Eucalyptus Eucalyptus TRSH 1 0.35971223
88
AlbaSanchez_a23 Juniperus t. Juniperus-type TRSH 12 4.3165468

‎samples/AlbaSanchez_a31.tsv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
samplename original_varname acc_varname groupid count percentage
2-
AlbaSanchez_a31 Daphne gnidium t. Daphne DWAR 19 12.5
2+
AlbaSanchez_a31 Daphne gnidium t. Daphne TRSH 19 12.5
33
AlbaSanchez_a31 Juniperus t. Juniperus-type TRSH 1 0.65789474
44
AlbaSanchez_a31 Olea Olea europaea TRSH 11 7.2368421
55
AlbaSanchez_a31 Pinus pinea/halepensis t. Pinus halepensis-type TRSH 26 17.105263

0 commit comments

Comments
 (0)