From 792e12235ca78d78e34f24230eb6229722644ad9 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 16 Mar 2019 23:13:09 +0100 Subject: [PATCH 001/119] Added test data --- samples/test_a1.tsv | 4 ++++ test.tsv | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 samples/test_a1.tsv create mode 100644 test.tsv diff --git a/samples/test_a1.tsv b/samples/test_a1.tsv new file mode 100644 index 000000000..f4c26bed3 --- /dev/null +++ b/samples/test_a1.tsv @@ -0,0 +1,4 @@ +var_ samplename count acc_var_ original_varname acc_varname groupid groupname higher_groupid included_in_percent_sum make_percent higher_groupname group_order sums percentage +5 test_a1 45 2 Pinus Pinus TRSH Trees & Shrubs TRSH TRUE TRUE Trees & Shrubs 0 100 45 +6 test_a1 6 3 Exotic Exotics (counted) NOPO Nonpollen NOPO FALSE FALSE Nonpollen 30 100 NaN +10 test_a1 55 10 Umbelliferae Apiaceae HERB Herbs HERB TRUE TRUE Herbs 5 100 55 \ No newline at end of file diff --git a/test.tsv b/test.tsv new file mode 100644 index 000000000..2d3ce2247 --- /dev/null +++ b/test.tsv @@ -0,0 +1,2 @@ +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 +test_a1 orig_test_a1 somewhere France 10 50 340 X Actually in Germany 30 \ No newline at end of file From 6b4918692fd27ac985f1a1a7d2ad7d05134ed82a Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 16 Mar 2019 23:21:04 +0100 Subject: [PATCH 002/119] added temperature and precip --- test.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.tsv b/test.tsv index 2d3ce2247..7882002d6 100644 --- a/test.tsv +++ b/test.tsv @@ -1,2 +1,2 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 -test_a1 orig_test_a1 somewhere France 10 50 340 X Actually in Germany 30 \ No newline at end of file +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 +test_a1 orig_test_a1 somewhere France 10 50 340 "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" X Actually in Germany 30 \ No newline at end of file From 8d9c65e893b08fc51334f95801617efb4ee232c1 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 18 Mar 2019 02:44:02 +0100 Subject: [PATCH 003/119] Use pytest.ini from tests submodule --- pytest.ini | 3 --- tests | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 11b682856..000000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -python_files = tests/*.py -python_functions = test_* fix_* diff --git a/tests b/tests index 818291c45..28596e3f4 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 818291c45fd6db518aa309293a970f5a4482a760 +Subproject commit 28596e3f4795e6bfa3db22912dce4d5d079fbcc1 From 85042e0071086e3c2ece801a90e015c44d9077a6 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 18 Mar 2019 03:58:00 +0100 Subject: [PATCH 004/119] Updated tests --- tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests b/tests index 28596e3f4..f5565a00e 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 28596e3f4795e6bfa3db22912dce4d5d079fbcc1 +Subproject commit f5565a00e660708d1c8a390819aa8dc3940b8871 From c6b5673c590f88907ce871743c83a367db42463c Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 18 Mar 2019 15:22:29 +0100 Subject: [PATCH 005/119] Updated tests --- tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests b/tests index f5565a00e..ab127be47 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit f5565a00e660708d1c8a390819aa8dc3940b8871 +Subproject commit ab127be474d4b9815b9cd3f75d0cb5d14cfea894 From 0d12e84e56d33850fad44f34237970d7e69e3344 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Tue, 19 Mar 2019 13:07:27 +0100 Subject: [PATCH 006/119] replaced spaces with tabs --- meta.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.tsv b/meta.tsv index cf7fa26c8..bb2184731 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1 +1 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 From 4aa3dfa6a71f849ee6e1c0bf349424fc639ec1ae Mon Sep 17 00:00:00 2001 From: Chilipp Date: Tue, 19 Mar 2019 13:07:47 +0100 Subject: [PATCH 007/119] updated tests --- tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests b/tests index ab127be47..f3377532d 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit ab127be474d4b9815b9cd3f75d0cb5d14cfea894 +Subproject commit f3377532dec0e0f2815b500d7ce7709741e5139d From 33685e7f0077137f2f7d7141025ff4037a6f88cc Mon Sep 17 00:00:00 2001 From: Chilipp Date: Tue, 19 Mar 2019 15:41:24 +0100 Subject: [PATCH 008/119] Added okexcept column --- meta.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.tsv b/meta.tsv index bb2184731..9c9cfeb6f 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1 +1 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept From f82bc52a950223f646c9a35b9f21e85468042491 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Tue, 19 Mar 2019 17:05:47 +0100 Subject: [PATCH 009/119] Updated submodule --- tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests b/tests index f3377532d..3fdc3bf6a 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit f3377532dec0e0f2815b500d7ce7709741e5139d +Subproject commit 3fdc3bf6a0fbdc3c730d28d7db2209b6477ae96b From 5c3d1c54d5d6ac0a6318f95e117dfb698a470b3d Mon Sep 17 00:00:00 2001 From: u14652 Date: Tue, 19 Mar 2019 17:09:52 +0000 Subject: [PATCH 010/119] Accept wrong Country for all samples --- test.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.tsv b/test.tsv index 7882002d6..0419b9fa8 100644 --- a/test.tsv +++ b/test.tsv @@ -1,2 +1,2 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 -test_a1 orig_test_a1 somewhere France 10 50 340 "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" X Actually in Germany 30 \ No newline at end of file +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country, From e6bae7d7a6b239ce0663e56c625b5b60bc307f97 Mon Sep 17 00:00:00 2001 From: u14652 Date: Tue, 19 Mar 2019 17:11:15 +0000 Subject: [PATCH 011/119] Accept wrong Country for all samples --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 0419b9fa8..770e119d4 100644 --- a/test.tsv +++ b/test.tsv @@ -1,2 +1,2 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country, +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country,Country, From 763ea384335f0bf4e4ca9cd2144e1bffd7e017e4 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Tue, 19 Mar 2019 18:20:42 +0100 Subject: [PATCH 012/119] Revert "Accept wrong Country for all samples" This reverts commit e6bae7d7a6b239ce0663e56c625b5b60bc307f97. --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 770e119d4..0419b9fa8 100644 --- a/test.tsv +++ b/test.tsv @@ -1,2 +1,2 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country,Country, +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country, From 12039ab36bfc21ccdea33d35becba773d594cb19 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Tue, 19 Mar 2019 18:20:44 +0100 Subject: [PATCH 013/119] Revert "Accept wrong Country for all samples" This reverts commit 5c3d1c54d5d6ac0a6318f95e117dfb698a470b3d. --- test.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.tsv b/test.tsv index 0419b9fa8..7882002d6 100644 --- a/test.tsv +++ b/test.tsv @@ -1,2 +1,2 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country, +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 +test_a1 orig_test_a1 somewhere France 10 50 340 "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" X Actually in Germany 30 \ No newline at end of file From f79a6072128923b24e955fa7da9a6d48c9c24ba9 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 20 Mar 2019 07:42:32 +0100 Subject: [PATCH 014/119] added second test sample --- samples/test_a2.tsv | 4 ++++ test.tsv | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 samples/test_a2.tsv diff --git a/samples/test_a2.tsv b/samples/test_a2.tsv new file mode 100644 index 000000000..f4c26bed3 --- /dev/null +++ b/samples/test_a2.tsv @@ -0,0 +1,4 @@ +var_ samplename count acc_var_ original_varname acc_varname groupid groupname higher_groupid included_in_percent_sum make_percent higher_groupname group_order sums percentage +5 test_a1 45 2 Pinus Pinus TRSH Trees & Shrubs TRSH TRUE TRUE Trees & Shrubs 0 100 45 +6 test_a1 6 3 Exotic Exotics (counted) NOPO Nonpollen NOPO FALSE FALSE Nonpollen 30 100 NaN +10 test_a1 55 10 Umbelliferae Apiaceae HERB Herbs HERB TRUE TRUE Herbs 5 100 55 \ No newline at end of file diff --git a/test.tsv b/test.tsv index 7882002d6..fccc6ba78 100644 --- a/test.tsv +++ b/test.tsv @@ -1,2 +1,3 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 -test_a1 orig_test_a1 somewhere France 10 50 340 "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" "5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5" X Actually in Germany 30 \ No newline at end of file +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country From 9bd06c1a9d74088327460d3aa2076973a9f40166 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Thu, 21 Mar 2019 23:34:15 +0100 Subject: [PATCH 015/119] Minr --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index c2fb8d818..c51ac3656 100644 --- a/.dockerignore +++ b/.dockerignore @@ -103,3 +103,4 @@ venv.bak/ # mypy .mypy_cache/ +.git From 9347f51d7db5e0ff3526b59190ff9842d8393bfa Mon Sep 17 00:00:00 2001 From: Chilipp Date: Thu, 21 Mar 2019 23:34:38 +0100 Subject: [PATCH 016/119] Added postgres into the framework --- .travis.yml | 3 +- Dockerfile | 2 + postgres/EMPD2.sql | 949 ++++++++++++++++++++++++++ postgres/README.md | 3 + postgres/scripts/FixedTables.xlsx | Bin 0 -> 27369 bytes postgres/scripts/create_empd2.sql | 307 +++++++++ postgres/scripts/import_into_empd2.py | 379 ++++++++++ postgres/scripts/makeFixedTables.py | 123 ++++ 8 files changed, 1765 insertions(+), 1 deletion(-) create mode 100644 postgres/EMPD2.sql create mode 100644 postgres/README.md create mode 100644 postgres/scripts/FixedTables.xlsx create mode 100644 postgres/scripts/create_empd2.sql create mode 100644 postgres/scripts/import_into_empd2.py create mode 100644 postgres/scripts/makeFixedTables.py diff --git a/.travis.yml b/.travis.yml index d8318f0c7..d812c75bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,5 @@ install: - docker build . -t empd2/empd-data script: - - docker run empd2/empd-data /bin/bash -c "cd /opt/empd-data && py.test -v --empd-meta=${METAFILE}" + - docker run empd2/empd-data /bin/bash -c "cd /opt/empd-data && $PYTEST -v --empd-meta=${METAFILE}" + - docker run empd2/empd-data /bin/bash -c "start_pg_server && createdb -U postgres EMPD2 && cd /opt/empd-data && psql EMPD2 -U postgres -f postgres/EMPD2.sql && python postgres/scripts/import_into_empd2.py meta.tsv -d \${DATABASE_URL}/EMPD2" diff --git a/Dockerfile b/Dockerfile index 309161870..013f44177 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,6 @@ FROM chilipp/empd-admin-base ADD ./ /opt/empd-data +USER postgres + CMD [ "/bin/bash" ] diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql new file mode 100644 index 000000000..c076182e2 --- /dev/null +++ b/postgres/EMPD2.sql @@ -0,0 +1,949 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 9.5.15 +-- Dumped by pg_dump version 9.5.15 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ageuncertainties ( + ageuncertainty character varying(1) NOT NULL, + description character varying(21) NOT NULL, + age character varying(14) +); + + +ALTER TABLE public.ageuncertainties OWNER TO postgres; + +-- +-- Name: climate; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.climate ( + samplename character varying(25) NOT NULL, + t_jan double precision, + t_feb double precision, + t_mar double precision, + t_apr double precision, + t_may double precision, + t_jun double precision, + t_jul double precision, + t_aug double precision, + t_sep double precision, + t_oct double precision, + t_nov double precision, + t_dec double precision, + t_djf double precision, + t_mam double precision, + t_jja double precision, + t_son double precision, + t_ann double precision, + p_jan double precision, + p_feb double precision, + p_mar double precision, + p_apr double precision, + p_may double precision, + p_jun double precision, + p_jul double precision, + p_aug double precision, + p_sep double precision, + p_oct double precision, + p_nov double precision, + p_dec double precision, + p_djf double precision, + p_mam double precision, + p_jja double precision, + p_son double precision, + p_ann double precision +); + + +ALTER TABLE public.climate OWNER TO postgres; + +-- +-- Name: countries; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.countries ( + country character varying(42) NOT NULL +); + + +ALTER TABLE public.countries OWNER TO postgres; + +-- +-- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ecosystems ( + samplename character varying(25) NOT NULL, + realm character(16), + biome character(62), + ecoregion character(65) +); + + +ALTER TABLE public.ecosystems OWNER TO postgres; + +-- +-- Name: groupid; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.groupid ( + groupid character varying(9) NOT NULL, + groupname character varying(60) NOT NULL, + higher_groupid character varying(4) NOT NULL +); + + +ALTER TABLE public.groupid OWNER TO postgres; + +-- +-- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.locationreliabilities ( + locationreliability character varying(1) NOT NULL, + description character varying(55) NOT NULL, + error character varying(5) +); + + +ALTER TABLE public.locationreliabilities OWNER TO postgres; + +-- +-- Name: metadata; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metadata ( + samplename character varying(25) NOT NULL, + originalsamplename character varying(70), + sitename character varying(70), + country character varying(42), + longitude double precision, + latitude double precision, + elevation double precision, + locationreliability character varying(1), + locationnotes text, + areaofsite double precision, + samplecontext character varying(40), + sitedescription text, + vegdescription text, + sampletype character varying(14), + samplemethod character varying(31), + agebp double precision, + ageuncertainty character varying(1), + ispercent boolean DEFAULT false, + notes text, + empd_version character varying(5) +); + + +ALTER TABLE public.metadata OWNER TO postgres; + +-- +-- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metapubli ( + samplename character varying(25) NOT NULL, + publiid integer NOT NULL +); + + +ALTER TABLE public.metapubli OWNER TO postgres; + +-- +-- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metaworker ( + samplename character varying(25) NOT NULL, + workerid integer NOT NULL, + workerrole character varying(3) +); + + +ALTER TABLE public.metaworker OWNER TO postgres; + +-- +-- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_counts ( + samplename character varying(25) NOT NULL, + var_ integer NOT NULL, + count double precision NOT NULL +); + + +ALTER TABLE public.p_counts OWNER TO postgres; + +-- +-- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_vars ( + var_ integer NOT NULL, + acc_var_ integer, + original_varname character varying(100) NOT NULL, + acc_varname character varying(100), + groupid character varying(9), + notes text +); + + +ALTER TABLE public.p_vars OWNER TO postgres; + +-- +-- Name: publications; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.publications ( + publiid integer NOT NULL, + doi character varying(50), + reference text NOT NULL +); + + +ALTER TABLE public.publications OWNER TO postgres; + +-- +-- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplecontexts ( + samplecontext character varying(40) NOT NULL +); + + +ALTER TABLE public.samplecontexts OWNER TO postgres; + +-- +-- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplemethods ( + samplemethod character varying(31) NOT NULL +); + + +ALTER TABLE public.samplemethods OWNER TO postgres; + +-- +-- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.sampletypes ( + sampletype character varying(14) NOT NULL, + notes character varying(13) +); + + +ALTER TABLE public.sampletypes OWNER TO postgres; + +-- +-- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workerroles ( + workerrole character varying(3) NOT NULL, + description character varying(35) NOT NULL +); + + +ALTER TABLE public.workerroles OWNER TO postgres; + +-- +-- Name: workers; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workers ( + workerid integer NOT NULL, + firstname character varying(25), + lastname character varying(25), + initials character varying(10), + address1 text, + email1 character varying(50), + phone1 character varying(25), + address2 text, + email2 character varying(50), + phone2 character varying(25) +); + + +ALTER TABLE public.workers OWNER TO postgres; + +-- +-- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP +\. + + +-- +-- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +\. + + +-- +-- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.countries (country) FROM stdin; +Albania +Algeria +Andorra +Armenia +Austria +Azerbaijan +Bahrain +Belarus +Belgium +Bosnia And Herzegovina +Bulgaria +Cape Verde +Croatia +Cyprus +Czech Republic +Denmark +Egypt +Eritrea +Estonia +Ethiopia +Faroe Islands +Finland +France +Georgia +Germany +Gibraltar +Greece +Greenland +Guernsey +Hungary +Iceland +Iran, Islamic Republic Of +Iraq +Ireland +Isle Of Man +Israel +Italy +India +Japan +Jersey +Kazakhstan +Kuwait +Kyrgyzstan +Latvia +Lebanon +Libyan Arab Jamahiriya +Liechtenstein +Lithuania +Luxembourg +Macedonia, The Former Yugoslav Republic Of +Malta +Moldova, Republic Of +Monaco +Morocco +Netherlands +Norway +Oman +Palestinian Territory, Occupied +Poland +Portugal +Qatar +Romania +Russian Federation +San Marino +Saudi Arabia +Serbia And Montenegro +Slovakia +Slovenia +Spain +Sweden +Switzerland +Syrian Arab Republic +Tunisia +Turkey +Ukraine +United Arab Emirates +United Kingdom +Yemen +Jordan +Black Sea +Dead Sea +Turkmenistan, Republic Of +Adriatic Sea +Svalbard and Jan Mayen +China, People s Republic Of +\. + + +-- +-- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; +\. + + +-- +-- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; +ALGA Algae AQUA +AQBR Aquatic Bryophytes NOPO +AQPT Aquatic Pteridophyta AQUA +AQUA Aquatics AQUA +AQVP Aquatic Vascular Plants AQUA +BRYO Bryophytes NOPO +BRYO/VACR TO BE REMOVED TBR +CHAR Charcoal CHAR +CHLO Chlorophytes NOPO +CYAN Cyanobacteria NOPO +DINO Dinoflagellates NOPO +DWAR Dwarf shrubs TRSH +FUNG Fungi NOPO +HELO Helophytes AQUA +HERB Herbs HERB +INUN Indeterminables and unknowns INUN +INVE Invertebrates NOPO +LIAN Liana TRSH +MACR Macrofossils NOPO +NOPO Nonpollen NOPO +PLAT Platyhelminthes (flatworms) NOPO +PREQ Pre-Quaternary type NOPO +RHIZ Rhizopods NOPO +ROTI Rotifera (Rotifers) NOPO +SPOR Taxonomically undifferentiated spores INUN +TEAM Testate amoebae NOPO +TRSH Trees and shrubs TRSH +UNPP Undifferentiated NPP INUN +UPBR Upland Bryophytes NOPO +UPHE Upland herbs HERB +VACR Vascular cryptogams (Pteridophytes) NOPO +VEMI Vegetative microfossils NOPO +NEMA Nematoda (roundworms) NOPO +ACRI Acritarchs NOPO +EUMY Eumycete NOPO +HEMI Hemi-parasitic NOPO +MOSS Moss NOPO +MICR Microcrustaceans NOPO +\. + + +-- +-- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! \N +\. + + +-- +-- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; +\. + + +-- +-- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metapubli (samplename, publiid) FROM stdin; +\. + + +-- +-- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; +\. + + +-- +-- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_counts (samplename, var_, count) FROM stdin; +\. + + +-- +-- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +\. + + +-- +-- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.publications (publiid, doi, reference) FROM stdin; +\. + + +-- +-- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplecontexts (samplecontext) FROM stdin; +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological +\. + + +-- +-- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplemethods (samplemethod) FROM stdin; +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer +\. + + +-- +-- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.sampletypes (sampletype, notes) FROM stdin; +dung \N +litter \N +moss \N +pollen trap To be removed +sediment submerged +spider web \N +soil terrestrial +sphagnum peat \N +peat \N +lichen \N +submerged \N +core_top \N +tussock \N +epiphytic moss \N +\. + + +-- +-- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workerroles (workerrole, description) FROM stdin; +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) +\. + + +-- +-- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; +\. + + +-- +-- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ageuncertainties + ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); + + +-- +-- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.countries + ADD CONSTRAINT countries_pkey PRIMARY KEY (country); + + +-- +-- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); + + +-- +-- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.groupid + ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); + + +-- +-- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.locationreliabilities + ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); + + +-- +-- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); + + +-- +-- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); + + +-- +-- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); + + +-- +-- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); + + +-- +-- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); + + +-- +-- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); + + +-- +-- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_doi_key UNIQUE (doi); + + +-- +-- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); + + +-- +-- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplecontexts + ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); + + +-- +-- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplemethods + ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); + + +-- +-- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.sampletypes + ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); + + +-- +-- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workerroles + ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); + + +-- +-- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workers + ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); + + +-- +-- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); + + +-- +-- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); + + +-- +-- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); + + +-- +-- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); + + +-- +-- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); + + +-- +-- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); + + +-- +-- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); + + +-- +-- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); + + +-- +-- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); + + +-- +-- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); + + +-- +-- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); + +-- +-- PostgreSQL database dump complete +-- diff --git a/postgres/README.md b/postgres/README.md new file mode 100644 index 000000000..2c4af0b45 --- /dev/null +++ b/postgres/README.md @@ -0,0 +1,3 @@ +# Scripts for building a postgres database out of the EMPD + +These files have mainly been developed by Manuel Chevalier ([@mchevalier2](https://github.com/mchevalier2)) diff --git a/postgres/scripts/FixedTables.xlsx b/postgres/scripts/FixedTables.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..c74c7f34da3ad7d7e17f575ec2eab155fde58316 GIT binary patch literal 27369 zcmeEt19xUkmv-E-ZQFJ_wyloYvD2|_+eyc^ZQFLzu{+6kcR%l}nOXDBFPQIMD>+rQ zYhN2xwdCHcomf?sg`Q zy7X?=R)qOrAe6a4Ab|e=|M!2`0~5*PHhqkUqEAV$2yxA5Of$)I;XU~=CJ`#Hz&5-@ zNYU2rutD8d1H3A#RFHlPR)%Dc8y?INt@x42Md9c>HH_Q_SuK;qGo^7s+2j{Sw{vIp zkexG1lm$cvG+)`VzkP!xr-8v#Y#a(a+*E3YJ=ggjmt1y2IlcpOtPH_Dyq!CidQm*%Hae!%+(*kI9?~ z#m>J`X+DG*$C0&9&1GeGAa*AlWS#g%B#Ke z7iw~cWHI}S-2o(*#fQbmG;t|+N|(+U>hhMKdD5fXBx3V7BDILq^y=79C`Guz=zM8@ zT0^p$TZYe7z>7l4mzBXa%^bOB3DX%q%gM#3Z~~!RG8c1cXd{k>W~)`6BUVHYFW9Qe z=3EvvhS?6>#O`{=Ry}tj8Qm!Fo^*1VBTB@q$j{8v;=^RQ4?bEAtT*FX?gJc9{pBOq zqy7;DKeqsT*#BH4ufE6Oi~tJR7a$;PAV^?0D~5m8iL0%{cSBp-?|=5U|5!7?fK3k2 z_y6{CX0Vxt< zlS?N`?5$sY8G=B+W>%_(Gg1SHdRer9x_4`8I)V;nMoM)=jRw&Q$T;NU#^4jAvcU=R zym;7G1r=INdtJaj+8r)yK;bEuQrU7|QaZ2Hjo~h$!k0oZ(cw-zzyxJ-M+MSU4t!An zflu{ATRX)hp6Uu96k-gdf3@SyKb`>`{1H>2k{{QAQVK z-H%p7YZeeuTQB5UMNqID3`4Oo<@J1O*?5|*xU25LtM5J6qsPf!p#N^Z?ewjfSb_lo ziNgQ^Apj;4V7~p=nO3M~ZCk*J_~AYG{ssP09hw~i#IjBbv^8QOyR6Gfpi0NiU$rOG zeodtO{fZ;_tGU+ck4;J@rpL&fG>1vKG^*Xl4C8F1)k@{UjKNibx<}nN@7KoeNzSCG zbDA2H!s{T=Vmfv#x2f5pn4{D(q$FN7d{=o6uy~;K2)sx9fS)LO=Q0YUvZZE28@WLj zUB0|Vy^qu){W$dB)F$whL8&G4IN}Zlr9SmRP7*k|jTwu*8(%|IC_j_}n|do`x9tg@ zTzyH$xX>2Y4On{EZdiz;mMO@Ml47pXU}pnO4aayq3))!1(Eip#?VuAV>%l(rqJ@p6 zJ#U@&%Xg&n$I8hY(c`@w;ZTQyjG9Vq&`;FsYyIVFhpNTS5pc$h7K2Z#R^wQW9)=n; zv2?Nf+SX!fj;6ZQ^hYnyp>;Cj4Hi1iI$MqYpE8a-t!>e@S&E{UNYbb4iO&SQMWyWc zauM8to1e<^Knhq7Ivr|rzMtjK$b0tEIYit&eiyMFG?(5_9!X=6UZ1;i5**evFTW`;N!}^ZoXT&7cN))1Pc6%`>2+syyZtlG$LmQV38r?9 zPLD@ts^1qN&XJ#@>Qjy+koeeK5IV*N#uowcuTAq2yks~XE`)iuWEoriLCK@!CZuw>KGdl*}d3B`}~;l?Eh7sCk7xz_w@u=bH(%&^| zDmbW`AVaEi)!D_z{`ha0X#KBKt$fHH^5Wvy@SWZ+JJW=Cfdq+L&5+IyvKIF13DYl$ zLq;ROv-ZrsueLJHGjOaX5HC$mR|~lyIX_G;NbFcKKmzW zZa~^XHv+8*UF(-p>3v*O-)##Mw(<_?(}gK?2Z!o%aZdWIE6;&WYol5=+dUr{072(} zZiCNXN8C}sfq>S~fq*doGb}lpo0vE`GW@-;{@DpLQvrj@jySwc^S~qG9x}k*>g&4* zRU*C_B2`|t=>^~W)hRxeoGV3j@oLLnBh<`LTAU^eW+Er;W@TS5aye(~e9R-sTD{D4 zzux`CqmF7J&q@yG^Qt@BwDii^YYX>Qip^o)Cnf9C;_aN_oMVyS)r`smftYo3NZ{$+ z1Px5rKG}I|a#55$7gtsLn6JZHUeMLT#2Yj7jC#ia^NaI0_7AqcvUdqsXWrx>PnAJw zx{MF(!X(wL{^xc__fwnMvptgCvkous)(L1;J;H&mim{J3#D%9b!-toJpX}S}3Qs)R z2SHC;eF!0X_R=gX6lrI5=Bg#p@3_QYmYL+o^&rLx8GgkTKlo6>YJ*jJ(mjAg=RHru zg^50Cc*2r|r4m>z_v-Q|&Oo*Os7@M=`s`8Tsa+tz;=gH3w9UF% zoA+v0KhuQCG2qRUBcstgfxmTlexi1*D;Udhp$;UcpvGlPp6njzgsG&pXqw5M2SitlL#T*Vaww|%vFsd(CJRu{%fYQcx8>>SBis_V0Z zh(6DSmZZapX)EjCvTj>iJh)8eOr_YPOozqrmR2&u!f~re#gjvige>U*oC~KP`(D<= zn8LKk2@4W(lmfNQj+K{`Mxo+{{-~rBsHQ9Hi@O_gEY|F55lvXhC{ir6?PW(U=@#{9 zL7*(mv=oV^a<-n24!B*{rx8_b;toOi{eQL z35yUTt|C^Ft>}>Ant7RwsIt>q#Br)3tQ!~A%V+MeZuz<4kgouYU=#Ha7@o-n_X_ST z&!j=-O`VnXG=QQ8b25Pq{b9UmcAj;PvMpJreT8X|6xP9qqa1D%Lh6=0lYx>fNOI)3 z4k=CQ0LR}*xbt6h1cveLDz|V*-|Y1cof^7-n1=h z20U;7n?O3a)YNhd4Ft483Iv4uPat(PH*hd9R(5i*urYJ|8$`7x9dKHax=N_8d{1KI z%SDuumvJa+JBul$@@14zeGc3hw0B@R1MDC$K{00+sIFqajF83Da&${(lM;yCi$y>* zR&9{N&Y0g{EpK|h_aE2kyY^|6E@lKAU+#mFtK=(})^U7^cyT{p1g?)RPsGp)PAXEM*Ex13hfU?^3(RTT?Q0NDZX1lJ z86BMkQ6U+pUL2a)gvbq-c(@zlb=ieX&dPttGhD-=oKD`+i0MBZ&4qFdcpVzf`u zhF)R;0}O2?0_Um^hXJ0q?Q$M|#_}WhZH6GO~%MI;qf;#zw@H zI8LWt@%=eo_a^>$ZiKG3J};1nC$9p9)ex>|{9Gd!b16r(b)j)oXh@tFskC{6gnb&E zM;ulx_f^9TyUG12QBG_t8j?%5@D+Zbt4&KJjik~o|5UO7rHvPz9)tXh+{l_e)BG2< zDZyO!wU7qIjnv>#Ur#{eDt_%bX(6VrN?%} zxC?_*15LF9&$7U;U>Y4stwRYtA3NB;pV>w<%CnLl&?Pz%dtE1bQxQoB8lKopd-TMs zBwXAfxo2|c%3&wTok=@ju4Q(1gp1GzwU+q z)M)AE7vQJQ37YjwA-DFP%BWJPh#`#bQfsHoLbEd1U#sQh5}XslnZjbwzV{6 z!0??TkP5~w3km}c9<8~GeSm;dNlZ}9u~!$rD<7hmB)D2O-cfk9iZfnc{FH)I>KU)q zjU+P$-l{ary}}Jx8a_=4x7vS=rQbRrv=5F-L9_+pDp00>QAwLvLt9cry9sIBu6Z#& z&{6x4SJ`=#dW3-&QTzNfA2BK;GPax*$>^7f;;1p6eai5*MnsbHZXt)lx(f9HSz}$2 z!~u1Dy;@DS2U0a7;WE!a-Se>2j|1#a2+r-oa`Vl)g`6;68aX6)dZHY!MtSlTklZEC z0e5U`aqMtq9*e*fyd1gIc`FdJC{!H+prAsbyx=kvq7-ZJ$h27VPOv1qhnv}i6p!dG zUYk7XDBL9pW7D!y0l9vU;=cUA2IQ`B@}3M-*i6ggh-5pQJM=|S+O0XpM%z|lp=%oo z9$wJvV@OX@8fTt>*y9l4xw?MfubBz1#15870*5QRLX}Bdln$mu^6C`lw)3;nUY4TZdIcRu~+v!WlDqAD|L{D1aEB_5&-cl zCZh*ZWEZ;yc;SEZ8w#ZzIkS0^bmGvgba2v#j51BS;5$qn>SGyz>$7j3G`)=VD-zg} zd;f6#>L$uVRBA@qlCNPCACXV&iaNQ){z~`U60<`m{uyfCH7rZY;Q6&ys2&{J8e!rj zM7F*&%If3#T~`-`StXyckns{tlE)Q}=CZK$P4~!oS&s4otfmBQDyG)3rsJvniaDc1 z6bHn*vdGsntkXGxyd(Gw^JYyV9_QTqa+!Av7MZr+L9%Z|{5H zm->%TyvL#WRV4qe4x=uW)zSOV2Z|K0dyrzY=@p-k8~Mk*_I-iw-w)AlpGG29_CPz- zx|b_wTlQ27<@1%a@=UPlWR~uk`lVf;k={3Kds}0C??3Gg z5!4-1Y8Le`50kl8fULuke=Ez9=-}p{Bue?22Atne_Pl9${f5N!*&p98lE{xU zanUv3i*C2|khx-l*;cRp(Wog#O7|rZH zI}OpB6%`B#C5#{cxaQtyOU*<>{h<0Eu zd+!EL(Bg9n4~I0$L)wBh*uiSSGa=;7TMD`Y+%_QQfuzIPR5mJcR$!zYBY&huOemMM zb}T0q7gUzII0P+m@NB$`j!wUrRkA_|ouFCsIVI3!ehp)=HyKlaEzpNv%>gBQ=^|rb zE2bIfI_|n!3gxO=5OtF#wPPqNE^pixb-~&3P}%QJ^)_TwUch1?+7s_0{fTSiYTd)b zs(t7kHC4T8kt6$7aK6r8dUMR08U03rj6D34JxeYR?SLm~XmN|xFE^%>?kA145;VDJ zotVLl5JGsvHt%3gCix+oX>D?1Oq@;r)sSfN?sEF0lJAG7?PCe1r44-~5qku84c24G z0NRVJ$jyFby+` z)Uq8R7|o|6?6IS-Om?keV4|=s`rRbyl&T*v%x~{*T)c22)dkG+xv3in-E}+L z(_tZJNgoU%lwLKl6j~ZLO-WctCot(Xa^vRPL#Hs6yw3W7;s+Vo=XXxwESt|}v zmPu`~z-QQ3Ekl`^z}a$BnbsmHYhLoRo&w;(W)gxG1$sZGdqOn(K>QyqnB?+-Hf?+I zvPs=b$g=LOH>*@CL9T4{PI)Xj+8=e-;SIX3D!YBZdVZAOb4p_ z=U<(0*Ny~nE?v(_qaz-=PT}JWTJX2}X9f6O)*qF-y;hq;W@BM5tFV|p<{e#mHJjca z`JCQNpI;q**IYh{D{Q#7+x*UxELimRct4-dBS?S4bMjHwtk?A-xMt}6TyQB^XZW$^ zx}GQS5VMb#x@`CS}_Q<2O-Sc%J#3`G-LLX8nicG>QI zj?XGzV6V9v;koog`!~_lX~x|{_sOH!+o)~-<>bo7_Cve3M$|PeZBY19GspLjJK-<# zgb(F;UK=6=ix0YrbJF)TADfNaa`4~$E~VSQHycIt{xoNM9PVz6+1G7fkF&?B9I3y0 z`=Ckt2*(&?#k3a6VqLHGu-^U_9j#XKneHuHVya_oIeZemQy(3?9lYq}Xi0P?u-X3{ zX+U^}zczei!tqh?`MBKV@dJNWD>mG^yZ`ydtMQ6bXg77z_5Ch`Np7#4S8KDm+{ix|=$jmL54abBN&J{HmnC&64|fgOg>x$5pDsL_QW; z9-$`J+yV}GtoT~{uo4TBLd#E`)z(=`5tXXC=2~-9dTMnIMLr(vpa^Oh9>67(TKc&S zKSFFbe{3$;A8=8Tsj~H#>POThq#{S|u`|f6yV1}6s-p&- ziAhY?>(WbZ0oPyTHwF>v2V!AiWkxj%K*nBY1wmHC_T%fWw4{tp#`p&qX3Nu@Qlqh> z1~}fFGFELidinyF6*gXtYRmgioeCZifMhW1?on(I${M@EYN+4By0Q+$q(Dd~dqG zox*grao(Q|D$;+gI=;6y74Jthr*API>bF}C;ekU40bN+snpGqNu+ze39#R5;Jh$vL1P!RSeV3c$?Q>>nDlUu>|bF~CB_YTq$z#m zMjuPYO8*5-+XR4C#HNC-`iYI%U|h?!pDlfVd^tq6qy+E>R*Hro@frZ27zq@74w}9;888eq zEd$WVv!ZYp*4Ti-sXRia8H@F6N77i^1WbaEiq-*W>RB7GA~YUph~Nrr+Dn^(9y^Pg zW{ZKIYt?dQ`dVyEQo4QQ??r#bs40ZeTb9L3S3m%O0jz)KO{Xjg1hg6e_XkCH8E0c;>k(fIzy zu?}HqD*3ER%Q}W^fH^uzrWPAI2l>YaM$bjN{?zXkwwxlkX}5fO0sZRHBhTK&y~uu8 z>RM>O2qAt+jU-F|wQo!c73Dz*YA2nyvcPTfCsWr6Q%Y0|)#oP8tJMcrolsRb zv5WnR=D3N*8VF7U3BpflP$Qz>*Qj2^3XhU%_mE_6Y$o-!XP68&8FHc(q&#uR0K~DX z7dgtjTL=u79YPu`StmqQYM;@MOx-X{%%uwq%(G`e->--C$g+Z&Q-VoLM!!#0a|SUPQl^Do+BlytQp%xfqHCAYk89a9Of1>>YmelR{wTDX zPR^ia?FaE<6zTa0wyLcvz8h=ixUaHj{ zMn5X`?`Y=6JXn95i5ZW>E2Y=~#yC!*vj&pUq&hApj~GjmR~8r4LWDq76!+KW#*twm z$`}*?JW}t*P52$)kyMAjkc2xmOd7{E0ahLz77uMs+O`BnXOU{@R zJ?{S>N{97?pSM!a11uaw(*;=AVm`>GOPoNKrem+Fj)hB#SnLV1{hunkk2UJz!chP~ z4$tUkM_iZ*cSwx@1JH~1E$FT{j}0qTuH=R-xkA5J^&56V1$jU~%s&H(rv3ws(nJyh zEYlbky+S{)Wvv_*9$7F zlhuiG`5E`%@LSWubDS0j5(o@I0Z-n;LleEtQuQ<8Hu|T**rQw2@K&RW{aA7OJA-k3 z*o4{Z&vu>?12K5@yV!!HkfbsTo#0F>(*wEwNL;yNB5|4o3TJP2hq|;l9yk^U&cYDf zN|~y?v&c*<_PCFp;$#Oi?0eqHk+@C11M{q6GA;tH)rk?=eljkHWI)#hnnW{DiUh1? zibOXelb`?q#$oV@xewA~7-pn`HQnmV1@h4kKQM z`mYR1&kZc<7}uSIdzb6a4JgmwQo?OWo+HgH?MED$Gx?U8xG${naUuS=w>HRBSrThv6SSCdt3Qn$7slt$*Yl zUu<&9#nd3{bSJ3#_@!uBX&Rt+0*0Il9o8FPq?|6N?KF&E9H5v^v0vm|?4iIvSrt<1 zX^{osIwd~;*HrIIl0osw@cOf5sI{%Cu81yj(~v2_j{cGeUuk>OlxBcH>2e7YVvvkPp%vf z|32`TOIPvGiV;DRxqF95p$$*8RpXFs%I=imr2R^qSK4AXX4)-NGeSwuGYH)C;%)ZB z|Cn8|TW{^=u!Wm_*yVZdJ(_~1xA`rvKVba+Zj5~vX&*P;CQ@zM{IheIz$GVolYA_m zyBr3$BT#$y)XJ#2^EW6TctlR0%bU-?eG}=Q>)*Hq$Y(<*`(GapnEyFM2Rt0Gqk0k4 zd=j4U)l!DNLSCTCh8uceDpah$R!}<7sxh6V)3MFLlUVU)pe_6C+@c-%mRD*+XR%JVM>&SR zUbY#u&^JhEQJ`17-u5!H{IqV$p-3V;uZr?2V6*mqqP4yhf7G;ied+C^_O#BI-eS@2 z@@j(v7hvsmb854UuqYdH=dEO+`b8e}WAHhL6U*aw&|Fhxi1)?vJeSrtg&e(X?^a3- z)k(TeULQpkYF*e#;qMT3Hq_qQDTzl-d#$wxQBfuvsjWKwgI*r((4W!p0%p%@MF|n9ykS^eHg(UDmr`j*q=F73#Yl&h}Yp*iC&o z)_0uz{eHEVvfDl^;5jo{Bb02q$~d6lXkGZ}=f&r0yGe63A9_uS0h$X6x?i%JKWDYx z?+`asi~{g|tZhr++9I^inl#m)Ht)4RpQY!wtjTd3y(-?|JNwOQtnJ@+({0Ns-rgmQ zvw{V^1mxbA8ckn&!F1H6d!GDajgP6N3uwiTKSMCTZ!8y@#LCbuktG05!y5ZaHAEwf z!`C8v{lE@NQjc41lda-pn5NPGP*qXQzOW0Xk)Iv0z|APL(P!Ex1_>>;*gXPIt_w<0 zObLMnm2umXgG{cAQZr;lK|-tE&0#^Y+ENV0xu_9ScIXjbPJpF1XY7$W*bq~y$Pyou z&va~}o=27t!4x?PA8iDp|2{4dx(PJSQ)pD+CDS}{J>I$HeFT zLW`^p0GV%+DJHm`P|&I@@Y{cS(r|gBpq3@$_6Xj|?^s6fppU6IvW3{eybrO-a|HoN z#hHDAci9=Yk;>L1r4qjZO{Olv9fF(nDFX5-yio!+D0P_f)A2jGq%9(jKKMeb z0x>&CDSWTzqr^1qAgWP@>&VC<;uAtOIje(|O#L1HIQXO+zQ`f^IJ`!M>)6*O^aDPE zB>E)0Mgc3yoh!WNSQl*=cm9SVu2<)nm#3y$=yxFcf4fO zJ7R8tzDx`S0Vh{AtmznYP%ue~X*ux#PA=J_L8m^YL*s99UuJ#ar5`$SaQ2^2Vf9mE&B&*~8gdGRD13YSJjn zv~T%ixxr%UF(UxjAG8Z(cH02U^!wAds6WJ-+?ewcCw6>kN73H!LKBj!Wmr(UR*PLE*JL!<~Rpyhg6{FYRHY$EDH z?5@J;`glUS+pA7$@6HT+l_}?{Lpz@{ujoD38S^QMe&@pYV z0J!MS6U7;?mF9BBsPTr0+m^udng|Qa^AvEzJb2HBF~P+~*Y@hm2CmHTRWL(#*X!uw zai4#ir25INLH-8VEwcfc+$8_JRQ{7R_^<4ofAUrTR=ED`jDNFLJLY<&K$#GIciVS( z0$bZx)exyBn?k7XcSO%XBUe{t&6%Mjn;L9we<0GLE=b21#QSs!Czltv?tP_N?}0Xz zh4I%2*}DC?WAyZT4NHu1fUm2sWS0+H{_Df$MGjutD={ica{<~6yg0EXmifCdz9R~; zeHJW{JMBZN7?$FdJt>#%Iz^f{dv3dVxOFw5^_w_iyK+T_k8OxSO**@ve0GbDCXp;1 znVGv@L8=%vG_RI=Uw$~k32o63iTuIRzWdC{PIq-zP#5*L=pf_*Q1>jJZn$M6W@Qw( z*3@N`oYEtoxmD}JoK%{s;ehu5d|7!(i?_t*dReuC-mVPl4Y z?LIqla5sV7XYch>l&E51A)#cgd2I^_UoNMPM-Je)F|x5Tvw~C`^=aES7%54V+z9GjHx_qXv9(^gKI&5yiw z__q(27jYkr54)$Q?#&K_{pEhgZFv~G6#|=->G^wyPA$Bp>Gvug9XN(}{Eu7C$aWil zz2TC@U6YIUkQwu4-6YXn^>&+jJ+YAU8xP*#;o-@)PYaFiU$5_|vP0hMvaz4L?spCn zHti2jE0=9I=jY=k4>caZ5pR<&EbmA%DG{5mug4|tj4n^wCr=#8&gX;8NFPbp+r`aL zHy2qh84sMe!NaG}nd(f(9dDOqy3XGn#@;@4*^9NJ-q#@zc4o^uXvy|R%6ZdQVJ3Og zcI~2;t208@4|4{GmnohtuC;GUYo0F;&V9z)GkQj=_ctdOHa|J7LhPA4KE06Q*tIBj)(T8`)Itk@2^5oI#YvF^j?QghrQc_Ftks|<2?mn^=^s; zgoDAm+`BKAM;7t}2yEYn$1|{}feuD+BU#8#wD|Ld*) z-XtI@CF0@S0Ndr1j*-j3?;8Zh#icm(IoTu1E~C#e6|F@ZB%LXPpNWZ8(w3R z5Y=krQmYpHHl?Pk_<~(TUKu{7eyucl+gNhTCMBzt{ijBRU}Bhl2+)5~=>_^5^#P9X z;rG9$PTnd47?uBnahV(d$Cgwr8=Kx>Nv+o_9ppK$^cOVwFU)qbN|#uv1Ptt4lx$)O z!9Y{~Um0jg)qi}no<6n=P;ODBtLcw}if$X>09HyL(zuT**;oF+aodk)yb{?NPW~sl2pKlw;EZXx*v7oS>1M%`VsE z?I&7A64_#N&44ceToR4GC2y(l=;u(7tlUD`0ya$CRVB9?c}`oZqXY+ zmb#B~gt+Rb8{K!bB5mPs`CG)yYI^i{3v3`WUovrj*SozmfYx6#a$4HDy}V)S@x1@j zcyqZ6X85-eP_oJT%QM|P%$?N#rJp54%MKAJ5-gk87`xZcY~}zF+fS&Y^x#LFqXsnY z-XzI!Y=Qc}N{< zF1=a_okUxdCgFM(>O$M+EY5Xxne%G40iz(r-!-0$Y zCXgi!0=xZK77K84>FiO&><1zJx`Q;>-L!xn(r**3-wtH7{3JVk9o5@B^BX`ClFgU{ z%{mcN2h0{)aUIAjdr0Yp?BQQTEQ(lIb#tZ4+lIIZ7AARL$hW9c1SibBwIPgXj@j4>9^}x|oVQ@vu0bz3#bB1`vJAur| zQCc~WXY~~B1I%CLgp}E%W?qi)jK2p-XmOa}I*|EM+O?nN#Fg-^?Ando%X{|`K*;RB zCk~tzpM#OvLxw+&m5=+TKX)e+d`^;i32>wjx%lG-*n+Pm_g`Kx1a5FK{PfyNK5{mn8&(O>wVzp8Jk4LAPYL!Z z!uA=iC65t9L~r_9+AQ}mI>z4_dntl~1o->S?maYQ=&STG2|^5{x_3&lPL)4hKLKB0 z{mC!+ zip0>Uy`nr)Tu-gS$S;o2Wk<;GU@=TTQ>DW#R_xn-TYyM1?R(iv|ld$qh=}jRoB>zJ+IlLhm-R8S%nm znczZ&B6E($(^$mr7!I@M8c&7kL>4&>jtB0LgFJV*C>oL<7kpb z*+myf`H5}7Q2>k1PtupW0|H>VZh1xK>|wTe{0{8qF2jj@GT3R1jncg~{_3%NS85FF zHpk+2MypTEw2H_U>5@mFE^F;$Ypq<_#|=L(KRe%eM{IlI1gun`g4Xg;QiZGrCPH2q z)PyntzJmK_i>9DS@r-Q*wDQ}xmCl{sp}Ng|C}uH9(nbv#RRPL+hqa8C`831%Ihm!_ zIiYu7CdVNXps2D%jh>$x*a%e7;HV4~5k?~MLAs0jiXzu zeWfLp%bA5H=S8@3-MjLP+tI-n&1^$BBfD?Q9BMeYn?JJ1@LF^!~Uad|i!)kLEx_ zU%|EnmB2vH+;}J3XzZ=@ZDSv6HOs>t+sD|I>pKK)DgOzh4HF3Jq;kC|1qXEn_k#|L zLJ!!7%v-WXod8DnD|j@Y67Y@|z9c+brIFivJ;YAuF(-NJK@&slV%cZifYF9);y`p6 zC$~rn%$Pedk5vmd+ZaQk=3N|WpBWl`bf~U{OC88(9up7S{3`_S9Pe@82VZOCG-cqO zZ|;dyGPfZ987Q7tJE^+a-2610v-svVdzlz~WxKP^Du2a76#i)cqeRAGjX(K!p&&UW zCkg~~)MAM;XYHM(OoSz{u9t52sfkA1<=IZ0`Gv_eqjQdgu4<3v<)+j0XdcJRE6XY+ zyhJG^{(?lWUZhvNsp!o3`_erGRi>19Nov2o`I&eTq~!Xp`DY+Iy+8y{Hl08O4F`~% zZOjPE80lPF9mA(bF2muNb&*)_Ebk4XkTdI1-Zb1PwQ8a-?$tygN*l!?)Xz+#-y@LR z%49CpOflx^(-w+P$0FKj=!?}mHZxY&7eNN+m9emX-mCTInoEBZuoGE=(op=>MlmfziB3}4`bm3?lRSo(88M9oE4YhM$>qnLEnC&O zo@RUQ7R0upMXgDKVxVsO%tU&;G07>Kmh$;KJNK@ zaFD64U7at1=)GW zeNk_@GQ#2Gu#>ux#z0o^JcvI_qfP8A^H;=c%gpC13rlPj8}71)&Rja}J;L0HUUK&b zbGPE-QYPdsHsivikbu2+cA9BIjzmxf6kh^1>6bC;iQPE7$)gF#t>%cbxdys%4jP?> z*DQk}{(eD(5Mm0dNd~mim#n}zWXGCrF0|ufqw?G78P(KI-6=7-au$V)@VXrWy3#7u z7b}kns{VH5-`+WGoXcBSo>tGUZPr?zHOi_7k?i+96g^g3*7$7%>3Lyy*%l|CCFaFd zqa%%oP@YPw0%mgy=dBkMX?F?6-ty#qgQh8l-80=m_;s<3QgFkLprA#2)bD6$&S&kP;^xDEB4JZ^|ZAEIX3jirq%Tfxws;btYYvmeCbn766Oui zJ{Si}9UI^CU0f5ppnB1*>IPg2Nn3vs!}T9ZAuA=r@Y(16{Hr zkA#ut+4!7DGDIC~QjtV*m9Yh-G>hcfdYu!UG@WK!T#>S7TT+o60~?W)*UufGm_Vsq zL|)F==}>TJAKtP;&e(gY{7_NsIp8{MG9~6%xa7(&Bc{GqdPt+TcGd&7er4VTW(Qhd z_7us~sZzg{K6&_7h=nD#O+btrj%m(N-`g(zl6^=;KO7ot0n(>7P3mtqtv%#37AT zGlCyU@VGJ5RLSH%F0hj?O~r8CMmj#e*g`vmudhn^iq#&pSqhGX*+e=Czfe{N^RPCJ*qFUWQM+ z{Utci{@Gjq8{yvtp&uiTmGR?QPGS3d`OEUApz~o{uXDGcj@p)X zh!4?+xC^#rcUoy>x!i*#&qORPW`=Ew9+;?tTC`L9hpPc0Jok&zISglh9{~##@-Vcx z#(hA|LT^KR+lynT=IdwZ{5kNpAi+R@#KZ2zLv^>7LZ;`rT11|Xk6P4O2QUc?1Nd{V zB~r_L1~!Rc_15`;%c)P`3x$Le#@US-o!%SEI9@wiJ8a9MVD3^CPjq zc=2o@n9=7IN>Z$;8sCDJ-IPzE#^;jzd0mS6t=@y_NuGl&DC7<`Y_v{YI5v-o8C-Ae1#KXKDl^ldpye zhYLypRpV-kgbR+%?THW%68gXKn04K%wKRz||Q^Fs>nv)7-`xH4x2v@DhpdmvT6N3k5I^FFheiLeQWG zJ#@z5swHY{50i3LsQh!`ZS0DXWY&NRTzV37&`>n>iiyEZ#8ty?4dDYwp02z?iHB8# zuKj=Qon=>?OS6W71a}DT4#C|acyMkOVjr&ikJ1 z&CdP@hY$3cuDLIoXZ7mUU0qerlaj9dL^%#$?O@VcDz5vrPM$1Ch!atoL>`RMqLUuG z4OB|DVg-F+!;!v7LT`C@-LWDT^a$p6w8*&2hj$h;$b(gzbkafN3>$yyo8MXV1C6g) zM>}R42$yUHABTYH18^DmUZ7bwU#Q(t<Pmb#}$E0_Z#DRwz zCSY2}^rRDvFum?0J5z_~;CR-7mhCsSGVbChQk*5U_FO;f8Ja zlsk~v?AbHf9)LAvk8l=>p?{dlS$XCe(RWyO*35ayh}XE zL~#6RVFW*vmy=rcfQk@eml-3}s#cbLjZY?ONA5w(wdHR7UF7m+f??9e>8Q%#NU#t= zDgPMIJ;vCa<-4w3&;+ z|JT)$<9}C6o9O&s`Pk|QZf{!GK8!deY&$o8R6Y?aRIWmL#* zL>*OUlDD#*-FZ=q14Y@OPPBS3=z(gZJ~*{#0>9?cQySB9?yS-B{^!FjiA!eXn0!lX z996Yzd-&)*@bZtd-}w=m*OCL><9eGhi}ggb6nFR|+If>?ImZt+`I~s`vd2b5 zNDiS;mTF%AIDv7={G{Q-onf4AN6U4>iTO3~Vz!GY_)&ik>WGtn#$;kp?)V2}3_t(Z zrKz#j;URdDPyCN=1{!*%G6xzkX!+z48I(D_^!yCG=sX3#o7QH@4KE^}xe9E%G z-t0p6Bg#cZkCw!9O@bSUtus~iJhZ?;VyfV1tTXB|%Oimr7{^Dug4Z^ls+HX^ML)+< z4guO`>244~Kh>KLqnY%Yid_w}vp+s?-rj{%Bli=QS3_XdfKsnpl#>r0{Og9+PA^n> zFPu9!$D_F;OlUtnIF)+o7IV3^l?M|tP$OaN;AE-R6*wG7zg=2o0+{ zjFSjUzB!u~#iIbnfE4M$k%D+V5R}525WsfYO>%$_>b((}=a!Ok>=T*yC?E$5-2iXPo5+E9)@>XZc>CY zN|t>lj;U;TdJ22TOQowXEG+H=oUEj*0-8cKO(`tskFru=3=-2UDVs`_C0$4)1{D=p zO1_~eDk!3cEg4Rjeg%^C$x6;V$wol3P@2+T|0gR2y9hTCWV@KGY@0b89h+)mXBE3r zAFCj<%T*pMnbGIcav3YQi1fulY)d#gmYGE6=~o(t2ed{duuRA4a6@%P9ZjXx5&16aRN(eWA;G(i1~ zzMv8spn1lZU(NKiE@mEO2dU|6#;G(2c$uTXQb8MmA$TbF!Er3AdA|QKLH~u4ral8l}v~=O@1!l5?NcU%v~T4dlHv)XKFT&aQtVQ`ds_x4`5~` zvwc@E(1s?dG^i?bS5%L{mXWZ^(dWJ#!#RnB(}FtPMpY})7SADu(RX`R8C&92^s~5% z-7=0~dn;KFopEStteZ(zg zKi+SH{5U;3rD)pfBFp8$5p@XB#&&L3I%RFV#V;t^zkci?7jL;)KHUAaV`j_1le_=B zPHM$bvAX@F9;iK{>P}o(Tov%S`p{-u)yQ>OTQh*XIoHacP&``NLzs8Wh}Kk7u_`kT z@Lb5@m(`yjlj@RbUuUs4L2iL`Ln1U#mg``h19Z?F7k&$7j6o<{v&$3Yl#A!gyH|Vo zfQXGwYKhDF4qjO%*xpLb{g;QjgUKf$J>oUD}O5fh{%2Q+@`4R zH`nzG9QQsL4`)B_RdJxU3hpW_WvkRT)cJ#mW(6OBUL4)s?&F?t#KMA=UEOR1Sc+`3 z&?}{9i})_c60W9u(=P$R-fV=>6%8;hHVHirgf3c zwAqCowsN_n1hmwhx?C*X-TCxpig<-XWT!Z$)Ddu@blYy~PDXM#=p%lJ6_g?ALET8N z1MX~r_pB9MYwx`{gcAt#4zJpr{G8eL06Qn0L6seR<*1adJ(Wh1qHo$+h+S!0? z0lcAl%hPbXr#9!B1_>vSby;D%aft_h-4B?*h}2ll+A-D4-u1w}c9-xX^ozu0NBAaP zn~5~eNM1X`do?fDp0=CqTq`Iup0<=bT@)zP@ot>mKDn!Ke;VFLV9eenGf?QBORnhR z%+y|-r74=g7z1lal*g!~Nkg$iAIPbqBhw=(;3QYmS#ZgYO1+>@>8<-xibF%?GYbU8 z58)oy*=AbbIK;tuEGnUa_DgQrlwpt+E{%=4IVS=hfKRoMlkk}fjRJ323Vj07sV^N6>Ksi~Vaycm_SObT>9h4Gun9xHRqvEXFwZ{V9i9uII39F8Me$NF#rkgaBc+Zq7Y$FsOii=Xnr#QMa!}c zL4idFiBkiTfDEAT`XMxnJ{iy`u*%U`jHwp%rGj*Fjr}2U90TJJ)r`MjNHM?&{@ngj z`!%1&qb5Z0V!}^Hv!`UCBxE2taLS8~#0(RHooM7sdf=N833x{{5{23ca?1CZau*W; z__#i5hPgy(MiSqW}#`3MQ8iq zixvW}<)g=GK(|(D?#%BV<~J5@Kxv4HJoH21o!LY$MVj}n zt|orwsWc*&Eo2h4NVrbl1{H{HvbpQw1_- za}ygA#;4y;u`u5>N5b(qU$tZ12_iVT{bb*YrdVAcvyNXOH_lAJuW$IKsK&|?*NTS@ zLBV~cL|ssrENsW0F!|;MV%udjEP35N(FE1VJXysKdAhZfmMdbo#6w$g@pYDu=Y{jx zWLt`-L<)RYToT~3!a09N#C!_*RMLAZE^0O}`L_uXs3x$sxA1g7rI)gO98qlsVEX4@ z+=^P~CJ{+;FTKX>g$asn>M-T-cW0QLbwiCl7Xpcg@Y9HyozoSJ*RXNh782F(t1ves`>p+S^Vy{QuL zeJCOeYhYq2NYvF2)xcBw?rzvKAk*-ZHfM&Dm@$k0-NWhrMQ}U+`^&xCZa__Q{cCAk zXbNTJDxm%0^x%whf12v7tEc_v?ND<2;~CEtaB~@h@w4~&`F=dsn)lVN zd^Ex05i8ACC5WwAoR3}54v1{M_o(! z8hwzheC<{XpNP(y$)i%M#H%5fjUrsJ-FVi^(uPh??XzT7;xC#*TSCWUt&SO_%E7L8 zS##7kl@=m_CSMW5&_av2IwzIet3@IRKDLO2?9B1AY{~#bFON~G{6nTFL>%=glo;wn z3`2&xOo#&(&qr2ms=7<8X865~$qYc_VDbqm-RT=LXHfj6#4XCqPVhP0c$>xXeWwIv zB_p}0XBf@Joe^Dt^jDtXD_o&=ID{>vAB4I+9C6Z)dy~!K*sQVquJe%!d`^opabJ3T zD3M6SV%_{DvAm)ect!a|@D*y9mm6Uly6#O$n;^F>(-Npc?o37rs#_yc+la87{ea;n zi+m09^&_FIygV1BiHj}R%FYpe*zXzzoHWJ_Sci6-C)7#x6A8XamD7G1RlUm42RDf; zvzn!GOopK~xjsv<0h-x4__^P`saaYFihItTKLSK?}I=<(Z;yz0m5r`hF#3ncC{*J4ktQzLzE6 zTD`+cl5ji(+1n!Q=KZpDA_AJJWJ&Q8RkJ1dDCtcadkWoh0vY|51EnAp%VP96f|iKM zN=HVR%$15eD`k#RL9h!gM)r8c54j8X=*THKv*z+oX{GRxakuU?I*}sDQbl`O{3ojw z(}zBLDsGy|P}B=k*akADc#1CAiU1_xS9)dvdJCx<6V4hF3J1x#TH+2na7;EB816gY z*Tvd)f`t}^f=uxfc8xHK>rz`ID&a;`#XaL$OH;uW3>jI8qn(u(J~?<;x>%I8i51Y3 zML5?+GEFl1VQtXyjwJoi?E91&Jk28CY}kzF$a)ZojA|uDO4Z~`NvS;E!)tq6Zwb@* znvBir7JU!9d(IuInQU4cD^Iz}6-P45_QuU12?whtSTuo0Cthw4ia9prT|y5%fc||V zBHILho(-+~2NVIVt+Y3^MClGF&b(jTSvtuqf>y8JQ!j2O$TDc?%cX<>JOUFwT4Y{Cq#jA0~B2IQjD{@ z*h4FaC*)~~u;mxhI&%*h_)^MVd~cIq;UtnLo*WEjE7X~#|qvJ%~rYYu-aP1r8-tR7zFugjvE^naW4L| zN-DuGnZsHxZ|DxCGH>n%ms@8{q1}A>Jni3e6)Wv22-_u-QQuACeM#WZ4_}VgjD%Th z<))&4^r5kaUXhxoPrx;3M^W)t%0+GX80d;=w4R^(Wt&|7__tY} zK)niW(BrC@;gA!q-my^|#*)IsCttG!I#G(Zq!I5v=bo6>NPmaM z+xRoswPj1g;~!&rsU{%wJwS21F`yime>q2RJ)I*GrR_GEP(GZ{9=!*8LHDENw;m#gT#b} zF;`lCj{SId%k-)13y)f%%)V65jfjR~Yb2VMw3oOoaFHn3R9F~4GT z8T(5?%dXEIQ(gp_hnyBfQ-N z_Rl0kvv*e8ORg1H>FRt!T)x-j3G1JIQT+SWmSGDGf4XTUp3Hh+C;{|UwO?AkD=?z5_22} za_Zq4g+RM%#n__r=!zP{8=SU{9DNwEo?Jj#ca!rnOqP>7Q#ZR&K61oH*yRI*vcS6L zZm7yPV2VJ5t|mSL?)L5-n!gRMt`(wt8?A~IjbSaf`CWyV0BQ`nllaKDyY?>2(yV*E zml(T%|MVwGex6FY*nVr2tTyl94>O}Hmv_@nxMSR0n0k5Boh$GC05KOTE~oRpSTK@-^mG-Xji z^^uM3j1(O0?41~m>>N#g%~Mb*@&7N240?24af^3l2KsBP_7Y^kg0%56+KJYgB&$d9NwxfkErQTIii2r+ZQArbW0m|$FN zn7hq0Jq);h=$B;ah2ujXW?cp|nnItzp>dC9tEbogu#PRIkki&}EP5I}=dXG^=BtLo zm3T>o&Xc)?wBYAvcp|{FfQKC1y%*EKEbQaAras1KGox`}s*bM^3oF+Q`IT0?{JJKg zPn0oHNuQN3CXGF5_*QlPM$ONi3xl@Es^3z(1>sTcMPtQ8r;sQK_K~GdLgVF`MXhKr z7fCHwYQJ#EPT5d*Ur3FL6(ym}6^M|P1qEIz6;(yiDEMubT30Qh60xoyDi?lGsVYyhjP@|+Z0 z)MfI5Fq~Be4U?8V2|KR3I3Nn*mGm8xINu!ccQwQKLb`pxf|flbt7R;P4!8b*^oJFQ z>vQ_%ItC-?4E#4a)v@vNf3{~?W)c##2GZsSX$M>mB=8^xD2DNgeX&ci~tKate6Ie?j ztmaPmu++)HPn-*$jkSOur>7=c9~JLN3Mo!{HO|j_Oxpi%3jz*752{D`=XbjNci#NB z{lgnx6r}$S`1`vh{ww^|HiODy{^d;*&xL=lxBpk+ucY69ufG4B^SqAwZ;~7++2>hR z_2=T})sBCQcR(Rh{}KPA_VIJd^HQI`DGVUL3W)NL!k^Cx&kG^_CMXg9aU4%&k)BhY zXH5U5*uDNE8$J^K;4{GtQm^pC?}Z2DY<22R=RPC)d++%JcZ&-xMFd zC(7?=;OElkVQ;^sT|$2xz*7+1bIS9$mERNz@jp_YqFA0&p5N#GrnpG|k@9rA`<(Lp z4&pZjTJevRryGgql;;k>-;{CHC(3ho;B&z9Bg}7ryzU=|@pPbhPWk)J@;3tv>;V7< q_K$t$bNSyl1Amo=8U987pRIv{G$d$w`Q`9L2LlIfA0|zHefl5uur|a1 literal 0 HcmV?d00001 diff --git a/postgres/scripts/create_empd2.sql b/postgres/scripts/create_empd2.sql new file mode 100644 index 000000000..dc91622f9 --- /dev/null +++ b/postgres/scripts/create_empd2.sql @@ -0,0 +1,307 @@ +-- +-- PostgreSQL database dump +-- + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +--SET client_min_messages = warning; + + +-- +-- Name: metadata; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE metadata ( + sampleName character varying(25) NOT NULL, -- PK + originalSampleName character varying(70), + siteName character varying(70), + country character varying(42), -------------- FK + longitude double precision, + latitude double precision, + elevation double precision, + locationReliability character varying(1), --- FK + locationNotes text, + areaOfSite double precision, + sampleContext character varying(40), -------- FK + siteDescription text, + vegDescription text, + sampleType character varying(14), ----------- FK + sampleMethod character varying(31), --------- FK + ageBP double precision, + ageUncertainty character varying(1), -------- FK + isPercent boolean DEFAULT FALSE, + notes text, + empd_version character varying(5) +); +ALTER TABLE metadata OWNER TO postgres; +ALTER TABLE ONLY metadata ADD CONSTRAINT metadata_pkey PRIMARY KEY (sampleName); + + +-- +-- Name: publication; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE publications ( + publiID integer NOT NULL, ------------------- PK + DOI character varying(50) UNIQUE, + reference text NOT NULL +); +ALTER TABLE publications OWNER TO postgres; +ALTER TABLE ONLY publications ADD CONSTRAINT publications_pkey PRIMARY KEY (publiID); + + +-- +-- Name: workers; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE workers ( + workerID integer NOT NULL, ------------------ PK + firstName character varying(25), + lastName character varying(25), + initials character varying(10), + address1 text, + email1 character varying(50), + phone1 character varying(25), + address2 text, + email2 character varying(50), + phone2 character varying(25) +); +ALTER TABLE workers OWNER TO postgres; +ALTER TABLE ONLY workers ADD CONSTRAINT workers_pkey PRIMARY KEY (workerID); + + +-- +-- Name: p_counts; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE p_counts ( + sampleName character varying(25) NOT NULL, -- PK, FK + var_ integer NOT NULL, ---------------------- PK, FK + count double precision NOT NULL +); +ALTER TABLE p_counts OWNER TO postgres; +ALTER TABLE ONLY p_counts ADD CONSTRAINT p_counts_pkey PRIMARY KEY (sampleName, var_); + + +-- +-- Name: p_vars; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE p_vars ( + var_ integer NOT NULL, ---------------------- PK + acc_var_ integer, + original_varname character varying(100) NOT NULL, + acc_varname character varying(100), + groupid character varying(9), ---------------- FK + notes text +); +ALTER TABLE p_vars OWNER TO postgres; +ALTER TABLE ONLY p_vars ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); + + +-- +-- Name: countries; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE countries ( + country character varying(42) NOT NULL ------- PK +); +ALTER TABLE countries OWNER TO postgres; +ALTER TABLE ONLY countries ADD CONSTRAINT countries_pkey PRIMARY KEY (country); + +-- +-- Name: wwf_terr_ecos; Type: TABLE; Schema: public; Owner: epd; Tablespace: +-- OK + +CREATE TABLE ecosystems ( + sampleName character varying(25) NOT NULL, -- PK + realm char(16), + biome char(62), + ecoregion char(65) +); +ALTER TABLE ecosystems OWNER TO postgres; +ALTER TABLE ONLY ecosystems ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (sampleName); + + + +-- +-- Name: locationReliabilities; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE locationReliabilities ( + locationReliability character varying(1), --- PK + description character varying(55) NOT NULL, + error character varying(5) +); +ALTER TABLE locationReliabilities OWNER TO postgres; +ALTER TABLE ONLY locationReliabilities ADD CONSTRAINT locationReliabilities_pkey PRIMARY KEY (locationReliability); + + +-- +-- Name: sampleContexts; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE sampleContexts ( + sampleContext character varying(40) --------- PK +); +ALTER TABLE sampleContexts OWNER TO postgres; +ALTER TABLE ONLY sampleContexts ADD CONSTRAINT sampleContexts_pkey PRIMARY KEY (sampleContext); + + +-- +-- Name: sampleTypes; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE sampleTypes ( + sampleType character varying(14), ----------- PK + notes character varying(13) +); +ALTER TABLE sampleTypes OWNER TO postgres; +ALTER TABLE ONLY sampleTypes ADD CONSTRAINT sampleTypes_pkey PRIMARY KEY (sampleType); + + +-- +-- Name: collectionMethods; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE sampleMethods ( + sampleMethod character varying(31) ---------- PK +); +ALTER TABLE sampleMethods OWNER TO postgres; +ALTER TABLE ONLY sampleMethods ADD CONSTRAINT sampleMethods_pkey PRIMARY KEY (sampleMethod); + + +-- +-- Name: ageUncertainties; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE ageUncertainties ( + ageUncertainty character varying(1), -------- PK + description character varying(21) NOT NULL, + age character varying(14) +); +ALTER TABLE ageUncertainties OWNER TO postgres; +ALTER TABLE ONLY ageUncertainties ADD CONSTRAINT ageUncertainties_pkey PRIMARY KEY (ageUncertainty); + + +-- +-- Name: workerRoles; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE workerRoles ( + workerRole character varying(3), ------------ PK + description character varying(35) NOT NULL +); +ALTER TABLE workerRoles OWNER TO postgres; +ALTER TABLE ONLY workerRoles ADD CONSTRAINT workerRoles_pkey PRIMARY KEY (workerRole); + + +-- +-- Name: groupID; Type: TABLE; Schema: public; +-- OK + +CREATE TABLE groupID ( + groupid character varying(9) NOT NULL, ------ PK + groupname character varying(60) NOT NULL, + higher_groupid character varying(4) NOT NULL +); +ALTER TABLE groupID OWNER TO postgres; +ALTER TABLE ONLY groupID ADD CONSTRAINT groupID_pkey PRIMARY KEY (groupid); + + +-- +-- Name: metadataPubli; Type: TABLE; Schema: public +-- OK + +CREATE TABLE metaPubli ( + sampleName character varying(25) NOT NULL, -- PK, FK + publiID integer NOT NULL -------------------- PK, FK +); +ALTER TABLE metaPubli OWNER TO postgres; +ALTER TABLE ONLY metaPubli ADD CONSTRAINT metaPubli_pkey PRIMARY KEY (sampleName, publiID); + + +-- +-- Name: metaWorker; Type: TABLE; Schema: public +-- OK + +CREATE TABLE metaWorker ( + sampleName character varying(25) NOT NULL, -- PK, FK + workerID integer NOT NULL, ------------------ PK, FK + workerRole character varying(3) ------------- FK +); +ALTER TABLE metaWorker OWNER TO postgres; +ALTER TABLE ONLY metaWorker ADD CONSTRAINT metaWorker_pkey PRIMARY KEY (sampleName, workerID); + + +-- +-- Name: climate; Type: TABLE; Schema: public +-- OK + +CREATE TABLE climate ( + sampleName character varying(25) NOT NULL, -- PK, FK + T_jan double precision, + T_feb double precision, + T_mar double precision, + T_apr double precision, + T_may double precision, + T_jun double precision, + T_jul double precision, + T_aug double precision, + T_sep double precision, + T_oct double precision, + T_nov double precision, + T_dec double precision, + T_djf double precision, + T_mam double precision, + T_jja double precision, + T_son double precision, + T_ann double precision, + P_jan double precision, + P_feb double precision, + P_mar double precision, + P_apr double precision, + P_may double precision, + P_jun double precision, + P_jul double precision, + P_aug double precision, + P_sep double precision, + P_oct double precision, + P_nov double precision, + P_dec double precision, + P_djf double precision, + P_mam double precision, + P_jja double precision, + P_son double precision, + P_ann double precision +); +ALTER TABLE climate OWNER TO postgres; +ALTER TABLE ONLY climate ADD CONSTRAINT metaWorkerclimate_pkey PRIMARY KEY (sampleName); + + + +-- FOREIGN KEYS +ALTER TABLE ONLY metadata ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES countries(country); +ALTER TABLE ONLY metadata ADD CONSTRAINT metadata_locationReliability_fk FOREIGN KEY (locationReliability) REFERENCES locationReliabilities(locationReliability); +ALTER TABLE ONLY metadata ADD CONSTRAINT metadata_sampleContext_fk FOREIGN KEY (sampleContext) REFERENCES sampleContexts(sampleContext); +ALTER TABLE ONLY metadata ADD CONSTRAINT metadata_sampleType_fk FOREIGN KEY (sampleType) REFERENCES sampleTypes(sampleType); +ALTER TABLE ONLY metadata ADD CONSTRAINT metadata_sampleMethod_fk FOREIGN KEY (sampleMethod) REFERENCES sampleMethods(sampleMethod); +ALTER TABLE ONLY metadata ADD CONSTRAINT metadata_ageUncertainty_fk FOREIGN KEY (ageUncertainty) REFERENCES ageUncertainties(ageUncertainty); + +ALTER TABLE ONLY p_counts ADD CONSTRAINT p_counts_sampleName_fk FOREIGN KEY (sampleName) REFERENCES metadata(sampleName); +ALTER TABLE ONLY p_counts ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES p_vars(var_); + +ALTER TABLE ONLY p_vars ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES groupid(groupid); + +ALTER TABLE ONLY metaPubli ADD CONSTRAINT metaPubli_sampleName_fk FOREIGN KEY (sampleName) REFERENCES metadata(sampleName); +ALTER TABLE ONLY metaPubli ADD CONSTRAINT metaPubli_publiID_fk FOREIGN KEY (publiID) REFERENCES publications(publiID); + +ALTER TABLE ONLY metaWorker ADD CONSTRAINT metaWorker_sampleName_fk FOREIGN KEY (sampleName) REFERENCES metadata(sampleName); +ALTER TABLE ONLY metaWorker ADD CONSTRAINT metaWorker_workerID_fk FOREIGN KEY (workerID) REFERENCES workers(workerID); +ALTER TABLE ONLY metaWorker ADD CONSTRAINT metaWorker_workerRole_fk FOREIGN KEY (workerRole) REFERENCES workerRoles(workerRole); + +ALTER TABLE ONLY climate ADD CONSTRAINT climates_sampleName_fk FOREIGN KEY (sampleName) REFERENCES metadata(sampleName); +ALTER TABLE ONLY ecosystems ADD CONSTRAINT ecosystems_sampleName_fk FOREIGN KEY (sampleName) REFERENCES metadata(sampleName); diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py new file mode 100644 index 000000000..e93095660 --- /dev/null +++ b/postgres/scripts/import_into_empd2.py @@ -0,0 +1,379 @@ +"""Import new data into the EMPD2""" +import psycopg2 as psql +import pandas as pd +import numpy as np +import os +import requests +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument( + 'meta', help="The path to the meta file. Default: %(default)s", nargs='?', + default=os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', 'meta.tsv'))) +parser.add_argument( + '-db', '--database-url', default=os.getenv('DATABASE_URL'), + help="The url to connect to the database. Default: %(default)s") + +args = parser.parse_args() + +meta = args.meta +db_url = args.database_url + +samples_dir = os.path.join(os.path.dirname(meta), 'samples') +base_meta = os.path.join(os.path.dirname(meta), 'meta.tsv') + +conn = psql.connect(db_url) +cursor = conn.cursor() + + +def is_null_str(x): + return 'NULL' if x == "" else "'" + x.strip().replace("'", " ").replace( + '\n', ' ').replace('\r', ' ')+"'" + + +def is_null_nb(x, nodata=np.nan): + return 'NULL' if x == nodata else str(round(x, 2)) + + +def listdir_fullpath(d): + return [os.path.join(d, f) for f in os.listdir(d)] + + +# script for returning elevation from lat, long, based on open elevation data +# which in turn is based on SRTM +def get_elevation(lat, long): + query = ('https://api.open-elevation.com/api/v1/lookup' + f'?locations={lat},{long}') + # json object, various ways you can extract value + r = requests.get(query).json() + # one approach is to use pandas json functionality: + elevation = pd.io.json.json_normalize(r, 'results')['elevation'].values[0] + return elevation + + +def clean_doi(doi): + DOI = doi.replace('https://doi.org/', '') + DOI = DOI.replace('http://dx.doi.org/', '') + DOI = DOI.replace('doi: ', '') + DOI = DOI.replace('doi.org/', '') + DOI = DOI.replace('DOI: ', '') + DOI = DOI.replace('DOI ', '') + DOI = DOI.replace('doi:', '') + DOI = DOI.replace(' ', '') + DOI = DOI.replace('https://link.springer.com/article/', '') + return DOI + + +PUBLI_ID = 1 # TODO: Has to be set from EMPD2 +WORKER_ID = 1 # TODO: Has to be set from EMPD2 + +err = 0 +list_of_errors = [] +missing_elevation = {} + + +METADATA = pd.read_csv(meta, sep='\t') +base_meta = METADATA +for col in set(base_meta.columns) - set(METADATA.columns): + METADATA[col] = '' + +METADATA.replace(np.nan, '', inplace=True) +PUBLI = METADATA[['Publication1', 'DOI1']] +tmp = METADATA[['Publication2', 'DOI2']] +tmp.columns = list(PUBLI.columns) +PUBLI = PUBLI.append(tmp, ignore_index=True) +tmp = METADATA[['Publication3', 'DOI3']] +tmp.columns = list(PUBLI.columns) +PUBLI = PUBLI.append(tmp, ignore_index=True) +tmp = METADATA[['Publication4', 'DOI4']] +tmp.columns = list(PUBLI.columns) +PUBLI = PUBLI.append(tmp, ignore_index=True) +tmp = METADATA[['Publication5', 'DOI5']] +tmp.columns = list(PUBLI.columns) +PUBLI = PUBLI.append(tmp, ignore_index=True) +PUBLI = PUBLI.drop_duplicates() + +# ----- +for x in range(PUBLI.shape[0]): + duplicate_doi = 0 + if PUBLI.iloc[x][1] != '': + duplicate_doi = PUBLI.query('DOI1 == %s' % is_null_str( + PUBLI.iloc[x][1])).shape[0] + if duplicate_doi > 1: + print(PUBLI.query('DOI1 == %s' % is_null_str(PUBLI.iloc[x][1]))) + print('DOI %s duplicated %d times.' % (PUBLI.iloc[x][1], + duplicate_doi)) + if PUBLI.iloc[x][0] != '' and duplicate_doi <= 1: + DOI = clean_doi(PUBLI.iloc[x][1]) + cursor.execute( + "SELECT * FROM publications WHERE (DOI IS NULL OR DOI=%s) AND " + "(reference =%s)" % (is_null_str(DOI), + is_null_str(PUBLI.iloc[x][0]))) + res = cursor.fetchall() + if len(res) == 0: + try: + cursor.execute( + "INSERT INTO publications (publiID, DOI, reference) " + "VALUES (%d, %s, %s)" % (PUBLI_ID, is_null_str(DOI), + is_null_str(PUBLI.iloc[x][0]))) + conn.commit() + except Exception: + print( + "ERROR: " + "INSERT INTO publications (publiID, DOI, reference) " + "VALUES (%d, %s, %s)" % (PUBLI_ID, is_null_str(DOI), + is_null_str(PUBLI.iloc[x][0]))) + conn = psql.connect(db_url) + cursor = conn.cursor() + PUBLI_ID += 1 + else: + pass + elif PUBLI.iloc[x][1] != '' and duplicate_doi <= 1: + print('Empty paper description for DOI %s' % PUBLI.iloc[x][1]) +# --- +# --- +# --- +_worker_attrs = ['FirstName', 'Initials', 'LastName', 'Address1', 'Email1', + 'Phone1', 'Address2', 'Email2', 'Phone2'] +WORKERS = METADATA[list('Worker1_' + a for a in _worker_attrs)] + +tmp = METADATA[list('Worker2_' + a for a in _worker_attrs)] +tmp.columns = list(WORKERS.columns) +WORKERS = WORKERS.append(tmp, ignore_index=True) + +tmp = METADATA[list('Worker3_' + a for a in _worker_attrs)] +tmp.columns = list(WORKERS.columns) +WORKERS = WORKERS.append(tmp, ignore_index=True) + +tmp = METADATA[list('Worker4_' + a for a in _worker_attrs)] +tmp.columns = list(WORKERS.columns) + +WORKERS = WORKERS.append(tmp, ignore_index=True) +WORKERS = WORKERS.drop_duplicates() +WORKERS.replace('', np.nan, inplace=True) +WORKERS = WORKERS.dropna(axis='index', how='all') +WORKERS.replace(np.nan, '', inplace=True) + +# --- +# --- +# --- +for x in range(WORKERS.shape[0]): + cursor.execute( + "SELECT * FROM Workers WHERE " + "(firstname IS NULL OR firstName=%s) AND " + "(initials ISNULL OR initials=%s) AND " + "(lastname ISNULL OR lastName=%s) AND " + "(address1 ISNULL OR address1=%s) AND " + "(email1 ISNULL OR email1=%s) AND " + "(phone1 ISNULL OR phone1=%s) AND " + "(address2 ISNULL OR address2=%s) AND " + "(email2 ISNULL OR email2=%s) AND " + "(phone2 ISNULL OR phone2=%s)" % ( + is_null_str(WORKERS.iloc[x][0]), is_null_str(WORKERS.iloc[x][1]), + is_null_str(WORKERS.iloc[x][2]), is_null_str(WORKERS.iloc[x][3]), + is_null_str(WORKERS.iloc[x][4]), + is_null_str(str(WORKERS.iloc[x][5])), + is_null_str(WORKERS.iloc[x][6]), is_null_str(WORKERS.iloc[x][7]), + is_null_str(str(WORKERS.iloc[x][8])))) + res = cursor.fetchall() + if len(res) == 0: + cursor.execute( + "INSERT INTO Workers " + "(workerID, firstName, initials, lastName, " + " address1, email1, phone1, address2, email2, phone2) VALUES " + "(%d, %s, %s, %s, %s, %s, %s, %s, %s, %s)" % ( + WORKER_ID, is_null_str(WORKERS.iloc[x][0]), + is_null_str(WORKERS.iloc[x][1]), + is_null_str(WORKERS.iloc[x][2]), + is_null_str(WORKERS.iloc[x][3]), + is_null_str(WORKERS.iloc[x][4]), + is_null_str(str(WORKERS.iloc[x][5])), + is_null_str(WORKERS.iloc[x][6]), + is_null_str(WORKERS.iloc[x][7]), + is_null_str(str(WORKERS.iloc[x][8])))) + conn.commit() + WORKER_ID += 1 + else: + pass +# --- +# --- +# --- +for x in range(METADATA.shape[0]): + elevation = METADATA.iloc[x][6] + elev_notes = str(METADATA.iloc[x][8]) + if elevation == '': + if elev_notes == '': + elev_notes = ('Elevation estimated from Google Earth from the ' + 'coordinates.') + else: + elev_notes += ('; Elevation estimated from Google Earth from the ' + 'coordinates') + if METADATA.iloc[x][0] in missing_elevation.keys(): + elevation = missing_elevation[METADATA.iloc[x][0]] + else: + try: + print(METADATA.iloc[x]['SampleName']) + elevation = get_elevation(METADATA.iloc[x]['Latitude'], + METADATA.iloc[x]['Longitude']) + f_missing_elev = open("missing_elevation.csv", "a") + f_missing_elev.write( + METADATA.iloc[x][0] + "," + str(elevation) + "\n") + f_missing_elev.close() + except Exception: + elevation = -9999 + try: + cursor.execute( + "INSERT INTO metadata " + "(sampleName, originalSampleName, siteName, country, longitude, " + " latitude, elevation, locationReliability, locationNotes, " + " areaOfSite, sampleContext, siteDescription, vegDescription, " + " sampleType, sampleMethod, ageBP, ageUncertainty, notes, " + " empd_version) VALUES " + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " + " %s, %s, 'EMPD2')" % ( + is_null_str(METADATA.iloc[x]['SampleName']), + is_null_str(str(METADATA.iloc[x]['OriginalSampleName'])), + is_null_str(str(METADATA.iloc[x]['SiteName'])), + is_null_str(METADATA.iloc[x]['Country']), + is_null_str(str(METADATA.iloc[x]['Longitude'])), + is_null_str(str(METADATA.iloc[x]['Latitude'])), + is_null_str(str(elevation)), + is_null_str(METADATA.iloc[x]['LocationReliability']), + is_null_str(elev_notes), + is_null_str(str(METADATA.iloc[x]['AreaOfSite'])), + is_null_str(METADATA.iloc[x]['SampleContext'].lower()), + is_null_str(METADATA.iloc[x]['SiteDescription']), + is_null_str(METADATA.iloc[x]['VegDescription']), + is_null_str(METADATA.iloc[x][ + 'SampleType'].split(' (to be ')[0].lower()), + is_null_str(METADATA.iloc[x]['SampleMethod'].lower()), + is_null_str(str(METADATA.iloc[x]['AgeBP'])), + is_null_str(METADATA.iloc[x]['AgeUncertainty']), + is_null_str(METADATA.iloc[x]['Notes']))) + conn.commit() + except psql.IntegrityError as e: + conn = psql.connect(db_url) + cursor = conn.cursor() + if str(e) not in list_of_errors: + list_of_errors.append(str(e)) + print('IntegrityError ', x, e) + err += 1 + except psql.DataError as e: + if str(e) not in list_of_errors: + list_of_errors.append(str(e)) + print('DataError ', x, e) + conn = psql.connect(db_url) + cursor = conn.cursor() + err += 1 + except AttributeError as e: + if str(e) not in list_of_errors: + list_of_errors.append(str(e)) + print('AttributeError ', x, e) + conn = psql.connect(db_url) + cursor = conn.cursor() + err += 1 + except Exception as e: + if str(e) not in list_of_errors: + list_of_errors.append(str(e)) + print('Error ', x, e) + conn = psql.connect(db_url) + cursor = conn.cursor() + err += 1 + + temperature = METADATA.iloc[x]['Temperature'] + precip = METADATA.iloc[x]['Precipitation'] +# try: + temperature = temperature or ','.join(['NULL'] * 17) + precip = precip or ','.join(['NULL'] * 17) + + cursor.execute( + "INSERT INTO climate VALUES (%s,%s,%s)" % ( + is_null_str(METADATA.iloc[x]['SampleName']), temperature, precip)) + conn.commit() + + for _worker in map('Worker{}_'.format, '1234'): + if METADATA.iloc[x][_worker + 'LastName'] != '': + cursor.execute( + "SELECT workerID from workers WHERE lastname= '%s' AND " + "(firstname ISNULL or firstname='%s')" % ( + METADATA.iloc[x][_worker + 'LastName'].strip(), + METADATA.iloc[x][_worker + 'FirstName'].strip())) + workerID = cursor.fetchall()[0][0] + cursor.execute( + "INSERT INTO metaworker (sampleName, workerID, workerRole) " + "VALUES (%s, %d, %s)" % ( + is_null_str(METADATA.iloc[x]['SampleName']), workerID, + is_null_str(METADATA.iloc[x][_worker + 'Role']))) + conn.commit() + for i in '1234': + _pub = 'Publication' + i + _doi = 'DOI' + i + if METADATA.iloc[x][_pub] != '': + cursor.execute( + "SELECT publiID from publications WHERE reference= %s AND " + "(DOI ISNULL or DOI='%s')" % ( + is_null_str(METADATA.iloc[x][_pub]), + clean_doi(METADATA.iloc[x][_doi]))) + publiID = cursor.fetchall()[0][0] + cursor.execute( + "INSERT INTO metapubli (sampleName, publiID) VALUES " + "(%s, %d)" % (is_null_str(METADATA.iloc[x]['SampleName']), + publiID)) + conn.commit() + +cursor.execute("SELECT MAX(var_) FROM p_vars") +res = cursor.fetchall() +TAXON_ID = (res[0][0] or 0) + 1 + +for samplename in METADATA.SampleName: + COUNTS = pd.read_csv(os.path.join(samples_dir, samplename + '.tsv'), + sep='\t') + for x, row in COUNTS.iterrows(): + ACCVARNAME = row.acc_varname + ORIVARNAME = row.original_varname + GROUPID = row.groupid # should take EPD or Neotoma groupids here! + NOTES = '' + cursor.execute( + "SELECT var_ FROM p_vars WHERE original_varname='%s' AND " + "acc_varname='%s' AND (groupID ISNULL OR groupID='%s')" % ( + ORIVARNAME, ACCVARNAME, GROUPID)) + res = cursor.fetchall() + if len(res) == 0: + cursor.execute( + "INSERT INTO p_vars " + "(var_, acc_var_, acc_varname, original_varname, groupID, " + " notes) VALUES (%d, NULL, '%s', '%s', '%s', '%s')" % ( + TAXON_ID, ACCVARNAME, ORIVARNAME, GROUPID, + is_null_str(NOTES))) + VAR_ = TAXON_ID + TAXON_ID += 1 + conn.commit() + else: + VAR_ = res[0][0] + samplename = row.samplename + try: + if row['count'] > 0: + val = round(row['count']) + try: + cursor.execute( + "INSERT INTO p_counts (sampleName, var_, count) " + "VALUES ('%s', %d, %d)" % ( + samplename, VAR_, val)) + conn.commit() + except psql.IntegrityError as e: + conn = psql.connect(db_url) + cursor = conn.cursor() + if 'duplicate key value violates unique constraint "p_counts_pkey"' in str(e): + cursor.execute( + "SELECT count FROM p_counts WHERE " + "sampleName = '%s' AND var_ = %d" % ( + samplename, VAR_)) + new_val = cursor.fetchall()[0][0] + val + cursor.execute( + "UPDATE p_counts SET count=%d WHERE " + "sampleName = '%s' AND var_ = %d" % ( + new_val, samplename, VAR_)) + conn.commit() + except Exception: + print(samplename, VAR_, "!" + str(row['count']) + "!") diff --git a/postgres/scripts/makeFixedTables.py b/postgres/scripts/makeFixedTables.py new file mode 100644 index 000000000..c2dd00832 --- /dev/null +++ b/postgres/scripts/makeFixedTables.py @@ -0,0 +1,123 @@ +import sys +import os.path as osp +import psycopg2 as psql +import pandas as pd +import numpy as np + +db_url = sys.argv[-1] + +conn = psql.connect(db_url) +cursor = conn.cursor() + + +def is_null_str(x): + return 'NULL' if x == "" else "'" + x.strip().replace("'", " ") + "'" + + +tables = osp.join(osp.dirname(__file__), "FixedTables.xlsx") + + +# --- +countries = pd.read_excel(tables, sheet_name="Country") +countries.replace(np.nan, '', inplace=True) + +for x in range(countries.shape[0]): + cursor.execute( + "INSERT INTO countries (country) VALUES ('%s')" % countries.iloc[x][0]) + +conn.commit() + + +# --- +LocationReliability = pd.read_excel(tables, sheet_name="LocationReliability") +LocationReliability.replace(np.nan, '', inplace=True) + +for x in range(LocationReliability.shape[0]): + cursor.execute( + "INSERT INTO LocationReliabilities " + "(locationReliability, description, error) " + "VALUES ('%s', '%s', %s)" % ( + LocationReliability.iloc[x][0], LocationReliability.iloc[x][1], + is_null_str(LocationReliability.iloc[x][2]))) + +conn.commit() + + +# --- +sampleContext = pd.read_excel(tables, sheet_name="SampleContext") +sampleContext.replace(np.nan, '', inplace=True) + +for x in range(sampleContext.shape[0]): + cursor.execute( + "INSERT INTO sampleContexts (sampleContext) VALUES ('%s')" % ( + sampleContext.iloc[x][0])) + +conn.commit() + + +# --- +sampleType = pd.read_excel(tables, sheet_name="SampleType") +sampleType.replace(np.nan, '', inplace=True) + +for x in range(sampleType.shape[0]): + cursor.execute( + "INSERT INTO sampleTypes (sampleType, notes) " + "VALUES ('%s', %s)" % (sampleType.iloc[x][0], + is_null_str(sampleType.iloc[x][1]))) + +conn.commit() + + +# --- +SampleMethod = pd.read_excel(tables, sheet_name="SampleMethod") +SampleMethod.replace(np.nan, '', inplace=True) + +for x in range(SampleMethod.shape[0]): + cursor.execute( + "INSERT INTO sampleMethods (sampleMethod) VALUES ('%s')" % ( + SampleMethod.iloc[x][0])) + +conn.commit() + + +# --- +ageUncertainties = pd.read_excel(tables, sheet_name="AgeUncertainty") +ageUncertainties.replace(np.nan, '', inplace=True) + +for x in range(ageUncertainties.shape[0]): + cursor.execute( + "INSERT INTO ageUncertainties (ageUncertainty, description, age) " + "VALUES ('%s', '%s', '%s')" % ( + ageUncertainties.iloc[x][0], ageUncertainties.iloc[x][1], + ageUncertainties.iloc[x][2])) + +conn.commit() + + +# --- +workerRoles = pd.read_excel(tables, sheet_name="WorkerRole") +workerRoles.replace(np.nan, '', inplace=True) + +for x in range(workerRoles.shape[0]): + cursor.execute( + "INSERT INTO workerRoles (workerRole, description) VALUES " + "('%s', '%s')" % (workerRoles.iloc[x][0], workerRoles.iloc[x][1])) + +conn.commit() + + +# --- +groupIDs = pd.read_excel(tables, sheet_name="GroupID") +groupIDs.replace(np.nan, '', inplace=True) + +for x in range(groupIDs.shape[0]): + cursor.execute( + "INSERT INTO groupID (groupID, groupname, higher_groupid) VALUES " + "(%s, %s, %s)" % ( + is_null_str(groupIDs.iloc[x][0]), is_null_str(groupIDs.iloc[x][1]), + is_null_str(groupIDs.iloc[x][2]))) + +conn.commit() + + +conn.close() From 0e8fcb03311b79e517c912ba6aec16c8595e72ed Mon Sep 17 00:00:00 2001 From: Chilipp Date: Thu, 21 Mar 2019 23:35:14 +0100 Subject: [PATCH 017/119] Updated tests --- tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests b/tests index 3fdc3bf6a..f2cd245e1 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 3fdc3bf6a0fbdc3c730d28d7db2209b6477ae96b +Subproject commit f2cd245e16206faa0332e4641f93bc184de18a37 From 0dbfef0934b8be570da60b232ce19a8b0ae2b65e Mon Sep 17 00:00:00 2001 From: Chilipp Date: Fri, 22 Mar 2019 06:33:23 +0100 Subject: [PATCH 018/119] use PYTEST env var in docker run --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d812c75bb..c53db1a9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,5 +13,5 @@ install: - docker build . -t empd2/empd-data script: - - docker run empd2/empd-data /bin/bash -c "cd /opt/empd-data && $PYTEST -v --empd-meta=${METAFILE}" + - docker run empd2/empd-data /bin/bash -c "cd /opt/empd-data && \$PYTEST -v --empd-meta=${METAFILE}" - docker run empd2/empd-data /bin/bash -c "start_pg_server && createdb -U postgres EMPD2 && cd /opt/empd-data && psql EMPD2 -U postgres -f postgres/EMPD2.sql && python postgres/scripts/import_into_empd2.py meta.tsv -d \${DATABASE_URL}/EMPD2" From cf6ed4d1c732d9a83542d8e572cba1d23d7926ae Mon Sep 17 00:00:00 2001 From: Chilipp Date: Fri, 22 Mar 2019 09:28:47 +0100 Subject: [PATCH 019/119] Update tests --- tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests b/tests index f2cd245e1..1b104af08 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit f2cd245e16206faa0332e4641f93bc184de18a37 +Subproject commit 1b104af08a17f9cee2c765248abddc4091d0c551 From a466760b91318dfbb291db4ca92c0e19a7d27fd4 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Fri, 22 Mar 2019 16:26:04 +0100 Subject: [PATCH 020/119] Allow update of existing samples --- postgres/scripts/import_into_empd2.py | 99 ++++++++++++++++++--------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index e93095660..8e06933e6 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -5,6 +5,7 @@ import os import requests import argparse +from itertools import product parser = argparse.ArgumentParser() parser.add_argument( @@ -74,10 +75,15 @@ def clean_doi(doi): METADATA = pd.read_csv(meta, sep='\t') -base_meta = METADATA -for col in set(base_meta.columns) - set(METADATA.columns): +base_cols = pd.read_csv(base_meta, nrows=1, sep='\t').columns + +for col in set(base_cols) - set(METADATA.columns): METADATA[col] = '' +# get the existing samples +cursor.execute('SELECT samplename FROM metadata') +existing_samples = [r[0] for r in cursor.fetchall()] + METADATA.replace(np.nan, '', inplace=True) PUBLI = METADATA[['Publication1', 'DOI1']] tmp = METADATA[['Publication2', 'DOI2']] @@ -200,6 +206,7 @@ def clean_doi(doi): # --- # --- for x in range(METADATA.shape[0]): + to_update = METADATA.iloc[x]['SampleName'] in existing_samples elevation = METADATA.iloc[x][6] elev_notes = str(METADATA.iloc[x][8]) if elevation == '': @@ -223,34 +230,47 @@ def clean_doi(doi): except Exception: elevation = -9999 try: - cursor.execute( - "INSERT INTO metadata " - "(sampleName, originalSampleName, siteName, country, longitude, " - " latitude, elevation, locationReliability, locationNotes, " - " areaOfSite, sampleContext, siteDescription, vegDescription, " - " sampleType, sampleMethod, ageBP, ageUncertainty, notes, " - " empd_version) VALUES " - "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " - " %s, %s, 'EMPD2')" % ( - is_null_str(METADATA.iloc[x]['SampleName']), - is_null_str(str(METADATA.iloc[x]['OriginalSampleName'])), - is_null_str(str(METADATA.iloc[x]['SiteName'])), - is_null_str(METADATA.iloc[x]['Country']), - is_null_str(str(METADATA.iloc[x]['Longitude'])), - is_null_str(str(METADATA.iloc[x]['Latitude'])), - is_null_str(str(elevation)), - is_null_str(METADATA.iloc[x]['LocationReliability']), - is_null_str(elev_notes), - is_null_str(str(METADATA.iloc[x]['AreaOfSite'])), - is_null_str(METADATA.iloc[x]['SampleContext'].lower()), - is_null_str(METADATA.iloc[x]['SiteDescription']), - is_null_str(METADATA.iloc[x]['VegDescription']), - is_null_str(METADATA.iloc[x][ - 'SampleType'].split(' (to be ')[0].lower()), - is_null_str(METADATA.iloc[x]['SampleMethod'].lower()), - is_null_str(str(METADATA.iloc[x]['AgeBP'])), - is_null_str(METADATA.iloc[x]['AgeUncertainty']), - is_null_str(METADATA.iloc[x]['Notes']))) + if to_update: + query = ( + "UPDATE metadata SET " + "sampleName = %s, originalSampleName = %s, siteName = %s, " + "country = %s, longitude = %s, latitude = %s, elevation = %s, " + "locationReliability = %s, locationNotes = %s, " + "areaOfSite = %s, sampleContext = %s, siteDescription = %s, " + "vegDescription = %s, sampleType = %s, sampleMethod = %s, " + "ageBP = %s, ageUncertainty = %s, notes = %s " + "WHERE sampleName = {}").format( + is_null_str(METADATA.iloc[x]['SampleName'])) + else: + query = ( + "INSERT INTO metadata " + "(sampleName, originalSampleName, siteName, country, longitude, " + " latitude, elevation, locationReliability, locationNotes, " + " areaOfSite, sampleContext, siteDescription, vegDescription, " + " sampleType, sampleMethod, ageBP, ageUncertainty, notes, " + " empd_version) VALUES " + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " + " %s, %s, 'EMPD2')") + cursor.execute(query % ( + is_null_str(METADATA.iloc[x]['SampleName']), + is_null_str(str(METADATA.iloc[x]['OriginalSampleName'])), + is_null_str(str(METADATA.iloc[x]['SiteName'])), + is_null_str(METADATA.iloc[x]['Country']), + is_null_str(str(METADATA.iloc[x]['Longitude'])), + is_null_str(str(METADATA.iloc[x]['Latitude'])), + is_null_str(str(elevation)), + is_null_str(METADATA.iloc[x]['LocationReliability']), + is_null_str(elev_notes), + is_null_str(str(METADATA.iloc[x]['AreaOfSite'])), + is_null_str(METADATA.iloc[x]['SampleContext'].lower()), + is_null_str(METADATA.iloc[x]['SiteDescription']), + is_null_str(METADATA.iloc[x]['VegDescription']), + is_null_str(METADATA.iloc[x][ + 'SampleType'].split(' (to be ')[0].lower()), + is_null_str(METADATA.iloc[x]['SampleMethod'].lower()), + is_null_str(str(METADATA.iloc[x]['AgeBP'])), + is_null_str(METADATA.iloc[x]['AgeUncertainty']), + is_null_str(METADATA.iloc[x]['Notes']))) conn.commit() except psql.IntegrityError as e: conn = psql.connect(db_url) @@ -287,9 +307,22 @@ def clean_doi(doi): temperature = temperature or ','.join(['NULL'] * 17) precip = precip or ','.join(['NULL'] * 17) - cursor.execute( - "INSERT INTO climate VALUES (%s,%s,%s)" % ( - is_null_str(METADATA.iloc[x]['SampleName']), temperature, precip)) + if to_update: + seasons = [ + 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', + 'oct', 'nov', 'dec', 'djf', 'mam', 'jja', 'son', 'ann'] + update_str = ', '.join( + '%s_%s = %s' % (v, s, is_null_str(val)) for (v, s), val in zip( + product('tp', seasons), np.r_[temperature.split(','), + precip.split(',')])) + cursor.execute( + "UPDATE climate SET %s WHERE sampleName = %s" % ( + update_str, is_null_str(METADATA.iloc[x]['SampleName']))) + else: + cursor.execute( + "INSERT INTO climate VALUES (%s,%s,%s)" % ( + is_null_str(METADATA.iloc[x]['SampleName']), temperature, + precip)) conn.commit() for _worker in map('Worker{}_'.format, '1234'): From db953e8a1992d920d2a25ea3d1f5f33693eff681 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Fri, 22 Mar 2019 12:34:40 +0100 Subject: [PATCH 021/119] Added postgres dump for test.tsv --- postgres/test.sql | 971 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 971 insertions(+) create mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql new file mode 100644 index 000000000..958492799 --- /dev/null +++ b/postgres/test.sql @@ -0,0 +1,971 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 9.5.15 +-- Dumped by pg_dump version 9.5.15 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ageuncertainties ( + ageuncertainty character varying(1) NOT NULL, + description character varying(21) NOT NULL, + age character varying(14) +); + + +ALTER TABLE public.ageuncertainties OWNER TO postgres; + +-- +-- Name: climate; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.climate ( + samplename character varying(25) NOT NULL, + t_jan double precision, + t_feb double precision, + t_mar double precision, + t_apr double precision, + t_may double precision, + t_jun double precision, + t_jul double precision, + t_aug double precision, + t_sep double precision, + t_oct double precision, + t_nov double precision, + t_dec double precision, + t_djf double precision, + t_mam double precision, + t_jja double precision, + t_son double precision, + t_ann double precision, + p_jan double precision, + p_feb double precision, + p_mar double precision, + p_apr double precision, + p_may double precision, + p_jun double precision, + p_jul double precision, + p_aug double precision, + p_sep double precision, + p_oct double precision, + p_nov double precision, + p_dec double precision, + p_djf double precision, + p_mam double precision, + p_jja double precision, + p_son double precision, + p_ann double precision +); + + +ALTER TABLE public.climate OWNER TO postgres; + +-- +-- Name: countries; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.countries ( + country character varying(42) NOT NULL +); + + +ALTER TABLE public.countries OWNER TO postgres; + +-- +-- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ecosystems ( + samplename character varying(25) NOT NULL, + realm character(16), + biome character(62), + ecoregion character(65) +); + + +ALTER TABLE public.ecosystems OWNER TO postgres; + +-- +-- Name: groupid; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.groupid ( + groupid character varying(9) NOT NULL, + groupname character varying(60) NOT NULL, + higher_groupid character varying(4) NOT NULL +); + + +ALTER TABLE public.groupid OWNER TO postgres; + +-- +-- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.locationreliabilities ( + locationreliability character varying(1) NOT NULL, + description character varying(55) NOT NULL, + error character varying(5) +); + + +ALTER TABLE public.locationreliabilities OWNER TO postgres; + +-- +-- Name: metadata; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metadata ( + samplename character varying(25) NOT NULL, + originalsamplename character varying(70), + sitename character varying(70), + country character varying(42), + longitude double precision, + latitude double precision, + elevation double precision, + locationreliability character varying(1), + locationnotes text, + areaofsite double precision, + samplecontext character varying(40), + sitedescription text, + vegdescription text, + sampletype character varying(14), + samplemethod character varying(31), + agebp double precision, + ageuncertainty character varying(1), + ispercent boolean DEFAULT false, + notes text, + empd_version character varying(5) +); + + +ALTER TABLE public.metadata OWNER TO postgres; + +-- +-- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metapubli ( + samplename character varying(25) NOT NULL, + publiid integer NOT NULL +); + + +ALTER TABLE public.metapubli OWNER TO postgres; + +-- +-- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metaworker ( + samplename character varying(25) NOT NULL, + workerid integer NOT NULL, + workerrole character varying(3) +); + + +ALTER TABLE public.metaworker OWNER TO postgres; + +-- +-- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_counts ( + samplename character varying(25) NOT NULL, + var_ integer NOT NULL, + count double precision NOT NULL +); + + +ALTER TABLE public.p_counts OWNER TO postgres; + +-- +-- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_vars ( + var_ integer NOT NULL, + acc_var_ integer, + original_varname character varying(100) NOT NULL, + acc_varname character varying(100), + groupid character varying(9), + notes text +); + + +ALTER TABLE public.p_vars OWNER TO postgres; + +-- +-- Name: publications; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.publications ( + publiid integer NOT NULL, + doi character varying(50), + reference text NOT NULL +); + + +ALTER TABLE public.publications OWNER TO postgres; + +-- +-- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplecontexts ( + samplecontext character varying(40) NOT NULL +); + + +ALTER TABLE public.samplecontexts OWNER TO postgres; + +-- +-- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplemethods ( + samplemethod character varying(31) NOT NULL +); + + +ALTER TABLE public.samplemethods OWNER TO postgres; + +-- +-- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.sampletypes ( + sampletype character varying(14) NOT NULL, + notes character varying(13) +); + + +ALTER TABLE public.sampletypes OWNER TO postgres; + +-- +-- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workerroles ( + workerrole character varying(3) NOT NULL, + description character varying(35) NOT NULL +); + + +ALTER TABLE public.workerroles OWNER TO postgres; + +-- +-- Name: workers; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workers ( + workerid integer NOT NULL, + firstname character varying(25), + lastname character varying(25), + initials character varying(10), + address1 text, + email1 character varying(50), + phone1 character varying(25), + address2 text, + email2 character varying(50), + phone2 character varying(25) +); + + +ALTER TABLE public.workers OWNER TO postgres; + +-- +-- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP +\. + + +-- +-- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +\. + + +-- +-- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.countries (country) FROM stdin; +Albania +Algeria +Andorra +Armenia +Austria +Azerbaijan +Bahrain +Belarus +Belgium +Bosnia And Herzegovina +Bulgaria +Cape Verde +Croatia +Cyprus +Czech Republic +Denmark +Egypt +Eritrea +Estonia +Ethiopia +Faroe Islands +Finland +France +Georgia +Germany +Gibraltar +Greece +Greenland +Guernsey +Hungary +Iceland +Iran, Islamic Republic Of +Iraq +Ireland +Isle Of Man +Israel +Italy +India +Japan +Jersey +Kazakhstan +Kuwait +Kyrgyzstan +Latvia +Lebanon +Libyan Arab Jamahiriya +Liechtenstein +Lithuania +Luxembourg +Macedonia, The Former Yugoslav Republic Of +Malta +Moldova, Republic Of +Monaco +Morocco +Netherlands +Norway +Oman +Palestinian Territory, Occupied +Poland +Portugal +Qatar +Romania +Russian Federation +San Marino +Saudi Arabia +Serbia And Montenegro +Slovakia +Slovenia +Spain +Sweden +Switzerland +Syrian Arab Republic +Tunisia +Turkey +Ukraine +United Arab Emirates +United Kingdom +Yemen +Jordan +Black Sea +Dead Sea +Turkmenistan, Republic Of +Adriatic Sea +Svalbard and Jan Mayen +China, People s Republic Of +\. + + +-- +-- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; +\. + + +-- +-- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; +ALGA Algae AQUA +AQBR Aquatic Bryophytes NOPO +AQPT Aquatic Pteridophyta AQUA +AQUA Aquatics AQUA +AQVP Aquatic Vascular Plants AQUA +BRYO Bryophytes NOPO +BRYO/VACR TO BE REMOVED TBR +CHAR Charcoal CHAR +CHLO Chlorophytes NOPO +CYAN Cyanobacteria NOPO +DINO Dinoflagellates NOPO +DWAR Dwarf shrubs TRSH +FUNG Fungi NOPO +HELO Helophytes AQUA +HERB Herbs HERB +INUN Indeterminables and unknowns INUN +INVE Invertebrates NOPO +LIAN Liana TRSH +MACR Macrofossils NOPO +NOPO Nonpollen NOPO +PLAT Platyhelminthes (flatworms) NOPO +PREQ Pre-Quaternary type NOPO +RHIZ Rhizopods NOPO +ROTI Rotifera (Rotifers) NOPO +SPOR Taxonomically undifferentiated spores INUN +TEAM Testate amoebae NOPO +TRSH Trees and shrubs TRSH +UNPP Undifferentiated NPP INUN +UPBR Upland Bryophytes NOPO +UPHE Upland herbs HERB +VACR Vascular cryptogams (Pteridophytes) NOPO +VEMI Vegetative microfossils NOPO +NEMA Nematoda (roundworms) NOPO +ACRI Acritarchs NOPO +EUMY Eumycete NOPO +HEMI Hemi-parasitic NOPO +MOSS Moss NOPO +MICR Microcrustaceans NOPO +\. + + +-- +-- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! \N +\. + + +-- +-- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +\. + + +-- +-- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metapubli (samplename, publiid) FROM stdin; +\. + + +-- +-- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; +\. + + +-- +-- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_counts (samplename, var_, count) FROM stdin; +test_a1 1 45 +test_a1 2 6 +test_a1 3 55 +\. + + +-- +-- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL +\. + + +-- +-- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.publications (publiid, doi, reference) FROM stdin; +\. + + +-- +-- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplecontexts (samplecontext) FROM stdin; +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological +\. + + +-- +-- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplemethods (samplemethod) FROM stdin; +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer +\. + + +-- +-- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.sampletypes (sampletype, notes) FROM stdin; +dung \N +litter \N +moss \N +pollen trap To be removed +sediment submerged +spider web \N +soil terrestrial +sphagnum peat \N +peat \N +lichen \N +submerged \N +core_top \N +tussock \N +epiphytic moss \N +\. + + +-- +-- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workerroles (workerrole, description) FROM stdin; +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) +\. + + +-- +-- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; +\. + + +-- +-- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ageuncertainties + ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); + + +-- +-- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.countries + ADD CONSTRAINT countries_pkey PRIMARY KEY (country); + + +-- +-- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); + + +-- +-- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.groupid + ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); + + +-- +-- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.locationreliabilities + ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); + + +-- +-- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); + + +-- +-- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); + + +-- +-- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); + + +-- +-- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); + + +-- +-- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); + + +-- +-- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); + + +-- +-- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_doi_key UNIQUE (doi); + + +-- +-- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); + + +-- +-- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplecontexts + ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); + + +-- +-- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplemethods + ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); + + +-- +-- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.sampletypes + ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); + + +-- +-- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workerroles + ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); + + +-- +-- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workers + ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); + + +-- +-- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); + + +-- +-- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); + + +-- +-- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); + + +-- +-- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); + + +-- +-- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); + + +-- +-- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); + + +-- +-- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); + + +-- +-- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); + + +-- +-- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); + + +-- +-- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); + + +-- +-- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); + + +-- +-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: psommer +-- + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM psommer; +GRANT ALL ON SCHEMA public TO psommer; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- + From 1775d56866dea7e1998af864e82e46594221cda4 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Fri, 22 Mar 2019 16:32:31 +0100 Subject: [PATCH 022/119] Revert "Added postgres dump for test.tsv" This reverts commit db953e8a1992d920d2a25ea3d1f5f33693eff681. --- postgres/test.sql | 971 ---------------------------------------------- 1 file changed, 971 deletions(-) delete mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql deleted file mode 100644 index 958492799..000000000 --- a/postgres/test.sql +++ /dev/null @@ -1,971 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 9.5.15 --- Dumped by pg_dump version 9.5.15 - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET client_min_messages = warning; -SET row_security = off; - --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: --- - -COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - -SET default_tablespace = ''; - -SET default_with_oids = false; - --- --- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ageuncertainties ( - ageuncertainty character varying(1) NOT NULL, - description character varying(21) NOT NULL, - age character varying(14) -); - - -ALTER TABLE public.ageuncertainties OWNER TO postgres; - --- --- Name: climate; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.climate ( - samplename character varying(25) NOT NULL, - t_jan double precision, - t_feb double precision, - t_mar double precision, - t_apr double precision, - t_may double precision, - t_jun double precision, - t_jul double precision, - t_aug double precision, - t_sep double precision, - t_oct double precision, - t_nov double precision, - t_dec double precision, - t_djf double precision, - t_mam double precision, - t_jja double precision, - t_son double precision, - t_ann double precision, - p_jan double precision, - p_feb double precision, - p_mar double precision, - p_apr double precision, - p_may double precision, - p_jun double precision, - p_jul double precision, - p_aug double precision, - p_sep double precision, - p_oct double precision, - p_nov double precision, - p_dec double precision, - p_djf double precision, - p_mam double precision, - p_jja double precision, - p_son double precision, - p_ann double precision -); - - -ALTER TABLE public.climate OWNER TO postgres; - --- --- Name: countries; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.countries ( - country character varying(42) NOT NULL -); - - -ALTER TABLE public.countries OWNER TO postgres; - --- --- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ecosystems ( - samplename character varying(25) NOT NULL, - realm character(16), - biome character(62), - ecoregion character(65) -); - - -ALTER TABLE public.ecosystems OWNER TO postgres; - --- --- Name: groupid; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.groupid ( - groupid character varying(9) NOT NULL, - groupname character varying(60) NOT NULL, - higher_groupid character varying(4) NOT NULL -); - - -ALTER TABLE public.groupid OWNER TO postgres; - --- --- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.locationreliabilities ( - locationreliability character varying(1) NOT NULL, - description character varying(55) NOT NULL, - error character varying(5) -); - - -ALTER TABLE public.locationreliabilities OWNER TO postgres; - --- --- Name: metadata; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metadata ( - samplename character varying(25) NOT NULL, - originalsamplename character varying(70), - sitename character varying(70), - country character varying(42), - longitude double precision, - latitude double precision, - elevation double precision, - locationreliability character varying(1), - locationnotes text, - areaofsite double precision, - samplecontext character varying(40), - sitedescription text, - vegdescription text, - sampletype character varying(14), - samplemethod character varying(31), - agebp double precision, - ageuncertainty character varying(1), - ispercent boolean DEFAULT false, - notes text, - empd_version character varying(5) -); - - -ALTER TABLE public.metadata OWNER TO postgres; - --- --- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metapubli ( - samplename character varying(25) NOT NULL, - publiid integer NOT NULL -); - - -ALTER TABLE public.metapubli OWNER TO postgres; - --- --- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metaworker ( - samplename character varying(25) NOT NULL, - workerid integer NOT NULL, - workerrole character varying(3) -); - - -ALTER TABLE public.metaworker OWNER TO postgres; - --- --- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_counts ( - samplename character varying(25) NOT NULL, - var_ integer NOT NULL, - count double precision NOT NULL -); - - -ALTER TABLE public.p_counts OWNER TO postgres; - --- --- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_vars ( - var_ integer NOT NULL, - acc_var_ integer, - original_varname character varying(100) NOT NULL, - acc_varname character varying(100), - groupid character varying(9), - notes text -); - - -ALTER TABLE public.p_vars OWNER TO postgres; - --- --- Name: publications; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.publications ( - publiid integer NOT NULL, - doi character varying(50), - reference text NOT NULL -); - - -ALTER TABLE public.publications OWNER TO postgres; - --- --- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplecontexts ( - samplecontext character varying(40) NOT NULL -); - - -ALTER TABLE public.samplecontexts OWNER TO postgres; - --- --- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplemethods ( - samplemethod character varying(31) NOT NULL -); - - -ALTER TABLE public.samplemethods OWNER TO postgres; - --- --- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.sampletypes ( - sampletype character varying(14) NOT NULL, - notes character varying(13) -); - - -ALTER TABLE public.sampletypes OWNER TO postgres; - --- --- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workerroles ( - workerrole character varying(3) NOT NULL, - description character varying(35) NOT NULL -); - - -ALTER TABLE public.workerroles OWNER TO postgres; - --- --- Name: workers; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workers ( - workerid integer NOT NULL, - firstname character varying(25), - lastname character varying(25), - initials character varying(10), - address1 text, - email1 character varying(50), - phone1 character varying(25), - address2 text, - email2 character varying(50), - phone2 character varying(25) -); - - -ALTER TABLE public.workers OWNER TO postgres; - --- --- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; -A Modern sample 0BP to Present -B Within last 100 years 0-50BP -C Within last 250 years 50-200BP -\. - - --- --- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -\. - - --- --- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.countries (country) FROM stdin; -Albania -Algeria -Andorra -Armenia -Austria -Azerbaijan -Bahrain -Belarus -Belgium -Bosnia And Herzegovina -Bulgaria -Cape Verde -Croatia -Cyprus -Czech Republic -Denmark -Egypt -Eritrea -Estonia -Ethiopia -Faroe Islands -Finland -France -Georgia -Germany -Gibraltar -Greece -Greenland -Guernsey -Hungary -Iceland -Iran, Islamic Republic Of -Iraq -Ireland -Isle Of Man -Israel -Italy -India -Japan -Jersey -Kazakhstan -Kuwait -Kyrgyzstan -Latvia -Lebanon -Libyan Arab Jamahiriya -Liechtenstein -Lithuania -Luxembourg -Macedonia, The Former Yugoslav Republic Of -Malta -Moldova, Republic Of -Monaco -Morocco -Netherlands -Norway -Oman -Palestinian Territory, Occupied -Poland -Portugal -Qatar -Romania -Russian Federation -San Marino -Saudi Arabia -Serbia And Montenegro -Slovakia -Slovenia -Spain -Sweden -Switzerland -Syrian Arab Republic -Tunisia -Turkey -Ukraine -United Arab Emirates -United Kingdom -Yemen -Jordan -Black Sea -Dead Sea -Turkmenistan, Republic Of -Adriatic Sea -Svalbard and Jan Mayen -China, People s Republic Of -\. - - --- --- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; -\. - - --- --- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; -ALGA Algae AQUA -AQBR Aquatic Bryophytes NOPO -AQPT Aquatic Pteridophyta AQUA -AQUA Aquatics AQUA -AQVP Aquatic Vascular Plants AQUA -BRYO Bryophytes NOPO -BRYO/VACR TO BE REMOVED TBR -CHAR Charcoal CHAR -CHLO Chlorophytes NOPO -CYAN Cyanobacteria NOPO -DINO Dinoflagellates NOPO -DWAR Dwarf shrubs TRSH -FUNG Fungi NOPO -HELO Helophytes AQUA -HERB Herbs HERB -INUN Indeterminables and unknowns INUN -INVE Invertebrates NOPO -LIAN Liana TRSH -MACR Macrofossils NOPO -NOPO Nonpollen NOPO -PLAT Platyhelminthes (flatworms) NOPO -PREQ Pre-Quaternary type NOPO -RHIZ Rhizopods NOPO -ROTI Rotifera (Rotifers) NOPO -SPOR Taxonomically undifferentiated spores INUN -TEAM Testate amoebae NOPO -TRSH Trees and shrubs TRSH -UNPP Undifferentiated NPP INUN -UPBR Upland Bryophytes NOPO -UPHE Upland herbs HERB -VACR Vascular cryptogams (Pteridophytes) NOPO -VEMI Vegetative microfossils NOPO -NEMA Nematoda (roundworms) NOPO -ACRI Acritarchs NOPO -EUMY Eumycete NOPO -HEMI Hemi-parasitic NOPO -MOSS Moss NOPO -MICR Microcrustaceans NOPO -\. - - --- --- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; -A Good for high resolution remote sensing (<100m) <100m -B Good for lower resolution remote sensing (<1km) <1km -C Ok for climate reconstruction or regional scale mapping <5km -D Ok for mapping at continental scale <20km -X Do not use! \N -\. - - --- --- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -\. - - --- --- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metapubli (samplename, publiid) FROM stdin; -\. - - --- --- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -\. - - --- --- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_counts (samplename, var_, count) FROM stdin; -test_a1 1 45 -test_a1 2 6 -test_a1 3 55 -\. - - --- --- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; -1 \N Pinus Pinus TRSH NULL -2 \N Exotic Exotics (counted) NOPO NULL -3 \N Umbelliferae Apiaceae HERB NULL -\. - - --- --- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.publications (publiid, doi, reference) FROM stdin; -\. - - --- --- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplecontexts (samplecontext) FROM stdin; -arable -blanket bog -bog -cave -cirque lake -closed forest -coastal -coastal lake -coastal wetland -drained_lake -ephemeral lake -ephemeral lake/pond -estuarine -fallow -fallow - scattered trees/shrubs -fen -floodplain mire -fluvial -forest -forest undefined -heath -ice from glacier -kettle lake -lagoon -lake -maquis -marine -marsh -marshland -mire -mor humus -moss-shrubby tundra -natural grassland -natural grasslands -natural open water -open forest -open forest/orchard -open forest/scattered trees/shrubs -orchard -orchard/scattered trees/shrubs -pasture -peatbog -pond -riverine -salt lake -salt marsh -scattered trees -scattered trees/shrubs -shrubs -shrubs/scrubland -soil -soligenous (mineraltrophic) Mire -sparse/no vegetation -swamp -tectonic lake -terrestrial -treeless vegetation -treeless vegetation/natural grassland -treeless vegetation/sparse vegetation -urban -valley mire -wetland -wetland bog -wetland bog/scattered trees/shrubs -wetland bog/treeless vegetation -archaeological -\. - - --- --- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplemethods (samplemethod) FROM stdin; -hand picking -hand picking - multiple samples -unspecified -auger corer -freeze corer -gravity corer -piston corer -russian corer -box -spade -hiller corer -gouge auger -core -corer unspecified -eckman-grab -box corer -\. - - --- --- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.sampletypes (sampletype, notes) FROM stdin; -dung \N -litter \N -moss \N -pollen trap To be removed -sediment submerged -spider web \N -soil terrestrial -sphagnum peat \N -peat \N -lichen \N -submerged \N -core_top \N -tussock \N -epiphytic moss \N -\. - - --- --- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workerroles (workerrole, description) FROM stdin; -R Responsible Person -R/A Both Responsible Person and Analyst -A Analyst -A/D Analyst (Deceased) -A/I Analysis (Inactive) -A/U Analyst (Unknown) -\. - - --- --- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; -\. - - --- --- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ageuncertainties - ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); - - --- --- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.countries - ADD CONSTRAINT countries_pkey PRIMARY KEY (country); - - --- --- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); - - --- --- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.groupid - ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); - - --- --- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.locationreliabilities - ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); - - --- --- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); - - --- --- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); - - --- --- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); - - --- --- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); - - --- --- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); - - --- --- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); - - --- --- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_doi_key UNIQUE (doi); - - --- --- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); - - --- --- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplecontexts - ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); - - --- --- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplemethods - ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); - - --- --- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.sampletypes - ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); - - --- --- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workerroles - ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); - - --- --- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workers - ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); - - --- --- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); - - --- --- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); - - --- --- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); - - --- --- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); - - --- --- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); - - --- --- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); - - --- --- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); - - --- --- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); - - --- --- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); - - --- --- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); - - --- --- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); - - --- --- Name: SCHEMA public; Type: ACL; Schema: -; Owner: psommer --- - -REVOKE ALL ON SCHEMA public FROM PUBLIC; -REVOKE ALL ON SCHEMA public FROM psommer; -GRANT ALL ON SCHEMA public TO psommer; -GRANT ALL ON SCHEMA public TO PUBLIC; - - --- --- PostgreSQL database dump complete --- - From 2e4812be1c50b2d2db9e9e46904938914a61f9b8 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 06:33:04 +0100 Subject: [PATCH 023/119] Added mapping from EMPD country to Natural Earth country --- postgres/scripts/FixedTables.xlsx | Bin 27369 -> 27978 bytes postgres/scripts/create_empd2.sql | 1 + postgres/scripts/makeFixedTables.py | 3 ++- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/postgres/scripts/FixedTables.xlsx b/postgres/scripts/FixedTables.xlsx index c74c7f34da3ad7d7e17f575ec2eab155fde58316..12e6d198e8d3b620669af0d405fb243563b9f226 100644 GIT binary patch delta 9987 zcmZ8{WmFtZ*DVY#LvT%Shv4oG!R5ha210O`06_+~;O-XO-9m78cbDKEAZWfJ&%NLK z-aCJ)>(n`W@3XsRt?FKVI0rkt3|no0g0lAUI@cZ^21YCX6AB%~9+q2|#&1>HN<|aY zNTyxs2w(({e_oaQNa`~BpSjI3B z{zZ`c+)jJGmn)yrRYu;ZuEYa8{Uq@C+;nZuLQ2L5;Sv*nUu0JK56((sK7U!L&m*O$x*-1+>W`q$6m z(j0$ysukbbEtsuj^Pt>Z%hm1dgEhkW@ui5eTLV79%9lR2~+Yy9tdfttI` z$mzE>IoFA79E}v;)z<*Ao;TI($do6?nPL$hB`L*WCJIJOn_E+rpgn^!&s1k{v*Xyy z5}v=f32X@1cPy%9JGG4!P+=2_g$Q^v zn8!2ouqn~NXq9xhjg(6|76n_>#!NK#JeA>Ed35r!GxCHMWh^${B8=^+A}gB)(ED>T zm@O_4AwutMfpuh}_`zc)e&Rl%KRE_2FL?;&sDph?%y}|_Xm>@vT{{5P>nO1%x^oH) z4+%2LC9}mw;FEZtxMXA~$1NB_mv9Jh;*HR1CkXNjifgg=fKAgu+YL%;QdGCcx<6aZ zJXkI`SGy$j$Ucr#HkDobB*BZudUkHk3bZwheXl_5ySvS15oQH9%>K_V|Mg^;(8vcv zUxBb{?n?uk_dno^d{Sr+yUE0#Z@#LmEQ0%WER_RW@d8D-S7uHw=(1)XL@Va;`sI4Igu>1v?`pB znsM*B_8RA1eSa;=tezQ)JQBo<%iz{s13w_3YJ$@w1<_z&NNM9`aOfemWm4+5dX-8h zo=R#7QtA2hFN2px2Ak-70S<_SaD-E{4CgUOgEY}Kyd591sL6tVCE)|D3|nG{IhiYt?w!CW(TV-sw{y(FKj_p-#FUm%k_Y0 zTkSmiiRSH%+L^)4g7N@_wuQdM_E?^Nw7j07r3V!6YV!$R#g-+I>FfA)e0*|doPujF zwjtBq!$IFjvTV85mGk>A-($&?Rif70YOlta?h1)GRkR*uTPq2)jN z4>>7DRuEhspriZ3AJ6N(K%$mqb8q-RNfIp%84Iaw;B{miW!D}*h-C=6>sK1>xcCZ$ zVwqsg(#YI0Am7*W%8IX8Ed*vmd!l@aXcJ0`26>x>jZ)!xgQkXY#JT5g-vM%vX zi}McY9lJ<1dnYRxLezX%ny37MHn5Tul;2~9qeEm32rIFjwiIAJbCC>C3F9eg9T<`) zK?7mf<84KXBga+A$AqX&I^oTZ6ET>kvn|Gn!oDG8?qw2TA3I!vT4=!|?4N{6zBN~<*uACNB zA7gF<3AzQs^dXEq1c>YpQ`dvNZd`pGp14iI^+Z=C0J zq)_k03c)0<73euZW>LX{9dpkdmJz03Bl5agPnT#_eG^M0pVK)~PL%F!b|H z3-XznR4{}I_Hhw@zE|EzvsvSWvglXyHQc~y3EP=(BrY@V1c<9 z3qhv~8NTy(i|RNxI+uX7w;Tn0c0uM7=uJRMRDyglye-0V91ftYf^8VsmS)R#Nrt&+ zPD^2X+&1xAagL^pyu(4#ro8B@1eY0yMESH-L`$+XO}uNb0qJ-RKGSjchpL>$p??OI ztUf`QVh1&;%_?$)?lVrMKNs(WwGOK7Uo@VINmmk$C}ho&ir)_oIOvT!VTWb$T$w zXu|{tCR-*UIel}Uj$Pc>JW6xiv1Oj4_bEbB7fIiIq>S1Jl&D#}#A+qJ zAz<6S89nGN)Lob1+8FDoP!J22#|^@T6#k$z-mf$+$mBBb7<*c2z(Ey+J+TbFc4j7G zK7T$~8U1NF6)dK2%|H&yvtP@1tlDypaJQuni}a}mD*ur5iTI!?+ne@P@XVe}y^rLC zc6rdf`jTy{t6bDX<;%|D386A*ojtqsZcD!tg z_+GlL3uT2<6RVklf@ps>XA9ki1+RJV{UnQCJuOvO zul3}l@buLFCw5$P^2bbB*M!a&@?AmGGHwok3ePV==z|7Q&$}3xVLAN_5 zT3Q2J)vx4}NAkExi(<}SlfEDpy#N9Npg|+F9MVU=x|qKH^hbP0_=QghKM=N z=7GNuO8E7JN2MQ1W(*+f$MQ4t^Zy98NF1gMFa62e-GBP3MGT~v4R0iE9m4xe-j~eB z@kuT?mM!RH!Y|HiEiRdO4KHuQ8RTZ&`xY`Q?JnQz$XZ%h>bfX(iw5@m)d$gh?#(GG z<}{M%ePi=Mm4EeY#Mh0!akz(7pu^1Eo98sT(*BWpAX~#t;4B&vs_>R-#YLd6jC{R> z>*YiV3;Eo!8vaH(J;DD&YkEZ zxCnQu`nD`jMn73Ainn+<*jix&zf|GDFi;U*pUAMbeDeb{q$&xKtVyaxq$i3yeJUDr z{BlguX4P%ssfkPk&a0k#MM+KZIt`0Lq{s996s^VT!S_O?mV)SI9mQE+S-5a}+4UL< zL%_#kaYhql;aP_%#Vq8tfBMLd86hJyX}_FyzG?i%U7u~93>x>cp^n)o~lt1Iy?&2p$F<#rFf7GQG5U?CwxAC-rM+A(h11M zuln;wNyrh|025o)Sjxt5)xu5d632}G^Ld`y(iaPmpk8MfBMM2}a4J*k;Ivq+|Byv% z@Nx2)G0rnqc8ADjQJ(!}2w#!?_`;vO6_QiB4qy}SM`y^w`GYZccU?6uqu$1P`TfM3 zcS3%%Rl&E8INtv(xJiFtovQlnw%`L1bBgjP-{5T2(CKB}?aM~+oVT+T#kS~L&u?^u(AtpO z8jq#OKL;Jp4X0UIvi0~}N46b3?w8&T=k%0Yp<`~(*Xiua+hsy}YfVL$&aztZG33xk z`*=d<@;sh_iMp2Is&LA$U*V~XE|KnU9}lae(lA8dXQmyj3l_euvyhRdm(z->cPWIq zlJmUNv7!Z_5+pJS@)8Gy4r_iU{VKDS=M;oDSNBb=xwy6(I)oP2238%{Sm3ghGTYKa z2l>9;2ijsT^jL&A6G~Wg0z^iVHxNYBd>XucG+;nJP2w1_yrf@!eytiTquR8p$8cS~ zaZJ5Xg9n^fUN5XC2?-*qRzs%^RDz zc{T&$VjEvV`k18t&q>%&ti|h<(qMq4N7rBAw#~7SV2How2pS1_7&MG` z`g?R-yocN}ar-m`I#1k&nSw&$XEvD=2g|{QDN!K?D+K%iV!|OzPxp?imqNvHHCR&5 zsgl1}vi0>vg^pG3uj-Ptc_QRGvL{! zLwx@`_TZZuDfW;?eOqKZ22u2;;jp1H9u@M=H722z@V`Y-f| z2^1Q;AU1C^ywp#;Ov^zDk2-G|`NS4_hjbbsX~(5O!Q#_amF50Y^(216NTI=7)AHP! z+T7@|TBRlJrN+&Y{^3mJP&+>vn7rqPI^q}xMOcF(5VJ$oY+_rNK<|XVLOD5>;cVx6 zAr|!hSh3XPp%XQe_!X>064E;irYV%?V4upU>ebYlf(J{xrq5*1(UTE9(;?6i)WEod zUFIh~&B0Qm^)C-X;z3>^V@t0UIHo@C6&bpDR=7-Zz;jS!tyh~{dQq{}i2sk?qd`7FNB-@iM;pi+(bE5=pJID3;ryQXZM&0H{~aeV33;@6y3_%Q-9cNT z&})AI9M&08Rw2c_oJzk+lWL~0Q;o;mvMryGS1$oSgM|`_z*KBf*~%CY0AHGNT$w|` ztRs$jA&wG21Hh^L0bwj~Oz8|PtCFG;)BpfPSd6Qsz~gv21zZ?7DbBzo3#msu%FSwa z9m?umfpT@?5ESMXYz3~`YZ~%RI9fzBcv^PtkBMF40OR2hMRfdLa%8?CLT0QK2POPO zpKfC)#jt-c6tde@mlEgH_@6lEPqz_0@IMXGX^3^>U|}J2L;*BRI*#7>=6w=q^`=J| zMpVqYUE$XCQp}9gjBl~GmpJ$=5R8Wfi2@J+;+-G*!Ck1^)bK^70X^`A1{In)v8SeD zui>!H45miLIX*@3IJf>JW>PtLTvMMio+h^jx{LEBbU9DH(MKc$Li+=woJJb2gzQ}h>0KkMr){^za^m9fRub@b;ZJP~b_?U|Zf zm6Fn9T(a{O^&MHOH7R6fuZXk2`Gu9%X5EhM056FCQwYP#4mYXif_fiAy47t*%or_4 z_Tjg)vy5OOeuFW|C@oU0(n3cfr`pt5K_I6SUuMYL3dOA1`@X>;Gnq54(ZQ`4Qm1;3 zmhdt(r`jc{2e>r7;Y`cPYc49;&v!B3T+@4}My|tsKvrX1=2XBvt8(+m|_eq?reHZD(x-KEpp(p8XUq%)ToI=g zulCR(Up%g3Zn8C|ZqPnzf&cn^S3x;OsY~q_X2q6TLh-fet7#_XS5vRMo~9k+n@<9u z+v`K03cJ!9D}mFWMbC$OKF<@7rz>NfDwdJkU-iPHMTb7mk1o1@E-v;(*Ss?)vRSHT z>dPT8TP`i24wjLXQ#0qKAIbx#duBfA%Bt5w+0iQqFCIQi2J7Q9w?kY4q&%*guIW3U zA$t0OE7y16(TR!2wO`k5k6szdUlI6KLQVmCBpFTMx=seg=MP-%BrJs-dR?|Vt>9bF>eq*%gSjc^wUS}PcarVo#LsRZik+3`9*2gs~>wEzs zTDlS~8Nw*D1A06>risz^y}dhRJ554~P&>wZxZDwfNc4eni@Q}w5CT`vlAeVQ`&(g=88>_qf+ELj%6s{;$ zFeMQ6>G@_2-h0MXY3>;-rqPX!{s)}lCuKCOKH|XX8`=icwqY2e4`B5&VhOn*&ir;HUiM5*XhI39`BG~v7J>|{)_#o`Zi?oc-8Gp6=rBHzh=pJVX;{8*;mGSP z*Ai4gVva2<^KognlMd5;K4^PaEp=w39eM4>B;PGDClFl~Ngt}&>cj#fQ8P)Z%ou@7 z8r(Z>bY`)aR>M4{FK?npm9bXEU5=z@hYqWe?mu)}k1|K})xh{Le929Doy>-U$ss|6 zQenD!g2@XqxG8fO<9m<>EE=Im)5 zPZws>U_BgAmp0rYWSW?w4$3YE_(FA}jvmpQpF-)%zLn!uOR;Q3CN$}(nasa}liwFj%eym@VSRd*G+xxePZvFnKCK}fHN4BqcvNsE5Hl8Q$BMVgm;1v2m{XYg)- zS`FhsLc(Mu8un50uncM#cS1WLiGDa*_+Jeg4+6w$Yck*}*KF}0-(kf1u1 z1cYD;K&FP?{R5VU!!H@SL@oT(c~$(cgS9=wrA0zoy1J%%DyhtQVXL5$aHl0VU6Zs@ z`g8=Y1O%K?t6}rRMCro0gel6j2*lEYjKA3my6^MC@;cOo_ZrS?2kWyq^rhmbU@8v} zCfA1{*fTJGQKuEm7Y(h2Pxw)Us`#l|8o_85^a_gB3a zv1waFF<13+nF+a@f9A|}4#%m%!MC>5*7kBK-B(V_;HMTc>_~GhS>@;td%TKiNVowI zL$gSp8Y+08^j)A;092gPUX@)1Gdl!`8H%WdAEEc{`KAJzZ3*#1v#6cRv3$m>d@X*# zCYUtkvOpXTc*O!HpuiCi`=i+6?R5_ize-EowQVV0SBCxnnm)t8?3#+r$)QO?HfrjgsN zGa-m!7n8VG$ZcaEZ}L%N@V=EG^AncB`9k$xAb~Lf{qvRTnr(aMNd4OPV2=)b>jN|D zOkmCoT_-GOcIWgXH#(2CLSK+ zqJ*l5PGbmX8ti~Nq7kQqYOb1EI|m?SFyS>&NyJy|?f@e4ljIxp{m1}kPxyc5Qvc5X zcik@THGiGmLG^MIT0f8=JV4ECb|JivecpTx`|sSgN9^@ z{;-UEqC%@bf=G$1F4tl37!~}{Z_It`kB`^r{U<5X$gctSr|z_< zAqTrBjTftNkj>F0+Ec~#6L((?1Uj8XN%HNMc|Wlsuvhi-N%~zhDAc~0Zah+D7&LYd z`Kl!BJCUcjxp-2a>X?~Kz6igfuK;boi1$<9mtFAwo= zl|wiWz>Xy+l}^0Ppd3A3%$Rv9WDru5b`F;={G|F$vF5`HTt{^uT{3TORtctqw0X9%M0mf)M96;T7M@Xb6b zMjE62AxM0)D6}~)Eb->{>9tLJxMJ@%{3KjD_t8^P*!s%hgamSwC;hryM%Wcx-^#C%Hips&>*xI z{te@Z`Ynnqr<)XkRfq$}COq{yqYF~J1GW9^3Z179J)4uzRmjH}GQEdl6HX*T$94pd zmB%IKfw&V-Ap)DxRlFmp3TA7^999x*MRv>RTAbY>AY9!evI*aTJ>m~gqa)0jM&D;x zEx5U5C|Gj4w3A%muVFBq6u#8*# z|5p2oK{vI|)}s(aiG7D4`ps)km$0Cvy?#1FKTlG!>XHH0UL-H;_=bb%pum2xZ(cMGq@U1i+m)I}6AhGuon>hkUfcDlQbS!+&KNuJ-JN+L1 z+biigj$JLmLXXva7NgIMl^9pci`7Dx!4{h3Xhs25{mzou9EdLyRz9bM??Z#~F+frK zIXhHTq@&fEm^OT=CvPfh-0tmHk`Jp^u18*6bl(XVnplxW^N$Ue7Ug0OQzbE?$Azvz#=CTJB zC0H03+d4cj6)Z%dhYdGq1L2S4kY{?~TuWjjT%s^Pp^lmt+Rz%l&2)^^_h-+U&t-{W zG0D3bUi|xUULG;77YO~~GI*g&H_D)wZd{%ROhNjt2{F5`%MtD5bjT8E=YV~J48vF^hHtI z7=%Z$p~`aOGWjoDu5UKcC4txpbtzKtf85^@TK;L9WWp)JJuCJu(pY6kuM>Z}hQuWE zaivi${g&O!$A4p9Wx{(-wb)JzT{YD?i5+g1yKwbH`h7h{770Vwt~;gc$MOBwA^RTk-;@jg3X0zmJ^e0{im9rca1gb>{DTaZ`G70F+FQ%um||LaSy3NvNE5ilMWgE*eR9Fycf zpJKqk;KKa(k27AvobK&^-Ymnw5dQZvOb#g&1TQ|#oE1>d`1f+toE3I39@B#7?SG!r z{{Jd?&cCS_90>7V7L0&>k@#W@egKoy-{@}(RzS7f-%AloR=~Pae5?X-e3&I8z(M10 zbQl`t*NK0$6a_Sc{ziNa5#qJ1FyH>~{sF+iVEsQj%}wI{t%L!`rtw`?l!%-buZ#Z! DFg@;A delta 9373 zcmZ9yWmp`+)~<~QcY-?%1cJLKxDD>xw(O373t->BJz}6a|pltUoTi77Lz{n(8pfG@u;B4yU;oBnTzm<2} ziB%go1!zLDoL40)o-g=9K3MA=dv7puaNI@Zq}GpAXWID%9ey zY`AzXE>ac;xJjfcqTc44sk=^*{5n0m- zYvX2h?1uJ4o++ofDLJAZi6FLf+h&W2Cz>-_D^p(fx6aoUp*>>Kp9*;$At4hhj~Rq#`Z9)ri?>^i^YKiI1-a4koYgwKJn(4|gm)*w>x+c?Mi3?! zgCL*}@ARV{39?_?MR^OEoUl3tc0K?CY(j zuXb(baXJt%TP=ZfhV)HcTQL(~bA4LI-52o(@Zb&_l#S!6g$Lbs#7@r>y&G@SM$jx8YBwi{jE^^xS@TcSw=NIP) z_b}gTV!=Fe_m(#sg9co133HXi?SNsC zRfyrvKM>`z3_NK|d;I1=LMueJ8R}&}_33~)+hbn+i-e8-qBAK)>b3r%8v~JsXg&%{ z89w2ks3{@8&BI0NA!(kTF#6(GVr_}tYv=MAeFE=n9SajT^Y&RYBp3__hw5`lkNWJZ zPGOGgqFXmQeDBy`5_d%L8y+y^5K#vgn_I6@VPKZ&5*6?m!3h69&G1|FX8!UHt&)ywMn-b6lnoi4yO#8e~^>V@;Y(Kp5N^L%>R>S|u($kM!< z`1sS{HdSdD0B&P!vp|EW!~y`cg&=3__E>bXzcN4w*NXRRg?N)O+Jo|AS^VpWGA2$ zQp4V9f@mD6RHRAbP)Czoq{3=&=mtPf`S}JCM|u1l8GV(Z3VGs8Z%Ll;s9zcEAiy@%Km-7=BhBjleFM6V8u2&2D#whkJL*U)P65D;dPE zRnTcL-^a*|{5-Dr(bU{44#IyFiR3w>N*k;O=&=qptm!NHw9C-`wgiL*i)tr{;bDS7 zh?uwQ`H`qw{A)pt&(y`E;Qb5>-F5e-lboFecw~cIb3z4@&O^HUuEiG6Wk``pCA9Rw9 zc!AdBN@pKN<$Kr3YkT4t*`&-(x5tw78BFu#GRf~9wd_Z;BZny;NPKYBZ_~E#E|x(HQs-`wVhZO74NlsZy_l~$tv#Y*qO3W= zQ?I|N4SF8r+onld_Z{PnF7i47a7V#>hBcXlbioKnc&+bd!Z?bRGD5fQrOiEYYr(O= zZD8e|W-#9%#dU&_NAqDjIl&NlZG1o|oZy!(8tLDAX|pf-tyhX%MdTG#5Ae6Fu`s$* zPa{^i)rsQpJzfG-dO-$s`(R4WR4m57ZsYtaVT?8riFR1C$=;+Hy>Nn$zJGCq%1i9+(3#+@0 zGL5haW9y$D=VM1_M#WWd<9zyQp*m_#?wmTjr4yO#wo}NbvZlebN8ePREDPRaN@&ok z{pN#H!%np%G*JIAEbqNX@`}X2RajxYQ9qv(F3hZiG$UopkVlP7y zp)LdpN+r)xN}IETvx>$ypo9r76weDO$D>ZQM~q63v+hDjcDlWqNlf*L=@xd#V~QqQ zlr^_3FB4Ph_bKkn4{F5i9tQ&;nfT~gHiwZZPNcuyE=axKoMmrvY?BbbbYKt?hQB;S z_N8NX6AFww43(Iz?}z=6mFPj^VuK^Lzq}(}mAuL5Vp+X1%aDyL8#I*XjiQ(_gLMg} zEQ1vcBV^t)%>T({ibq>PbarA%7p1vbLm|2 zFQ1H0g|H}&({i~xbpOe+3WI68LA=2^-A5|JLR3K8;Mv|-UrDhVZ{giY1$WT87sclg zI7VZeH|jQ7h9Mfi&@aoKKGC{gZ~Qu}iCOkJ+7CNFZa#~{)A#WcYL=Pb1u)tcmRV99Jh|Ep5d_UGvG27+_X$-gPgnX$kk#(IGWXS zavW+rBlR&fj4EOLnNv$B(`aC|F+f3UHRJJi94#+k}*9L3frRJ`OW?#SMKo~Uvge?8lkCoW7HHM(~2 z{4;E?Gwe@0xF4p`;2ruzzg``M=A$jT-?PhkVr z&Do91<)?YxPbzaBc|nh$*mHLd)jake>d_$*eM%Er+K;>e*&Q9^co4>P-pr$~$EH|P zQ5_8}>j0h*NxOifqoYFWi+zzs2J9Qg#c?$IJN9)6Fqa{lYQX@&-0G;N8uc6O^k7D) z6{&;P`)^2-Gm%u&Cu;aU%Mho?W^be;eE%9;JT`oAbJ^iZ=xms*MJ^0l7GxQapG>`t zcHaAI;D{h0r*MBrry{gHM28ol4lxT!*}Ao$JJ53-X%0pqf=6S$igX!G-ZjcQE%Lj1 zNn7Us?6Obc#%(N{aI>s>d~r-8%g|#z+U8YB6HS2d~ zdN!V7=7H|R?u*4Rf$DiFPX#(Bw({bNrY$LV(rq7&-JUc*Gj{cPVm9g@GTn5gWDXwo z5Fu{;eZT0*niWtE@TuT*O|Nhc~`Xto;J%ABQ(v_z1WkyEN{>mmAr?s&Ge&--zns!SoN9#J*~jtnOY z4B*XJm!kg?CDR(~rXaJ_KrB}kU(IyD8kE}PggH@}7N(7llW+Ha;O!6)#p?mxtnMvm z2s0<1{E`zD0`sdY`l0Jr#cy@0K}iyhZ+}hDO=|kXqWvMs?js@<$|aBZq&ZJ?Iy-rV zWVq(!cswlbCI=>@#_QFRPJPepiCa-o1n!O1&o{{VF_1BF6pKB1nVG)ao>Tp}<@bPk zIXYvEPCr;UjdXDGN3%=z=}5|JsVz-->D9&34Gx3RkfX-k&hQ?F0mx*VFn ze=d79mtPWZl&Nkj^1iG~QZC#Ys?I9hFu!ax zZGn|Ajr`9QI&51$MfR3j^V|*hG{x>3cC5ua6IYEiTs*xnr@urH4ssLOi)VAM)%n@pNMVrKSY^QolvX}A6W3c6ujOoHT!r| z%;?2M*!T25TzzS}U=-g;oA7x4mC2#>qXI0fx6x8^fH>KOtQuehpS5BF9PnVvSEx%A^Z;+ZPVA9QKzZ17B<0}F-~hHX6lTL8$Chm z-KO8)`*)bwP?YJ6mrt>27{K$@0w$Pa%~IWE>u`@g|U_Frm_ z|5%+&@TZr_Qr9+u20r*VHa5b-G3kmF%6g@6qw6)%>?eeUz=5R3q;N10E3`EK`Bh<) zDeVFkTb==J+X)mrcJ`i_hz28tH=2lu72}Tg13JF!xS5awBUowlu4Jt2f6(unp=ec- zcc|5+Bw(UO8^k4zk2K}ckWJ_o1!z{ZRqI(3(N!IcG094H;4Hg-wI*V%(olEP?8gQb zWt>!ygb2}2Qfi}UZm(OKPq9)!@!wgFV883zP#5rL2bQb;Aq`*Vi@h!d!`1vCJ;)U4oSts^0# zW7);EE&5kSOC_AuraVER5(!F%=KeiAT|inSR4pjkwg_r-hC(G3PSOANtJFGRV-m)# zv-X~Q>wuo^KPvM-X=~hOVCZQdrC-#8s-RERvHk0KDXMEff*0< zq)yOl#f8lx|5d^6yWrHHW?SjVFG-euEn*zluN^b;;8)y>>rbq$NAQyh>8Jci(#0Q= z_c&?JP03+WLU_R)baQqfvgQDKZG&*NB&{$LL7Kce6GYALn%WjlaUTf4t{Vj0L5R%A zaH5Q+wUQ=%P1;4=n0RSUx5?J#R`MSp!}RDW$lu%GDw2jw;asbGaigt!#1Y7NkrWV8 z3_>;KciH{vwavn%-MbMGd?5oS{t&tYn@Uc8St`ZHxbWTpA_Q0l;y`*hyz0^XS!I4j zlRiz|Y3vka#a31Y^L!C-l)Ovz_wF5be}N^-aOo8D4-mN_lTlPHgPcL#x@kC;bitA( zWCg9-A0#X|kr9okp+TO_=$LvR)2@|XHVEq4$-bDHZy%80ofpsKi_4k;9+vbetvoEuM1jIC%|05zq-Q;^CQ zYD${RKPJ_7Er$*bdv?sIB}piXVo6|qZv6Rw`U{8n3ck+-kpb9qV#Wjhx9f07#AzGT z98}{Vfgx05tMwp{Ax$EEx`DH%HZd6;cCjzq)_;5U8f!8n17k2iDf=g^Gb8StRNHi) zpnK3uU=;kTH;;!nO{wIHC#BM)SJQ|ju@Vp%82g{iB+&LIV6>1!LdY^le_Lsi*ScDP zj)|*dqHZgO_^H+e@gszXI1sG2z3S_!1jfEp=i+VQlX$fcp3)6?_-uEiUU4EgIQ-N+ z{}8VS=EH$Qqbv~0yS;5@by%!UU{}P>HED zruMJ6g5=QTa*#ntmYwCEQhyYg(jm1Bb0UMAAFoS&db|(@mkWPkC|Q+a^^cRNEIZ!# z7f5l6ixtU@@We>`hRB|Eb}_vGrNGMf5nw+(SimI(x;BwH$qJq!k+=oax>u~MVk}6i zm+e1nwP((i=~80K%Peb0;nJvdS~fX4YAOAb`};^)`AH-mx6sXr-Px+m&_(liAsu8% zZlM>?pPU*M*=SDs2jma5nr(RzPyUCftY*pRcB#KG|IEm1cUMvgmGp#ZdAH?kQr3J% zArKza$jyIhu2exzaT9bXV5qup$Bw1T*|SZp(oQbbrn65!>2%C?)Nvsr ztN_}Ho$}1mjZ{>kk~i`8CEn zgR@JP;Si-YW&PSUOzEBzvjG@O5UfBW>kQK0Ikx-M()9;k1TiwF&;1GfpBI3AOe)Bz z@0wuC!ckyg1V8+J+5vO0dfM9+W|}(g^5TZ{P&&Q#UfxGbsTLMe$<$9}+lll#Tw9uTQ>eub7|jV*m=8a%5~Bap~2CVbqbbsHo(Fw-dUFvt;Qbu!HrgTW1 zn!9o7(j#A-daf2Sz+ii(c)8|}>Tn1!9xhqfu{i4powjZ-@mJU6=@(~Uh z9-e4_1?lws{QZm%9P(QO#=Y)%{c@3Y=(xRKK5xG|JsmH(t@VM8e423Qdd5*qjokSC z`>^Dh-Thww=pM|c?shuZg7cDmxmDbPa&?yNo_Whp7BYN{nx)Ng*!gr`Zs=y~GWPUp z$Xl!z{k(>RwLMeb`JR4vq(V4j1#Ln&ea9(!sU|aYZ9ivlc!}Wwbg6$;R{L~2iVZ@lu`g*tP<>N+ugcX9J*$QKi{`-*V(Yz97~bWF{exIG-&Rp?{(+q$_3 z#p_B7Nj3gGd_3&e5lpCm1W51|LpQ!E5|ao)?Dpz8UmBUu52SQ_9v;smnGAUyx`zdN znam~49SixP+`J01{_K#z%6^zSF!c_hfau;$i@DdV9UJ#QiT$3K`-LzQck8_@-hKI{ zZf%DEK7Xfa}0%m1W@qh#}W#6z7oD@6YovPZ* zmaj<)d6qe`MKWDc6j|yq8qjp3t8r+r&`=vAmC232Me({+qc#@IiHlX^3HQ!eB$pqw z<_RgjgMxTM0QXdE%Hwa{#6{($^i1NQYvNN_@UESx|E8`6bz(s8ZE>%8Wr2E_V28XP z1n`|2BWuO-BQj-j6=k^B+ePBtOshe@NfC_@Dr(C8{Wo>>tkjf60G89glNtNW#&mlH z^M(6YcYN#78XuXYE3Qx;nV`66<&nTtR4=Qs|0B5ok&g@Ik&y?qB1ny8)b8WycnlDQ zuHBrq_l4@8vB?PPM1%S<2iP-=OD)+4CjV{tEfKVYMxiDFidQMqN2nnYLBgJ8mR=WS zmeFd7DJA6u(4L=CP8OlyCXKd$)auVX!>c^9QhH4SS~VYyCbbMzA4!S;RS{#_mueH& zO(oYn@<6?BfA6TVEX*?Yp#m1v9>J(aOnamf`?mkYPF$-(Gu8h$a|r-I{!vr@akf%4x~0ox(a>k3fk~;91I_vWiV!5%{B>z9V{8f9-GWAU^IwaK zuA4BRG8OijJ&)hMi%dAt)_Ibw0?VS4TW52gjLUq7%#v1VNLR^Mm};}li#fK4$s&OG|Hf)FJIe!4k-Ts@FBVcJiR)XIxE#^_JP;=E-$j`L7 z<0~$1zmup?t0=3N;@behtbJ~@rfdT6IOSSAl@e4Y(=W8t4t#~e!Eywwt$B-u2c<*7 zK&AQec{0tmqvDPFd_lsy^rT=`!K^dAV|hdPHL}k%? z5nIRQJm(O!HNHMKMKxLd^m%df`uvIm;(POV@alXA!R+54G;+v>=74p5rdoytll$NF zbA{?TVZ%fre4{ZZ>Gijo-oqyG7w;_F^Nx4bK_&Y!L3QG~v>rc~)r(Q$!Ix11*47|@ySRyLObtSk zO2Z%eH)!9>L*LN_vmsO}P>f-VdeqLIO-h9C51idJ!!7DiI_<`Bfe----$t>&-HMyk z@1>*fqz6LijJ~%S?J4T{%XNNr)o%CAZ-h%svEmG}>cUPNu$pfpa{*LA=vc*_F&`yC zMQEZ^bk#BwVdV$_mML@OoLC8Zc90i9!t;QNJv^D35AzW$tc#*(hBzXy_sjf$=KOlG zzm|G3N_aX@tAsxhDT;}E@}~5q`#%w7Nhn#+??cpe_cgR;KOo=7TeVPD&BId28aQOp zvSewsri@^G6qVEJ18H7;2)TSTb(+oQ#xhTaKMtBhMFUq8KGOY*e21w2Hmt-vybC7k z<9f5x=pr0LE$oWz3Bh2lCKQNKf+686<`4Bv_zX9##AxRN$c7Z|LJe5qM^^m7#JLpd zn{We{*y{3KU{BGzta~^8vp}Lz`K2e|i{H*2bRw(&h9+oAW)@BH2QKDeoN~O8$?Px1 zkjwXP{N<*&h@mC@<<=9RJIe(=Mg%yY#=DLj%CWH@L#=t}By55d9{!KpKk=NpM$#*x zYs9`fBk%koql2dy6|8Zz0qq3H)KA7&B9=Oz$DB%v0vF-tTcw8u_U6o4J{bOz4Y@os zK~-52PWUS;cSytvDxO;xO=WkJ6WUK=G>qE>Epgvr7uXu1l$2r=JEUlEjkS=f`w zIW%k$r%I!KcM^*WnMf4$o@_A03a6hTIv9TWlF00#u+Hi1QkS099+{Fz2qfl@HkgP$2K zELgdZ9Y1&r_y;6nmNS% z0=BfL*z6Z{PAs*_RN_&T+7J(z^(*h|lZGVOlGRMOgyS-JQocWGESMjDHSKJgSlgkC-^>gpt;3d ziaAp-fE0i7nv2f`6(_=8!7;9`FXa?GApQF;{%DSl@PGUj2NR?Ow~vuH%|wvs3L+r; z?+0@j7<}lN!+kug7XtjM6nfs>-+eRqif3sCnNnY%z=h(3W?LUl5lTS z|BW!)vB6DiCK}j@!6_U58>zP=Ap8Fh5F88)&i~chi*e$vod{fnSt5-+4brT|-^u?E D-Q>aa diff --git a/postgres/scripts/create_empd2.sql b/postgres/scripts/create_empd2.sql index dc91622f9..0a7a56f4c 100644 --- a/postgres/scripts/create_empd2.sql +++ b/postgres/scripts/create_empd2.sql @@ -108,6 +108,7 @@ ALTER TABLE ONLY p_vars ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); CREATE TABLE countries ( country character varying(42) NOT NULL ------- PK + natural_earth character varying(42) ); ALTER TABLE countries OWNER TO postgres; ALTER TABLE ONLY countries ADD CONSTRAINT countries_pkey PRIMARY KEY (country); diff --git a/postgres/scripts/makeFixedTables.py b/postgres/scripts/makeFixedTables.py index c2dd00832..26ebefbb6 100644 --- a/postgres/scripts/makeFixedTables.py +++ b/postgres/scripts/makeFixedTables.py @@ -23,7 +23,8 @@ def is_null_str(x): for x in range(countries.shape[0]): cursor.execute( - "INSERT INTO countries (country) VALUES ('%s')" % countries.iloc[x][0]) + "INSERT INTO countries (country, natural_earth) VALUES ('%s')" % ( + countries.iloc[x][0], is_null_str(countries.iloc[x][1]))) conn.commit() From 71bc1a9ea899efbf02ac35f14f6eb38b55db611e Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 06:33:49 +0100 Subject: [PATCH 024/119] removed tests submodule [skip ci] --- .gitmodules | 3 --- tests | 1 - 2 files changed, 4 deletions(-) delete mode 160000 tests diff --git a/.gitmodules b/.gitmodules index 37994f81e..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "tests"] - path = tests - url = https://github.com/EMPD2/EMPD-test.git diff --git a/tests b/tests deleted file mode 160000 index 1b104af08..000000000 --- a/tests +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1b104af08a17f9cee2c765248abddc4091d0c551 From b8329721e5e24c44467156ac9328c7c2c2dacc1d Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 06:34:47 +0100 Subject: [PATCH 025/119] added tests --- .gitignore | 104 +++++++++++++++++++ tests/conftest.py | 215 +++++++++++++++++++++++++++++++++++++++ tests/fixes.py | 96 +++++++++++++++++ tests/test_critical.py | 29 ++++++ tests/test_formatting.py | 26 +++++ tests/test_meta.py | 94 +++++++++++++++++ 6 files changed, 564 insertions(+) create mode 100644 .gitignore create mode 100644 tests/conftest.py create mode 100644 tests/fixes.py create mode 100644 tests/test_critical.py create mode 100644 tests/test_formatting.py create mode 100644 tests/test_meta.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..894a44cc0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..7e6fd90f3 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,215 @@ +# configuration module for pytest +from collections import OrderedDict +import pathlib +import os.path as osp +import pytest +from functools import partial + +all_reports = OrderedDict() + + +report_path = '' + +_meta_file = pathlib.Path( + osp.join(osp.dirname(osp.abspath(__file__)), '..', 'meta.tsv')).resolve() + +_base_meta_file = pathlib.Path(str(_meta_file.absolute())) + +_commit_fixes = False + + +def _meta(fname=None): + import pandas as pd + fname = fname or _meta_file + return pd.read_csv(str(fname), sep='\t', index_col='SampleName') + + +def _data_files(): + import pandas as pd + df = pd.read_csv(str(_meta_file), sep='\t', index_col='SampleName') + base = osp.join(osp.dirname(_meta_file), 'samples') + return [osp.join(base, sample + '.tsv') for sample in df.index.values] + + +@pytest.fixture +def skip_ci(request): + return request.config.getoption("--skip-ci") + + +@pytest.fixture +def counts(): + import pandas as pd + read_tsv = partial(pd.read_csv, sep='\t') + return pd.concat(list(map(read_tsv, _data_files()))) + + +@pytest.fixture(scope='session') +def meta_file(request): + """The meta data file as PosixPath""" + return pathlib.Path(request.config.getoption( + '--empd-meta')).expanduser().resolve() + + +@pytest.fixture(scope='session') +def meta(meta_file): + """The meta data as pandas DataFrame""" + return _meta(meta_file) + + +@pytest.fixture(scope='session') +def base_meta_file(): + return _base_meta_file + + +@pytest.fixture(scope='session') +def base_meta(): + return _meta(_base_meta_file) + + +@pytest.fixture(scope='session') +def base_counts(): + """A pd.DataFrame mapping from varname to acc_varname""" + import pandas as pd + import glob + base = osp.join(osp.dirname(_meta_file), 'samples') + data_files = _data_files() + read_tsv = partial(pd.read_csv, sep='\t') + files = filter( + lambda f1: not any(osp.samefile(f1, f2) for f2 in data_files), + glob.glob(osp.join(base, '*.tsv'))) + return pd.concat(list(map(read_tsv, files)), ignore_index=True) + + +@pytest.fixture(scope='session') +def samples_dir(meta_file): + return osp.join(osp.dirname(meta_file), 'samples') + + +@pytest.fixture +def data_file(request): + """The path to a data file (set with metafunc.parameterize down below)""" + return request.param + + +@pytest.fixture +def data_frame(data_file): + import pandas as pd + return pd.read_csv(data_file, sep='\t') + + +@pytest.fixture +def meta_row(request): + """One row of the meta data""" + return request.param + + +@pytest.fixture(scope='session') +def local_repo(): + try: + from git import Repo + except ImportError: + return osp.dirname(str(_meta_file)) + else: + return Repo(osp.dirname(str(_meta_file))) + + +@pytest.fixture(scope='session') +def commit_fixes(): + return _commit_fixes + + +def pytest_generate_tests(metafunc): + if 'data_file' in metafunc.fixturenames: + files = _data_files() + metafunc.parametrize('data_file', files, + ids=list(map(osp.basename, files)), indirect=True) + if 'meta_row' in metafunc.fixturenames: + meta = _meta() + metafunc.parametrize('meta_row', [row for key, row in meta.iterrows()], + ids=meta.index.tolist(), indirect=True) + + +def get_priority(report): + if report.passed: + return 0 + elif report.skipped: + return 1 + elif report.failed: + return 2 + + +def pytest_runtest_logreport(report): + if report_path: + all_reports.setdefault(report.nodeid, []).append(report) + + +def pytest_sessionfinish(session): + md = '' + for key, reports in all_reports.items(): + reports.sort(key=get_priority) + + d = OrderedDict() + report = reports[-1] + if report.longreprtext: + d['error'] = report.longreprtext + for attr in ['stderr', 'stdout', 'log']: + if getattr(report, 'cap' + attr): + d[attr] = getattr(report, 'cap' + attr) + if d and not any(report.skipped for report in reports): + md += '
%s..%s\n' % ( + report.nodeid, report.outcome.upper()) + if len(d) == 1: + message, val = next(iter(d.items())) + if message == 'error': + val = '\n\n```\n%s\n```' % val + md += '%s: %s\n' % (message, val) + else: + for key, s in sorted(d.items()): + details = ( + '
\n' + '%s\n\n' + '%s\n
') % ( + key, '```\n%s\n```' % s if key == 'error' else s) + md += '\n' + details + '\n' + md += '
\n' + + if report_path: + report_path.write_text(md, encoding='utf-8') + + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting") + group.addoption( + "--markdown-report", action="store", metavar="path", + default=None, help="create markdown report file at given path.") + group = parser.getgroup('EMPD') + group.addoption( + "--fix-db", action='store_true', default=False, + help="Run database fixes") + group.addoption( + "--commit", action='store_true', default=False, + help="Commit the changes after having fixed the open issues.") + group.addoption( + "--skip-ci", action='store_true', default=False, + help=("Do not build the commits with the continous integration. " + "Has only an effect if the `--commit` argument is passed as " + "well.")) + group.addoption('--empd-meta', default=str(_meta_file), + help="The path to the meta file") + + +def pytest_configure(config): + global report_path, _meta_file, _base_meta_file, _commit_fixes + if config.option.markdown_report: + report_path = pathlib.Path( + config.option.markdown_report).expanduser().resolve() + if config.option.commit: + _commit_fixes = True + _meta_file = pathlib.Path(config.option.empd_meta).expanduser().resolve() + _base_meta_file = pathlib.Path( + osp.join(osp.dirname(_meta_file), 'meta.tsv')) + if not config.option.fix_db: + if getattr(config.option, 'markexpr', None): + config.option.markexpr += ' and not dbfix' + else: + config.option.markexpr = 'not dbfix' diff --git a/tests/fixes.py b/tests/fixes.py new file mode 100644 index 000000000..32c479f70 --- /dev/null +++ b/tests/fixes.py @@ -0,0 +1,96 @@ +# Module for fixing errors in the meta data +from latlon_utils import get_country, get_climate +import requests +from itertools import starmap +import pandas as pd +import pytest + + +def get_elevation(points): + query = ('https://api.open-elevation.com/api/v1/lookup?locations=' + + '|'.join(starmap('{},{}'.format, points))) + r = requests.get(query).json() + # json object, various ways you can extract value + # one approach is to use pandas json functionality: + elevation = pd.io.json.json_normalize(r, 'results')['elevation'].values + return elevation + + +@pytest.mark.dbfix +def fix_country(meta, meta_row, meta_file, commit_fixes, local_repo, + skip_ci): + country = get_country(meta_row.Latitude, meta_row.Longitude) + current = getattr(meta_row, 'Country', '') + if country != current: + message = "Changed country from %s to %s" % (current, country) + print(message) + meta.loc[meta_row.name, 'Country'] = country + meta.to_csv(str(meta_file), sep='\t') + if commit_fixes: + local_repo.index.add([meta_file.name]) + local_repo.index.commit( + ("[%s]: %s" % (meta_row.name, message)) + + ('\n\n[skip ci]' if skip_ci else '')) + + +@pytest.mark.dbfix +def fix_elevation(meta, meta_file, commit_fixes, local_repo, skip_ci): + try: + samples = meta[meta.Elevation.isnull()].index.values + except AttributeError: + samples = meta.index.values + if len(samples): + message = "Set elevation for %s" % samples + print(message) + points = meta.loc[samples, ['Latitude', 'Longitude']].values + elev = get_elevation(points) + meta.loc[samples, 'Elevation'] = elev + meta.to_csv(str(meta_file), sep='\t') + if commit_fixes: + local_repo.index.add([meta_file.name]) + local_repo.index.commit( + message + ('\n\n[skip ci]' if skip_ci else '')) + else: + pytest.skip("All elevations already specified") + + +@pytest.mark.dbfix +def fix_temperature(meta, meta_file, commit_fixes, local_repo, skip_ci): + try: + samples = meta[meta.Temperature.isnull()].index.values + except AttributeError: + samples = meta.index.values + if len(samples): + message = "Set temperature from WorldClim v2 for %s" % samples + print(message) + temperature = get_climate( + *meta.loc[samples, ['Latitude', 'Longitude']].values.T, + variables=['tavg']) + meta.loc[samples, 'Temperature'] = list(map( + ','.join, temperature.values.astype(str))) + meta.to_csv(str(meta_file), sep='\t') + if commit_fixes: + local_repo.index.add([meta_file.name]) + local_repo.index.commit( + message + ('\n\n[skip ci]' if skip_ci else '')) + + +@pytest.mark.dbfix +def fix_precipitation(meta, meta_file, commit_fixes, local_repo, skip_ci): + try: + samples = meta[meta.Precipitation.isnull()].index.values + except AttributeError: + samples = meta.index.values + if len(samples): + message = "Set temperature from WorldClim v2 for %s" % samples + print(message) + precip = get_climate( + *meta.loc[samples, ['Latitude', 'Longitude']].values.T, + variables=['prec']) + meta.loc[samples, 'Precipitation'] = list(map( + ','.join, precip.values.astype(str))) + meta.to_csv(str(meta_file), sep='\t') + if commit_fixes: + local_repo.index.add([meta_file.name]) + local_repo.index.commit( + message + ('\n\n[skip ci]' if skip_ci else '')) diff --git a/tests/test_critical.py b/tests/test_critical.py new file mode 100644 index 000000000..dcf765edc --- /dev/null +++ b/tests/test_critical.py @@ -0,0 +1,29 @@ +# Critical tests for the EMPD data. If they do not run, you can abort the tests +import pytest + + +@pytest.mark.critical +def test_columns(meta, base_meta): + required_cols = { + 'OriginalSampleName', 'Longitude', 'Latitude', + 'Worker1_Role', 'Worker1_LastName', 'Worker1_Initials', + 'Worker1_FirstName', 'Worker1_Address1', + 'Worker1_Email1', 'Worker1_Phone1'} + assert required_cols <= set(meta.columns) + assert meta.columns.all() # make sure every column has a title + + +@pytest.mark.critical +def test_samplename_duplicates(meta): + assert not meta.index.has_duplicates, "Duplicated samples: " + str( + meta.index[meta.index.duplicated(keep=False)]) + + +@pytest.mark.critical +def test_data_columns(data_frame): + ref_cols = {'var_', 'samplename', 'count', 'acc_var_', + 'original_varname', 'acc_varname', 'groupid', 'groupname', + 'higher_groupid', 'included_in_percent_sum', 'make_percent', + 'higher_groupname', 'percentage'} + assert ref_cols <= set(data_frame.columns) + assert data_frame.columns.all() # make sure every column has a title diff --git a/tests/test_formatting.py b/tests/test_formatting.py new file mode 100644 index 000000000..69dbfa6a3 --- /dev/null +++ b/tests/test_formatting.py @@ -0,0 +1,26 @@ +# Test the formatting of the data +import pytest + +def test_trailing_spaces(meta_row): + meta_row = meta_row.astype(str) + assert (meta_row.str.strip() == meta_row).all() + + +def test_newline_chars(meta_row): + meta_row = meta_row.astype(str) + assert (meta_row.str.replace('\n', ' ').str.replace('\r', ' ') == + meta_row).all() + + +@pytest.mark.parametrize('doi_field', ['DOI1', 'DOI2', 'DOI3', 'DOI4', 'DOI5']) +def test_doi(meta_row, doi_field): + if meta_row.notnull()[doi_field]: + doi = meta_row.copy(True)[doi_field] + + for patt in ['https://doi.org/', 'http://dx.doi.org/', 'doi: ', + 'doi.org/', 'DOI: ', 'DOI ', 'doi:', ' ', + 'https://link.springer.com/article/']: + doi = doi.replace(patt, '') + assert doi == meta_row.loc[doi_field] + else: + pytest.skip("%s field empty specified" % doi_field) diff --git a/tests/test_meta.py b/tests/test_meta.py new file mode 100644 index 000000000..0f51bfb06 --- /dev/null +++ b/tests/test_meta.py @@ -0,0 +1,94 @@ +# Test functions for the meta data +import os.path as osp +import glob +from latlon_utils import get_country +from itertools import starmap +import numpy as np + + +def okexcept(meta_row, col): + okexcept = getattr(meta_row, 'okexcept', []) + try: + return col in okexcept + except TypeError: + return False + + +def test_lat_lon(meta_row): + assert okexcept(meta_row, 'Latitude') or ( + meta_row.Latitude >= -90 and meta_row.Latitude <= 90) + assert okexcept(meta_row, 'Longitude') or ( + meta_row.Longitude >= -180 and meta_row.Longitude <= 360) + + +def test_country(meta_row): + assert okexcept(meta_row, 'Country') or meta_row.Country == get_country( + *meta_row[['Latitude', 'Longitude']]) + + +def test_elevation(meta_row): + assert okexcept(meta_row, 'Elevation') or ~np.isnan(meta_row.Elevation) + + +def test_temperature(meta_row): + assert meta_row.Temperature + temperature = np.array(meta_row.Temperature.split(',')).astype(float) + assert len(temperature) == 17 + assert ((temperature > -100) & (temperature < 100)).all() + + +def test_precip(meta_row): + assert meta_row.Precipitation + precip = np.array(meta_row.Precipitation.split(',')).astype(float) + assert len(precip) == 17 + assert ((precip > 0) & (precip < 10000)).all() + + +def test_orig_varnames(data_frame): + assert data_frame.original_varname.all() + duplicated = data_frame.duplicated('original_varname') + assert not duplicated.any(), ( + 'Duplicated acc_varnames: ' + str( + data_frame[duplicated].original_varname)) + + +def test_acc_varnames(data_frame): + assert data_frame.acc_varname.all() + duplicated = data_frame.duplicated('acc_varname', keep=False) + if duplicated.any(): + names = data_frame[duplicated][['original_varname', 'acc_varname']] + print('Found several duplicated acc_varnames:\n\n- ' + + '\n - '.join(starmap('{}: {}'.format, names.values))) + + +def test_counts(data_frame): + assert data_frame['count'].notnull().all() + + +def test_groupid(data_frame): + assert data_frame.groupid.all() + + +def test_higher_groupid(data_frame): + assert data_frame.groupid.all() + + +def test_percentage(data_frame): + assert data_frame[data_frame.make_percent].percentage.notnull().all() + + +def test_samplename(data_frame): + assert data_frame.samplename.all() + assert (data_frame.samplename.unique() == data_frame.samplename[:1]).all() + + +def test_data_exists(meta_row, samples_dir): + fname = osp.join(samples_dir, meta_row.name) + '.tsv' + assert osp.exists(fname), f"Missing {fname}" + + +def test_meta_exists(samples_dir, meta, base_meta): + files = glob.glob(osp.join(samples_dir, '*.tsv')) + samples = map(lambda f: osp.splitext(osp.basename(f))[0], files) + missing = set(samples).difference(base_meta.index).difference(meta.index) + assert not missing, f"Missing meta information for samples {missing}" From c277f08e8a27a4cb0afb2e90e64a6dfb9f78627f Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Sat, 23 Mar 2019 14:07:33 +0000 Subject: [PATCH 026/119] Added postgres dump for test.tsv --- postgres/test.sql | 949 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 949 insertions(+) create mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql new file mode 100644 index 000000000..aa3fb52dd --- /dev/null +++ b/postgres/test.sql @@ -0,0 +1,949 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ageuncertainties ( + ageuncertainty character varying(1) NOT NULL, + description character varying(21) NOT NULL, + age character varying(14) +); + + +ALTER TABLE public.ageuncertainties OWNER TO postgres; + +-- +-- Name: climate; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.climate ( + samplename character varying(25) NOT NULL, + t_jan double precision, + t_feb double precision, + t_mar double precision, + t_apr double precision, + t_may double precision, + t_jun double precision, + t_jul double precision, + t_aug double precision, + t_sep double precision, + t_oct double precision, + t_nov double precision, + t_dec double precision, + t_djf double precision, + t_mam double precision, + t_jja double precision, + t_son double precision, + t_ann double precision, + p_jan double precision, + p_feb double precision, + p_mar double precision, + p_apr double precision, + p_may double precision, + p_jun double precision, + p_jul double precision, + p_aug double precision, + p_sep double precision, + p_oct double precision, + p_nov double precision, + p_dec double precision, + p_djf double precision, + p_mam double precision, + p_jja double precision, + p_son double precision, + p_ann double precision +); + + +ALTER TABLE public.climate OWNER TO postgres; + +-- +-- Name: countries; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.countries ( + country character varying(42) NOT NULL, + natural_earth character varying(42) +); + + +ALTER TABLE public.countries OWNER TO postgres; + +-- +-- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ecosystems ( + samplename character varying(25) NOT NULL, + realm character(16), + biome character(62), + ecoregion character(65) +); + + +ALTER TABLE public.ecosystems OWNER TO postgres; + +-- +-- Name: groupid; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.groupid ( + groupid character varying(9) NOT NULL, + groupname character varying(60) NOT NULL, + higher_groupid character varying(4) NOT NULL +); + + +ALTER TABLE public.groupid OWNER TO postgres; + +-- +-- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.locationreliabilities ( + locationreliability character varying(1) NOT NULL, + description character varying(55) NOT NULL, + error character varying(5) +); + + +ALTER TABLE public.locationreliabilities OWNER TO postgres; + +-- +-- Name: metadata; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metadata ( + samplename character varying(25) NOT NULL, + originalsamplename character varying(70), + sitename character varying(70), + country character varying(42), + longitude double precision, + latitude double precision, + elevation double precision, + locationreliability character varying(1), + locationnotes text, + areaofsite double precision, + samplecontext character varying(40), + sitedescription text, + vegdescription text, + sampletype character varying(14), + samplemethod character varying(31), + agebp double precision, + ageuncertainty character varying(1), + ispercent boolean DEFAULT false, + notes text, + empd_version character varying(5) +); + + +ALTER TABLE public.metadata OWNER TO postgres; + +-- +-- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metapubli ( + samplename character varying(25) NOT NULL, + publiid integer NOT NULL +); + + +ALTER TABLE public.metapubli OWNER TO postgres; + +-- +-- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metaworker ( + samplename character varying(25) NOT NULL, + workerid integer NOT NULL, + workerrole character varying(3) +); + + +ALTER TABLE public.metaworker OWNER TO postgres; + +-- +-- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_counts ( + samplename character varying(25) NOT NULL, + var_ integer NOT NULL, + count double precision NOT NULL +); + + +ALTER TABLE public.p_counts OWNER TO postgres; + +-- +-- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_vars ( + var_ integer NOT NULL, + acc_var_ integer, + original_varname character varying(100) NOT NULL, + acc_varname character varying(100), + groupid character varying(9), + notes text +); + + +ALTER TABLE public.p_vars OWNER TO postgres; + +-- +-- Name: publications; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.publications ( + publiid integer NOT NULL, + doi character varying(50), + reference text NOT NULL +); + + +ALTER TABLE public.publications OWNER TO postgres; + +-- +-- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplecontexts ( + samplecontext character varying(40) NOT NULL +); + + +ALTER TABLE public.samplecontexts OWNER TO postgres; + +-- +-- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplemethods ( + samplemethod character varying(31) NOT NULL +); + + +ALTER TABLE public.samplemethods OWNER TO postgres; + +-- +-- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.sampletypes ( + sampletype character varying(14) NOT NULL, + notes character varying(13) +); + + +ALTER TABLE public.sampletypes OWNER TO postgres; + +-- +-- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workerroles ( + workerrole character varying(3) NOT NULL, + description character varying(35) NOT NULL +); + + +ALTER TABLE public.workerroles OWNER TO postgres; + +-- +-- Name: workers; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workers ( + workerid integer NOT NULL, + firstname character varying(25), + lastname character varying(25), + initials character varying(10), + address1 text, + email1 character varying(50), + phone1 character varying(25), + address2 text, + email2 character varying(50), + phone2 character varying(25) +); + + +ALTER TABLE public.workers OWNER TO postgres; + +-- +-- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP +\. + + +-- +-- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +\. + + +-- +-- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.countries (country, natural_earth) FROM stdin; +Albania Albania +Algeria Algeria +Andorra Andorra +Armenia Armenia +Austria Austria +Azerbaijan Azerbaijan +Bahrain Bahrain +Belarus Belarus +Belgium Belgium +Bosnia And Herzegovina Bosnia and Herzegovina +Bulgaria Bulgaria +Cape Verde Cape Verde +Croatia Croatia +Cyprus Cyprus +Czech Republic Czech Republic +Denmark Denmark +Egypt Egypt +Eritrea Eritrea +Estonia Estonia +Ethiopia Ethiopia +Faroe Islands Faroe Islands +Finland Finland +France France +Georgia Georgia +Germany Germany +Gibraltar Gibraltar +Greece Greece +Greenland Greenland +Guernsey Guernsey +Hungary Hungary +Iceland Iceland +Iran, Islamic Republic Of Iran +Iraq Iraq +Ireland Ireland +Isle Of Man Isle of Man +Israel Israel +Italy Italy +India India +Japan Japan +Jersey Jersey +Kazakhstan Kazakhstan +Kuwait Kuwait +Kyrgyzstan Kyrgyzstan +Latvia Latvia +Lebanon Lebanon +Libyan Arab Jamahiriya Libya +Liechtenstein Liechtenstein +Lithuania Lithuania +Luxembourg Luxembourg +Macedonia, The Former Yugoslav Republic Of Macedonia +Malta Malta +Moldova, Republic Of Moldova +Monaco Monaco +Morocco Morocco +Netherlands Netherlands +Norway Norway +Oman Oman +Palestinian Territory, Occupied Palestine +Poland Poland +Portugal Portugal +Qatar Qatar +Romania Romania +Russian Federation Russia +San Marino San Marino +Saudi Arabia Saudi Arabia +Serbia And Montenegro Montenegro +Slovakia Slovakia +Slovenia Slovenia +Spain Spain +Sweden Sweden +Switzerland Switzerland +Syrian Arab Republic Syria +Tunisia Tunisia +Turkey Turkey +Ukraine Ukraine +United Arab Emirates United Arab Emirates +United Kingdom United Kingdom +Yemen Yemen +Jordan Jordan +Black Sea \N +Dead Sea \N +Turkmenistan, Republic Of Turkmenistan +Adriatic Sea \N +Svalbard and Jan Mayen Norway +China, People s Republic Of China +\. + + +-- +-- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; +\. + + +-- +-- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; +ALGA Algae AQUA +AQBR Aquatic Bryophytes NOPO +AQPT Aquatic Pteridophyta AQUA +AQUA Aquatics AQUA +AQVP Aquatic Vascular Plants AQUA +BRYO Bryophytes NOPO +BRYO/VACR TO BE REMOVED TBR +CHAR Charcoal CHAR +CHLO Chlorophytes NOPO +CYAN Cyanobacteria NOPO +DINO Dinoflagellates NOPO +DWAR Dwarf shrubs TRSH +FUNG Fungi NOPO +HELO Helophytes AQUA +HERB Herbs HERB +INUN Indeterminables and unknowns INUN +INVE Invertebrates NOPO +LIAN Liana TRSH +MACR Macrofossils NOPO +NOPO Nonpollen NOPO +PLAT Platyhelminthes (flatworms) NOPO +PREQ Pre-Quaternary type NOPO +RHIZ Rhizopods NOPO +ROTI Rotifera (Rotifers) NOPO +SPOR Taxonomically undifferentiated spores INUN +TEAM Testate amoebae NOPO +TRSH Trees and shrubs TRSH +UNPP Undifferentiated NPP INUN +UPBR Upland Bryophytes NOPO +UPHE Upland herbs HERB +VACR Vascular cryptogams (Pteridophytes) NOPO +VEMI Vegetative microfossils NOPO +NEMA Nematoda (roundworms) NOPO +ACRI Acritarchs NOPO +EUMY Eumycete NOPO +HEMI Hemi-parasitic NOPO +MOSS Moss NOPO +MICR Microcrustaceans NOPO +\. + + +-- +-- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! \N +\. + + +-- +-- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +\. + + +-- +-- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metapubli (samplename, publiid) FROM stdin; +\. + + +-- +-- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; +\. + + +-- +-- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_counts (samplename, var_, count) FROM stdin; +test_a1 1 45 +test_a1 2 6 +test_a1 3 55 +\. + + +-- +-- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL +\. + + +-- +-- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.publications (publiid, doi, reference) FROM stdin; +\. + + +-- +-- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplecontexts (samplecontext) FROM stdin; +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological +\. + + +-- +-- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplemethods (samplemethod) FROM stdin; +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer +\. + + +-- +-- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.sampletypes (sampletype, notes) FROM stdin; +dung \N +litter \N +moss \N +pollen trap To be removed +sediment submerged +spider web \N +soil terrestrial +sphagnum peat \N +peat \N +lichen \N +submerged \N +core_top \N +tussock \N +epiphytic moss \N +\. + + +-- +-- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workerroles (workerrole, description) FROM stdin; +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) +\. + + +-- +-- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; +\. + + +-- +-- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ageuncertainties + ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); + + +-- +-- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.countries + ADD CONSTRAINT countries_pkey PRIMARY KEY (country); + + +-- +-- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); + + +-- +-- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.groupid + ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); + + +-- +-- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.locationreliabilities + ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); + + +-- +-- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); + + +-- +-- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); + + +-- +-- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); + + +-- +-- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); + + +-- +-- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); + + +-- +-- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); + + +-- +-- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_doi_key UNIQUE (doi); + + +-- +-- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); + + +-- +-- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplecontexts + ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); + + +-- +-- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplemethods + ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); + + +-- +-- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.sampletypes + ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); + + +-- +-- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workerroles + ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); + + +-- +-- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workers + ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); + + +-- +-- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); + + +-- +-- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); + + +-- +-- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); + + +-- +-- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); + + +-- +-- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); + + +-- +-- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); + + +-- +-- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); + + +-- +-- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); + + +-- +-- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); + + +-- +-- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); + + +-- +-- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); + + +-- +-- PostgreSQL database dump complete +-- + From 2eb9ad3916d76f3811adb06a2830506b94e035dc Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 11:15:48 +0100 Subject: [PATCH 027/119] minor --- postgres/scripts/makeFixedTables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/scripts/makeFixedTables.py b/postgres/scripts/makeFixedTables.py index 26ebefbb6..3c4c3abce 100644 --- a/postgres/scripts/makeFixedTables.py +++ b/postgres/scripts/makeFixedTables.py @@ -23,7 +23,7 @@ def is_null_str(x): for x in range(countries.shape[0]): cursor.execute( - "INSERT INTO countries (country, natural_earth) VALUES ('%s')" % ( + "INSERT INTO countries (country, natural_earth) VALUES ('%s', '%s')" % ( countries.iloc[x][0], is_null_str(countries.iloc[x][1]))) conn.commit() From b4eb52b222435594701833e801fdd643e294948c Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 14:29:54 +0100 Subject: [PATCH 028/119] typo --- postgres/scripts/create_empd2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/scripts/create_empd2.sql b/postgres/scripts/create_empd2.sql index 0a7a56f4c..64d13583b 100644 --- a/postgres/scripts/create_empd2.sql +++ b/postgres/scripts/create_empd2.sql @@ -107,7 +107,7 @@ ALTER TABLE ONLY p_vars ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); -- OK CREATE TABLE countries ( - country character varying(42) NOT NULL ------- PK + country character varying(42) NOT NULL, ------- PK natural_earth character varying(42) ); ALTER TABLE countries OWNER TO postgres; From 6f35ac79c3f25ffaa146d3a5150350364a6c38cd Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 14:34:49 +0100 Subject: [PATCH 029/119] minor --- postgres/scripts/makeFixedTables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/scripts/makeFixedTables.py b/postgres/scripts/makeFixedTables.py index 3c4c3abce..eab8a002a 100644 --- a/postgres/scripts/makeFixedTables.py +++ b/postgres/scripts/makeFixedTables.py @@ -23,7 +23,7 @@ def is_null_str(x): for x in range(countries.shape[0]): cursor.execute( - "INSERT INTO countries (country, natural_earth) VALUES ('%s', '%s')" % ( + "INSERT INTO countries (country, natural_earth) VALUES ('%s', %s)" % ( countries.iloc[x][0], is_null_str(countries.iloc[x][1]))) conn.commit() From f0ba25475a8fc471da6b621cc714e2053fcf48bd Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 15:24:10 +0100 Subject: [PATCH 030/119] added pytest.ini --- tests/pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/pytest.ini diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 000000000..eabbb4f0f --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +python_files = *.py +python_functions = test_* fix_* From 02d9c090e981491e810b6a86d1c5d1d503af63a8 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sat, 23 Mar 2019 16:10:49 +0100 Subject: [PATCH 031/119] split fixed tables into separate tsv files --- postgres/scripts/FixedTables.xlsx | Bin 27978 -> 0 bytes postgres/scripts/makeFixedTables.py | 17 ++-- postgres/scripts/tables/AgeUncertainty.tsv | 4 + postgres/scripts/tables/Country.tsv | 86 ++++++++++++++++++ postgres/scripts/tables/GroupID.tsv | 39 ++++++++ .../scripts/tables/LocationReliability.tsv | 6 ++ postgres/scripts/tables/SampleContext.tsv | 67 ++++++++++++++ postgres/scripts/tables/SampleMethod.tsv | 17 ++++ postgres/scripts/tables/SampleType.tsv | 15 +++ postgres/scripts/tables/WorkerRole.tsv | 7 ++ 10 files changed, 251 insertions(+), 7 deletions(-) delete mode 100644 postgres/scripts/FixedTables.xlsx create mode 100644 postgres/scripts/tables/AgeUncertainty.tsv create mode 100644 postgres/scripts/tables/Country.tsv create mode 100644 postgres/scripts/tables/GroupID.tsv create mode 100644 postgres/scripts/tables/LocationReliability.tsv create mode 100644 postgres/scripts/tables/SampleContext.tsv create mode 100644 postgres/scripts/tables/SampleMethod.tsv create mode 100644 postgres/scripts/tables/SampleType.tsv create mode 100644 postgres/scripts/tables/WorkerRole.tsv diff --git a/postgres/scripts/FixedTables.xlsx b/postgres/scripts/FixedTables.xlsx deleted file mode 100644 index 12e6d198e8d3b620669af0d405fb243563b9f226..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27978 zcmeEtgLkD*mu_r3>2z${w%M_bj-8He+qR94ZQJPBNyoaU`!{oE?)tv@3+A4+PU=+E z-p_vaeyi%Os#GaRgMy&}K>$Gk0Ra&L$@C}{`2qt0%>c5fKu{oB!nQU}CN@rb-`wp? z9Chg2tgQ(0!9XZ-fj|KA|Nry9I06&N<2HQ^NTQEPFNkq1=!`STbKyPtF(wh;UVv?Q z2$7?$-Qj||uLgL&sZc`sEm#?nK5Td}MYQ2XCKrWc?9?)F8DzCi63vvw1!a?+8{N*G zHb8aGC{Yv;8c++c;ppnZkx|29DK-rS9&D{cPBBjAp<#W2O?P41m)bp9lx;&lk&*Fp zUBAGSk9x@U8dt-y;&AhqnMjJsPWkOFyC*;xfwv4vT@wiXT_p}izR)x8fIekuh_HCf z4&ttF58Z%4_e|m9&APVCqhP63NQebvsyjTM;YsO+e%~}_Xkzc3h&7RHJRGf1@`%)# zKr3ATESjn5x|3QdbG<2#fCm{*^a}yL?P<5I{hmpP)br|4+wlQehze z4QP!rfPM)J=(u_gCRUF0bbtQ-{|@`VxR?Gn*DK>?<@*`ng0CdrLPj1JH)D~6q+JCi zeiDB3_LbZ~Y>Fx%`Lfwfj*IjSCjeC3r_=jwY<-g_>U4zgVUMLU0tJnmxY@NTIOWCO z8Ip?3Az946a(4jPW$}LTAx&J$ox-IvhN_~qG*5bTn^ zPjg6CW6SWV8hBAi`JyVgwuL?SG+{c!XF0j}1fD;XQ|5dw4SmGX&}_BZbHs}9{uxI_ z*__j&)-cSk-IG=>b3}=V1?7opT6~x^_ufadk>zGQ%YA?yy1!!N zdelDxzjO=m4*S0u_Md$Q81Rw< z$o}vCbR>;i_Wv2=gFXUg`I0Y41yfQ&=VZwx*`sv4c3 z&WeY9zoEgXYOV9zN4vvk4JbV3QYc%_OG@XJxzXQ6RQi%DCOX_{1(={t?tFtXl>=Yo zN90xc(9%jVN$6KRy824tQ3Ce4aOltKOR8u=RqL#ED_1k7Dbv1I0wTUW^nc*2ih$*ie?2DYfLn(1=ODN#n} z<=qcfLu(cgQCrVsSw+xr?DRviF%=EGs@Y#P+HhChgIC{su1AlOy+Hq6d)qX+%d-Uo z0#bzq0zw2dBtU!n?=7uR)!Mdz1L?zi?i~{0LJfw^7MN9o)Ne!7QeH)`8DFj19)wDN z%4JQY;{A#}SinN_|Iy`(19~J>Y+7i?1Z>a0dg`5?kJp1q!1}6QYqG&RM zOUY6CBng~GD)wN;L@8Sy=`%988}t!q$Dq_?aK5@~agpOyGwp%`a>4j{)O0WxnbXBg3#Jx9%Wm6a$&xCV&nXH#lTo7kxPHkn!l>V zyp4s3;Vqld2V@lVVpm-+XBP10r?R-;SE9?ZA4P${pUb1*foH)yg4e!8tBftnTwIrr z>ET#jpMGx#R60|_;~K%z)fz&N!-Rr<+Js`gLB|-s{o=fv3F8@;)bZsCt7W~>wxaC>EAdVx zJL$t0e7}#7Utbwgw~z~LM-Sxdga4gwjUghWXFfhhxiT|sb4p$v^q%y6@_nIB0Z1Rz zHpt61tnO$Z*b02`<5&2LjcWx|kR!!+`qG50yhXlj*z_7yBb9hsmV@RcXW!o|VjGt_ ze6N`QZftrI7;#4h2Lf8d00P4Lud&I|+{DDmk^XPO^5=D!nW|N@$A&b#O?}TT;vO=< z*5>QG2wft+86s6tzUhU~E8rBLO2(O@vUs&+uO4b`_m-kY^?L<@2gL+qCS;*=q~;R*KbO-zO#O)8h4v{)~N*&()051CfYj zbBO=(-2@#>$3EG4YjRPPEf-fs>xj3*T3*oA!o(XZ^OS1G01MJtm+gbKul!vC&Y34U z$n)EvG;PKQPGOSDR{zsaNB0w(+0#AZ-O~;)@3sjT6+ME1uFA2GSEPlYi|9-c0JAh{YupI-Q9jQJ=Orq6YO()|}F-^9~sGL8&G*lctEX${n7b zQ1djqpU>;}Gvs<}7wyxJ+$v>DXL$|!l&Sbu%e^{$i8IjcKWdVOqdt37x$742vH5PA z5^b|?*5LQ$zX zBGzz8&BEJR%px82dH=;y7fC>ol!RtsdwGyt?KfD}0?qm#*h<7qnA|D!2huSrhe#4ECSq|C__hfbo0?zR*f$0AS=o?I7U9Y$ttl&`g~bv&8Uc!m zv=_#OLy?V%Zu{!;Qom}gS)CgzsRkdUvau&; zsjSZqBKbTOT9OPWrmd`l%erl8a^o_ZGnV0uG9DBoSX#*p3&*V@7f%j75HP0$aL%26 z?0Z=cV+qrsBrHhCQSjHbI96Ry7=?-(`lFGMqnWO(FYa#0F{>g?ua!|HTp0n&fiHJS2 z{Lf=$#yV7Vt7 z-7C4WJd*~QHnmsQ(*Q&b=41jJ`@?wB>^$ooWm~gM`wG*b$gP79M%mpagw!m#Cj%v! zk>$v6#}ju)GPubB(K6VRVt`R1QZ6J|%>p2_vVU2JtIj%(f=(uX#c7C@d%7RG{2}Q) zDt{Z(!J8y|q;d;Y^>xZXYW-|+<+@!hr56UiQH@vIV{F}iuxvPZg z%J(?7zCuLlL9<%c&_h-+Q6!^~;$!4SS9=GJBft&<3lwX1f$}OAa)dOdj=ftln*=}j zcQOXNrA}MilLZUT`w2`O3e(#``YI@FIZ$AjZ3;+3kmj?G-Qc;BjtS48dYHP`CP;Er`XTZ#TDRVnK0Z@cRD~WkEpQ?eR>h$K zA>?(nnTURseNK>lRP)S9q}3}$T1U~Fu65QQ-U?WlkmzUZSWJkxHdZx?!>%;2M;@d$ z3|0aWnH^=yjNWh`+2$j3}1Wvg0Q2kdsJI5DWcOG3+ovyH99X0IQC_2~%r0S4M>PQO0S?3##am@}bVw1l(_7fknEb0gF zy*D;5G20U=4Iqag;nZ{WpF`2fBLh#mWsQmPQfy*--fv|HwJRXTVN2!qj8_t8Fyg_& zHNdwP!r2rGn}?o$ z@Z5FkSIaD>T+HNg8Z$9z8e^zMFB|MA%M0jSsw&d*HF}M83LibNB6Sf=Uwl41NhH z@c2fjg>6umrXs=Az^^M{O9N86Dy_=rL?ypl??8ty$`~$vqtIy3W=__T8Rw?uDK5H0 zV5eD?*Xq{7`4!J@pCQRV8gv)SFeW&Ov6O zNCdv*{eYH9pM_rOep~?hIuMS&N9TU&Xn;ywfpvarIuNC@RgJFAR@3*YEdko3)IpLF zth;T93X~%u4cmt*ue{OA2OA?`AcA4+)cxY_#EM_%ZE}t6ls1(|sM???i^Z&a@@>5d6^0x5+#>kSkp`RQ z>iu|q;?QC?SU~GLC4pYP?Pj5U&8}mln>9&Tly@z>T&J*iq@;@YKzaoCr7gbF;Foi< zwGp@48;0H9N`6E6ANwcgSaNz>j5+1!CO;-#5-RvCf#>f^&bvR~Fkg2ENA=`R!%-;PA;q}Fu*!sZd6^8)dj_|YbKEZjFI zc+Y>C+BqLO9o`~l$Fx?d$^7JB^NDfpes;j<@v*nsfFHB!(J{`*tzf-MO!$H{ay6oi z=GmldYzT)&d zUgj&1#3w#Cg1$TY@TPt1)NJq@UFrx2` z>lROTI0=rTsa_D-VQUU|?hizbr|3Z9daxq*91Na#)ZCu*`ExF@Q&&Hq%7LzNLf_n;03mp$L07 z1*^@cA3|8^GOz5rTf4c^C->rLhnpKHLqFMJ=TchLBkB8@ih}OFGB}9Lf_i^rTgwYxqiYt>fVz%0uKZ@P0F;>T=UluK{bdoUAv zc&L4go8q78oUiDeR@{sfl+_DDIh+UHyvQ#QG8j0*0S^hxN~@~AwFcvMeWWJyRb;Zp zn1z_PfftQ@?QGc~V@(&a0=YUiFFD@X^ z4Lma#9&+@>MwJE+f=(fVVCGZ`4RS@mF(UZ~L@AnI4a!$~5})SLqf;GAkwPW~UvRlx zYgG3n!X4PE-n)TgbpM>flOgpA9bI9TFa9eaaslLM8*$r$JT?I~zNDi#OqWaWzro8m zHF+mQU&`jSx6Y*ImzKV9w+WqO@UDL5|2le$P{n~KTnKN*Yac~-As9^IU^0;ij$;Tq zpZ1IQ+>B4b*+kFZcGPqIK7i}9AnGPHU{7COT<*Lg@&K#v@q5rm<$WNxq?B1#XaLbl zQh{^ha@^OIqIs;Jo~zr`%!}Ooy8S2Wu z7VKUwBhg)tj80cr)T=aLUT_uY}w*V2>0g!o%D|kqPI9gsk#55@SQA z2{5P$3)+a7^PJ#8{Tps`j5zQ?jc*g9hzfBn8;)(s?5kl|)b*E3(S}x5Kra+(p27NYd0KL*TcFwDI*vvu4c3AWhv&%2;iP?BjlV8o%7!87HG(xZw=(g6w5BJ73S{^Q|Pq_Wk<)_eS$R z3|z=O*$=$o{rewYYkV!Qhfa(5>!2UED~PYA12z5gFHX2?hk{=&T+c|NBksFS5aJA4 z@wWPB`T1PdAC$YjR$D@5W8p5Uu^B(+9bI`en%^IIonB3!UL4+PFCN7eHe7$&yyZz2 zEP8vqpUvmtr@wx6@=?=h(DA~*rtke+a4A^d{@r-hEbMcNKTOR~Gnu+QRXG%C&Qt;*+V#IHPl3pK$H_NZRpmR&_9>1S)6T$jDM zIo((vhP#_$_H}-)&qj=!i)MRnKkA(nbsGjUI&WRB(LOJf_r@$gyg4A%$bGWDr^-!m zj4VWtUfeW1d~63VdO2DW?(uK-Kf!^A)`vXQxvxfj9(TVtU1VpAH)61yT6g!m-FP)! zQ4sBhPP)F|WiZO^Rq$wTwv^mBimT3AzB=H%b{_C0U^!Oivkgx-w2jq-6GcRTk(As( zK0myk)JCTx@Y80cA8&CN<2IO!iBd?YMmIVY1Kmn^JgQrgfxuuUQFF871ci>NxZp*I z?dCfKp)WQRD7Kc?*8-Bz()z%fvpRD$x^fz83P2(?`0ItNlm#vhJl>E97!DJH>I)(S zOd%=y5iEQ_A!*Vij+C%pV`04_FqPt*g8NuQp+RgTPm?>SXZ`@N$Cob zCs+bBOqLiTSUR8+9tjkDZf;;osT{zKrOAS1QGtL~*4oz>29Vac{UMjE#uinG15K@rOvU>V&goms zhx+Z#>Dz)12>3P9G&cm^9wl1&lxM>H13hjCfJP~b%ijqv_YF0%VeJz{9bmtILPOZvEm{v>Am(VKBqF6Nn*g2xQ2%=jzO7GXA)3ohL zCFVB@fz2aBM}bq98dkSc1q6VVr=C^k64Iy(B3%nY2atlGR(Dbr*{AjdR@8_Pb1Q>@ z@X;C6is(G809Y9F4+cQ?IcpH1UKo6fcQkp3><2s4 z&LG9nhzABH#||K+R=4-UFdGztZ8Sd3)Fr0T=nengC_+OuM}>>Bx5~s}4sI~UjU50E zBG@B2q~8m(M*>!29MBI|tX-v&8+TzW@TWMeCzHCFYLP=me~MLW1}V8xe@aD*D1kJs zF+F++2bZii9bwpCEZJCqAt-9dAAMr)#JBP|B49qj)l4MJUgR4%j6;D@f}Bq*%Dmyf>e6eg2w>@cXJV`3L8ka`R4 z`CpU|DVJUBSu*WXdlD;tiV&}l`uZkB_M-TKtGS_1L53_D>vLrdEvwi2gR8k)$keC9 zA@IpH6xpx9sA`|6s@M7l?OE0^aY*R<$yU9iMfUomg8?ra!+luzeZSk~ zeDCd+pIq74f_ra4=Tn2;o3G20FDpMo!@pWr+84Ln6oI1gjJyn7T3&B&)#%;}@YH$> zpXk=;&uYh%^Uln-)P1hXDvY5GJAr^Oej4g0ZPyXfb%b1y% z9o}{{u7T&5d6srw2Np-#M5}DSmd(q+Z!I(73g+u4|4b`WWdNQjco@Lyk_GP~Dd+0M zNEU#Gii(%LziEiBPNa_3Kd;V*s-VAk8 z(DegAN#k4+J@p9P$+ie}=Z5@Mj9NQPwqdy(9k&~5o$=094}ODF5|8ta+l+PxKiN&M z$ufU&PuiECj3vb+=MZ1%9^au;U0}}BqHk!*Go7;@&mjQ+;TDrkm$BiMf47m585*S_ z+Z!!79IEc-27H&Lo2#k0o9|q+f}ZpDTFtsjS4V}UhpJJzcz?=;YEJiQ6`kov*PZE0 zcJ2^=qrQe;cV@sn&^TVb(*MSn8ZX^Z2>2CtreB`Tbmp1%jv(eiLtaJuTSki_#?;a! z5;HTfS%%MOxmStxAmVwb|H`23%)p|aVcki%ce&xrfa2^mCESMiDbmbBj#h+ZlkFON znukVrWwgi}vEY!BVa`I2F^8c$L~+z)ja}wS*HNoGK+$^eW{Pyw8?vB1+^AzGD{Z^C z)LrUX`c8BhmM1}Wa_Y;Rzw2BcNX`vU5Um^eS0Vr=u9Zc9_Vd@s^sKB2|CpI9487@w z$B`cfWyjg!({6|QH*-pWTAK!rt+s`R?f&|-kTUSc!u$l=mFke)%TE5xvW{Pj#iql& znEv5ol6?EC*$ldF{Uh&qVv|!YrUqFjJ3%!^&qd2h(*WKHm~!7}v0r&3<#ae~r(yl# z06){o_lum1JrwvRt3%2>EwTWb?1i|Ku$Pjq>7KmPH98oXh1VWftlT#p*EYhW9P-l( z=WrfnW&qcDr_wG+jQ~GrZBE-XN3w&q`%71`AL8q_`;QC5QR87peGzh=$gH}i_3Y%| zBS}2oxvhc9R!0p|2raS;jKkSVys^3?Cs(p_juW+g212hD$QrlrLqg>EQ%~L+$?q5_ zzT0~^T4k?wv-c~za#Xpv&7fu?9f zGive{yg%aktax#`WKc+-R?^vBqu2zM)J1JRT;LyUt^fNCc>i4grY*psI1JMN zaZSMV&n7%zO~8icgutJP zT>^XxwC1bGuid->;?uyr+s*lASiE!ttJh7cZ%2w)EI-@3tzBI!u4A?V3km}Uxy8%o z(-)lvm~M8Xr_0-;oaC7IYwHb`NwnOc%~lxp`NnQ_nxP-_RuwlL+w9zlm9GX`vQN&f zT9L1LWj3@H>$H0mV;Jk@n?VbGg9H`@dNu3q&oj%9>!$3A#KQ9`s4x6BYwyRJ>r3&6 z&5PF;-ae|2>%8f$7C&8HY`(w;SbN=^*eoM1%7)x|D_N*O%7cCkKIL#=d%OkBHCKgr zpD)jIYU(QF=w*AiQDCY}(suIrC^A#&z)cE&hp@Au^43a8JZ#=;tNRrdWwMdlrrkg2 z`C>Im@7i*?&ka`=i0%->(;$hTW)dEdHY_Np;rS72Az&B1(vSMdKb*` zp?9WIZP&xuJ}V8Uxi821j)Sk?ukJ#2+lLuEXC`Zef>lTP3n(~x7hd{V@tNvw(p>F_ zUbA9=#)5(lWOmEvtmgY2(#AKV06ZUS+YB?-IsY!Gd1=a_>t`rZ2r<+G^50kAAVnM^w@UG-5}eAz0rxmJ3Z{ zWoVa36MWy$n)=GrMI(&E*CKoUzF7f!-i(DB_E7)Q6-}2&?CB- z084Ml*duYUA)@#uOLRmw)3J$m7Fj|FTjVHwxDkl)c2ppA6KI^L(4@das&VXkv~$bB zj+|JQU+su0D8EQzZzDy6R!$b?h-$uNBjvQaA)-)P4*M4;5v>d$lV=%sTQDiqO`R!L z1jwdQopP}w|NbUBF~7!Vrvw{R9_C^LGU;I>#f-`zu_%;hS)7?M}koXU)+-d{HM`Zp&i!2TRo3D~7Cb*r@Fe=Ol+kYgfIlWQQ%9C+>1n=Z`ETeZY z#=beShSB<*s>pxme>IRe zC~}BC3a|afdE{#o`hgHZ9DN*KtAL&4&KX{Nq=P<;JAXqS*Q<1p8qiMo!cDV;yNGU3 zR)1J)6Kbn*@T)cYUMiqy3>?x$__wP%ioLN~V9;3UL_}5~?=bcm4^Ifd4ms`ydgzCv zdOl}6b)S;SZ^nFLJoO*dB;K1`AO$^*400oC$zb!zn7+w5_sCsGS*js*Z?GeXWpb_tuHx`YG z9LLI+y)Ro!#<&+r&FbZu_N{+BH&{$PVgLa9gLZ+;ZW~~met*=B`r|EU2r@?0oxkqU z8YW;DrL0{W;oZo0`NiC7L(McHu)Swy_Y1MSt=UOYU77!Kd-0`*@~Uc&)*iArK3#-c zn_JJ9qZfsPQxkNOl6|h_8kPjRByg^wPGP<5`zdEo@ix%su#dZXB3@)Zs)ZWq^aut$ zB=WEV8qUW>-K2^m6Hyl;_ir4o563jSy=o-(PF)j&qZWAR$>izNu;a(~^MNLMvjd7t zG>c(}in}MNc<3L)=7xm#=(4z3tny;#r={0U1Lg(btL}Q6tlg^pIzPVzRBIFr9&c=< zfL>VS>J@w2e)XA`Uuo9gg;@8-apUF^KXBfbzr;K@VX*eLD1P;-t$CQdE_(N)#Y;7L z$s2^g|5;;(e)(#EQ|7IWMz5YGp!};gU`1iOsz{WR%^_63cSKJ?BUe{t&6%Jjn;UIye<0DIEl9^0#QSs! zCs!1>?g>z?_rRFS!uqR+Y~7ab7(KpR!x3Tr!qd@Lvdf37xcsnrmP3&CN{ouqSb#AD zFHUTYW%@3R=ZH#Vp9M$gPII3shOKyIPr_-tPM+q?miyB@+`5Lq`c)k1r*dV6k8OxS zZ91Eve0Hn02B9o1shPW8L8=%P43DN-Uw$~^F-_4BvHY*4efOE;o$i{hpe`!i=pd8= zQ1>kEZun(nCS_Fkw$x?RoU%ioxmD}JoK))S;ehu5W!<;(OTbRzzwR*_6V|0x0czeQ z|Ht_R<3D?h>o&yffA<&#XI|AOP4(c{1<3?!-PH+J&KhW1tPMz-)K1&LBHkaXh$e|N zNc_Sq8PduKBT6ESvxp*$Js2=eMuV0utu6D`|UmTcvXUHks z@#Ms;gMYYtuWD~i%|DE?aN)5zt2k%uyZC5o1+L#c>Q78hzi$4%>v;9dl#2l8ln=Q8 zdC=2+w8ac@+3-T#u=n`*{e01H!`B5ou>Vo3=gppbwrKDAdNyvsXN!+Gb7fn;da1fS zb^SB=v+*^OQohS+<7(>8#9P+}$lb_V?kB&^(4>pFgvDn!tQc#?-EWf^)Z@I&y<6^zv`3wY@%T(;+twPDQzMEDxM%~*LIO|xb zw45#OJ85{Y8tva(K0en^h(AKJL@VeM!Na`0-*19>%{$93z9YvryE0I8g6f#a!6Fah z1kT-)HNkX@0bxt(D^=hKft>tEc(L(>WQy-8?1E%x%wh%zl);#-w#OlZivg=1)d`@b zsv!Q_6iWmoQ^QHf;HFJ$&uvWk1C^@W@{lxf6&2o2T{fa&8t=y)Uu#7!b=9KoT&bn{ zgciACYN9AYRoWfs^u8z>CRb;UgC>t0oHaYr*@`M6T~JiEP{4?NS3p~fqF@9h8&PhZ z8ttc9<64UP7(g5;E%`k(I@~ECL0BST8k*cG%VKD$iwIK(O~|CCO|@S2)DZe%+Pa8Q zj_%Y_6F4|Wx4xtf&&4?ZrN4KduyVZA)`3oHda~5UqaT*vFGbI z2KTmkeh0AYKi2MI-Mcq+F1H8x=BEL)qrDXl(vUvZ#$%k6s-%}w3E~T&2{Um@VRiwq zR#8xiUMoVk9hKOksc5()kw&dQs-$+{s74GBNt)J;a(gFvfG?_1O-x->c!5#Rr`UE) z>;j0Yil0#Hhq1B6lC@LV0%A^FRP)DcLB$u#zh?70&}$y6^74K!hNCp#Na;cT0i^W$ z`p0I;?>{#ABmV*|N&ErQ_O7J#Y68R=$nMJ*YF+*?BjJ+@K)6^FyFijD#$tH?p`iT2 zg`>!z2{2Xwh<((b%JbOza!Q||2KhwrU_y{AWzFXoWM#YGV&qAx7->ri7!f1&{bS`N zJgOQR#+veJH2Gm0dS&4btF9V`>E#p|;4F#YsO6SpW=Tn+#fyovMCp+b=}z!1O;nvtFIqmk$tY7 z6T?U=rrZ~0S+&g78gu`{r77_q1P37X<*liT4Pd=jZvzCtP4=M1riz9U9w`(;6(dr! z;QhV|AZ+2|2Wwuxm}~KlUj0Y+0|j%&n8h4oEZ`3md?GX`&X_;6HTpr{*k9&b^y&cj zi#@Tb9~J*tCKdx&2Ev^DKo5u$Ep|>srxr+_&HYM2uL=kvn}TViITMakxPy_2Nti`e z{yEl+!bg8sbKP|B1aosv%lO0YC6A9zFSVd&$V2lWd&+5Z@R21q%^R}x)pG4YAf^E( z#rv(orgc}(c;n{JVE1l}@5iPjS@5~@X;eeCRFo?uZjI$-b^rnz-2 z!Kxc?wciCzzz~{Edg>DWsM**IN)l@#dkmPv3Z9VVeMz2-eaId=2Z`?8Oj zYFUQUw$XDV7;N4hQ%+hk+ge0qJy904@||u&$BTZtsu3LE)H&W>Y)Gy+ZumlN^oE^2@1+~ z{}eRhd*+V7Uu2sFr!cae%a-lO)#(lFTh-kg>tLM^AiDsa?As?sJH5?y zP;>Yu`ZY!`)V7s;DoTD#Z}K@pme<>_yOnv$8giag-SNXvIYw`%X6kMkt>NbAukb%P zW={an%@g>?9=3yxr_x_DI8X8?pza|17R$sHoYuuEKq z0_h@u7z8q%Cq$O<6NH?RN)dBGU7u06({yMAZCKCT4Nd4PzgQ4OAZ7#Sn-jWCJVGE1 zg5`cA%k&0S4M5h7E8|P96SR$cV1My~UoIQp z(#Sk!jLj-o3qT`!m>#T^OLG$abeQ_m4x8Z^UT zB`6P9uH1}Y9~rV9>o308>Rwtz#FYitTqL};zEKsa-DG8~j2~|xLF+t0vW#B{8Sa)whdqTCAl3|&ZLvI6Z;u9>|n3;UQ@8wc1{ z;rqa+oB@jDSh;g>N0b{yXqTKpiXnVRe|Rcm;Um>T9|H=8_c4=!Jn3W8fRu3&kwV1L z{ps_Lp4Us>F5Eo@zS?@7coR-<^Xd4#{t71iWvlQdicc5kP>g(&|B;7Jf4;{H25jhO z{{|$WM&_5jqhX2>e_we0xHmA5Bzw`uXr+0F>PC>fb zN8yk$JU$T++V+Rs{rA>0a_xKDf_YwRIm{>Csj3lf*H&tIZi3BKDq(d4YK9zzQJ5e; zrY*fMuxLYr3K8_8k8<~kiSZ_CRgkPW(#{d9>Rg{~*Hcbv*6+qYoHV``o0+R@5-d(X zK=!Tp(A;HEhP1#ZS-zS!cJENf+=1|47b!f$@S9rrvjZ=}%go;K&|l&|34cF^WaoVY zoFn+xDgNrW8Uk#<<%fO1%?J4ZIJ4{IZe{ZKuPp_CPYS2P~lS`gPL_&Md%M^OPuig`=;n|GSP}1V* z_q_{9a>5vKPKGl27Dhrk!jvB-qXts6{MZoVtO+b#{K^Rp%{i`4!v~}nFvtjfQ{A4u zi!Hcf2QqbJU$5QY*kU#PoR*0juy01xUlJ9yIg?q;0mkOPV%FNk=Z1MOVxXoSqWBFvTlNcMN-&%O9NA6vz zF)Z8ci`yA(J~7kZM7BtlJOXuC>K@wagN$5Xw+qg;V9WY8QhvkII&xisQ7%!LE2-e*;>#Otl`U;C+%v_NR^;(LuyPg?G zx7GMcODb0|2~Ez6aOS#q{O%$yzmhSRDcIXkI1Y9Zft*1lB4exAz8!oz5c;vbJB%^s$TOpY;Pq8?K21 z(d8UmA}O$A?nK;Htz4{Q^o1ICacF&J=ycJcIuZIROy zfp@;S$5P2$f_SH(U%h^ksF}^pPs2NlZ*H@diy>6~bk<(wt6Ye}8|{CP$T+C=C;KiG zB&Xy=j);L)EK%;PwX>9oxCGYq-0eO!(S*A^+xcaFVKU9=j6I>N#$$Q8`6NA>+cERX zvRdhDq7*V;L84bL^2^t$=*;-vWqXJ!j4APwRDON)Gw~wG$qilePe68hfry^0+JT7b z4j?((SP_;n(z&+UhK~=NhQl%IBC+0C-W!A=r`Dr9X}D9WHH47vHH0Bb8^s}1PmH7A zBaq$7WiC`rG3V*h7K%^CBHF3xiq$$cGgj9ZK?diT3OR_E(*;Du+?hEPY=bfZmso^! z3VMk9>}!kn>byDU(qH-QM3$h{6))S#r)4NGh^yK@X^y@mkA2OIn8t<^+{LWo^yA8w zt?pb;vpw^2oLkPKK0nRoZd=qb<7(@?=bdLO+15fmQJW}O1o|2bmdrdtAyegUVO91f z+`u-ci5T>9IuE~Kb9TAPdpFc1v)Ro=yLkfc)w%uC!Fjvsd_9{vt*vW)Zj;y1#u_JC zMW(0d_EMn79Q{QVy@E>ULInzi#|6vaTO33Br(EPGV5I!lF|>`knL-|Lxs??#fc_5` z$o_6FWvXe_%KR7dy?ojCg6C`Fvq!j;&_JUG~tPOQ-pbICrd< z-2K7St@yB%3AKyEurMjaZ||L*W}1*A5tISVn}9Zgqeg~hn ztXk#S%A=C9|0l|ucTPLU@)owI)st(xwWepSvdXVWw%cBTY!qo=U6yW^)T?ZRg}^cL~Pc@??F3rYVNqGu=UW^|4J-@WYOvphbJ*j#k9= z3bDc7XwTVqI(k#tJw?aCj1`O=eqW^J(*a%6Mlb`eZiKsWccsG{3}7<9vQw=8S5 ztfX^9TXQO}&o_Owa8nIkY){t1k*lzYfn3dkV9~;9$&gsBl|oHagUM8=tOhu$YlO8zvJt>Jc%9Q!kiTp+obJ_oQo6(kzF%hb~HIj zO+xM?VWe?7J|~h4QO}Z8B#~TgY(XK-EP1+K??fw2tI-};q^!}NR3t~wN+{)3x&ssw zD3y!E!x1|j3J&ALQ(njs`&%kMR1{|pxE_a8i76I7xe9W`)YnQ6dDPaf>s&3h%hi|1=SYrDG#P|}ZF|$B(P3W(d<8S>1z{kw9U3=0r(&(U~R~{M_ zh1R4G0ii;8x{29G;`Q(dZ=u#8Ul;Pz6BJ2V$vc&D66#DOoDCASWaoW7cm>!%jbFnh z@_coVyUP?==e57Q@CN$z9PZ2y|Qw^(Xzr(4a zudaCbMe_8AnAM;kGHF=OAce;2EC-=eWA?SPLS)~BbDy1woA(YjR2!X4guQVqFib#? zR0D;x7VMJeXkrU1GTF9S^+Wc&V?;P(paWK3dkN|r58*XgfS>d- zLgMYE;K0w%-i9~81^Ry-gn|&xFrEN|P~?~YaS&qtyH}g05wp(*H@pq|n;XkJNB#E& z!Kfi3TstB-hG>l0up)}lIYkriaQ?9?y{M=I0%R&(0QM2S#!Fbf1+yUr{?^)%N1Tpc zMgLq#+BL6x=|VBzdGZH-H$UI6|I^-8f5o{hTOh$5f(8ptaMyvL32wnTKCy1Qz3`z-pFzB1PyS<=1h z{7+I`Wa__27hQ@cJw2#b_+eONbh}DnuR{e)Rc&f9m<j}Fg zV4N21wAf9sQj#S*G?m$ug&Dqu4tkYhM2l~l;*lk4&2pqa zK|!oN1v*d@44U7un4!$kP_JB%_S?r9G7vWDxyAQWRVpV-oaf(S9gCZ%@nJjf%VBZW+Y*O^wK+Y%UT|yXQkM9*_P7t!Yg?BQQp@rxq{et#_ z)-HU!hz*}|1`?V*dL}vo@Fwk0PD63@_EWekPdy^~_DfHjxqmRfY+PyjVdY_5O5$j2 zL0%{;9xS)Uyo56ULqUf%GtAsdP-~m`p*!OxOxp4qMjx`*I(9Oe^Rk!A*0DBGhG5aK zVDu%p3Hy+R_~_I82vI0MH?7JZ4GGLPD^93ott{uNfK1eu+`Xnt%k7wF{HyJ`zG+?kFya`$>C{-Jd@Np|T!mhRXpdEgp-k4| zsdmxgDu5P58&zkVyS$mzd0tBZN8O-Cy0SOuj%lqnIJsbqyz1Oj64P?#q~7BD>;C4I zb4KM?`Ic4?O|?r$_~;$@=B8-6%qpuT52Ni_fO4ZtwD&3lpo%^!ceEwYss}OZz!*~W z?`4gyJ6lJ-nxm}azI0_+H98bmhB$NyKxtl03UmYYHscoQifSru0U|p16J@!__BH@b ze70F%M?_!k!(lAey#9HN;GFSE-G?{bD9x6h=a?J!d*Jy@7isW=-VWR$H(=U$d{FM_ zCu|G=@O^ReYwPe3vgjwkgR8#!PW?u~dptX_Q+H8=Sp8E>+~I&;xWcY!rmtHzhxgxr zy67hZQ)v%tG<()(x3kOTEb{FQX)0N^q{dWQMZv=#r8~84YtM$)br4gsM_0^yeGYw$ zhU5j5Wx>5!1#XAb3ySV731=F_*D&8sRn&5^0td-xf}`f9KM=8Z6>|8(zI;-yo>?GZtGuSMs0rKBACMCaV| zDWMbC;&y1a^I(Jsx+)>Y&y#P9%szlVgiX9UfQDdEgTe3+vxv~kN3)pdoM`;>_Xp_u z=!#eh<^ll#F1%rX_(jVg23JEaFozE@ZUS5Kp&(gUmai$-V`SXiq6$jW{dbD z@K4E2h*C$%a!!L-N{6Q=@wdF*clCvZfj%I~O3Er=DOA&yAVPkWoeF1=jBZicM5;9L z{B1%|VZMdr8>+(mLR!S);rOW+5LlnAYa3Vauf}xhjhA*s#K&doWTrZZ}+S) zx5T~I%j79?O+Q5Jtz_SmV=JvOe0ixNXME++K!cNJf^&6jgj1=iWkZ=@J?o0~wQmvn zxIuoWjIf0BXtxcvY-(mw(WKQ`mdBkdY9FSJ2LZTy#1HjDHVrBYIYO4koJ(KJ7IodMbPKsLz_)iBhQc8ngR5U*;YV&(P&8zN$wFd zR#Q#Iip&_$V?G-ot2a&|)g{xh#%5)V-U922Mxw7Q_nCbb_?hmg;2@Ye2Bmb>HdlyS zE{;3*PWAo+Dn5RRmGqoXw@zdI;+F>7^BUI-iaFnRJPH8Ltp!wfToYEAd9oRB|6Ee< zA#{Sh3A6qApX-^JH~paEAn|KCL^-_r;by(S(w_zp5xJ|D z(-hVH=Bj?4>&^$~{`BXa3IW`={M!l(*($YlH9!#QjL<#Ui>tfaZOj9SOhl-%tDA!u zPmyB=e!1jyfyf#Ct&7Rd)N^34HwOuPMFWDfb$pLKiF4_$D4ss6ok8+)WywWG2`T@w zJkytKEl%P4Z#leCf|_cMUCtJ6ZUVZ~h5RBRvXfksYAA$oI&If=$0ONXj1fP@^Gnfm z;jX3EfVVc_J6U-hWpopcmR&;fw@llO-Br1-l{m#c_$$sitRKiI_{VYT^Ox! zaj=bW2Vc{?<7>FtQJr;3g+&s~{83@E{^KR|RX=dw&r7Ys>UV3OdTwbmsry#H5|qqxRN$F2$w( z9&(IF^;4wBWu}?l4}>~6hesnk(D8#;HhCCgMM~wMZO)EB1`^TCXUDhmU{SpsmI4xm zf>0Ann4`o7U?-c-dhH>wVLTe~0XRoXOddz2cq{O*x4lxlHVbwrb5xv_0&Y7GffQG= zG#e4bRPJTZ1~K75@Nrfwk>2(W@m4G}%%h4moncEnZwFwJ?avG$T~wk>3N@YSD9vvM z;ON;lVW{vJU-ItOXO<`P)@TIlNp`OrX%>Bw8K1QfvB3QT-A= zk^MI(G?WZp>bcFjE0&+CR8G^owO`3G8@cQ0K8v&)#aTsjA=U9Cj-XxJA>l) zAuN=#;-+=!letH>vTQQEn!_({n7;z1?y_7NaO)fhq93wX{&_dB6Az|HMN_S;eYJ{PrZmckC#r8aP*i1S z1GT=yhN0rUP@>H*ND{FH#813=hT8Tc8j-SYmvo$FWR9X@i!#khO49{3{Ox^PQPEW< zMB~KibfPWU<83l>7bp?fu5boOkC;oQoJ{n!l*Sa4K z6g<7zUQryNCP?{umbA*Rv&M>`M$oRRl~z;bRDbKP5eW(0Q92@iwQ#1N$;wZ%;W=?d zi-qN5c0+Zyjn~MGJFZGCwgsmlYLorI)-0pvr9SR>w%chjL5AS3=~)1LR;=BBv45x8 z(J)a2ZN~92p|D|&Z$H5%Mjut-b~>j;?O0_9cV!HyQ`<~j8XY*B{oWTGM2hlhO&qyO zm`+f;J2=Cn$aH@;`vp*|dsg~TSE2ETrGl8GLsJk_V7-NF^8$`h*G?4n3W~pQ8;I_! zVhn28fs(R}Iyv{2X2cPjAk+sKB}&PSfnF5(F@&0?a9ZBAo<*uj1T0&~%381KJsql) z%5{~X-+duPSOW`Neu9o(sQOE#OE-g_0hxwh^x4zgWXzcW&-;_z^WYAE?~k3EZeUGw z{cC9(cq(P(qv>ei!`&U;3b5n;Wbc%FcZ%k;tEc1F%}`Ru!ztedcw-5Nx!wEhY&Q;X z)%#*wKAL!8k%&0?&al^*-2Efey?0Po$^Klu;>F64kIvh7m6Ku6%2ysY53xc9}BEap%pUEumwvmPbrd zimn`{og zoo}ftnJL9Q!syO#4H*KYzw-rO5DIr7p?pL8Nutxk1(J5ynP?8jXO9KA%ta~)I4;P5 zzV!G|qrDQ3b@i9T^NOD57XyeADb%noH6k{2-IWXh z1cw_h2sF&qkA$-G^PQI@EVSS&J4N*2do~I>s*mZj4{bM(tC8y^5FbdD(|;LNxk%T8 zHU^bi&d@m|A(O>ONzy5Z7>Vy@=j5Q;5J>F{i&zxT+9LX%zNgxMn944j75)hwaY^yetUFE*mdLFcd963g`zRAfezb ze+Q$?Lg&@Gwv#my3^MWKTAy}bp@-8M=ev%HTy0uFN7^pTD|-RsgKWg zGUZ#Vx7c6B9}U6wwurcTKW`n6fM+RQRQyEKYymw=ex1sh%&-(s!T8OdT8M^iA$kl& zQ`C6*b9$J}#d|k)>TJXOU}t)qtg(upa_4T*k&|+!&E=m`OORnfH*R#=k)labg*%#n zjwDXhr`Z6Xj6`k=FfoLKxbWH>d2>SvR<5iYYC;T;(Pg0?AC9Yghe50=$}Fv3Nf1IJrSM*4Sjv_#Q?e zqi+H#$2d{0HNDyg3_;Crsc+~>)9f*v_`kTZbyAoIE#oU@%8~l5D_7dxm-em_ms8FY zb+wU3Mz=lV+vdj1S664IhmQ*V5bp1=H^SXD7fS% z8)bHJhE@)b%hQwM%g?8F=Ik>GB$qzBY?EK+CY>I1gfstcTG;z~Z^|idH<*7Dwd_oE zs^)cp&7B;hL!~8CGaA;kyHxA?ly~^EP8P8fOjnJ2jVNK^Z4Es_U*SCX9tw&{@GROU*IH6Y6(hy!Thi!E7iCbir30x&v$PIlo3<(~DKqMw{Hfh1S*| z6L&GM>Yrsg`nJV55{MZdE(09Kt6#s}XxE4))739>xcwt+YJ_s56AR}zF;r-evIsF zN)^)5g>EU8`WC>;e#vq^`x&k=i~0xee#8JM-uIDY_9lI+0s_y*FFZ@eRPpJZaSZv; zSg>$rOUlpiA8v10K6QQJQ!S9$mFl?`Rrkv9dcZxbbFU4=K!eVyVP#4$Qwvbn4FSzp zSBk@?%A|oH!T`M0^_UFSTnS+^ANAyL*`OM8#g9tx_83kO`&tg&hVUk^NXa3u zVsr`rOMc`H+Z~k#Rv-U_@qJd;SPXu3!aY=U@xZJ|Zfz7NZ&x8OV1z%JIZ4CXi11rS z=RIL(+^rT1QsVtg%l$f~#qH)iWvt;oe?^@z8Xt1hM+1;pzH$*3U#Dqw#1{iEQME>7 z+j0C~i3X;gmYa(%6&Go00>V6gSCsK%+$W)go=h9R2y3g1a~=+|9)n`vR% zz1B;HpAR_sh2m8Tz%4xtafw>ME!SZ3VTl_WB&TB2gZlD|Nr{ikVoeQQk3pyc@ek@`ye{tnN~R6oY3$r z0lGx>z=CtytAMNmF~0dj z>li(D(Tk&E?;_q`3 z9f&j5p=G|L>)doBcQ!09akdmXt$_0JZM6gP3A9wP2|x*)tk>iWYPidil9b`Rj!Pcy zcEbd@i@$SWANsQFiMKX;$8=yvCeM0L!_>ut8dn!)TT6LiDI)M}kO=4%a|Ssa{HG7P zf@~bHApS=S*$~J0yASHy+5Nu@LVWLEuk`njvRM`sFG$(!+D%(^t#dCfEmN407CeVm zO!SdP!mfHT0|6Fx&)r2{`Lwet#pg@vSkp5vfiN9XdK}`IeXE^G*G zxq-}ry|JQ|j+GHX=1U*Kt7;o<=O$7K8Kmqm6UJ-^lR_>iCDUnhHf+SqWmCH5!YFhO znOMbN!45~xsb#aeXBDa;5@9-_-H0#aM2PkdKtD8^*#vBFA92gd2C%y*&q^Uh{YaV@ zL9$GzW6`vG#Yw0p0gQrqA$`jtAuvmJscH~cz_1IP*R+FWw}{2j=G7aJ{;&*lb;j6S z$7BefPV^?bIyO$8a?O|sUJw#WJnNbj}@r0|QmBr77go>wO!)H% zT7%|TyB*sSi?EOtm_9E^E8xdKJRfR+Vwix$cM>a-%{*B^ob}2b=Dd~xKK=_UZVUD2 zp%oQC{*ntJ_YG#eVHp+2WrbY+Wy!mphKs&o-mY_-47mAEo45{|{We#hpjrwLHMYWs zrH=Q05uEd_uLk@)IWhiLrg;0Rfa)hh}P+!FAe&)-~7A%!xcgb z(*H^DpO=^X8~9h-43S*@+f^n{f&Y1-!{32_6@>ry>m8o*JQeEy%_N8T583{w;HOI3 zzrlNu)Tn=H{{KT)`zgy)8QI?~?2!Ew2+Kc2WuG!U)yDeGkWKQ(aXczzJ!N@XP5+yv z?)4v89&78LvOKLn{LS)(^(o6As}Y|PJ}t`oP57MsDdFR@e%x|;%JMYt_czP7;3JFU z|78I`g+5Jp`wgWR`{Mu}li!}QJk3}6%>sV+N0!Izm8UFE&vAdVtUz?p{@`(sr@BvB zo}NAYX0cQLBg^AS#8Z~18-Kr9c-0?So^As^C3w1Y`AyIS{Npel7cx&-{xhQd%>e~v tY5)cGkNEN_{6B+%zr**9{s#Y7XrLes3z=U2+UUfFf`)_->}G#``X8eZX;%OM diff --git a/postgres/scripts/makeFixedTables.py b/postgres/scripts/makeFixedTables.py index eab8a002a..d0d5d257b 100644 --- a/postgres/scripts/makeFixedTables.py +++ b/postgres/scripts/makeFixedTables.py @@ -3,6 +3,7 @@ import psycopg2 as psql import pandas as pd import numpy as np +from functools import partial db_url = sys.argv[-1] @@ -14,11 +15,12 @@ def is_null_str(x): return 'NULL' if x == "" else "'" + x.strip().replace("'", " ") + "'" -tables = osp.join(osp.dirname(__file__), "FixedTables.xlsx") +tables = osp.join(osp.dirname(__file__), "tables", '') +read_tsv = partial(pd.read_csv, sep='\t') # --- -countries = pd.read_excel(tables, sheet_name="Country") +countries = read_tsv(tables + "Country.tsv") countries.replace(np.nan, '', inplace=True) for x in range(countries.shape[0]): @@ -30,7 +32,7 @@ def is_null_str(x): # --- -LocationReliability = pd.read_excel(tables, sheet_name="LocationReliability") +LocationReliability = read_tsv(tables + "LocationReliability.tsv") LocationReliability.replace(np.nan, '', inplace=True) for x in range(LocationReliability.shape[0]): @@ -45,7 +47,7 @@ def is_null_str(x): # --- -sampleContext = pd.read_excel(tables, sheet_name="SampleContext") +sampleContext = read_tsv(tables + "SampleContext.tsv") sampleContext.replace(np.nan, '', inplace=True) for x in range(sampleContext.shape[0]): @@ -57,7 +59,7 @@ def is_null_str(x): # --- -sampleType = pd.read_excel(tables, sheet_name="SampleType") +sampleType = read_tsv(tables + "SampleType.tsv") sampleType.replace(np.nan, '', inplace=True) for x in range(sampleType.shape[0]): @@ -70,7 +72,7 @@ def is_null_str(x): # --- -SampleMethod = pd.read_excel(tables, sheet_name="SampleMethod") +SampleMethod = read_tsv(tables + "SampleMethod.tsv") SampleMethod.replace(np.nan, '', inplace=True) for x in range(SampleMethod.shape[0]): @@ -82,7 +84,7 @@ def is_null_str(x): # --- -ageUncertainties = pd.read_excel(tables, sheet_name="AgeUncertainty") +ageUncertainties = read_tsv(tables + "AgeUncertainty.tsv") ageUncertainties.replace(np.nan, '', inplace=True) for x in range(ageUncertainties.shape[0]): @@ -96,6 +98,7 @@ def is_null_str(x): # --- +workerRoles = read_tsv(tables + "WorkerRole.tsv") workerRoles = pd.read_excel(tables, sheet_name="WorkerRole") workerRoles.replace(np.nan, '', inplace=True) diff --git a/postgres/scripts/tables/AgeUncertainty.tsv b/postgres/scripts/tables/AgeUncertainty.tsv new file mode 100644 index 000000000..3afa6d5c9 --- /dev/null +++ b/postgres/scripts/tables/AgeUncertainty.tsv @@ -0,0 +1,4 @@ +Age Uncertainty Description Age +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP diff --git a/postgres/scripts/tables/Country.tsv b/postgres/scripts/tables/Country.tsv new file mode 100644 index 000000000..f4572c6e6 --- /dev/null +++ b/postgres/scripts/tables/Country.tsv @@ -0,0 +1,86 @@ +Country Natural Earth +Albania Albania +Algeria Algeria +Andorra Andorra +Armenia Armenia +Austria Austria +Azerbaijan Azerbaijan +Bahrain Bahrain +Belarus Belarus +Belgium Belgium +Bosnia And Herzegovina Bosnia and Herzegovina +Bulgaria Bulgaria +Cape Verde Cape Verde +Croatia Croatia +Cyprus Cyprus +Czech Republic Czech Republic +Denmark Denmark +Egypt Egypt +Eritrea Eritrea +Estonia Estonia +Ethiopia Ethiopia +Faroe Islands Faroe Islands +Finland Finland +France France +Georgia Georgia +Germany Germany +Gibraltar Gibraltar +Greece Greece +Greenland Greenland +Guernsey Guernsey +Hungary Hungary +Iceland Iceland +Iran, Islamic Republic Of Iran +Iraq Iraq +Ireland Ireland +Isle Of Man Isle of Man +Israel Israel +Italy Italy +India India +Japan Japan +Jersey Jersey +Kazakhstan Kazakhstan +Kuwait Kuwait +Kyrgyzstan Kyrgyzstan +Latvia Latvia +Lebanon Lebanon +Libyan Arab Jamahiriya Libya +Liechtenstein Liechtenstein +Lithuania Lithuania +Luxembourg Luxembourg +Macedonia, The Former Yugoslav Republic Of Macedonia +Malta Malta +Moldova, Republic Of Moldova +Monaco Monaco +Morocco Morocco +Netherlands Netherlands +Norway Norway +Oman Oman +Palestinian Territory, Occupied Palestine +Poland Poland +Portugal Portugal +Qatar Qatar +Romania Romania +Russian Federation Russia +San Marino San Marino +Saudi Arabia Saudi Arabia +Serbia And Montenegro Montenegro +Slovakia Slovakia +Slovenia Slovenia +Spain Spain +Sweden Sweden +Switzerland Switzerland +Syrian Arab Republic Syria +Tunisia Tunisia +Turkey Turkey +Ukraine Ukraine +United Arab Emirates United Arab Emirates +United Kingdom United Kingdom +Yemen Yemen +Jordan Jordan +Black Sea +Dead Sea +Turkmenistan, Republic Of Turkmenistan +Adriatic Sea +Svalbard and Jan Mayen Norway +China, People s Republic Of China diff --git a/postgres/scripts/tables/GroupID.tsv b/postgres/scripts/tables/GroupID.tsv new file mode 100644 index 000000000..0f2fef9d6 --- /dev/null +++ b/postgres/scripts/tables/GroupID.tsv @@ -0,0 +1,39 @@ +groupid groupname higher_groupid +ALGA Algae AQUA +AQBR Aquatic Bryophytes NOPO +AQPT Aquatic Pteridophyta AQUA +AQUA Aquatics AQUA +AQVP Aquatic Vascular Plants AQUA +BRYO Bryophytes NOPO +BRYO/VACR TO BE REMOVED TBR +CHAR Charcoal CHAR +CHLO Chlorophytes NOPO +CYAN Cyanobacteria NOPO +DINO Dinoflagellates NOPO +DWAR Dwarf shrubs TRSH +FUNG Fungi NOPO +HELO Helophytes AQUA +HERB Herbs HERB +INUN Indeterminables and unknowns INUN +INVE Invertebrates NOPO +LIAN Liana TRSH +MACR Macrofossils NOPO +NOPO Nonpollen NOPO +PLAT Platyhelminthes (flatworms) NOPO +PREQ Pre-Quaternary type NOPO +RHIZ Rhizopods NOPO +ROTI Rotifera (Rotifers) NOPO +SPOR Taxonomically undifferentiated spores INUN +TEAM Testate amoebae NOPO +TRSH Trees and shrubs TRSH +UNPP Undifferentiated NPP INUN +UPBR Upland Bryophytes NOPO +UPHE Upland herbs HERB +VACR Vascular cryptogams (Pteridophytes) NOPO +VEMI Vegetative microfossils NOPO +NEMA Nematoda (roundworms) NOPO +ACRI Acritarchs NOPO +EUMY Eumycete NOPO +HEMI Hemi-parasitic NOPO +MOSS Moss NOPO +MICR Microcrustaceans NOPO diff --git a/postgres/scripts/tables/LocationReliability.tsv b/postgres/scripts/tables/LocationReliability.tsv new file mode 100644 index 000000000..cbc539bea --- /dev/null +++ b/postgres/scripts/tables/LocationReliability.tsv @@ -0,0 +1,6 @@ +Location Reliability Description Error +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! diff --git a/postgres/scripts/tables/SampleContext.tsv b/postgres/scripts/tables/SampleContext.tsv new file mode 100644 index 000000000..9d8aead53 --- /dev/null +++ b/postgres/scripts/tables/SampleContext.tsv @@ -0,0 +1,67 @@ +Sample Context +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological diff --git a/postgres/scripts/tables/SampleMethod.tsv b/postgres/scripts/tables/SampleMethod.tsv new file mode 100644 index 000000000..a8a4a9116 --- /dev/null +++ b/postgres/scripts/tables/SampleMethod.tsv @@ -0,0 +1,17 @@ +CollectionMethod +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer diff --git a/postgres/scripts/tables/SampleType.tsv b/postgres/scripts/tables/SampleType.tsv new file mode 100644 index 000000000..60469c752 --- /dev/null +++ b/postgres/scripts/tables/SampleType.tsv @@ -0,0 +1,15 @@ +SampleType notes +dung +litter +moss +pollen trap To be removed +sediment submerged +spider web +soil terrestrial +sphagnum peat +peat +lichen +submerged +core_top +tussock +epiphytic moss diff --git a/postgres/scripts/tables/WorkerRole.tsv b/postgres/scripts/tables/WorkerRole.tsv new file mode 100644 index 000000000..568312e50 --- /dev/null +++ b/postgres/scripts/tables/WorkerRole.tsv @@ -0,0 +1,7 @@ +WorkerRole Description +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) From 83be78bdcd6b604d9f980be56134ad2f972885ce Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 06:14:54 +0100 Subject: [PATCH 032/119] Added handling of okexcept in postgres --- postgres/scripts/create_empd2.sql | 1 + postgres/scripts/import_into_empd2.py | 48 ++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/postgres/scripts/create_empd2.sql b/postgres/scripts/create_empd2.sql index 64d13583b..14cb93bac 100644 --- a/postgres/scripts/create_empd2.sql +++ b/postgres/scripts/create_empd2.sql @@ -34,6 +34,7 @@ CREATE TABLE metadata ( ageUncertainty character varying(1), -------- FK isPercent boolean DEFAULT FALSE, notes text, + okexcept text, empd_version character varying(5) ); ALTER TABLE metadata OWNER TO postgres; diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index 8e06933e6..8794b232b 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -6,6 +6,7 @@ import requests import argparse from itertools import product +from collections import defaultdict parser = argparse.ArgumentParser() parser.add_argument( @@ -84,6 +85,42 @@ def clean_doi(doi): cursor.execute('SELECT samplename FROM metadata') existing_samples = [r[0] for r in cursor.fetchall()] +# check okexcept column and update fixed tables +okexcept = defaultdict(set) +for key, row in METADATA[METADATA.okexcept.astype(bool)].iterrows(): + if row.okexcept and str(row.okexcept) != 'nan': + row_okexcept = row.okexcept.split(',') + for col in row_okexcept[:]: + col = col.strip() + if col in ['Country', 'GroupID', 'SampleContext', 'SampleMethod', + 'SampleType']: + okexcept[col].add(row[col]) + row_okexcept.remove(col) + METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) + +table_map = { + 'Country': 'countries', + 'GroupID': 'groupID', + 'SampleContext': 'sampleContexts', + 'SampleType': 'sampleTypes', + 'SampleMethod': 'sampleMethods', + } + +for col, vals in okexcept.items(): + fname = os.path.join(os.path.dirname(__file__), 'tables', col + '.tsv') + df = pd.read_csv(fname, sep='\t') + new_vals = np.sort(set(vals) - set(df.iloc[:, 0]))[:, np.newaxis] + new_vals = np.tile(new_vals, (1, df.shape[1])) + new_vals[:, 1:] = '' + df = pd.concat([df, pd.DataFrame(new_vals, columns=df.columns)], + ignore_index=True) + df.to_csv(fname, index=False, sep='\t') + cursor.execute('INSERT INTO %s VALUES %s' % ( + table_map[col], ', '.join( + '({})'.format(', '.join(map(is_null_str, v))) for v in new_vals))) + conn.commit() + + METADATA.replace(np.nan, '', inplace=True) PUBLI = METADATA[['Publication1', 'DOI1']] tmp = METADATA[['Publication2', 'DOI2']] @@ -238,7 +275,7 @@ def clean_doi(doi): "locationReliability = %s, locationNotes = %s, " "areaOfSite = %s, sampleContext = %s, siteDescription = %s, " "vegDescription = %s, sampleType = %s, sampleMethod = %s, " - "ageBP = %s, ageUncertainty = %s, notes = %s " + "ageBP = %s, ageUncertainty = %s, notes = %s, okexcept = %s" "WHERE sampleName = {}").format( is_null_str(METADATA.iloc[x]['SampleName'])) else: @@ -248,9 +285,9 @@ def clean_doi(doi): " latitude, elevation, locationReliability, locationNotes, " " areaOfSite, sampleContext, siteDescription, vegDescription, " " sampleType, sampleMethod, ageBP, ageUncertainty, notes, " - " empd_version) VALUES " - "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " - " %s, %s, 'EMPD2')") + " okexcept, empd_version) VALUES " + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " + " %s, %s, %s, 'EMPD2')") cursor.execute(query % ( is_null_str(METADATA.iloc[x]['SampleName']), is_null_str(str(METADATA.iloc[x]['OriginalSampleName'])), @@ -270,7 +307,8 @@ def clean_doi(doi): is_null_str(METADATA.iloc[x]['SampleMethod'].lower()), is_null_str(str(METADATA.iloc[x]['AgeBP'])), is_null_str(METADATA.iloc[x]['AgeUncertainty']), - is_null_str(METADATA.iloc[x]['Notes']))) + is_null_str(METADATA.iloc[x]['Notes']), + is_null_str(METADATA.iloc[x]['okexcept']))) conn.commit() except psql.IntegrityError as e: conn = psql.connect(db_url) From 58728f61431da92d5cfcf4e19709e4669891b9b3 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 06:27:06 +0100 Subject: [PATCH 033/119] some fixes with postgres import --- postgres/scripts/import_into_empd2.py | 27 +++++++++++++++------------ postgres/scripts/makeFixedTables.py | 3 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index 8794b232b..55dc92107 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -109,16 +109,19 @@ def clean_doi(doi): for col, vals in okexcept.items(): fname = os.path.join(os.path.dirname(__file__), 'tables', col + '.tsv') df = pd.read_csv(fname, sep='\t') - new_vals = np.sort(set(vals) - set(df.iloc[:, 0]))[:, np.newaxis] - new_vals = np.tile(new_vals, (1, df.shape[1])) - new_vals[:, 1:] = '' - df = pd.concat([df, pd.DataFrame(new_vals, columns=df.columns)], - ignore_index=True) - df.to_csv(fname, index=False, sep='\t') - cursor.execute('INSERT INTO %s VALUES %s' % ( - table_map[col], ', '.join( - '({})'.format(', '.join(map(is_null_str, v))) for v in new_vals))) - conn.commit() + new_vals = set(vals) - set(df.iloc[:, 0]) + if new_vals: + new_vals = np.sort(list(new_vals))[:, np.newaxis] + new_vals = np.tile(new_vals, (1, df.shape[1])) + new_vals[:, 1:] = '' + df = pd.concat([df, pd.DataFrame(new_vals, columns=df.columns)], + ignore_index=True) + df.to_csv(fname, index=False, sep='\t') + cursor.execute('INSERT INTO %s VALUES %s' % ( + table_map[col], ', '.join( + '({})'.format(', '.join(map(is_null_str, v))) + for v in new_vals))) + conn.commit() METADATA.replace(np.nan, '', inplace=True) @@ -275,7 +278,7 @@ def clean_doi(doi): "locationReliability = %s, locationNotes = %s, " "areaOfSite = %s, sampleContext = %s, siteDescription = %s, " "vegDescription = %s, sampleType = %s, sampleMethod = %s, " - "ageBP = %s, ageUncertainty = %s, notes = %s, okexcept = %s" + "ageBP = %s, ageUncertainty = %s, notes = %s, okexcept = %s " "WHERE sampleName = {}").format( is_null_str(METADATA.iloc[x]['SampleName'])) else: @@ -287,7 +290,7 @@ def clean_doi(doi): " sampleType, sampleMethod, ageBP, ageUncertainty, notes, " " okexcept, empd_version) VALUES " "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " - " %s, %s, %s, 'EMPD2')") + " %s, %s, %s, %s, 'EMPD2')") cursor.execute(query % ( is_null_str(METADATA.iloc[x]['SampleName']), is_null_str(str(METADATA.iloc[x]['OriginalSampleName'])), diff --git a/postgres/scripts/makeFixedTables.py b/postgres/scripts/makeFixedTables.py index d0d5d257b..6ea60851e 100644 --- a/postgres/scripts/makeFixedTables.py +++ b/postgres/scripts/makeFixedTables.py @@ -99,7 +99,6 @@ def is_null_str(x): # --- workerRoles = read_tsv(tables + "WorkerRole.tsv") -workerRoles = pd.read_excel(tables, sheet_name="WorkerRole") workerRoles.replace(np.nan, '', inplace=True) for x in range(workerRoles.shape[0]): @@ -111,7 +110,7 @@ def is_null_str(x): # --- -groupIDs = pd.read_excel(tables, sheet_name="GroupID") +groupIDs = read_tsv(tables + "GroupID.tsv") groupIDs.replace(np.nan, '', inplace=True) for x in range(groupIDs.shape[0]): From 8ede07f491a8d50b0df2fadb3958a4d781659817 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 07:12:41 +0100 Subject: [PATCH 034/119] Include natural_earth and okexcept in postgres dump --- postgres/EMPD2.sql | 189 ++++++++++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 88 deletions(-) diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql index c076182e2..359334330 100644 --- a/postgres/EMPD2.sql +++ b/postgres/EMPD2.sql @@ -95,7 +95,8 @@ ALTER TABLE public.climate OWNER TO postgres; -- CREATE TABLE public.countries ( - country character varying(42) NOT NULL + country character varying(42) NOT NULL, + natural_earth character varying(42) ); @@ -165,6 +166,7 @@ CREATE TABLE public.metadata ( ageuncertainty character varying(1), ispercent boolean DEFAULT false, notes text, + okexcept text, empd_version character varying(5) ); @@ -327,92 +329,92 @@ COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul -- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres -- -COPY public.countries (country) FROM stdin; -Albania -Algeria -Andorra -Armenia -Austria -Azerbaijan -Bahrain -Belarus -Belgium -Bosnia And Herzegovina -Bulgaria -Cape Verde -Croatia -Cyprus -Czech Republic -Denmark -Egypt -Eritrea -Estonia -Ethiopia -Faroe Islands -Finland -France -Georgia -Germany -Gibraltar -Greece -Greenland -Guernsey -Hungary -Iceland -Iran, Islamic Republic Of -Iraq -Ireland -Isle Of Man -Israel -Italy -India -Japan -Jersey -Kazakhstan -Kuwait -Kyrgyzstan -Latvia -Lebanon -Libyan Arab Jamahiriya -Liechtenstein -Lithuania -Luxembourg -Macedonia, The Former Yugoslav Republic Of -Malta -Moldova, Republic Of -Monaco -Morocco -Netherlands -Norway -Oman -Palestinian Territory, Occupied -Poland -Portugal -Qatar -Romania -Russian Federation -San Marino -Saudi Arabia -Serbia And Montenegro -Slovakia -Slovenia -Spain -Sweden -Switzerland -Syrian Arab Republic -Tunisia -Turkey -Ukraine -United Arab Emirates -United Kingdom -Yemen -Jordan -Black Sea -Dead Sea -Turkmenistan, Republic Of -Adriatic Sea -Svalbard and Jan Mayen -China, People s Republic Of +COPY public.countries (country, natural_earth) FROM stdin; +Albania Albania +Algeria Algeria +Andorra Andorra +Armenia Armenia +Austria Austria +Azerbaijan Azerbaijan +Bahrain Bahrain +Belarus Belarus +Belgium Belgium +Bosnia And Herzegovina Bosnia and Herzegovina +Bulgaria Bulgaria +Cape Verde Cape Verde +Croatia Croatia +Cyprus Cyprus +Czech Republic Czech Republic +Denmark Denmark +Egypt Egypt +Eritrea Eritrea +Estonia Estonia +Ethiopia Ethiopia +Faroe Islands Faroe Islands +Finland Finland +France France +Georgia Georgia +Germany Germany +Gibraltar Gibraltar +Greece Greece +Greenland Greenland +Guernsey Guernsey +Hungary Hungary +Iceland Iceland +Iran, Islamic Republic Of Iran +Iraq Iraq +Ireland Ireland +Isle Of Man Isle of Man +Israel Israel +Italy Italy +India India +Japan Japan +Jersey Jersey +Kazakhstan Kazakhstan +Kuwait Kuwait +Kyrgyzstan Kyrgyzstan +Latvia Latvia +Lebanon Lebanon +Libyan Arab Jamahiriya Libya +Liechtenstein Liechtenstein +Lithuania Lithuania +Luxembourg Luxembourg +Macedonia, The Former Yugoslav Republic Of Macedonia +Malta Malta +Moldova, Republic Of Moldova +Monaco Monaco +Morocco Morocco +Netherlands Netherlands +Norway Norway +Oman Oman +Palestinian Territory, Occupied Palestine +Poland Poland +Portugal Portugal +Qatar Qatar +Romania Romania +Russian Federation Russia +San Marino San Marino +Saudi Arabia Saudi Arabia +Serbia And Montenegro Montenegro +Slovakia Slovakia +Slovenia Slovenia +Spain Spain +Sweden Sweden +Switzerland Switzerland +Syrian Arab Republic Syria +Tunisia Tunisia +Turkey Turkey +Ukraine Ukraine +United Arab Emirates United Arab Emirates +United Kingdom United Kingdom +Yemen Yemen +Jordan Jordan +Black Sea \N +Dead Sea \N +Turkmenistan, Republic Of Turkmenistan +Adriatic Sea \N +Svalbard and Jan Mayen Norway +China, People s Republic Of China \. @@ -487,7 +489,7 @@ X Do not use! \N -- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres -- -COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; +COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; \. @@ -944,6 +946,17 @@ ALTER TABLE ONLY public.p_counts ALTER TABLE ONLY public.p_vars ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); + +-- +-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres +-- + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM postgres; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + -- -- PostgreSQL database dump complete -- From 6164bf0661dd9fa93737dedc5838d6572949765c Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 07:18:41 +0100 Subject: [PATCH 035/119] added tab-delimited postgres dump --- tab-delimited/.csv | 0 tab-delimited/.tsv | 0 tab-delimited/ageuncertainties.tsv | 4 ++ tab-delimited/climate.tsv | 1 + tab-delimited/countries.tsv | 86 +++++++++++++++++++++++++ tab-delimited/ecosystems.tsv | 1 + tab-delimited/groupid.tsv | 39 +++++++++++ tab-delimited/locationreliabilities.tsv | 6 ++ tab-delimited/metadata.tsv | 1 + tab-delimited/metapubli.tsv | 1 + tab-delimited/metaworker.tsv | 1 + tab-delimited/p_counts.tsv | 1 + tab-delimited/p_vars.tsv | 1 + tab-delimited/publications.tsv | 1 + tab-delimited/samplecontexts.tsv | 67 +++++++++++++++++++ tab-delimited/samplemethods.tsv | 17 +++++ tab-delimited/sampletypes.tsv | 15 +++++ tab-delimited/workerroles.tsv | 7 ++ tab-delimited/workers.tsv | 1 + 19 files changed, 250 insertions(+) create mode 100644 tab-delimited/.csv create mode 100644 tab-delimited/.tsv create mode 100644 tab-delimited/ageuncertainties.tsv create mode 100644 tab-delimited/climate.tsv create mode 100644 tab-delimited/countries.tsv create mode 100644 tab-delimited/ecosystems.tsv create mode 100644 tab-delimited/groupid.tsv create mode 100644 tab-delimited/locationreliabilities.tsv create mode 100644 tab-delimited/metadata.tsv create mode 100644 tab-delimited/metapubli.tsv create mode 100644 tab-delimited/metaworker.tsv create mode 100644 tab-delimited/p_counts.tsv create mode 100644 tab-delimited/p_vars.tsv create mode 100644 tab-delimited/publications.tsv create mode 100644 tab-delimited/samplecontexts.tsv create mode 100644 tab-delimited/samplemethods.tsv create mode 100644 tab-delimited/sampletypes.tsv create mode 100644 tab-delimited/workerroles.tsv create mode 100644 tab-delimited/workers.tsv diff --git a/tab-delimited/.csv b/tab-delimited/.csv new file mode 100644 index 000000000..e69de29bb diff --git a/tab-delimited/.tsv b/tab-delimited/.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tab-delimited/ageuncertainties.tsv b/tab-delimited/ageuncertainties.tsv new file mode 100644 index 000000000..c69d814c5 --- /dev/null +++ b/tab-delimited/ageuncertainties.tsv @@ -0,0 +1,4 @@ +ageuncertainty description age +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP diff --git a/tab-delimited/climate.tsv b/tab-delimited/climate.tsv new file mode 100644 index 000000000..f30354209 --- /dev/null +++ b/tab-delimited/climate.tsv @@ -0,0 +1 @@ +samplename t_jan t_feb t_mar t_apr t_may t_jun t_jul t_aug t_sep t_oct t_nov t_dec t_djf t_mam t_jja t_son t_ann p_jan p_feb p_mar p_apr p_may p_jun p_jul p_aug p_sep p_oct p_nov p_dec p_djf p_mam p_jja p_son p_ann diff --git a/tab-delimited/countries.tsv b/tab-delimited/countries.tsv new file mode 100644 index 000000000..14b117a44 --- /dev/null +++ b/tab-delimited/countries.tsv @@ -0,0 +1,86 @@ +country natural_earth +Albania Albania +Algeria Algeria +Andorra Andorra +Armenia Armenia +Austria Austria +Azerbaijan Azerbaijan +Bahrain Bahrain +Belarus Belarus +Belgium Belgium +Bosnia And Herzegovina Bosnia and Herzegovina +Bulgaria Bulgaria +Cape Verde Cape Verde +Croatia Croatia +Cyprus Cyprus +Czech Republic Czech Republic +Denmark Denmark +Egypt Egypt +Eritrea Eritrea +Estonia Estonia +Ethiopia Ethiopia +Faroe Islands Faroe Islands +Finland Finland +France France +Georgia Georgia +Germany Germany +Gibraltar Gibraltar +Greece Greece +Greenland Greenland +Guernsey Guernsey +Hungary Hungary +Iceland Iceland +Iran, Islamic Republic Of Iran +Iraq Iraq +Ireland Ireland +Isle Of Man Isle of Man +Israel Israel +Italy Italy +India India +Japan Japan +Jersey Jersey +Kazakhstan Kazakhstan +Kuwait Kuwait +Kyrgyzstan Kyrgyzstan +Latvia Latvia +Lebanon Lebanon +Libyan Arab Jamahiriya Libya +Liechtenstein Liechtenstein +Lithuania Lithuania +Luxembourg Luxembourg +Macedonia, The Former Yugoslav Republic Of Macedonia +Malta Malta +Moldova, Republic Of Moldova +Monaco Monaco +Morocco Morocco +Netherlands Netherlands +Norway Norway +Oman Oman +Palestinian Territory, Occupied Palestine +Poland Poland +Portugal Portugal +Qatar Qatar +Romania Romania +Russian Federation Russia +San Marino San Marino +Saudi Arabia Saudi Arabia +Serbia And Montenegro Montenegro +Slovakia Slovakia +Slovenia Slovenia +Spain Spain +Sweden Sweden +Switzerland Switzerland +Syrian Arab Republic Syria +Tunisia Tunisia +Turkey Turkey +Ukraine Ukraine +United Arab Emirates United Arab Emirates +United Kingdom United Kingdom +Yemen Yemen +Jordan Jordan +Black Sea +Dead Sea +Turkmenistan, Republic Of Turkmenistan +Adriatic Sea +Svalbard and Jan Mayen Norway +China, People s Republic Of China diff --git a/tab-delimited/ecosystems.tsv b/tab-delimited/ecosystems.tsv new file mode 100644 index 000000000..40926bfb3 --- /dev/null +++ b/tab-delimited/ecosystems.tsv @@ -0,0 +1 @@ +samplename realm biome ecoregion diff --git a/tab-delimited/groupid.tsv b/tab-delimited/groupid.tsv new file mode 100644 index 000000000..0f2fef9d6 --- /dev/null +++ b/tab-delimited/groupid.tsv @@ -0,0 +1,39 @@ +groupid groupname higher_groupid +ALGA Algae AQUA +AQBR Aquatic Bryophytes NOPO +AQPT Aquatic Pteridophyta AQUA +AQUA Aquatics AQUA +AQVP Aquatic Vascular Plants AQUA +BRYO Bryophytes NOPO +BRYO/VACR TO BE REMOVED TBR +CHAR Charcoal CHAR +CHLO Chlorophytes NOPO +CYAN Cyanobacteria NOPO +DINO Dinoflagellates NOPO +DWAR Dwarf shrubs TRSH +FUNG Fungi NOPO +HELO Helophytes AQUA +HERB Herbs HERB +INUN Indeterminables and unknowns INUN +INVE Invertebrates NOPO +LIAN Liana TRSH +MACR Macrofossils NOPO +NOPO Nonpollen NOPO +PLAT Platyhelminthes (flatworms) NOPO +PREQ Pre-Quaternary type NOPO +RHIZ Rhizopods NOPO +ROTI Rotifera (Rotifers) NOPO +SPOR Taxonomically undifferentiated spores INUN +TEAM Testate amoebae NOPO +TRSH Trees and shrubs TRSH +UNPP Undifferentiated NPP INUN +UPBR Upland Bryophytes NOPO +UPHE Upland herbs HERB +VACR Vascular cryptogams (Pteridophytes) NOPO +VEMI Vegetative microfossils NOPO +NEMA Nematoda (roundworms) NOPO +ACRI Acritarchs NOPO +EUMY Eumycete NOPO +HEMI Hemi-parasitic NOPO +MOSS Moss NOPO +MICR Microcrustaceans NOPO diff --git a/tab-delimited/locationreliabilities.tsv b/tab-delimited/locationreliabilities.tsv new file mode 100644 index 000000000..358c68139 --- /dev/null +++ b/tab-delimited/locationreliabilities.tsv @@ -0,0 +1,6 @@ +locationreliability description error +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! diff --git a/tab-delimited/metadata.tsv b/tab-delimited/metadata.tsv new file mode 100644 index 000000000..f31e489d1 --- /dev/null +++ b/tab-delimited/metadata.tsv @@ -0,0 +1 @@ +samplename originalsamplename sitename country longitude latitude elevation locationreliability locationnotes areaofsite samplecontext sitedescription vegdescription sampletype samplemethod agebp ageuncertainty ispercent notes okexcept empd_version diff --git a/tab-delimited/metapubli.tsv b/tab-delimited/metapubli.tsv new file mode 100644 index 000000000..87ecd1f3c --- /dev/null +++ b/tab-delimited/metapubli.tsv @@ -0,0 +1 @@ +samplename publiid diff --git a/tab-delimited/metaworker.tsv b/tab-delimited/metaworker.tsv new file mode 100644 index 000000000..51501704a --- /dev/null +++ b/tab-delimited/metaworker.tsv @@ -0,0 +1 @@ +samplename workerid workerrole diff --git a/tab-delimited/p_counts.tsv b/tab-delimited/p_counts.tsv new file mode 100644 index 000000000..8b51c97bb --- /dev/null +++ b/tab-delimited/p_counts.tsv @@ -0,0 +1 @@ +samplename var_ count diff --git a/tab-delimited/p_vars.tsv b/tab-delimited/p_vars.tsv new file mode 100644 index 000000000..e0f7475b9 --- /dev/null +++ b/tab-delimited/p_vars.tsv @@ -0,0 +1 @@ +var_ acc_var_ original_varname acc_varname groupid notes diff --git a/tab-delimited/publications.tsv b/tab-delimited/publications.tsv new file mode 100644 index 000000000..3e1bf798d --- /dev/null +++ b/tab-delimited/publications.tsv @@ -0,0 +1 @@ +publiid doi reference diff --git a/tab-delimited/samplecontexts.tsv b/tab-delimited/samplecontexts.tsv new file mode 100644 index 000000000..9c818983e --- /dev/null +++ b/tab-delimited/samplecontexts.tsv @@ -0,0 +1,67 @@ +samplecontext +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological diff --git a/tab-delimited/samplemethods.tsv b/tab-delimited/samplemethods.tsv new file mode 100644 index 000000000..d29857875 --- /dev/null +++ b/tab-delimited/samplemethods.tsv @@ -0,0 +1,17 @@ +samplemethod +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer diff --git a/tab-delimited/sampletypes.tsv b/tab-delimited/sampletypes.tsv new file mode 100644 index 000000000..ba0274210 --- /dev/null +++ b/tab-delimited/sampletypes.tsv @@ -0,0 +1,15 @@ +sampletype notes +dung +litter +moss +pollen trap To be removed +sediment submerged +spider web +soil terrestrial +sphagnum peat +peat +lichen +submerged +core_top +tussock +epiphytic moss diff --git a/tab-delimited/workerroles.tsv b/tab-delimited/workerroles.tsv new file mode 100644 index 000000000..20effabb3 --- /dev/null +++ b/tab-delimited/workerroles.tsv @@ -0,0 +1,7 @@ +workerrole description +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) diff --git a/tab-delimited/workers.tsv b/tab-delimited/workers.tsv new file mode 100644 index 000000000..5c2b22268 --- /dev/null +++ b/tab-delimited/workers.tsv @@ -0,0 +1 @@ +workerid firstname lastname initials address1 email1 phone1 address2 email2 phone2 From eb44889c0594496a14f9173e5b76dc35fa96a7c7 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 08:06:40 +0100 Subject: [PATCH 036/119] minor --- postgres/scripts/import_into_empd2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index 55dc92107..518bd47aa 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -38,6 +38,10 @@ def is_null_nb(x, nodata=np.nan): return 'NULL' if x == nodata else str(round(x, 2)) +def notnull(x): + return str(x) not in ['', 'nan'] + + def listdir_fullpath(d): return [os.path.join(d, f) for f in os.listdir(d)] @@ -93,7 +97,7 @@ def clean_doi(doi): for col in row_okexcept[:]: col = col.strip() if col in ['Country', 'GroupID', 'SampleContext', 'SampleMethod', - 'SampleType']: + 'SampleType'] and notnull(row[col]): okexcept[col].add(row[col]) row_okexcept.remove(col) METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) From 9b71d2d78472891517a008fdd6a2581ab3bf72af Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 10:09:26 +0100 Subject: [PATCH 037/119] Added more tests for metadata --- tests/conftest.py | 60 +++++++++++++++++++++++++++++ tests/test_meta.py | 95 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 141 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7e6fd90f3..74dfb8d8b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,6 +80,66 @@ def base_counts(): return pd.concat(list(map(read_tsv, files)), ignore_index=True) +@pytest.fixture(scope='session') +def countries(meta_file): + """The countries in the database with the natural_earth column as index""" + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', 'countries.tsv'), + sep='\t', index_col=1).iloc[:, 0] + + +@pytest.fixture(scope='session') +def samplecontexts(meta_file): + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', + 'samplecontexts.tsv'), + sep='\t').iloc[:, 0] + + +@pytest.fixture(scope='session') +def sampletypes(meta_file): + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', 'sampletypes.tsv'), + sep='\t').iloc[:, 0] + + +@pytest.fixture(scope='session') +def samplemethods(meta_file): + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', 'samplemethods.tsv'), + sep='\t').iloc[:, 0] + + +@pytest.fixture(scope='session') +def workerroles(meta_file): + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', 'workerroles.tsv'), + sep='\t').iloc[:, 0] + + +@pytest.fixture(scope='session') +def locationreliabilities(meta_file): + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', + 'locationreliabilities.tsv'), + sep='\t').iloc[:, 0] + + +@pytest.fixture(scope='session') +def ageuncertainties(meta_file): + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', + 'ageuncertainties.tsv'), + sep='\t').iloc[:, 0] + + @pytest.fixture(scope='session') def samples_dir(meta_file): return osp.join(osp.dirname(meta_file), 'samples') diff --git a/tests/test_meta.py b/tests/test_meta.py index 0f51bfb06..3f3742614 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,9 +1,11 @@ # Test functions for the meta data import os.path as osp import glob +import pytest from latlon_utils import get_country from itertools import starmap import numpy as np +import pandas as pd def okexcept(meta_row, col): @@ -14,6 +16,25 @@ def okexcept(meta_row, col): return False +def notnull(val): + return not isnull(val) + + +def isnull(val): + val = str(val) + return val == '' or val == 'nan' or val == 'None' + + +# ---- meta.tsv tests + + +def test_meta_exists(samples_dir, meta, base_meta): + files = glob.glob(osp.join(samples_dir, '*.tsv')) + samples = map(lambda f: osp.splitext(osp.basename(f))[0], files) + missing = set(samples).difference(base_meta.index).difference(meta.index) + assert not missing, f"Missing meta information for samples {missing}" + + def test_lat_lon(meta_row): assert okexcept(meta_row, 'Latitude') or ( meta_row.Latitude >= -90 and meta_row.Latitude <= 90) @@ -26,11 +47,61 @@ def test_country(meta_row): *meta_row[['Latitude', 'Longitude']]) +def test_samplecontext(meta_row, samplecontexts): + samplecontext = str(getattr(meta_row, 'SampleContext', '')) + if isnull(samplecontext): + return pytest.skip("No SampleContext specified.") + assert (okexcept(meta_row, 'SampleContext') or + samplecontext in samplecontexts) + + +def test_sampletype(meta_row, sampletypes): + sampletype = str(getattr(meta_row, 'SampleType', '')) + if isnull(sampletype): + return pytest.skip("No SampleType specified.") + assert (okexcept(meta_row, 'SampleType') or sampletype in sampletypes) + + +def test_samplemethod(meta_row, samplemethods): + samplemethod = str(getattr(meta_row, 'SampleMethod', '')) + if isnull(samplemethod): + return pytest.skip("No samplemethod specified.") + assert (okexcept(meta_row, 'SampleMethod') or + samplemethod in samplemethods) + + +@pytest.mark.parametrize('worker', ['Worker1', 'Worker2', 'Worker3', + 'Worker4']) +def test_workerrole(meta_row, worker, workerroles): + if (isnull(getattr(meta_row, worker + '_LastName', '')) or + isnull(getattr(meta_row, worker + '_Role', ''))): + return pytest.skip(worker + "_Role not specified.") + assert meta_row[worker + '_Role'] in workerroles + + +def test_locationreliability(meta_row, locationreliabilities): + locationreliability = str(getattr(meta_row, 'LocationReliability', '')) + if isnull(locationreliability): + return pytest.skip("No LocationReliability specified.") + assert (okexcept(meta_row, 'LocationReliability') or + locationreliability in locationreliabilities) + + +def test_ageuncertainty(meta_row, ageuncertainties): + ageuncertainty = str(getattr(meta_row, 'AgeUncertainty', '')) + if isnull(ageuncertainty): + return pytest.skip("No AgeUncertainty specified.") + assert (okexcept(meta_row, 'AgeUncertainty') or + ageuncertainty in ageuncertainties) + + def test_elevation(meta_row): assert okexcept(meta_row, 'Elevation') or ~np.isnan(meta_row.Elevation) def test_temperature(meta_row): + if okexcept(meta_row, 'Temperature'): + return assert meta_row.Temperature temperature = np.array(meta_row.Temperature.split(',')).astype(float) assert len(temperature) == 17 @@ -38,12 +109,17 @@ def test_temperature(meta_row): def test_precip(meta_row): + if okexcept(meta_row, 'Precipitation'): + return assert meta_row.Precipitation precip = np.array(meta_row.Precipitation.split(',')).astype(float) assert len(precip) == 17 assert ((precip > 0) & (precip < 10000)).all() +# --- sample file tests + + def test_orig_varnames(data_frame): assert data_frame.original_varname.all() duplicated = data_frame.duplicated('original_varname') @@ -69,26 +145,17 @@ def test_groupid(data_frame): assert data_frame.groupid.all() -def test_higher_groupid(data_frame): - assert data_frame.groupid.all() - - def test_percentage(data_frame): assert data_frame[data_frame.make_percent].percentage.notnull().all() -def test_samplename(data_frame): - assert data_frame.samplename.all() - assert (data_frame.samplename.unique() == data_frame.samplename[:1]).all() +def test_samplename(data_file): + samplename = osp.splitext(osp.basename(data_file))[0] + df = pd.read_csv(data_file, sep='\t') + samplenames = df.samplename + assert set(samplenames) == {samplename} def test_data_exists(meta_row, samples_dir): fname = osp.join(samples_dir, meta_row.name) + '.tsv' assert osp.exists(fname), f"Missing {fname}" - - -def test_meta_exists(samples_dir, meta, base_meta): - files = glob.glob(osp.join(samples_dir, '*.tsv')) - samples = map(lambda f: osp.splitext(osp.basename(f))[0], files) - missing = set(samples).difference(base_meta.index).difference(meta.index) - assert not missing, f"Missing meta information for samples {missing}" From 6fec1061e9f9113c4177a0aa1bc479fd6e7f903c Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 06:14:02 +0100 Subject: [PATCH 038/119] added new test sample with new SampleContext --- samples/test_a2.tsv | 6 +++--- samples/test_a3.tsv | 4 ++++ test.tsv | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 samples/test_a3.tsv diff --git a/samples/test_a2.tsv b/samples/test_a2.tsv index f4c26bed3..206f8d136 100644 --- a/samples/test_a2.tsv +++ b/samples/test_a2.tsv @@ -1,4 +1,4 @@ var_ samplename count acc_var_ original_varname acc_varname groupid groupname higher_groupid included_in_percent_sum make_percent higher_groupname group_order sums percentage -5 test_a1 45 2 Pinus Pinus TRSH Trees & Shrubs TRSH TRUE TRUE Trees & Shrubs 0 100 45 -6 test_a1 6 3 Exotic Exotics (counted) NOPO Nonpollen NOPO FALSE FALSE Nonpollen 30 100 NaN -10 test_a1 55 10 Umbelliferae Apiaceae HERB Herbs HERB TRUE TRUE Herbs 5 100 55 \ No newline at end of file +5 test_a2 45 2 Pinus Pinus TRSH Trees & Shrubs TRSH TRUE TRUE Trees & Shrubs 0 100 45 +6 test_a2 6 3 Exotic Exotics (counted) NOPO Nonpollen NOPO FALSE FALSE Nonpollen 30 100 NaN +10 test_a2 55 10 Umbelliferae Apiaceae HERB Herbs HERB TRUE TRUE Herbs 5 100 55 diff --git a/samples/test_a3.tsv b/samples/test_a3.tsv new file mode 100644 index 000000000..736c45f9e --- /dev/null +++ b/samples/test_a3.tsv @@ -0,0 +1,4 @@ +var_ samplename count acc_var_ original_varname acc_varname groupid groupname higher_groupid included_in_percent_sum make_percent higher_groupname group_order sums percentage +5 test_a3 45 2 Pinus Pinus TRSH Trees & Shrubs TRSH TRUE TRUE Trees & Shrubs 0 100 45 +6 test_a3 6 3 Exotic Exotics (counted) NOPO Nonpollen NOPO FALSE FALSE Nonpollen 30 100 NaN +10 test_a3 55 10 Umbelliferae Apiaceae HERB Herbs HERB TRUE TRUE Herbs 5 100 55 diff --git a/test.tsv b/test.tsv index fccc6ba78..da027790b 100644 --- a/test.tsv +++ b/test.tsv @@ -1,3 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 I dont know SampleContext From 8a832d7e6a017abbd4a4253e50db1505f6e3cefe Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 06:25:46 +0100 Subject: [PATCH 039/119] Minor --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index da027790b..2650801dc 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 I dont know SampleContext +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 2c269cc16f59088128e84cd0ec451acea6d4311e Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 17:43:25 +0100 Subject: [PATCH 040/119] Improved meta tests --- tests/conftest.py | 15 ++++++++------- tests/test_meta.py | 11 ++++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 74dfb8d8b..75db3f2c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,7 +86,8 @@ def countries(meta_file): import pandas as pd return pd.read_csv( osp.join(osp.dirname(meta_file), 'tab-delimited', 'countries.tsv'), - sep='\t', index_col=1).iloc[:, 0] + sep='\t').dropna(subset=['natural_earth']).set_index( + 'natural_earth').iloc[:, 0] @pytest.fixture(scope='session') @@ -95,7 +96,7 @@ def samplecontexts(meta_file): return pd.read_csv( osp.join(osp.dirname(meta_file), 'tab-delimited', 'samplecontexts.tsv'), - sep='\t').iloc[:, 0] + sep='\t').iloc[:, 0].values @pytest.fixture(scope='session') @@ -103,7 +104,7 @@ def sampletypes(meta_file): import pandas as pd return pd.read_csv( osp.join(osp.dirname(meta_file), 'tab-delimited', 'sampletypes.tsv'), - sep='\t').iloc[:, 0] + sep='\t').iloc[:, 0].values @pytest.fixture(scope='session') @@ -111,7 +112,7 @@ def samplemethods(meta_file): import pandas as pd return pd.read_csv( osp.join(osp.dirname(meta_file), 'tab-delimited', 'samplemethods.tsv'), - sep='\t').iloc[:, 0] + sep='\t').iloc[:, 0].values @pytest.fixture(scope='session') @@ -119,7 +120,7 @@ def workerroles(meta_file): import pandas as pd return pd.read_csv( osp.join(osp.dirname(meta_file), 'tab-delimited', 'workerroles.tsv'), - sep='\t').iloc[:, 0] + sep='\t').iloc[:, 0].values @pytest.fixture(scope='session') @@ -128,7 +129,7 @@ def locationreliabilities(meta_file): return pd.read_csv( osp.join(osp.dirname(meta_file), 'tab-delimited', 'locationreliabilities.tsv'), - sep='\t').iloc[:, 0] + sep='\t').iloc[:, 0].values @pytest.fixture(scope='session') @@ -137,7 +138,7 @@ def ageuncertainties(meta_file): return pd.read_csv( osp.join(osp.dirname(meta_file), 'tab-delimited', 'ageuncertainties.tsv'), - sep='\t').iloc[:, 0] + sep='\t').iloc[:, 0].values @pytest.fixture(scope='session') diff --git a/tests/test_meta.py b/tests/test_meta.py index 3f3742614..75c24aabd 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -42,17 +42,18 @@ def test_lat_lon(meta_row): meta_row.Longitude >= -180 and meta_row.Longitude <= 360) -def test_country(meta_row): - assert okexcept(meta_row, 'Country') or meta_row.Country == get_country( - *meta_row[['Latitude', 'Longitude']]) +def test_country(meta_row, countries): + assert okexcept(meta_row, 'Country') or meta_row.Country in countries.loc[ + [get_country(*meta_row[['Latitude', 'Longitude']])]].values def test_samplecontext(meta_row, samplecontexts): samplecontext = str(getattr(meta_row, 'SampleContext', '')) if isnull(samplecontext): return pytest.skip("No SampleContext specified.") - assert (okexcept(meta_row, 'SampleContext') or - samplecontext in samplecontexts) + assert (samplecontext.lower() == samplecontext and + (okexcept(meta_row, 'SampleContext') or + samplecontext in samplecontexts)) def test_sampletype(meta_row, sampletypes): From 423c69deaa7f734a92d5cb3592f94dbfdf33de15 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 24 Mar 2019 22:31:02 +0100 Subject: [PATCH 041/119] added a script to update fixed tables --- postgres/scripts/updateFixedTables.py | 65 +++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 postgres/scripts/updateFixedTables.py diff --git a/postgres/scripts/updateFixedTables.py b/postgres/scripts/updateFixedTables.py new file mode 100644 index 000000000..fbb32e4f5 --- /dev/null +++ b/postgres/scripts/updateFixedTables.py @@ -0,0 +1,65 @@ +import sys +import os.path as osp +import psycopg2 as psql +import pandas as pd +import numpy as np +from functools import partial + +db_url = sys.argv[1] +to_update = sys.argv[2:] + +conn = psql.connect(db_url) +cursor = conn.cursor() + + +def is_null_str(x): + return 'NULL' if x == "" else "'" + x.strip().replace("'", " ") + "'" + + +tables = osp.join(osp.dirname(__file__), "tables", '') + +read_tsv = partial(pd.read_csv, sep='\t') + +# --- +if 'Country' in to_update or 'all' in to_update: + countries = read_tsv(tables + "Country.tsv") + countries.replace(np.nan, '', inplace=True) + + for x in range(countries.shape[0]): + cursor.execute( + "UPDATE countries SET natural_earth = %s WHERE country = '%s'" % ( + is_null_str(countries.iloc[x][1]), countries.iloc[x][0])) + + conn.commit() + + +# --- +if 'SampleType' in to_update or 'all' in to_update: + sampleType = read_tsv(tables + "SampleType.tsv") + sampleType.replace(np.nan, '', inplace=True) + + for x in range(sampleType.shape[0]): + cursor.execute( + "UPDATE sampleTypes SET notes = %s WHERE sampleType = '%s'" % ( + is_null_str(sampleType.iloc[x][1]), sampleType.iloc[x][0])) + + conn.commit() + + +# --- +if 'GroupID' in to_update or 'all' in to_update: + groupIDs = read_tsv(tables + "GroupID.tsv") + groupIDs.replace(np.nan, '', inplace=True) + + for x in range(groupIDs.shape[0]): + cursor.execute( + "UPDATE groupID SET groupname = %s, higher_groupid = %s " + "WHERE groupID = %s" % ( + is_null_str(groupIDs.iloc[x][1]), + is_null_str(groupIDs.iloc[x][2]), + is_null_str(groupIDs.iloc[x][0]))) + + conn.commit() + + +conn.close() From ff84bf085ac3ac2611db9baba0383fcf0a899289 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 05:42:30 +0000 Subject: [PATCH 042/119] [test_a1]: Changed country from France to Germany --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 2650801dc..dbc448a22 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 919bffe15d5891c3a59f8134aef7f2fa9d3cf4d9 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 07:00:05 +0100 Subject: [PATCH 043/119] Revert "[test_a1]: Changed country from France to Germany" This reverts commit ff84bf085ac3ac2611db9baba0383fcf0a899289. --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index dbc448a22..2650801dc 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 40d0d79195ce9b3183bf63e8b51624f43ddea86c Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 06:07:44 +0000 Subject: [PATCH 044/119] [test_a1]: Changed country from France to Germany --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 2650801dc..dbc448a22 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 0cea95e97e7c9f99bc47c041b5d368b84f9c896f Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 06:08:43 +0000 Subject: [PATCH 045/119] Merged test.tsv into meta.tsv [skip ci] --- meta.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta.tsv b/meta.tsv index 9c9cfeb6f..dbc448a22 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From b698bb3a7b4b785c9994cd5349d8a9d6524c0faf Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 06:08:44 +0000 Subject: [PATCH 046/119] Removed postgres dump of test.tsv --- postgres/test.sql | 949 ---------------------------------------------- 1 file changed, 949 deletions(-) delete mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql deleted file mode 100644 index aa3fb52dd..000000000 --- a/postgres/test.sql +++ /dev/null @@ -1,949 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) --- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET client_min_messages = warning; -SET row_security = off; - -SET default_tablespace = ''; - -SET default_with_oids = false; - --- --- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ageuncertainties ( - ageuncertainty character varying(1) NOT NULL, - description character varying(21) NOT NULL, - age character varying(14) -); - - -ALTER TABLE public.ageuncertainties OWNER TO postgres; - --- --- Name: climate; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.climate ( - samplename character varying(25) NOT NULL, - t_jan double precision, - t_feb double precision, - t_mar double precision, - t_apr double precision, - t_may double precision, - t_jun double precision, - t_jul double precision, - t_aug double precision, - t_sep double precision, - t_oct double precision, - t_nov double precision, - t_dec double precision, - t_djf double precision, - t_mam double precision, - t_jja double precision, - t_son double precision, - t_ann double precision, - p_jan double precision, - p_feb double precision, - p_mar double precision, - p_apr double precision, - p_may double precision, - p_jun double precision, - p_jul double precision, - p_aug double precision, - p_sep double precision, - p_oct double precision, - p_nov double precision, - p_dec double precision, - p_djf double precision, - p_mam double precision, - p_jja double precision, - p_son double precision, - p_ann double precision -); - - -ALTER TABLE public.climate OWNER TO postgres; - --- --- Name: countries; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.countries ( - country character varying(42) NOT NULL, - natural_earth character varying(42) -); - - -ALTER TABLE public.countries OWNER TO postgres; - --- --- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ecosystems ( - samplename character varying(25) NOT NULL, - realm character(16), - biome character(62), - ecoregion character(65) -); - - -ALTER TABLE public.ecosystems OWNER TO postgres; - --- --- Name: groupid; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.groupid ( - groupid character varying(9) NOT NULL, - groupname character varying(60) NOT NULL, - higher_groupid character varying(4) NOT NULL -); - - -ALTER TABLE public.groupid OWNER TO postgres; - --- --- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.locationreliabilities ( - locationreliability character varying(1) NOT NULL, - description character varying(55) NOT NULL, - error character varying(5) -); - - -ALTER TABLE public.locationreliabilities OWNER TO postgres; - --- --- Name: metadata; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metadata ( - samplename character varying(25) NOT NULL, - originalsamplename character varying(70), - sitename character varying(70), - country character varying(42), - longitude double precision, - latitude double precision, - elevation double precision, - locationreliability character varying(1), - locationnotes text, - areaofsite double precision, - samplecontext character varying(40), - sitedescription text, - vegdescription text, - sampletype character varying(14), - samplemethod character varying(31), - agebp double precision, - ageuncertainty character varying(1), - ispercent boolean DEFAULT false, - notes text, - empd_version character varying(5) -); - - -ALTER TABLE public.metadata OWNER TO postgres; - --- --- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metapubli ( - samplename character varying(25) NOT NULL, - publiid integer NOT NULL -); - - -ALTER TABLE public.metapubli OWNER TO postgres; - --- --- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metaworker ( - samplename character varying(25) NOT NULL, - workerid integer NOT NULL, - workerrole character varying(3) -); - - -ALTER TABLE public.metaworker OWNER TO postgres; - --- --- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_counts ( - samplename character varying(25) NOT NULL, - var_ integer NOT NULL, - count double precision NOT NULL -); - - -ALTER TABLE public.p_counts OWNER TO postgres; - --- --- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_vars ( - var_ integer NOT NULL, - acc_var_ integer, - original_varname character varying(100) NOT NULL, - acc_varname character varying(100), - groupid character varying(9), - notes text -); - - -ALTER TABLE public.p_vars OWNER TO postgres; - --- --- Name: publications; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.publications ( - publiid integer NOT NULL, - doi character varying(50), - reference text NOT NULL -); - - -ALTER TABLE public.publications OWNER TO postgres; - --- --- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplecontexts ( - samplecontext character varying(40) NOT NULL -); - - -ALTER TABLE public.samplecontexts OWNER TO postgres; - --- --- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplemethods ( - samplemethod character varying(31) NOT NULL -); - - -ALTER TABLE public.samplemethods OWNER TO postgres; - --- --- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.sampletypes ( - sampletype character varying(14) NOT NULL, - notes character varying(13) -); - - -ALTER TABLE public.sampletypes OWNER TO postgres; - --- --- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workerroles ( - workerrole character varying(3) NOT NULL, - description character varying(35) NOT NULL -); - - -ALTER TABLE public.workerroles OWNER TO postgres; - --- --- Name: workers; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workers ( - workerid integer NOT NULL, - firstname character varying(25), - lastname character varying(25), - initials character varying(10), - address1 text, - email1 character varying(50), - phone1 character varying(25), - address2 text, - email2 character varying(50), - phone2 character varying(25) -); - - -ALTER TABLE public.workers OWNER TO postgres; - --- --- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; -A Modern sample 0BP to Present -B Within last 100 years 0-50BP -C Within last 250 years 50-200BP -\. - - --- --- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -\. - - --- --- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.countries (country, natural_earth) FROM stdin; -Albania Albania -Algeria Algeria -Andorra Andorra -Armenia Armenia -Austria Austria -Azerbaijan Azerbaijan -Bahrain Bahrain -Belarus Belarus -Belgium Belgium -Bosnia And Herzegovina Bosnia and Herzegovina -Bulgaria Bulgaria -Cape Verde Cape Verde -Croatia Croatia -Cyprus Cyprus -Czech Republic Czech Republic -Denmark Denmark -Egypt Egypt -Eritrea Eritrea -Estonia Estonia -Ethiopia Ethiopia -Faroe Islands Faroe Islands -Finland Finland -France France -Georgia Georgia -Germany Germany -Gibraltar Gibraltar -Greece Greece -Greenland Greenland -Guernsey Guernsey -Hungary Hungary -Iceland Iceland -Iran, Islamic Republic Of Iran -Iraq Iraq -Ireland Ireland -Isle Of Man Isle of Man -Israel Israel -Italy Italy -India India -Japan Japan -Jersey Jersey -Kazakhstan Kazakhstan -Kuwait Kuwait -Kyrgyzstan Kyrgyzstan -Latvia Latvia -Lebanon Lebanon -Libyan Arab Jamahiriya Libya -Liechtenstein Liechtenstein -Lithuania Lithuania -Luxembourg Luxembourg -Macedonia, The Former Yugoslav Republic Of Macedonia -Malta Malta -Moldova, Republic Of Moldova -Monaco Monaco -Morocco Morocco -Netherlands Netherlands -Norway Norway -Oman Oman -Palestinian Territory, Occupied Palestine -Poland Poland -Portugal Portugal -Qatar Qatar -Romania Romania -Russian Federation Russia -San Marino San Marino -Saudi Arabia Saudi Arabia -Serbia And Montenegro Montenegro -Slovakia Slovakia -Slovenia Slovenia -Spain Spain -Sweden Sweden -Switzerland Switzerland -Syrian Arab Republic Syria -Tunisia Tunisia -Turkey Turkey -Ukraine Ukraine -United Arab Emirates United Arab Emirates -United Kingdom United Kingdom -Yemen Yemen -Jordan Jordan -Black Sea \N -Dead Sea \N -Turkmenistan, Republic Of Turkmenistan -Adriatic Sea \N -Svalbard and Jan Mayen Norway -China, People s Republic Of China -\. - - --- --- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; -\. - - --- --- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; -ALGA Algae AQUA -AQBR Aquatic Bryophytes NOPO -AQPT Aquatic Pteridophyta AQUA -AQUA Aquatics AQUA -AQVP Aquatic Vascular Plants AQUA -BRYO Bryophytes NOPO -BRYO/VACR TO BE REMOVED TBR -CHAR Charcoal CHAR -CHLO Chlorophytes NOPO -CYAN Cyanobacteria NOPO -DINO Dinoflagellates NOPO -DWAR Dwarf shrubs TRSH -FUNG Fungi NOPO -HELO Helophytes AQUA -HERB Herbs HERB -INUN Indeterminables and unknowns INUN -INVE Invertebrates NOPO -LIAN Liana TRSH -MACR Macrofossils NOPO -NOPO Nonpollen NOPO -PLAT Platyhelminthes (flatworms) NOPO -PREQ Pre-Quaternary type NOPO -RHIZ Rhizopods NOPO -ROTI Rotifera (Rotifers) NOPO -SPOR Taxonomically undifferentiated spores INUN -TEAM Testate amoebae NOPO -TRSH Trees and shrubs TRSH -UNPP Undifferentiated NPP INUN -UPBR Upland Bryophytes NOPO -UPHE Upland herbs HERB -VACR Vascular cryptogams (Pteridophytes) NOPO -VEMI Vegetative microfossils NOPO -NEMA Nematoda (roundworms) NOPO -ACRI Acritarchs NOPO -EUMY Eumycete NOPO -HEMI Hemi-parasitic NOPO -MOSS Moss NOPO -MICR Microcrustaceans NOPO -\. - - --- --- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; -A Good for high resolution remote sensing (<100m) <100m -B Good for lower resolution remote sensing (<1km) <1km -C Ok for climate reconstruction or regional scale mapping <5km -D Ok for mapping at continental scale <20km -X Do not use! \N -\. - - --- --- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -\. - - --- --- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metapubli (samplename, publiid) FROM stdin; -\. - - --- --- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -\. - - --- --- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_counts (samplename, var_, count) FROM stdin; -test_a1 1 45 -test_a1 2 6 -test_a1 3 55 -\. - - --- --- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; -1 \N Pinus Pinus TRSH NULL -2 \N Exotic Exotics (counted) NOPO NULL -3 \N Umbelliferae Apiaceae HERB NULL -\. - - --- --- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.publications (publiid, doi, reference) FROM stdin; -\. - - --- --- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplecontexts (samplecontext) FROM stdin; -arable -blanket bog -bog -cave -cirque lake -closed forest -coastal -coastal lake -coastal wetland -drained_lake -ephemeral lake -ephemeral lake/pond -estuarine -fallow -fallow - scattered trees/shrubs -fen -floodplain mire -fluvial -forest -forest undefined -heath -ice from glacier -kettle lake -lagoon -lake -maquis -marine -marsh -marshland -mire -mor humus -moss-shrubby tundra -natural grassland -natural grasslands -natural open water -open forest -open forest/orchard -open forest/scattered trees/shrubs -orchard -orchard/scattered trees/shrubs -pasture -peatbog -pond -riverine -salt lake -salt marsh -scattered trees -scattered trees/shrubs -shrubs -shrubs/scrubland -soil -soligenous (mineraltrophic) Mire -sparse/no vegetation -swamp -tectonic lake -terrestrial -treeless vegetation -treeless vegetation/natural grassland -treeless vegetation/sparse vegetation -urban -valley mire -wetland -wetland bog -wetland bog/scattered trees/shrubs -wetland bog/treeless vegetation -archaeological -\. - - --- --- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplemethods (samplemethod) FROM stdin; -hand picking -hand picking - multiple samples -unspecified -auger corer -freeze corer -gravity corer -piston corer -russian corer -box -spade -hiller corer -gouge auger -core -corer unspecified -eckman-grab -box corer -\. - - --- --- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.sampletypes (sampletype, notes) FROM stdin; -dung \N -litter \N -moss \N -pollen trap To be removed -sediment submerged -spider web \N -soil terrestrial -sphagnum peat \N -peat \N -lichen \N -submerged \N -core_top \N -tussock \N -epiphytic moss \N -\. - - --- --- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workerroles (workerrole, description) FROM stdin; -R Responsible Person -R/A Both Responsible Person and Analyst -A Analyst -A/D Analyst (Deceased) -A/I Analysis (Inactive) -A/U Analyst (Unknown) -\. - - --- --- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; -\. - - --- --- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ageuncertainties - ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); - - --- --- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.countries - ADD CONSTRAINT countries_pkey PRIMARY KEY (country); - - --- --- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); - - --- --- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.groupid - ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); - - --- --- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.locationreliabilities - ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); - - --- --- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); - - --- --- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); - - --- --- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); - - --- --- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); - - --- --- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); - - --- --- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); - - --- --- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_doi_key UNIQUE (doi); - - --- --- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); - - --- --- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplecontexts - ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); - - --- --- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplemethods - ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); - - --- --- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.sampletypes - ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); - - --- --- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workerroles - ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); - - --- --- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workers - ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); - - --- --- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); - - --- --- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); - - --- --- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); - - --- --- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); - - --- --- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); - - --- --- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); - - --- --- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); - - --- --- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); - - --- --- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); - - --- --- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); - - --- --- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); - - --- --- PostgreSQL database dump complete --- - From 7337d04e7d8396b01f3eedde5380e3020898bd86 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 06:08:45 +0000 Subject: [PATCH 047/119] Removed test.tsv to finish the PR --- test.tsv | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 test.tsv diff --git a/test.tsv b/test.tsv deleted file mode 100644 index dbc448a22..000000000 --- a/test.tsv +++ /dev/null @@ -1,4 +0,0 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From d9a8162ed7dd6b42bdd700880dbebb152b7c35af Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 07:18:42 +0100 Subject: [PATCH 048/119] Revert "Removed test.tsv to finish the PR" This reverts commit 7337d04e7d8396b01f3eedde5380e3020898bd86. --- test.tsv | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test.tsv diff --git a/test.tsv b/test.tsv new file mode 100644 index 000000000..dbc448a22 --- /dev/null +++ b/test.tsv @@ -0,0 +1,4 @@ +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 6da4e49832e4494cb5617663a932571ca306b861 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 07:18:46 +0100 Subject: [PATCH 049/119] Revert "Removed postgres dump of test.tsv" This reverts commit b698bb3a7b4b785c9994cd5349d8a9d6524c0faf. --- postgres/test.sql | 949 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 949 insertions(+) create mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql new file mode 100644 index 000000000..aa3fb52dd --- /dev/null +++ b/postgres/test.sql @@ -0,0 +1,949 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ageuncertainties ( + ageuncertainty character varying(1) NOT NULL, + description character varying(21) NOT NULL, + age character varying(14) +); + + +ALTER TABLE public.ageuncertainties OWNER TO postgres; + +-- +-- Name: climate; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.climate ( + samplename character varying(25) NOT NULL, + t_jan double precision, + t_feb double precision, + t_mar double precision, + t_apr double precision, + t_may double precision, + t_jun double precision, + t_jul double precision, + t_aug double precision, + t_sep double precision, + t_oct double precision, + t_nov double precision, + t_dec double precision, + t_djf double precision, + t_mam double precision, + t_jja double precision, + t_son double precision, + t_ann double precision, + p_jan double precision, + p_feb double precision, + p_mar double precision, + p_apr double precision, + p_may double precision, + p_jun double precision, + p_jul double precision, + p_aug double precision, + p_sep double precision, + p_oct double precision, + p_nov double precision, + p_dec double precision, + p_djf double precision, + p_mam double precision, + p_jja double precision, + p_son double precision, + p_ann double precision +); + + +ALTER TABLE public.climate OWNER TO postgres; + +-- +-- Name: countries; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.countries ( + country character varying(42) NOT NULL, + natural_earth character varying(42) +); + + +ALTER TABLE public.countries OWNER TO postgres; + +-- +-- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ecosystems ( + samplename character varying(25) NOT NULL, + realm character(16), + biome character(62), + ecoregion character(65) +); + + +ALTER TABLE public.ecosystems OWNER TO postgres; + +-- +-- Name: groupid; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.groupid ( + groupid character varying(9) NOT NULL, + groupname character varying(60) NOT NULL, + higher_groupid character varying(4) NOT NULL +); + + +ALTER TABLE public.groupid OWNER TO postgres; + +-- +-- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.locationreliabilities ( + locationreliability character varying(1) NOT NULL, + description character varying(55) NOT NULL, + error character varying(5) +); + + +ALTER TABLE public.locationreliabilities OWNER TO postgres; + +-- +-- Name: metadata; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metadata ( + samplename character varying(25) NOT NULL, + originalsamplename character varying(70), + sitename character varying(70), + country character varying(42), + longitude double precision, + latitude double precision, + elevation double precision, + locationreliability character varying(1), + locationnotes text, + areaofsite double precision, + samplecontext character varying(40), + sitedescription text, + vegdescription text, + sampletype character varying(14), + samplemethod character varying(31), + agebp double precision, + ageuncertainty character varying(1), + ispercent boolean DEFAULT false, + notes text, + empd_version character varying(5) +); + + +ALTER TABLE public.metadata OWNER TO postgres; + +-- +-- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metapubli ( + samplename character varying(25) NOT NULL, + publiid integer NOT NULL +); + + +ALTER TABLE public.metapubli OWNER TO postgres; + +-- +-- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metaworker ( + samplename character varying(25) NOT NULL, + workerid integer NOT NULL, + workerrole character varying(3) +); + + +ALTER TABLE public.metaworker OWNER TO postgres; + +-- +-- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_counts ( + samplename character varying(25) NOT NULL, + var_ integer NOT NULL, + count double precision NOT NULL +); + + +ALTER TABLE public.p_counts OWNER TO postgres; + +-- +-- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_vars ( + var_ integer NOT NULL, + acc_var_ integer, + original_varname character varying(100) NOT NULL, + acc_varname character varying(100), + groupid character varying(9), + notes text +); + + +ALTER TABLE public.p_vars OWNER TO postgres; + +-- +-- Name: publications; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.publications ( + publiid integer NOT NULL, + doi character varying(50), + reference text NOT NULL +); + + +ALTER TABLE public.publications OWNER TO postgres; + +-- +-- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplecontexts ( + samplecontext character varying(40) NOT NULL +); + + +ALTER TABLE public.samplecontexts OWNER TO postgres; + +-- +-- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplemethods ( + samplemethod character varying(31) NOT NULL +); + + +ALTER TABLE public.samplemethods OWNER TO postgres; + +-- +-- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.sampletypes ( + sampletype character varying(14) NOT NULL, + notes character varying(13) +); + + +ALTER TABLE public.sampletypes OWNER TO postgres; + +-- +-- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workerroles ( + workerrole character varying(3) NOT NULL, + description character varying(35) NOT NULL +); + + +ALTER TABLE public.workerroles OWNER TO postgres; + +-- +-- Name: workers; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workers ( + workerid integer NOT NULL, + firstname character varying(25), + lastname character varying(25), + initials character varying(10), + address1 text, + email1 character varying(50), + phone1 character varying(25), + address2 text, + email2 character varying(50), + phone2 character varying(25) +); + + +ALTER TABLE public.workers OWNER TO postgres; + +-- +-- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP +\. + + +-- +-- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +\. + + +-- +-- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.countries (country, natural_earth) FROM stdin; +Albania Albania +Algeria Algeria +Andorra Andorra +Armenia Armenia +Austria Austria +Azerbaijan Azerbaijan +Bahrain Bahrain +Belarus Belarus +Belgium Belgium +Bosnia And Herzegovina Bosnia and Herzegovina +Bulgaria Bulgaria +Cape Verde Cape Verde +Croatia Croatia +Cyprus Cyprus +Czech Republic Czech Republic +Denmark Denmark +Egypt Egypt +Eritrea Eritrea +Estonia Estonia +Ethiopia Ethiopia +Faroe Islands Faroe Islands +Finland Finland +France France +Georgia Georgia +Germany Germany +Gibraltar Gibraltar +Greece Greece +Greenland Greenland +Guernsey Guernsey +Hungary Hungary +Iceland Iceland +Iran, Islamic Republic Of Iran +Iraq Iraq +Ireland Ireland +Isle Of Man Isle of Man +Israel Israel +Italy Italy +India India +Japan Japan +Jersey Jersey +Kazakhstan Kazakhstan +Kuwait Kuwait +Kyrgyzstan Kyrgyzstan +Latvia Latvia +Lebanon Lebanon +Libyan Arab Jamahiriya Libya +Liechtenstein Liechtenstein +Lithuania Lithuania +Luxembourg Luxembourg +Macedonia, The Former Yugoslav Republic Of Macedonia +Malta Malta +Moldova, Republic Of Moldova +Monaco Monaco +Morocco Morocco +Netherlands Netherlands +Norway Norway +Oman Oman +Palestinian Territory, Occupied Palestine +Poland Poland +Portugal Portugal +Qatar Qatar +Romania Romania +Russian Federation Russia +San Marino San Marino +Saudi Arabia Saudi Arabia +Serbia And Montenegro Montenegro +Slovakia Slovakia +Slovenia Slovenia +Spain Spain +Sweden Sweden +Switzerland Switzerland +Syrian Arab Republic Syria +Tunisia Tunisia +Turkey Turkey +Ukraine Ukraine +United Arab Emirates United Arab Emirates +United Kingdom United Kingdom +Yemen Yemen +Jordan Jordan +Black Sea \N +Dead Sea \N +Turkmenistan, Republic Of Turkmenistan +Adriatic Sea \N +Svalbard and Jan Mayen Norway +China, People s Republic Of China +\. + + +-- +-- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; +\. + + +-- +-- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; +ALGA Algae AQUA +AQBR Aquatic Bryophytes NOPO +AQPT Aquatic Pteridophyta AQUA +AQUA Aquatics AQUA +AQVP Aquatic Vascular Plants AQUA +BRYO Bryophytes NOPO +BRYO/VACR TO BE REMOVED TBR +CHAR Charcoal CHAR +CHLO Chlorophytes NOPO +CYAN Cyanobacteria NOPO +DINO Dinoflagellates NOPO +DWAR Dwarf shrubs TRSH +FUNG Fungi NOPO +HELO Helophytes AQUA +HERB Herbs HERB +INUN Indeterminables and unknowns INUN +INVE Invertebrates NOPO +LIAN Liana TRSH +MACR Macrofossils NOPO +NOPO Nonpollen NOPO +PLAT Platyhelminthes (flatworms) NOPO +PREQ Pre-Quaternary type NOPO +RHIZ Rhizopods NOPO +ROTI Rotifera (Rotifers) NOPO +SPOR Taxonomically undifferentiated spores INUN +TEAM Testate amoebae NOPO +TRSH Trees and shrubs TRSH +UNPP Undifferentiated NPP INUN +UPBR Upland Bryophytes NOPO +UPHE Upland herbs HERB +VACR Vascular cryptogams (Pteridophytes) NOPO +VEMI Vegetative microfossils NOPO +NEMA Nematoda (roundworms) NOPO +ACRI Acritarchs NOPO +EUMY Eumycete NOPO +HEMI Hemi-parasitic NOPO +MOSS Moss NOPO +MICR Microcrustaceans NOPO +\. + + +-- +-- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! \N +\. + + +-- +-- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +\. + + +-- +-- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metapubli (samplename, publiid) FROM stdin; +\. + + +-- +-- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; +\. + + +-- +-- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_counts (samplename, var_, count) FROM stdin; +test_a1 1 45 +test_a1 2 6 +test_a1 3 55 +\. + + +-- +-- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL +\. + + +-- +-- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.publications (publiid, doi, reference) FROM stdin; +\. + + +-- +-- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplecontexts (samplecontext) FROM stdin; +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological +\. + + +-- +-- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplemethods (samplemethod) FROM stdin; +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer +\. + + +-- +-- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.sampletypes (sampletype, notes) FROM stdin; +dung \N +litter \N +moss \N +pollen trap To be removed +sediment submerged +spider web \N +soil terrestrial +sphagnum peat \N +peat \N +lichen \N +submerged \N +core_top \N +tussock \N +epiphytic moss \N +\. + + +-- +-- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workerroles (workerrole, description) FROM stdin; +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) +\. + + +-- +-- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; +\. + + +-- +-- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ageuncertainties + ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); + + +-- +-- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.countries + ADD CONSTRAINT countries_pkey PRIMARY KEY (country); + + +-- +-- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); + + +-- +-- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.groupid + ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); + + +-- +-- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.locationreliabilities + ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); + + +-- +-- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); + + +-- +-- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); + + +-- +-- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); + + +-- +-- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); + + +-- +-- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); + + +-- +-- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); + + +-- +-- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_doi_key UNIQUE (doi); + + +-- +-- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); + + +-- +-- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplecontexts + ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); + + +-- +-- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplemethods + ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); + + +-- +-- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.sampletypes + ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); + + +-- +-- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workerroles + ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); + + +-- +-- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workers + ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); + + +-- +-- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); + + +-- +-- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); + + +-- +-- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); + + +-- +-- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); + + +-- +-- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); + + +-- +-- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); + + +-- +-- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); + + +-- +-- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); + + +-- +-- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); + + +-- +-- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); + + +-- +-- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); + + +-- +-- PostgreSQL database dump complete +-- + From 795280ce1cc083997205e045f79a76a722e5c8ba Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 07:25:43 +0100 Subject: [PATCH 050/119] Revert "Merged test.tsv into meta.tsv [skip ci]" This reverts commit 0cea95e97e7c9f99bc47c041b5d368b84f9c896f. --- meta.tsv | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta.tsv b/meta.tsv index dbc448a22..9c9cfeb6f 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1,4 +1 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From bf6081136b1de069d1042a6026d97cc289e65f83 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 07:35:18 +0100 Subject: [PATCH 051/119] Revert "[test_a1]: Changed country from France to Germany" This reverts commit 40d0d79195ce9b3183bf63e8b51624f43ddea86c. --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index dbc448a22..2650801dc 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 0e230bd20b060f945b2072c973978b4aa2ec0662 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 09:37:02 +0100 Subject: [PATCH 052/119] Merge branch 'upstream/master' into test-data From 340139917f3e36a6f7f6bdd51c76af5183b9939f Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 09:07:28 +0000 Subject: [PATCH 053/119] [test_a1]: Changed country from France to Germany --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 2650801dc..dbc448a22 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From e373a2ba741d5838fed5c16f3ef8abefe8c46a2e Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 09:08:00 +0000 Subject: [PATCH 054/119] Merge branch 'upstream/master' into test-data From e289b1824e7ef492e6db84828704780cb893babb Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 09:08:01 +0000 Subject: [PATCH 055/119] Merged test.tsv into meta.tsv [skip ci] --- meta.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta.tsv b/meta.tsv index 9c9cfeb6f..dbc448a22 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 9f3286355d2145966c57025f08e05f4214cac229 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 09:08:02 +0000 Subject: [PATCH 056/119] Added postgres dump for test.tsv --- postgres/EMPD2.sql | 114 +++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 60 deletions(-) diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql index 359334330..8c6961531 100644 --- a/postgres/EMPD2.sql +++ b/postgres/EMPD2.sql @@ -2,11 +2,12 @@ -- PostgreSQL database dump -- --- Dumped from database version 9.5.15 --- Dumped by pg_dump version 9.5.15 +-- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) SET statement_timeout = 0; SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); @@ -14,20 +15,6 @@ SET check_function_bodies = false; SET client_min_messages = warning; SET row_security = off; --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: --- - -COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - SET default_tablespace = ''; SET default_with_oids = false; @@ -322,6 +309,9 @@ C Within last 250 years 50-200BP -- COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 \. @@ -490,6 +480,9 @@ X Do not use! \N -- COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 +test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know \N \N \N \N \N \N f \N \N EMPD2 \. @@ -514,6 +507,12 @@ COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -- COPY public.p_counts (samplename, var_, count) FROM stdin; +test_a2 1 45 +test_a2 2 6 +test_a2 3 55 +test_a3 1 45 +test_a3 2 6 +test_a3 3 55 \. @@ -522,6 +521,9 @@ COPY public.p_counts (samplename, var_, count) FROM stdin; -- COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL \. @@ -604,6 +606,7 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological +i dont know \. @@ -676,7 +679,7 @@ COPY public.workers (workerid, firstname, lastname, initials, address1, email1, -- --- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ageuncertainties @@ -684,7 +687,7 @@ ALTER TABLE ONLY public.ageuncertainties -- --- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.countries @@ -692,7 +695,7 @@ ALTER TABLE ONLY public.countries -- --- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -700,7 +703,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.groupid @@ -708,7 +711,7 @@ ALTER TABLE ONLY public.groupid -- --- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.locationreliabilities @@ -716,7 +719,7 @@ ALTER TABLE ONLY public.locationreliabilities -- --- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -724,7 +727,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -732,7 +735,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -740,7 +743,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -748,7 +751,7 @@ ALTER TABLE ONLY public.climate -- --- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -756,7 +759,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -764,7 +767,7 @@ ALTER TABLE ONLY public.p_vars -- --- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -772,7 +775,7 @@ ALTER TABLE ONLY public.publications -- --- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -780,7 +783,7 @@ ALTER TABLE ONLY public.publications -- --- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplecontexts @@ -788,7 +791,7 @@ ALTER TABLE ONLY public.samplecontexts -- --- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplemethods @@ -796,7 +799,7 @@ ALTER TABLE ONLY public.samplemethods -- --- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.sampletypes @@ -804,7 +807,7 @@ ALTER TABLE ONLY public.sampletypes -- --- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workerroles @@ -812,7 +815,7 @@ ALTER TABLE ONLY public.workerroles -- --- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workers @@ -820,7 +823,7 @@ ALTER TABLE ONLY public.workers -- --- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -828,7 +831,7 @@ ALTER TABLE ONLY public.climate -- --- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -836,7 +839,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -844,7 +847,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -852,7 +855,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -860,7 +863,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -868,7 +871,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -876,7 +879,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -884,7 +887,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -892,7 +895,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -900,7 +903,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -908,7 +911,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -916,7 +919,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -924,7 +927,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -932,7 +935,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -940,23 +943,14 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); --- --- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres --- - -REVOKE ALL ON SCHEMA public FROM PUBLIC; -REVOKE ALL ON SCHEMA public FROM postgres; -GRANT ALL ON SCHEMA public TO postgres; -GRANT ALL ON SCHEMA public TO PUBLIC; - - -- -- PostgreSQL database dump complete -- + From ebfcb8d8c421bd755f6daf68a759f94732acf123 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 09:08:02 +0000 Subject: [PATCH 057/119] Removed postgres dump of test.tsv --- postgres/test.sql | 949 ---------------------------------------------- 1 file changed, 949 deletions(-) delete mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql deleted file mode 100644 index aa3fb52dd..000000000 --- a/postgres/test.sql +++ /dev/null @@ -1,949 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) --- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET client_min_messages = warning; -SET row_security = off; - -SET default_tablespace = ''; - -SET default_with_oids = false; - --- --- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ageuncertainties ( - ageuncertainty character varying(1) NOT NULL, - description character varying(21) NOT NULL, - age character varying(14) -); - - -ALTER TABLE public.ageuncertainties OWNER TO postgres; - --- --- Name: climate; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.climate ( - samplename character varying(25) NOT NULL, - t_jan double precision, - t_feb double precision, - t_mar double precision, - t_apr double precision, - t_may double precision, - t_jun double precision, - t_jul double precision, - t_aug double precision, - t_sep double precision, - t_oct double precision, - t_nov double precision, - t_dec double precision, - t_djf double precision, - t_mam double precision, - t_jja double precision, - t_son double precision, - t_ann double precision, - p_jan double precision, - p_feb double precision, - p_mar double precision, - p_apr double precision, - p_may double precision, - p_jun double precision, - p_jul double precision, - p_aug double precision, - p_sep double precision, - p_oct double precision, - p_nov double precision, - p_dec double precision, - p_djf double precision, - p_mam double precision, - p_jja double precision, - p_son double precision, - p_ann double precision -); - - -ALTER TABLE public.climate OWNER TO postgres; - --- --- Name: countries; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.countries ( - country character varying(42) NOT NULL, - natural_earth character varying(42) -); - - -ALTER TABLE public.countries OWNER TO postgres; - --- --- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ecosystems ( - samplename character varying(25) NOT NULL, - realm character(16), - biome character(62), - ecoregion character(65) -); - - -ALTER TABLE public.ecosystems OWNER TO postgres; - --- --- Name: groupid; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.groupid ( - groupid character varying(9) NOT NULL, - groupname character varying(60) NOT NULL, - higher_groupid character varying(4) NOT NULL -); - - -ALTER TABLE public.groupid OWNER TO postgres; - --- --- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.locationreliabilities ( - locationreliability character varying(1) NOT NULL, - description character varying(55) NOT NULL, - error character varying(5) -); - - -ALTER TABLE public.locationreliabilities OWNER TO postgres; - --- --- Name: metadata; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metadata ( - samplename character varying(25) NOT NULL, - originalsamplename character varying(70), - sitename character varying(70), - country character varying(42), - longitude double precision, - latitude double precision, - elevation double precision, - locationreliability character varying(1), - locationnotes text, - areaofsite double precision, - samplecontext character varying(40), - sitedescription text, - vegdescription text, - sampletype character varying(14), - samplemethod character varying(31), - agebp double precision, - ageuncertainty character varying(1), - ispercent boolean DEFAULT false, - notes text, - empd_version character varying(5) -); - - -ALTER TABLE public.metadata OWNER TO postgres; - --- --- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metapubli ( - samplename character varying(25) NOT NULL, - publiid integer NOT NULL -); - - -ALTER TABLE public.metapubli OWNER TO postgres; - --- --- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metaworker ( - samplename character varying(25) NOT NULL, - workerid integer NOT NULL, - workerrole character varying(3) -); - - -ALTER TABLE public.metaworker OWNER TO postgres; - --- --- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_counts ( - samplename character varying(25) NOT NULL, - var_ integer NOT NULL, - count double precision NOT NULL -); - - -ALTER TABLE public.p_counts OWNER TO postgres; - --- --- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_vars ( - var_ integer NOT NULL, - acc_var_ integer, - original_varname character varying(100) NOT NULL, - acc_varname character varying(100), - groupid character varying(9), - notes text -); - - -ALTER TABLE public.p_vars OWNER TO postgres; - --- --- Name: publications; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.publications ( - publiid integer NOT NULL, - doi character varying(50), - reference text NOT NULL -); - - -ALTER TABLE public.publications OWNER TO postgres; - --- --- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplecontexts ( - samplecontext character varying(40) NOT NULL -); - - -ALTER TABLE public.samplecontexts OWNER TO postgres; - --- --- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplemethods ( - samplemethod character varying(31) NOT NULL -); - - -ALTER TABLE public.samplemethods OWNER TO postgres; - --- --- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.sampletypes ( - sampletype character varying(14) NOT NULL, - notes character varying(13) -); - - -ALTER TABLE public.sampletypes OWNER TO postgres; - --- --- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workerroles ( - workerrole character varying(3) NOT NULL, - description character varying(35) NOT NULL -); - - -ALTER TABLE public.workerroles OWNER TO postgres; - --- --- Name: workers; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workers ( - workerid integer NOT NULL, - firstname character varying(25), - lastname character varying(25), - initials character varying(10), - address1 text, - email1 character varying(50), - phone1 character varying(25), - address2 text, - email2 character varying(50), - phone2 character varying(25) -); - - -ALTER TABLE public.workers OWNER TO postgres; - --- --- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; -A Modern sample 0BP to Present -B Within last 100 years 0-50BP -C Within last 250 years 50-200BP -\. - - --- --- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -\. - - --- --- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.countries (country, natural_earth) FROM stdin; -Albania Albania -Algeria Algeria -Andorra Andorra -Armenia Armenia -Austria Austria -Azerbaijan Azerbaijan -Bahrain Bahrain -Belarus Belarus -Belgium Belgium -Bosnia And Herzegovina Bosnia and Herzegovina -Bulgaria Bulgaria -Cape Verde Cape Verde -Croatia Croatia -Cyprus Cyprus -Czech Republic Czech Republic -Denmark Denmark -Egypt Egypt -Eritrea Eritrea -Estonia Estonia -Ethiopia Ethiopia -Faroe Islands Faroe Islands -Finland Finland -France France -Georgia Georgia -Germany Germany -Gibraltar Gibraltar -Greece Greece -Greenland Greenland -Guernsey Guernsey -Hungary Hungary -Iceland Iceland -Iran, Islamic Republic Of Iran -Iraq Iraq -Ireland Ireland -Isle Of Man Isle of Man -Israel Israel -Italy Italy -India India -Japan Japan -Jersey Jersey -Kazakhstan Kazakhstan -Kuwait Kuwait -Kyrgyzstan Kyrgyzstan -Latvia Latvia -Lebanon Lebanon -Libyan Arab Jamahiriya Libya -Liechtenstein Liechtenstein -Lithuania Lithuania -Luxembourg Luxembourg -Macedonia, The Former Yugoslav Republic Of Macedonia -Malta Malta -Moldova, Republic Of Moldova -Monaco Monaco -Morocco Morocco -Netherlands Netherlands -Norway Norway -Oman Oman -Palestinian Territory, Occupied Palestine -Poland Poland -Portugal Portugal -Qatar Qatar -Romania Romania -Russian Federation Russia -San Marino San Marino -Saudi Arabia Saudi Arabia -Serbia And Montenegro Montenegro -Slovakia Slovakia -Slovenia Slovenia -Spain Spain -Sweden Sweden -Switzerland Switzerland -Syrian Arab Republic Syria -Tunisia Tunisia -Turkey Turkey -Ukraine Ukraine -United Arab Emirates United Arab Emirates -United Kingdom United Kingdom -Yemen Yemen -Jordan Jordan -Black Sea \N -Dead Sea \N -Turkmenistan, Republic Of Turkmenistan -Adriatic Sea \N -Svalbard and Jan Mayen Norway -China, People s Republic Of China -\. - - --- --- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; -\. - - --- --- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; -ALGA Algae AQUA -AQBR Aquatic Bryophytes NOPO -AQPT Aquatic Pteridophyta AQUA -AQUA Aquatics AQUA -AQVP Aquatic Vascular Plants AQUA -BRYO Bryophytes NOPO -BRYO/VACR TO BE REMOVED TBR -CHAR Charcoal CHAR -CHLO Chlorophytes NOPO -CYAN Cyanobacteria NOPO -DINO Dinoflagellates NOPO -DWAR Dwarf shrubs TRSH -FUNG Fungi NOPO -HELO Helophytes AQUA -HERB Herbs HERB -INUN Indeterminables and unknowns INUN -INVE Invertebrates NOPO -LIAN Liana TRSH -MACR Macrofossils NOPO -NOPO Nonpollen NOPO -PLAT Platyhelminthes (flatworms) NOPO -PREQ Pre-Quaternary type NOPO -RHIZ Rhizopods NOPO -ROTI Rotifera (Rotifers) NOPO -SPOR Taxonomically undifferentiated spores INUN -TEAM Testate amoebae NOPO -TRSH Trees and shrubs TRSH -UNPP Undifferentiated NPP INUN -UPBR Upland Bryophytes NOPO -UPHE Upland herbs HERB -VACR Vascular cryptogams (Pteridophytes) NOPO -VEMI Vegetative microfossils NOPO -NEMA Nematoda (roundworms) NOPO -ACRI Acritarchs NOPO -EUMY Eumycete NOPO -HEMI Hemi-parasitic NOPO -MOSS Moss NOPO -MICR Microcrustaceans NOPO -\. - - --- --- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; -A Good for high resolution remote sensing (<100m) <100m -B Good for lower resolution remote sensing (<1km) <1km -C Ok for climate reconstruction or regional scale mapping <5km -D Ok for mapping at continental scale <20km -X Do not use! \N -\. - - --- --- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -\. - - --- --- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metapubli (samplename, publiid) FROM stdin; -\. - - --- --- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -\. - - --- --- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_counts (samplename, var_, count) FROM stdin; -test_a1 1 45 -test_a1 2 6 -test_a1 3 55 -\. - - --- --- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; -1 \N Pinus Pinus TRSH NULL -2 \N Exotic Exotics (counted) NOPO NULL -3 \N Umbelliferae Apiaceae HERB NULL -\. - - --- --- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.publications (publiid, doi, reference) FROM stdin; -\. - - --- --- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplecontexts (samplecontext) FROM stdin; -arable -blanket bog -bog -cave -cirque lake -closed forest -coastal -coastal lake -coastal wetland -drained_lake -ephemeral lake -ephemeral lake/pond -estuarine -fallow -fallow - scattered trees/shrubs -fen -floodplain mire -fluvial -forest -forest undefined -heath -ice from glacier -kettle lake -lagoon -lake -maquis -marine -marsh -marshland -mire -mor humus -moss-shrubby tundra -natural grassland -natural grasslands -natural open water -open forest -open forest/orchard -open forest/scattered trees/shrubs -orchard -orchard/scattered trees/shrubs -pasture -peatbog -pond -riverine -salt lake -salt marsh -scattered trees -scattered trees/shrubs -shrubs -shrubs/scrubland -soil -soligenous (mineraltrophic) Mire -sparse/no vegetation -swamp -tectonic lake -terrestrial -treeless vegetation -treeless vegetation/natural grassland -treeless vegetation/sparse vegetation -urban -valley mire -wetland -wetland bog -wetland bog/scattered trees/shrubs -wetland bog/treeless vegetation -archaeological -\. - - --- --- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplemethods (samplemethod) FROM stdin; -hand picking -hand picking - multiple samples -unspecified -auger corer -freeze corer -gravity corer -piston corer -russian corer -box -spade -hiller corer -gouge auger -core -corer unspecified -eckman-grab -box corer -\. - - --- --- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.sampletypes (sampletype, notes) FROM stdin; -dung \N -litter \N -moss \N -pollen trap To be removed -sediment submerged -spider web \N -soil terrestrial -sphagnum peat \N -peat \N -lichen \N -submerged \N -core_top \N -tussock \N -epiphytic moss \N -\. - - --- --- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workerroles (workerrole, description) FROM stdin; -R Responsible Person -R/A Both Responsible Person and Analyst -A Analyst -A/D Analyst (Deceased) -A/I Analysis (Inactive) -A/U Analyst (Unknown) -\. - - --- --- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; -\. - - --- --- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ageuncertainties - ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); - - --- --- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.countries - ADD CONSTRAINT countries_pkey PRIMARY KEY (country); - - --- --- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); - - --- --- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.groupid - ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); - - --- --- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.locationreliabilities - ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); - - --- --- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); - - --- --- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); - - --- --- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); - - --- --- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); - - --- --- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); - - --- --- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); - - --- --- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_doi_key UNIQUE (doi); - - --- --- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); - - --- --- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplecontexts - ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); - - --- --- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplemethods - ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); - - --- --- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.sampletypes - ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); - - --- --- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workerroles - ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); - - --- --- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workers - ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); - - --- --- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); - - --- --- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); - - --- --- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); - - --- --- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); - - --- --- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); - - --- --- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); - - --- --- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); - - --- --- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); - - --- --- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); - - --- --- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); - - --- --- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); - - --- --- PostgreSQL database dump complete --- - From a20dfc1f53f760c20883172b72fef34a8900d894 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 09:08:03 +0000 Subject: [PATCH 058/119] Updated tab-delimited files from EMPD2 postgres database --- tab-delimited/climate.tsv | 3 +++ tab-delimited/metadata.tsv | 3 +++ tab-delimited/p_counts.tsv | 6 ++++++ tab-delimited/p_vars.tsv | 3 +++ tab-delimited/samplecontexts.tsv | 1 + 5 files changed, 16 insertions(+) diff --git a/tab-delimited/climate.tsv b/tab-delimited/climate.tsv index f30354209..927f140e7 100644 --- a/tab-delimited/climate.tsv +++ b/tab-delimited/climate.tsv @@ -1 +1,4 @@ samplename t_jan t_feb t_mar t_apr t_may t_jun t_jul t_aug t_sep t_oct t_nov t_dec t_djf t_mam t_jja t_son t_ann p_jan p_feb p_mar p_apr p_may p_jun p_jul p_aug p_sep p_oct p_nov p_dec p_djf p_mam p_jja p_son p_ann +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 diff --git a/tab-delimited/metadata.tsv b/tab-delimited/metadata.tsv index f31e489d1..05b7cc8e8 100644 --- a/tab-delimited/metadata.tsv +++ b/tab-delimited/metadata.tsv @@ -1 +1,4 @@ samplename originalsamplename sitename country longitude latitude elevation locationreliability locationnotes areaofsite samplecontext sitedescription vegdescription sampletype samplemethod agebp ageuncertainty ispercent notes okexcept empd_version +test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 +test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know f EMPD2 diff --git a/tab-delimited/p_counts.tsv b/tab-delimited/p_counts.tsv index 8b51c97bb..1fbbceef4 100644 --- a/tab-delimited/p_counts.tsv +++ b/tab-delimited/p_counts.tsv @@ -1 +1,7 @@ samplename var_ count +test_a2 1 45 +test_a2 2 6 +test_a2 3 55 +test_a3 1 45 +test_a3 2 6 +test_a3 3 55 diff --git a/tab-delimited/p_vars.tsv b/tab-delimited/p_vars.tsv index e0f7475b9..a5d1fca17 100644 --- a/tab-delimited/p_vars.tsv +++ b/tab-delimited/p_vars.tsv @@ -1 +1,4 @@ var_ acc_var_ original_varname acc_varname groupid notes +1 Pinus Pinus TRSH NULL +2 Exotic Exotics (counted) NOPO NULL +3 Umbelliferae Apiaceae HERB NULL diff --git a/tab-delimited/samplecontexts.tsv b/tab-delimited/samplecontexts.tsv index 9c818983e..ce53754ae 100644 --- a/tab-delimited/samplecontexts.tsv +++ b/tab-delimited/samplecontexts.tsv @@ -65,3 +65,4 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological +i dont know From 1a07f0292aa9fd22dc060c0f64784b29b5ad7a61 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 09:08:03 +0000 Subject: [PATCH 059/119] Removed test.tsv to finish the PR --- test.tsv | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 test.tsv diff --git a/test.tsv b/test.tsv deleted file mode 100644 index dbc448a22..000000000 --- a/test.tsv +++ /dev/null @@ -1,4 +0,0 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 1bd5cba1b421a234bf42704b2a2335b1e9c0ce54 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:16:45 +0100 Subject: [PATCH 060/119] Revert "Removed test.tsv to finish the PR" This reverts commit 1a07f0292aa9fd22dc060c0f64784b29b5ad7a61. --- test.tsv | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test.tsv diff --git a/test.tsv b/test.tsv new file mode 100644 index 000000000..dbc448a22 --- /dev/null +++ b/test.tsv @@ -0,0 +1,4 @@ +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 25dc07b642417768167e641330725ed4b29b7026 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:16:50 +0100 Subject: [PATCH 061/119] Revert "Updated tab-delimited files from EMPD2 postgres database" This reverts commit a20dfc1f53f760c20883172b72fef34a8900d894. --- tab-delimited/climate.tsv | 3 --- tab-delimited/metadata.tsv | 3 --- tab-delimited/p_counts.tsv | 6 ------ tab-delimited/p_vars.tsv | 3 --- tab-delimited/samplecontexts.tsv | 1 - 5 files changed, 16 deletions(-) diff --git a/tab-delimited/climate.tsv b/tab-delimited/climate.tsv index 927f140e7..f30354209 100644 --- a/tab-delimited/climate.tsv +++ b/tab-delimited/climate.tsv @@ -1,4 +1 @@ samplename t_jan t_feb t_mar t_apr t_may t_jun t_jul t_aug t_sep t_oct t_nov t_dec t_djf t_mam t_jja t_son t_ann p_jan p_feb p_mar p_apr p_may p_jun p_jul p_aug p_sep p_oct p_nov p_dec p_djf p_mam p_jja p_son p_ann -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 diff --git a/tab-delimited/metadata.tsv b/tab-delimited/metadata.tsv index 05b7cc8e8..f31e489d1 100644 --- a/tab-delimited/metadata.tsv +++ b/tab-delimited/metadata.tsv @@ -1,4 +1 @@ samplename originalsamplename sitename country longitude latitude elevation locationreliability locationnotes areaofsite samplecontext sitedescription vegdescription sampletype samplemethod agebp ageuncertainty ispercent notes okexcept empd_version -test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 -test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know f EMPD2 diff --git a/tab-delimited/p_counts.tsv b/tab-delimited/p_counts.tsv index 1fbbceef4..8b51c97bb 100644 --- a/tab-delimited/p_counts.tsv +++ b/tab-delimited/p_counts.tsv @@ -1,7 +1 @@ samplename var_ count -test_a2 1 45 -test_a2 2 6 -test_a2 3 55 -test_a3 1 45 -test_a3 2 6 -test_a3 3 55 diff --git a/tab-delimited/p_vars.tsv b/tab-delimited/p_vars.tsv index a5d1fca17..e0f7475b9 100644 --- a/tab-delimited/p_vars.tsv +++ b/tab-delimited/p_vars.tsv @@ -1,4 +1 @@ var_ acc_var_ original_varname acc_varname groupid notes -1 Pinus Pinus TRSH NULL -2 Exotic Exotics (counted) NOPO NULL -3 Umbelliferae Apiaceae HERB NULL diff --git a/tab-delimited/samplecontexts.tsv b/tab-delimited/samplecontexts.tsv index ce53754ae..9c818983e 100644 --- a/tab-delimited/samplecontexts.tsv +++ b/tab-delimited/samplecontexts.tsv @@ -65,4 +65,3 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological -i dont know From 2a40e8d7e0497c48fca04d75fca26a49f730a8f9 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:16:54 +0100 Subject: [PATCH 062/119] Revert "Removed postgres dump of test.tsv" This reverts commit ebfcb8d8c421bd755f6daf68a759f94732acf123. --- postgres/test.sql | 949 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 949 insertions(+) create mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql new file mode 100644 index 000000000..aa3fb52dd --- /dev/null +++ b/postgres/test.sql @@ -0,0 +1,949 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ageuncertainties ( + ageuncertainty character varying(1) NOT NULL, + description character varying(21) NOT NULL, + age character varying(14) +); + + +ALTER TABLE public.ageuncertainties OWNER TO postgres; + +-- +-- Name: climate; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.climate ( + samplename character varying(25) NOT NULL, + t_jan double precision, + t_feb double precision, + t_mar double precision, + t_apr double precision, + t_may double precision, + t_jun double precision, + t_jul double precision, + t_aug double precision, + t_sep double precision, + t_oct double precision, + t_nov double precision, + t_dec double precision, + t_djf double precision, + t_mam double precision, + t_jja double precision, + t_son double precision, + t_ann double precision, + p_jan double precision, + p_feb double precision, + p_mar double precision, + p_apr double precision, + p_may double precision, + p_jun double precision, + p_jul double precision, + p_aug double precision, + p_sep double precision, + p_oct double precision, + p_nov double precision, + p_dec double precision, + p_djf double precision, + p_mam double precision, + p_jja double precision, + p_son double precision, + p_ann double precision +); + + +ALTER TABLE public.climate OWNER TO postgres; + +-- +-- Name: countries; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.countries ( + country character varying(42) NOT NULL, + natural_earth character varying(42) +); + + +ALTER TABLE public.countries OWNER TO postgres; + +-- +-- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ecosystems ( + samplename character varying(25) NOT NULL, + realm character(16), + biome character(62), + ecoregion character(65) +); + + +ALTER TABLE public.ecosystems OWNER TO postgres; + +-- +-- Name: groupid; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.groupid ( + groupid character varying(9) NOT NULL, + groupname character varying(60) NOT NULL, + higher_groupid character varying(4) NOT NULL +); + + +ALTER TABLE public.groupid OWNER TO postgres; + +-- +-- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.locationreliabilities ( + locationreliability character varying(1) NOT NULL, + description character varying(55) NOT NULL, + error character varying(5) +); + + +ALTER TABLE public.locationreliabilities OWNER TO postgres; + +-- +-- Name: metadata; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metadata ( + samplename character varying(25) NOT NULL, + originalsamplename character varying(70), + sitename character varying(70), + country character varying(42), + longitude double precision, + latitude double precision, + elevation double precision, + locationreliability character varying(1), + locationnotes text, + areaofsite double precision, + samplecontext character varying(40), + sitedescription text, + vegdescription text, + sampletype character varying(14), + samplemethod character varying(31), + agebp double precision, + ageuncertainty character varying(1), + ispercent boolean DEFAULT false, + notes text, + empd_version character varying(5) +); + + +ALTER TABLE public.metadata OWNER TO postgres; + +-- +-- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metapubli ( + samplename character varying(25) NOT NULL, + publiid integer NOT NULL +); + + +ALTER TABLE public.metapubli OWNER TO postgres; + +-- +-- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metaworker ( + samplename character varying(25) NOT NULL, + workerid integer NOT NULL, + workerrole character varying(3) +); + + +ALTER TABLE public.metaworker OWNER TO postgres; + +-- +-- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_counts ( + samplename character varying(25) NOT NULL, + var_ integer NOT NULL, + count double precision NOT NULL +); + + +ALTER TABLE public.p_counts OWNER TO postgres; + +-- +-- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_vars ( + var_ integer NOT NULL, + acc_var_ integer, + original_varname character varying(100) NOT NULL, + acc_varname character varying(100), + groupid character varying(9), + notes text +); + + +ALTER TABLE public.p_vars OWNER TO postgres; + +-- +-- Name: publications; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.publications ( + publiid integer NOT NULL, + doi character varying(50), + reference text NOT NULL +); + + +ALTER TABLE public.publications OWNER TO postgres; + +-- +-- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplecontexts ( + samplecontext character varying(40) NOT NULL +); + + +ALTER TABLE public.samplecontexts OWNER TO postgres; + +-- +-- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplemethods ( + samplemethod character varying(31) NOT NULL +); + + +ALTER TABLE public.samplemethods OWNER TO postgres; + +-- +-- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.sampletypes ( + sampletype character varying(14) NOT NULL, + notes character varying(13) +); + + +ALTER TABLE public.sampletypes OWNER TO postgres; + +-- +-- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workerroles ( + workerrole character varying(3) NOT NULL, + description character varying(35) NOT NULL +); + + +ALTER TABLE public.workerroles OWNER TO postgres; + +-- +-- Name: workers; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workers ( + workerid integer NOT NULL, + firstname character varying(25), + lastname character varying(25), + initials character varying(10), + address1 text, + email1 character varying(50), + phone1 character varying(25), + address2 text, + email2 character varying(50), + phone2 character varying(25) +); + + +ALTER TABLE public.workers OWNER TO postgres; + +-- +-- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP +\. + + +-- +-- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +\. + + +-- +-- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.countries (country, natural_earth) FROM stdin; +Albania Albania +Algeria Algeria +Andorra Andorra +Armenia Armenia +Austria Austria +Azerbaijan Azerbaijan +Bahrain Bahrain +Belarus Belarus +Belgium Belgium +Bosnia And Herzegovina Bosnia and Herzegovina +Bulgaria Bulgaria +Cape Verde Cape Verde +Croatia Croatia +Cyprus Cyprus +Czech Republic Czech Republic +Denmark Denmark +Egypt Egypt +Eritrea Eritrea +Estonia Estonia +Ethiopia Ethiopia +Faroe Islands Faroe Islands +Finland Finland +France France +Georgia Georgia +Germany Germany +Gibraltar Gibraltar +Greece Greece +Greenland Greenland +Guernsey Guernsey +Hungary Hungary +Iceland Iceland +Iran, Islamic Republic Of Iran +Iraq Iraq +Ireland Ireland +Isle Of Man Isle of Man +Israel Israel +Italy Italy +India India +Japan Japan +Jersey Jersey +Kazakhstan Kazakhstan +Kuwait Kuwait +Kyrgyzstan Kyrgyzstan +Latvia Latvia +Lebanon Lebanon +Libyan Arab Jamahiriya Libya +Liechtenstein Liechtenstein +Lithuania Lithuania +Luxembourg Luxembourg +Macedonia, The Former Yugoslav Republic Of Macedonia +Malta Malta +Moldova, Republic Of Moldova +Monaco Monaco +Morocco Morocco +Netherlands Netherlands +Norway Norway +Oman Oman +Palestinian Territory, Occupied Palestine +Poland Poland +Portugal Portugal +Qatar Qatar +Romania Romania +Russian Federation Russia +San Marino San Marino +Saudi Arabia Saudi Arabia +Serbia And Montenegro Montenegro +Slovakia Slovakia +Slovenia Slovenia +Spain Spain +Sweden Sweden +Switzerland Switzerland +Syrian Arab Republic Syria +Tunisia Tunisia +Turkey Turkey +Ukraine Ukraine +United Arab Emirates United Arab Emirates +United Kingdom United Kingdom +Yemen Yemen +Jordan Jordan +Black Sea \N +Dead Sea \N +Turkmenistan, Republic Of Turkmenistan +Adriatic Sea \N +Svalbard and Jan Mayen Norway +China, People s Republic Of China +\. + + +-- +-- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; +\. + + +-- +-- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; +ALGA Algae AQUA +AQBR Aquatic Bryophytes NOPO +AQPT Aquatic Pteridophyta AQUA +AQUA Aquatics AQUA +AQVP Aquatic Vascular Plants AQUA +BRYO Bryophytes NOPO +BRYO/VACR TO BE REMOVED TBR +CHAR Charcoal CHAR +CHLO Chlorophytes NOPO +CYAN Cyanobacteria NOPO +DINO Dinoflagellates NOPO +DWAR Dwarf shrubs TRSH +FUNG Fungi NOPO +HELO Helophytes AQUA +HERB Herbs HERB +INUN Indeterminables and unknowns INUN +INVE Invertebrates NOPO +LIAN Liana TRSH +MACR Macrofossils NOPO +NOPO Nonpollen NOPO +PLAT Platyhelminthes (flatworms) NOPO +PREQ Pre-Quaternary type NOPO +RHIZ Rhizopods NOPO +ROTI Rotifera (Rotifers) NOPO +SPOR Taxonomically undifferentiated spores INUN +TEAM Testate amoebae NOPO +TRSH Trees and shrubs TRSH +UNPP Undifferentiated NPP INUN +UPBR Upland Bryophytes NOPO +UPHE Upland herbs HERB +VACR Vascular cryptogams (Pteridophytes) NOPO +VEMI Vegetative microfossils NOPO +NEMA Nematoda (roundworms) NOPO +ACRI Acritarchs NOPO +EUMY Eumycete NOPO +HEMI Hemi-parasitic NOPO +MOSS Moss NOPO +MICR Microcrustaceans NOPO +\. + + +-- +-- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! \N +\. + + +-- +-- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 +\. + + +-- +-- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metapubli (samplename, publiid) FROM stdin; +\. + + +-- +-- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; +\. + + +-- +-- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_counts (samplename, var_, count) FROM stdin; +test_a1 1 45 +test_a1 2 6 +test_a1 3 55 +\. + + +-- +-- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL +\. + + +-- +-- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.publications (publiid, doi, reference) FROM stdin; +\. + + +-- +-- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplecontexts (samplecontext) FROM stdin; +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological +\. + + +-- +-- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplemethods (samplemethod) FROM stdin; +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer +\. + + +-- +-- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.sampletypes (sampletype, notes) FROM stdin; +dung \N +litter \N +moss \N +pollen trap To be removed +sediment submerged +spider web \N +soil terrestrial +sphagnum peat \N +peat \N +lichen \N +submerged \N +core_top \N +tussock \N +epiphytic moss \N +\. + + +-- +-- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workerroles (workerrole, description) FROM stdin; +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) +\. + + +-- +-- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; +\. + + +-- +-- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ageuncertainties + ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); + + +-- +-- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.countries + ADD CONSTRAINT countries_pkey PRIMARY KEY (country); + + +-- +-- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); + + +-- +-- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.groupid + ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); + + +-- +-- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.locationreliabilities + ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); + + +-- +-- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); + + +-- +-- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); + + +-- +-- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); + + +-- +-- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); + + +-- +-- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); + + +-- +-- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); + + +-- +-- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_doi_key UNIQUE (doi); + + +-- +-- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); + + +-- +-- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplecontexts + ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); + + +-- +-- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplemethods + ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); + + +-- +-- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.sampletypes + ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); + + +-- +-- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workerroles + ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); + + +-- +-- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workers + ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); + + +-- +-- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); + + +-- +-- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); + + +-- +-- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); + + +-- +-- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); + + +-- +-- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); + + +-- +-- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); + + +-- +-- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); + + +-- +-- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); + + +-- +-- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); + + +-- +-- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); + + +-- +-- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); + + +-- +-- PostgreSQL database dump complete +-- + From d35ad498eca3cfde87763ef3f7dca2ec3c88f815 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:16:58 +0100 Subject: [PATCH 063/119] Revert "Added postgres dump for test.tsv" This reverts commit 9f3286355d2145966c57025f08e05f4214cac229. --- postgres/EMPD2.sql | 114 ++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql index 8c6961531..359334330 100644 --- a/postgres/EMPD2.sql +++ b/postgres/EMPD2.sql @@ -2,12 +2,11 @@ -- PostgreSQL database dump -- --- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) --- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped from database version 9.5.15 +-- Dumped by pg_dump version 9.5.15 SET statement_timeout = 0; SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); @@ -15,6 +14,20 @@ SET check_function_bodies = false; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + SET default_tablespace = ''; SET default_with_oids = false; @@ -309,9 +322,6 @@ C Within last 250 years 50-200BP -- COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 \. @@ -480,9 +490,6 @@ X Do not use! \N -- COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 -test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know \N \N \N \N \N \N f \N \N EMPD2 \. @@ -507,12 +514,6 @@ COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -- COPY public.p_counts (samplename, var_, count) FROM stdin; -test_a2 1 45 -test_a2 2 6 -test_a2 3 55 -test_a3 1 45 -test_a3 2 6 -test_a3 3 55 \. @@ -521,9 +522,6 @@ test_a3 3 55 -- COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; -1 \N Pinus Pinus TRSH NULL -2 \N Exotic Exotics (counted) NOPO NULL -3 \N Umbelliferae Apiaceae HERB NULL \. @@ -606,7 +604,6 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological -i dont know \. @@ -679,7 +676,7 @@ COPY public.workers (workerid, firstname, lastname, initials, address1, email1, -- --- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ageuncertainties @@ -687,7 +684,7 @@ ALTER TABLE ONLY public.ageuncertainties -- --- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.countries @@ -695,7 +692,7 @@ ALTER TABLE ONLY public.countries -- --- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -703,7 +700,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.groupid @@ -711,7 +708,7 @@ ALTER TABLE ONLY public.groupid -- --- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.locationreliabilities @@ -719,7 +716,7 @@ ALTER TABLE ONLY public.locationreliabilities -- --- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -727,7 +724,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -735,7 +732,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -743,7 +740,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -751,7 +748,7 @@ ALTER TABLE ONLY public.climate -- --- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -759,7 +756,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -767,7 +764,7 @@ ALTER TABLE ONLY public.p_vars -- --- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -775,7 +772,7 @@ ALTER TABLE ONLY public.publications -- --- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -783,7 +780,7 @@ ALTER TABLE ONLY public.publications -- --- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplecontexts @@ -791,7 +788,7 @@ ALTER TABLE ONLY public.samplecontexts -- --- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplemethods @@ -799,7 +796,7 @@ ALTER TABLE ONLY public.samplemethods -- --- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.sampletypes @@ -807,7 +804,7 @@ ALTER TABLE ONLY public.sampletypes -- --- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workerroles @@ -815,7 +812,7 @@ ALTER TABLE ONLY public.workerroles -- --- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workers @@ -823,7 +820,7 @@ ALTER TABLE ONLY public.workers -- --- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -831,7 +828,7 @@ ALTER TABLE ONLY public.climate -- --- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -839,7 +836,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -847,7 +844,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -855,7 +852,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -863,7 +860,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -871,7 +868,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -879,7 +876,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -887,7 +884,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -895,7 +892,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -903,7 +900,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -911,7 +908,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -919,7 +916,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -927,7 +924,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -935,7 +932,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -943,7 +940,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -951,6 +948,15 @@ ALTER TABLE ONLY public.p_vars -- --- PostgreSQL database dump complete +-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres -- +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM postgres; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- From bdd58b775f469786b93d35c1952b21169c5cae1b Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:17:03 +0100 Subject: [PATCH 064/119] Revert "Merged test.tsv into meta.tsv [skip ci]" This reverts commit e289b1824e7ef492e6db84828704780cb893babb. --- meta.tsv | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta.tsv b/meta.tsv index dbc448a22..9c9cfeb6f 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1,4 +1 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From ada9ac52ba63ad50e0d8a541a3a2fb71fecf89f1 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:17:08 +0100 Subject: [PATCH 065/119] Revert "[test_a1]: Changed country from France to Germany" This reverts commit 340139917f3e36a6f7f6bdd51c76af5183b9939f. --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index dbc448a22..2650801dc 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 507b6868d949a2a4eae20356e2f138b23577e302 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 10:45:22 +0000 Subject: [PATCH 066/119] [test_a1]: Changed country from France to Germany --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 2650801dc..dbc448a22 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From f47f61b5b5a450f90d1380f2e977f4033f88bbb8 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 10:45:27 +0000 Subject: [PATCH 067/119] Merge branch 'upstream/master' into test-data From adc95619208fad17a731dacf6c9662b2750d29ec Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 10:45:27 +0000 Subject: [PATCH 068/119] Merged test.tsv into meta.tsv [skip ci] --- meta.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta.tsv b/meta.tsv index 9c9cfeb6f..dbc448a22 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 26c9b8e1822f222b6536af165e6e81356161977d Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 10:45:28 +0000 Subject: [PATCH 069/119] Added postgres dump for test.tsv --- postgres/EMPD2.sql | 114 +++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 60 deletions(-) diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql index 359334330..8c6961531 100644 --- a/postgres/EMPD2.sql +++ b/postgres/EMPD2.sql @@ -2,11 +2,12 @@ -- PostgreSQL database dump -- --- Dumped from database version 9.5.15 --- Dumped by pg_dump version 9.5.15 +-- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) SET statement_timeout = 0; SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); @@ -14,20 +15,6 @@ SET check_function_bodies = false; SET client_min_messages = warning; SET row_security = off; --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: --- - -COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - SET default_tablespace = ''; SET default_with_oids = false; @@ -322,6 +309,9 @@ C Within last 250 years 50-200BP -- COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 \. @@ -490,6 +480,9 @@ X Do not use! \N -- COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 +test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know \N \N \N \N \N \N f \N \N EMPD2 \. @@ -514,6 +507,12 @@ COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -- COPY public.p_counts (samplename, var_, count) FROM stdin; +test_a2 1 45 +test_a2 2 6 +test_a2 3 55 +test_a3 1 45 +test_a3 2 6 +test_a3 3 55 \. @@ -522,6 +521,9 @@ COPY public.p_counts (samplename, var_, count) FROM stdin; -- COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL \. @@ -604,6 +606,7 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological +i dont know \. @@ -676,7 +679,7 @@ COPY public.workers (workerid, firstname, lastname, initials, address1, email1, -- --- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ageuncertainties @@ -684,7 +687,7 @@ ALTER TABLE ONLY public.ageuncertainties -- --- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.countries @@ -692,7 +695,7 @@ ALTER TABLE ONLY public.countries -- --- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -700,7 +703,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.groupid @@ -708,7 +711,7 @@ ALTER TABLE ONLY public.groupid -- --- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.locationreliabilities @@ -716,7 +719,7 @@ ALTER TABLE ONLY public.locationreliabilities -- --- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -724,7 +727,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -732,7 +735,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -740,7 +743,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -748,7 +751,7 @@ ALTER TABLE ONLY public.climate -- --- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -756,7 +759,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -764,7 +767,7 @@ ALTER TABLE ONLY public.p_vars -- --- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -772,7 +775,7 @@ ALTER TABLE ONLY public.publications -- --- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -780,7 +783,7 @@ ALTER TABLE ONLY public.publications -- --- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplecontexts @@ -788,7 +791,7 @@ ALTER TABLE ONLY public.samplecontexts -- --- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplemethods @@ -796,7 +799,7 @@ ALTER TABLE ONLY public.samplemethods -- --- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.sampletypes @@ -804,7 +807,7 @@ ALTER TABLE ONLY public.sampletypes -- --- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workerroles @@ -812,7 +815,7 @@ ALTER TABLE ONLY public.workerroles -- --- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workers @@ -820,7 +823,7 @@ ALTER TABLE ONLY public.workers -- --- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -828,7 +831,7 @@ ALTER TABLE ONLY public.climate -- --- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -836,7 +839,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -844,7 +847,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -852,7 +855,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -860,7 +863,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -868,7 +871,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -876,7 +879,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -884,7 +887,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -892,7 +895,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -900,7 +903,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -908,7 +911,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -916,7 +919,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -924,7 +927,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -932,7 +935,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -940,23 +943,14 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); --- --- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres --- - -REVOKE ALL ON SCHEMA public FROM PUBLIC; -REVOKE ALL ON SCHEMA public FROM postgres; -GRANT ALL ON SCHEMA public TO postgres; -GRANT ALL ON SCHEMA public TO PUBLIC; - - -- -- PostgreSQL database dump complete -- + From fea50c0039b0e49b81bba5166c40f708f867279b Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 10:45:28 +0000 Subject: [PATCH 070/119] Removed postgres dump of test.tsv --- postgres/test.sql | 949 ---------------------------------------------- 1 file changed, 949 deletions(-) delete mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql deleted file mode 100644 index aa3fb52dd..000000000 --- a/postgres/test.sql +++ /dev/null @@ -1,949 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) --- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET client_min_messages = warning; -SET row_security = off; - -SET default_tablespace = ''; - -SET default_with_oids = false; - --- --- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ageuncertainties ( - ageuncertainty character varying(1) NOT NULL, - description character varying(21) NOT NULL, - age character varying(14) -); - - -ALTER TABLE public.ageuncertainties OWNER TO postgres; - --- --- Name: climate; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.climate ( - samplename character varying(25) NOT NULL, - t_jan double precision, - t_feb double precision, - t_mar double precision, - t_apr double precision, - t_may double precision, - t_jun double precision, - t_jul double precision, - t_aug double precision, - t_sep double precision, - t_oct double precision, - t_nov double precision, - t_dec double precision, - t_djf double precision, - t_mam double precision, - t_jja double precision, - t_son double precision, - t_ann double precision, - p_jan double precision, - p_feb double precision, - p_mar double precision, - p_apr double precision, - p_may double precision, - p_jun double precision, - p_jul double precision, - p_aug double precision, - p_sep double precision, - p_oct double precision, - p_nov double precision, - p_dec double precision, - p_djf double precision, - p_mam double precision, - p_jja double precision, - p_son double precision, - p_ann double precision -); - - -ALTER TABLE public.climate OWNER TO postgres; - --- --- Name: countries; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.countries ( - country character varying(42) NOT NULL, - natural_earth character varying(42) -); - - -ALTER TABLE public.countries OWNER TO postgres; - --- --- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.ecosystems ( - samplename character varying(25) NOT NULL, - realm character(16), - biome character(62), - ecoregion character(65) -); - - -ALTER TABLE public.ecosystems OWNER TO postgres; - --- --- Name: groupid; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.groupid ( - groupid character varying(9) NOT NULL, - groupname character varying(60) NOT NULL, - higher_groupid character varying(4) NOT NULL -); - - -ALTER TABLE public.groupid OWNER TO postgres; - --- --- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.locationreliabilities ( - locationreliability character varying(1) NOT NULL, - description character varying(55) NOT NULL, - error character varying(5) -); - - -ALTER TABLE public.locationreliabilities OWNER TO postgres; - --- --- Name: metadata; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metadata ( - samplename character varying(25) NOT NULL, - originalsamplename character varying(70), - sitename character varying(70), - country character varying(42), - longitude double precision, - latitude double precision, - elevation double precision, - locationreliability character varying(1), - locationnotes text, - areaofsite double precision, - samplecontext character varying(40), - sitedescription text, - vegdescription text, - sampletype character varying(14), - samplemethod character varying(31), - agebp double precision, - ageuncertainty character varying(1), - ispercent boolean DEFAULT false, - notes text, - empd_version character varying(5) -); - - -ALTER TABLE public.metadata OWNER TO postgres; - --- --- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metapubli ( - samplename character varying(25) NOT NULL, - publiid integer NOT NULL -); - - -ALTER TABLE public.metapubli OWNER TO postgres; - --- --- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.metaworker ( - samplename character varying(25) NOT NULL, - workerid integer NOT NULL, - workerrole character varying(3) -); - - -ALTER TABLE public.metaworker OWNER TO postgres; - --- --- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_counts ( - samplename character varying(25) NOT NULL, - var_ integer NOT NULL, - count double precision NOT NULL -); - - -ALTER TABLE public.p_counts OWNER TO postgres; - --- --- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.p_vars ( - var_ integer NOT NULL, - acc_var_ integer, - original_varname character varying(100) NOT NULL, - acc_varname character varying(100), - groupid character varying(9), - notes text -); - - -ALTER TABLE public.p_vars OWNER TO postgres; - --- --- Name: publications; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.publications ( - publiid integer NOT NULL, - doi character varying(50), - reference text NOT NULL -); - - -ALTER TABLE public.publications OWNER TO postgres; - --- --- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplecontexts ( - samplecontext character varying(40) NOT NULL -); - - -ALTER TABLE public.samplecontexts OWNER TO postgres; - --- --- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.samplemethods ( - samplemethod character varying(31) NOT NULL -); - - -ALTER TABLE public.samplemethods OWNER TO postgres; - --- --- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.sampletypes ( - sampletype character varying(14) NOT NULL, - notes character varying(13) -); - - -ALTER TABLE public.sampletypes OWNER TO postgres; - --- --- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workerroles ( - workerrole character varying(3) NOT NULL, - description character varying(35) NOT NULL -); - - -ALTER TABLE public.workerroles OWNER TO postgres; - --- --- Name: workers; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.workers ( - workerid integer NOT NULL, - firstname character varying(25), - lastname character varying(25), - initials character varying(10), - address1 text, - email1 character varying(50), - phone1 character varying(25), - address2 text, - email2 character varying(50), - phone2 character varying(25) -); - - -ALTER TABLE public.workers OWNER TO postgres; - --- --- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; -A Modern sample 0BP to Present -B Within last 100 years 0-50BP -C Within last 250 years 50-200BP -\. - - --- --- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -\. - - --- --- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.countries (country, natural_earth) FROM stdin; -Albania Albania -Algeria Algeria -Andorra Andorra -Armenia Armenia -Austria Austria -Azerbaijan Azerbaijan -Bahrain Bahrain -Belarus Belarus -Belgium Belgium -Bosnia And Herzegovina Bosnia and Herzegovina -Bulgaria Bulgaria -Cape Verde Cape Verde -Croatia Croatia -Cyprus Cyprus -Czech Republic Czech Republic -Denmark Denmark -Egypt Egypt -Eritrea Eritrea -Estonia Estonia -Ethiopia Ethiopia -Faroe Islands Faroe Islands -Finland Finland -France France -Georgia Georgia -Germany Germany -Gibraltar Gibraltar -Greece Greece -Greenland Greenland -Guernsey Guernsey -Hungary Hungary -Iceland Iceland -Iran, Islamic Republic Of Iran -Iraq Iraq -Ireland Ireland -Isle Of Man Isle of Man -Israel Israel -Italy Italy -India India -Japan Japan -Jersey Jersey -Kazakhstan Kazakhstan -Kuwait Kuwait -Kyrgyzstan Kyrgyzstan -Latvia Latvia -Lebanon Lebanon -Libyan Arab Jamahiriya Libya -Liechtenstein Liechtenstein -Lithuania Lithuania -Luxembourg Luxembourg -Macedonia, The Former Yugoslav Republic Of Macedonia -Malta Malta -Moldova, Republic Of Moldova -Monaco Monaco -Morocco Morocco -Netherlands Netherlands -Norway Norway -Oman Oman -Palestinian Territory, Occupied Palestine -Poland Poland -Portugal Portugal -Qatar Qatar -Romania Romania -Russian Federation Russia -San Marino San Marino -Saudi Arabia Saudi Arabia -Serbia And Montenegro Montenegro -Slovakia Slovakia -Slovenia Slovenia -Spain Spain -Sweden Sweden -Switzerland Switzerland -Syrian Arab Republic Syria -Tunisia Tunisia -Turkey Turkey -Ukraine Ukraine -United Arab Emirates United Arab Emirates -United Kingdom United Kingdom -Yemen Yemen -Jordan Jordan -Black Sea \N -Dead Sea \N -Turkmenistan, Republic Of Turkmenistan -Adriatic Sea \N -Svalbard and Jan Mayen Norway -China, People s Republic Of China -\. - - --- --- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; -\. - - --- --- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.groupid (groupid, groupname, higher_groupid) FROM stdin; -ALGA Algae AQUA -AQBR Aquatic Bryophytes NOPO -AQPT Aquatic Pteridophyta AQUA -AQUA Aquatics AQUA -AQVP Aquatic Vascular Plants AQUA -BRYO Bryophytes NOPO -BRYO/VACR TO BE REMOVED TBR -CHAR Charcoal CHAR -CHLO Chlorophytes NOPO -CYAN Cyanobacteria NOPO -DINO Dinoflagellates NOPO -DWAR Dwarf shrubs TRSH -FUNG Fungi NOPO -HELO Helophytes AQUA -HERB Herbs HERB -INUN Indeterminables and unknowns INUN -INVE Invertebrates NOPO -LIAN Liana TRSH -MACR Macrofossils NOPO -NOPO Nonpollen NOPO -PLAT Platyhelminthes (flatworms) NOPO -PREQ Pre-Quaternary type NOPO -RHIZ Rhizopods NOPO -ROTI Rotifera (Rotifers) NOPO -SPOR Taxonomically undifferentiated spores INUN -TEAM Testate amoebae NOPO -TRSH Trees and shrubs TRSH -UNPP Undifferentiated NPP INUN -UPBR Upland Bryophytes NOPO -UPHE Upland herbs HERB -VACR Vascular cryptogams (Pteridophytes) NOPO -VEMI Vegetative microfossils NOPO -NEMA Nematoda (roundworms) NOPO -ACRI Acritarchs NOPO -EUMY Eumycete NOPO -HEMI Hemi-parasitic NOPO -MOSS Moss NOPO -MICR Microcrustaceans NOPO -\. - - --- --- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; -A Good for high resolution remote sensing (<100m) <100m -B Good for lower resolution remote sensing (<1km) <1km -C Ok for climate reconstruction or regional scale mapping <5km -D Ok for mapping at continental scale <20km -X Do not use! \N -\. - - --- --- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere France 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N EMPD2 -\. - - --- --- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metapubli (samplename, publiid) FROM stdin; -\. - - --- --- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -\. - - --- --- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_counts (samplename, var_, count) FROM stdin; -test_a1 1 45 -test_a1 2 6 -test_a1 3 55 -\. - - --- --- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; -1 \N Pinus Pinus TRSH NULL -2 \N Exotic Exotics (counted) NOPO NULL -3 \N Umbelliferae Apiaceae HERB NULL -\. - - --- --- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.publications (publiid, doi, reference) FROM stdin; -\. - - --- --- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplecontexts (samplecontext) FROM stdin; -arable -blanket bog -bog -cave -cirque lake -closed forest -coastal -coastal lake -coastal wetland -drained_lake -ephemeral lake -ephemeral lake/pond -estuarine -fallow -fallow - scattered trees/shrubs -fen -floodplain mire -fluvial -forest -forest undefined -heath -ice from glacier -kettle lake -lagoon -lake -maquis -marine -marsh -marshland -mire -mor humus -moss-shrubby tundra -natural grassland -natural grasslands -natural open water -open forest -open forest/orchard -open forest/scattered trees/shrubs -orchard -orchard/scattered trees/shrubs -pasture -peatbog -pond -riverine -salt lake -salt marsh -scattered trees -scattered trees/shrubs -shrubs -shrubs/scrubland -soil -soligenous (mineraltrophic) Mire -sparse/no vegetation -swamp -tectonic lake -terrestrial -treeless vegetation -treeless vegetation/natural grassland -treeless vegetation/sparse vegetation -urban -valley mire -wetland -wetland bog -wetland bog/scattered trees/shrubs -wetland bog/treeless vegetation -archaeological -\. - - --- --- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.samplemethods (samplemethod) FROM stdin; -hand picking -hand picking - multiple samples -unspecified -auger corer -freeze corer -gravity corer -piston corer -russian corer -box -spade -hiller corer -gouge auger -core -corer unspecified -eckman-grab -box corer -\. - - --- --- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.sampletypes (sampletype, notes) FROM stdin; -dung \N -litter \N -moss \N -pollen trap To be removed -sediment submerged -spider web \N -soil terrestrial -sphagnum peat \N -peat \N -lichen \N -submerged \N -core_top \N -tussock \N -epiphytic moss \N -\. - - --- --- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workerroles (workerrole, description) FROM stdin; -R Responsible Person -R/A Both Responsible Person and Analyst -A Analyst -A/D Analyst (Deceased) -A/I Analysis (Inactive) -A/U Analyst (Unknown) -\. - - --- --- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres --- - -COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; -\. - - --- --- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ageuncertainties - ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); - - --- --- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.countries - ADD CONSTRAINT countries_pkey PRIMARY KEY (country); - - --- --- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); - - --- --- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.groupid - ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); - - --- --- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.locationreliabilities - ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); - - --- --- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); - - --- --- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); - - --- --- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); - - --- --- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); - - --- --- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); - - --- --- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); - - --- --- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_doi_key UNIQUE (doi); - - --- --- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.publications - ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); - - --- --- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplecontexts - ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); - - --- --- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.samplemethods - ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); - - --- --- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.sampletypes - ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); - - --- --- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workerroles - ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); - - --- --- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.workers - ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); - - --- --- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.climate - ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.ecosystems - ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); - - --- --- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); - - --- --- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); - - --- --- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); - - --- --- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); - - --- --- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metadata - ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); - - --- --- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); - - --- --- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metapubli - ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); - - --- --- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.metaworker - ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); - - --- --- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); - - --- --- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_counts - ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); - - --- --- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.p_vars - ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); - - --- --- PostgreSQL database dump complete --- - From d07562465a6385a42d81a81cd6b4cd82aca6ac6f Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 10:45:29 +0000 Subject: [PATCH 071/119] Updated tab-delimited files from EMPD2 postgres database --- tab-delimited/climate.tsv | 3 +++ tab-delimited/metadata.tsv | 3 +++ tab-delimited/p_counts.tsv | 6 ++++++ tab-delimited/p_vars.tsv | 3 +++ tab-delimited/samplecontexts.tsv | 1 + 5 files changed, 16 insertions(+) diff --git a/tab-delimited/climate.tsv b/tab-delimited/climate.tsv index f30354209..927f140e7 100644 --- a/tab-delimited/climate.tsv +++ b/tab-delimited/climate.tsv @@ -1 +1,4 @@ samplename t_jan t_feb t_mar t_apr t_may t_jun t_jul t_aug t_sep t_oct t_nov t_dec t_djf t_mam t_jja t_son t_ann p_jan p_feb p_mar p_apr p_may p_jun p_jul p_aug p_sep p_oct p_nov p_dec p_djf p_mam p_jja p_son p_ann +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 diff --git a/tab-delimited/metadata.tsv b/tab-delimited/metadata.tsv index f31e489d1..05b7cc8e8 100644 --- a/tab-delimited/metadata.tsv +++ b/tab-delimited/metadata.tsv @@ -1 +1,4 @@ samplename originalsamplename sitename country longitude latitude elevation locationreliability locationnotes areaofsite samplecontext sitedescription vegdescription sampletype samplemethod agebp ageuncertainty ispercent notes okexcept empd_version +test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 +test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know f EMPD2 diff --git a/tab-delimited/p_counts.tsv b/tab-delimited/p_counts.tsv index 8b51c97bb..1fbbceef4 100644 --- a/tab-delimited/p_counts.tsv +++ b/tab-delimited/p_counts.tsv @@ -1 +1,7 @@ samplename var_ count +test_a2 1 45 +test_a2 2 6 +test_a2 3 55 +test_a3 1 45 +test_a3 2 6 +test_a3 3 55 diff --git a/tab-delimited/p_vars.tsv b/tab-delimited/p_vars.tsv index e0f7475b9..a5d1fca17 100644 --- a/tab-delimited/p_vars.tsv +++ b/tab-delimited/p_vars.tsv @@ -1 +1,4 @@ var_ acc_var_ original_varname acc_varname groupid notes +1 Pinus Pinus TRSH NULL +2 Exotic Exotics (counted) NOPO NULL +3 Umbelliferae Apiaceae HERB NULL diff --git a/tab-delimited/samplecontexts.tsv b/tab-delimited/samplecontexts.tsv index 9c818983e..ce53754ae 100644 --- a/tab-delimited/samplecontexts.tsv +++ b/tab-delimited/samplecontexts.tsv @@ -65,3 +65,4 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological +i dont know From bcab1518ec0d2dd7dde5982861365028921d69f3 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 10:45:30 +0000 Subject: [PATCH 072/119] Removed test.tsv to finish the PR --- test.tsv | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 test.tsv diff --git a/test.tsv b/test.tsv deleted file mode 100644 index dbc448a22..000000000 --- a/test.tsv +++ /dev/null @@ -1,4 +0,0 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 010a3177e32516c22c258820c1aaeee3b3ae3d8f Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:47:55 +0100 Subject: [PATCH 073/119] Revert "Removed test.tsv to finish the PR" This reverts commit bcab1518ec0d2dd7dde5982861365028921d69f3. --- test.tsv | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test.tsv diff --git a/test.tsv b/test.tsv new file mode 100644 index 000000000..dbc448a22 --- /dev/null +++ b/test.tsv @@ -0,0 +1,4 @@ +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 8c88c624e67e5d160f81d360038f5fc16c2de41b Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:51:03 +0100 Subject: [PATCH 074/119] Revert "Updated tab-delimited files from EMPD2 postgres database" This reverts commit d07562465a6385a42d81a81cd6b4cd82aca6ac6f. --- tab-delimited/climate.tsv | 3 --- tab-delimited/metadata.tsv | 3 --- tab-delimited/p_counts.tsv | 6 ------ tab-delimited/p_vars.tsv | 3 --- tab-delimited/samplecontexts.tsv | 1 - 5 files changed, 16 deletions(-) diff --git a/tab-delimited/climate.tsv b/tab-delimited/climate.tsv index 927f140e7..f30354209 100644 --- a/tab-delimited/climate.tsv +++ b/tab-delimited/climate.tsv @@ -1,4 +1 @@ samplename t_jan t_feb t_mar t_apr t_may t_jun t_jul t_aug t_sep t_oct t_nov t_dec t_djf t_mam t_jja t_son t_ann p_jan p_feb p_mar p_apr p_may p_jun p_jul p_aug p_sep p_oct p_nov p_dec p_djf p_mam p_jja p_son p_ann -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 diff --git a/tab-delimited/metadata.tsv b/tab-delimited/metadata.tsv index 05b7cc8e8..f31e489d1 100644 --- a/tab-delimited/metadata.tsv +++ b/tab-delimited/metadata.tsv @@ -1,4 +1 @@ samplename originalsamplename sitename country longitude latitude elevation locationreliability locationnotes areaofsite samplecontext sitedescription vegdescription sampletype samplemethod agebp ageuncertainty ispercent notes okexcept empd_version -test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 -test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know f EMPD2 diff --git a/tab-delimited/p_counts.tsv b/tab-delimited/p_counts.tsv index 1fbbceef4..8b51c97bb 100644 --- a/tab-delimited/p_counts.tsv +++ b/tab-delimited/p_counts.tsv @@ -1,7 +1 @@ samplename var_ count -test_a2 1 45 -test_a2 2 6 -test_a2 3 55 -test_a3 1 45 -test_a3 2 6 -test_a3 3 55 diff --git a/tab-delimited/p_vars.tsv b/tab-delimited/p_vars.tsv index a5d1fca17..e0f7475b9 100644 --- a/tab-delimited/p_vars.tsv +++ b/tab-delimited/p_vars.tsv @@ -1,4 +1 @@ var_ acc_var_ original_varname acc_varname groupid notes -1 Pinus Pinus TRSH NULL -2 Exotic Exotics (counted) NOPO NULL -3 Umbelliferae Apiaceae HERB NULL diff --git a/tab-delimited/samplecontexts.tsv b/tab-delimited/samplecontexts.tsv index ce53754ae..9c818983e 100644 --- a/tab-delimited/samplecontexts.tsv +++ b/tab-delimited/samplecontexts.tsv @@ -65,4 +65,3 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological -i dont know From cf92024b528f9a34f12d4324582ed4ea24fbc3a3 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:51:12 +0100 Subject: [PATCH 075/119] Revert "Added postgres dump for test.tsv" This reverts commit 26c9b8e1822f222b6536af165e6e81356161977d. --- postgres/EMPD2.sql | 114 ++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql index 8c6961531..359334330 100644 --- a/postgres/EMPD2.sql +++ b/postgres/EMPD2.sql @@ -2,12 +2,11 @@ -- PostgreSQL database dump -- --- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) --- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped from database version 9.5.15 +-- Dumped by pg_dump version 9.5.15 SET statement_timeout = 0; SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); @@ -15,6 +14,20 @@ SET check_function_bodies = false; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + SET default_tablespace = ''; SET default_with_oids = false; @@ -309,9 +322,6 @@ C Within last 250 years 50-200BP -- COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 \. @@ -480,9 +490,6 @@ X Do not use! \N -- COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 -test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know \N \N \N \N \N \N f \N \N EMPD2 \. @@ -507,12 +514,6 @@ COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -- COPY public.p_counts (samplename, var_, count) FROM stdin; -test_a2 1 45 -test_a2 2 6 -test_a2 3 55 -test_a3 1 45 -test_a3 2 6 -test_a3 3 55 \. @@ -521,9 +522,6 @@ test_a3 3 55 -- COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; -1 \N Pinus Pinus TRSH NULL -2 \N Exotic Exotics (counted) NOPO NULL -3 \N Umbelliferae Apiaceae HERB NULL \. @@ -606,7 +604,6 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological -i dont know \. @@ -679,7 +676,7 @@ COPY public.workers (workerid, firstname, lastname, initials, address1, email1, -- --- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ageuncertainties @@ -687,7 +684,7 @@ ALTER TABLE ONLY public.ageuncertainties -- --- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.countries @@ -695,7 +692,7 @@ ALTER TABLE ONLY public.countries -- --- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -703,7 +700,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.groupid @@ -711,7 +708,7 @@ ALTER TABLE ONLY public.groupid -- --- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.locationreliabilities @@ -719,7 +716,7 @@ ALTER TABLE ONLY public.locationreliabilities -- --- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -727,7 +724,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -735,7 +732,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -743,7 +740,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -751,7 +748,7 @@ ALTER TABLE ONLY public.climate -- --- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -759,7 +756,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -767,7 +764,7 @@ ALTER TABLE ONLY public.p_vars -- --- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -775,7 +772,7 @@ ALTER TABLE ONLY public.publications -- --- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -783,7 +780,7 @@ ALTER TABLE ONLY public.publications -- --- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplecontexts @@ -791,7 +788,7 @@ ALTER TABLE ONLY public.samplecontexts -- --- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplemethods @@ -799,7 +796,7 @@ ALTER TABLE ONLY public.samplemethods -- --- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.sampletypes @@ -807,7 +804,7 @@ ALTER TABLE ONLY public.sampletypes -- --- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workerroles @@ -815,7 +812,7 @@ ALTER TABLE ONLY public.workerroles -- --- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workers @@ -823,7 +820,7 @@ ALTER TABLE ONLY public.workers -- --- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -831,7 +828,7 @@ ALTER TABLE ONLY public.climate -- --- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -839,7 +836,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -847,7 +844,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -855,7 +852,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -863,7 +860,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -871,7 +868,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -879,7 +876,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -887,7 +884,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -895,7 +892,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -903,7 +900,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -911,7 +908,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -919,7 +916,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -927,7 +924,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -935,7 +932,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -943,7 +940,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -951,6 +948,15 @@ ALTER TABLE ONLY public.p_vars -- --- PostgreSQL database dump complete +-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres -- +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM postgres; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- From 5e3e27bf20c1ae5216e1488f7322c612d6d1538d Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:51:18 +0100 Subject: [PATCH 076/119] Revert "Merged test.tsv into meta.tsv [skip ci]" This reverts commit adc95619208fad17a731dacf6c9662b2750d29ec. --- meta.tsv | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta.tsv b/meta.tsv index dbc448a22..9c9cfeb6f 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1,4 +1 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 6b7c9db1965f18aeb62a112d02b190da75416c61 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:51:23 +0100 Subject: [PATCH 077/119] Revert "[test_a1]: Changed country from France to Germany" This reverts commit 507b6868d949a2a4eae20356e2f138b23577e302. --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index dbc448a22..2650801dc 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From c0813a5eb147a10b4f364e153837951aa7a6fca0 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:52:28 +0100 Subject: [PATCH 078/119] Merge branch 'upstream/master' into test-data From 9128681cc04d13c442d11a424cbcedccd1b047df Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 11:01:39 +0000 Subject: [PATCH 079/119] [test_a1]: Changed country from France to Germany --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 2650801dc..dbc448a22 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 437cb16b4323afaad483f7a7d1cea97a3cc39544 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 11:52:10 +0100 Subject: [PATCH 080/119] export fixed tables --- postgres/scripts/import_into_empd2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index d0d7050c1..bfd3ce071 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -126,6 +126,7 @@ def clean_doi(doi): '({})'.format(', '.join(map(is_null_str, v))) for v in new_vals))) conn.commit() + df.to_csv(fname, sep='\t', index=False) METADATA.replace(np.nan, '', inplace=True) From a7976b54f260770ff91abc98722ed7210bfd9c13 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 11:01:47 +0000 Subject: [PATCH 081/119] Merged test.tsv into meta.tsv [skip ci] --- meta.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta.tsv b/meta.tsv index 9c9cfeb6f..dbc448a22 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From b8eeb67b4590da73c16b285b5d34a2027e6808ad Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 11:01:48 +0000 Subject: [PATCH 082/119] Updated fixed tables --- postgres/scripts/tables/SampleContext.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/postgres/scripts/tables/SampleContext.tsv b/postgres/scripts/tables/SampleContext.tsv index 9d8aead53..2c321e858 100644 --- a/postgres/scripts/tables/SampleContext.tsv +++ b/postgres/scripts/tables/SampleContext.tsv @@ -65,3 +65,4 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological +i dont know From 00fd0ff36a435b9a31bdb380241612ea87e8026c Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 11:01:48 +0000 Subject: [PATCH 083/119] Added postgres dump for test.tsv --- postgres/EMPD2.sql | 114 +++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 60 deletions(-) diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql index 359334330..8c6961531 100644 --- a/postgres/EMPD2.sql +++ b/postgres/EMPD2.sql @@ -2,11 +2,12 @@ -- PostgreSQL database dump -- --- Dumped from database version 9.5.15 --- Dumped by pg_dump version 9.5.15 +-- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) SET statement_timeout = 0; SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); @@ -14,20 +15,6 @@ SET check_function_bodies = false; SET client_min_messages = warning; SET row_security = off; --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: --- - -COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - SET default_tablespace = ''; SET default_with_oids = false; @@ -322,6 +309,9 @@ C Within last 250 years 50-200BP -- COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 \. @@ -490,6 +480,9 @@ X Do not use! \N -- COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 +test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know \N \N \N \N \N \N f \N \N EMPD2 \. @@ -514,6 +507,12 @@ COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -- COPY public.p_counts (samplename, var_, count) FROM stdin; +test_a2 1 45 +test_a2 2 6 +test_a2 3 55 +test_a3 1 45 +test_a3 2 6 +test_a3 3 55 \. @@ -522,6 +521,9 @@ COPY public.p_counts (samplename, var_, count) FROM stdin; -- COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL \. @@ -604,6 +606,7 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological +i dont know \. @@ -676,7 +679,7 @@ COPY public.workers (workerid, firstname, lastname, initials, address1, email1, -- --- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ageuncertainties @@ -684,7 +687,7 @@ ALTER TABLE ONLY public.ageuncertainties -- --- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.countries @@ -692,7 +695,7 @@ ALTER TABLE ONLY public.countries -- --- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -700,7 +703,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.groupid @@ -708,7 +711,7 @@ ALTER TABLE ONLY public.groupid -- --- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.locationreliabilities @@ -716,7 +719,7 @@ ALTER TABLE ONLY public.locationreliabilities -- --- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -724,7 +727,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -732,7 +735,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -740,7 +743,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -748,7 +751,7 @@ ALTER TABLE ONLY public.climate -- --- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -756,7 +759,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -764,7 +767,7 @@ ALTER TABLE ONLY public.p_vars -- --- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -772,7 +775,7 @@ ALTER TABLE ONLY public.publications -- --- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -780,7 +783,7 @@ ALTER TABLE ONLY public.publications -- --- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplecontexts @@ -788,7 +791,7 @@ ALTER TABLE ONLY public.samplecontexts -- --- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplemethods @@ -796,7 +799,7 @@ ALTER TABLE ONLY public.samplemethods -- --- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.sampletypes @@ -804,7 +807,7 @@ ALTER TABLE ONLY public.sampletypes -- --- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workerroles @@ -812,7 +815,7 @@ ALTER TABLE ONLY public.workerroles -- --- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workers @@ -820,7 +823,7 @@ ALTER TABLE ONLY public.workers -- --- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -828,7 +831,7 @@ ALTER TABLE ONLY public.climate -- --- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -836,7 +839,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -844,7 +847,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -852,7 +855,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -860,7 +863,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -868,7 +871,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -876,7 +879,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -884,7 +887,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -892,7 +895,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -900,7 +903,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -908,7 +911,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -916,7 +919,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -924,7 +927,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -932,7 +935,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -940,23 +943,14 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); --- --- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres --- - -REVOKE ALL ON SCHEMA public FROM PUBLIC; -REVOKE ALL ON SCHEMA public FROM postgres; -GRANT ALL ON SCHEMA public TO postgres; -GRANT ALL ON SCHEMA public TO PUBLIC; - - -- -- PostgreSQL database dump complete -- + From 56e09a1cf1a0e1b2f5dbdd4d9577d79906b43d78 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 11:01:50 +0000 Subject: [PATCH 084/119] Updated tab-delimited files from EMPD2 postgres database --- tab-delimited/climate.tsv | 3 +++ tab-delimited/metadata.tsv | 3 +++ tab-delimited/p_counts.tsv | 6 ++++++ tab-delimited/p_vars.tsv | 3 +++ tab-delimited/samplecontexts.tsv | 1 + 5 files changed, 16 insertions(+) diff --git a/tab-delimited/climate.tsv b/tab-delimited/climate.tsv index f30354209..927f140e7 100644 --- a/tab-delimited/climate.tsv +++ b/tab-delimited/climate.tsv @@ -1 +1,4 @@ samplename t_jan t_feb t_mar t_apr t_may t_jun t_jul t_aug t_sep t_oct t_nov t_dec t_djf t_mam t_jja t_son t_ann p_jan p_feb p_mar p_apr p_may p_jun p_jul p_aug p_sep p_oct p_nov p_dec p_djf p_mam p_jja p_son p_ann +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 diff --git a/tab-delimited/metadata.tsv b/tab-delimited/metadata.tsv index f31e489d1..05b7cc8e8 100644 --- a/tab-delimited/metadata.tsv +++ b/tab-delimited/metadata.tsv @@ -1 +1,4 @@ samplename originalsamplename sitename country longitude latitude elevation locationreliability locationnotes areaofsite samplecontext sitedescription vegdescription sampletype samplemethod agebp ageuncertainty ispercent notes okexcept empd_version +test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 +test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know f EMPD2 diff --git a/tab-delimited/p_counts.tsv b/tab-delimited/p_counts.tsv index 8b51c97bb..1fbbceef4 100644 --- a/tab-delimited/p_counts.tsv +++ b/tab-delimited/p_counts.tsv @@ -1 +1,7 @@ samplename var_ count +test_a2 1 45 +test_a2 2 6 +test_a2 3 55 +test_a3 1 45 +test_a3 2 6 +test_a3 3 55 diff --git a/tab-delimited/p_vars.tsv b/tab-delimited/p_vars.tsv index e0f7475b9..a5d1fca17 100644 --- a/tab-delimited/p_vars.tsv +++ b/tab-delimited/p_vars.tsv @@ -1 +1,4 @@ var_ acc_var_ original_varname acc_varname groupid notes +1 Pinus Pinus TRSH NULL +2 Exotic Exotics (counted) NOPO NULL +3 Umbelliferae Apiaceae HERB NULL diff --git a/tab-delimited/samplecontexts.tsv b/tab-delimited/samplecontexts.tsv index 9c818983e..ce53754ae 100644 --- a/tab-delimited/samplecontexts.tsv +++ b/tab-delimited/samplecontexts.tsv @@ -65,3 +65,4 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological +i dont know From 05cc0a3608fb11cb2425cff355fe4868369a006d Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 25 Mar 2019 11:01:50 +0000 Subject: [PATCH 085/119] Removed test.tsv to finish the PR --- test.tsv | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 test.tsv diff --git a/test.tsv b/test.tsv deleted file mode 100644 index dbc448a22..000000000 --- a/test.tsv +++ /dev/null @@ -1,4 +0,0 @@ -SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 60dbc57f772b0e4357ea8792ed52bf1919be05a2 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:12:40 +0100 Subject: [PATCH 086/119] Merge branch 'upstream/master' into test-data From f0f3fc9408eec88427787787cd3e6bf2354fe8cb Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:12:40 +0100 Subject: [PATCH 087/119] Revert "Removed test.tsv to finish the PR" This reverts commit 05cc0a3608fb11cb2425cff355fe4868369a006d. --- test.tsv | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test.tsv diff --git a/test.tsv b/test.tsv new file mode 100644 index 000000000..dbc448a22 --- /dev/null +++ b/test.tsv @@ -0,0 +1,4 @@ +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept +test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 3d03039993ba1eb4fbe73df5713cb80b9abab44b Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:12:49 +0100 Subject: [PATCH 088/119] Revert "Updated tab-delimited files from EMPD2 postgres database" This reverts commit 56e09a1cf1a0e1b2f5dbdd4d9577d79906b43d78. --- tab-delimited/climate.tsv | 3 --- tab-delimited/metadata.tsv | 3 --- tab-delimited/p_counts.tsv | 6 ------ tab-delimited/p_vars.tsv | 3 --- tab-delimited/samplecontexts.tsv | 1 - 5 files changed, 16 deletions(-) diff --git a/tab-delimited/climate.tsv b/tab-delimited/climate.tsv index 927f140e7..f30354209 100644 --- a/tab-delimited/climate.tsv +++ b/tab-delimited/climate.tsv @@ -1,4 +1 @@ samplename t_jan t_feb t_mar t_apr t_may t_jun t_jul t_aug t_sep t_oct t_nov t_dec t_djf t_mam t_jja t_son t_ann p_jan p_feb p_mar p_apr p_may p_jun p_jul p_aug p_sep p_oct p_nov p_dec p_djf p_mam p_jja p_son p_ann -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 diff --git a/tab-delimited/metadata.tsv b/tab-delimited/metadata.tsv index 05b7cc8e8..f31e489d1 100644 --- a/tab-delimited/metadata.tsv +++ b/tab-delimited/metadata.tsv @@ -1,4 +1 @@ samplename originalsamplename sitename country longitude latitude elevation locationreliability locationnotes areaofsite samplecontext sitedescription vegdescription sampletype samplemethod agebp ageuncertainty ispercent notes okexcept empd_version -test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 f EMPD2 -test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know f EMPD2 diff --git a/tab-delimited/p_counts.tsv b/tab-delimited/p_counts.tsv index 1fbbceef4..8b51c97bb 100644 --- a/tab-delimited/p_counts.tsv +++ b/tab-delimited/p_counts.tsv @@ -1,7 +1 @@ samplename var_ count -test_a2 1 45 -test_a2 2 6 -test_a2 3 55 -test_a3 1 45 -test_a3 2 6 -test_a3 3 55 diff --git a/tab-delimited/p_vars.tsv b/tab-delimited/p_vars.tsv index a5d1fca17..e0f7475b9 100644 --- a/tab-delimited/p_vars.tsv +++ b/tab-delimited/p_vars.tsv @@ -1,4 +1 @@ var_ acc_var_ original_varname acc_varname groupid notes -1 Pinus Pinus TRSH NULL -2 Exotic Exotics (counted) NOPO NULL -3 Umbelliferae Apiaceae HERB NULL diff --git a/tab-delimited/samplecontexts.tsv b/tab-delimited/samplecontexts.tsv index ce53754ae..9c818983e 100644 --- a/tab-delimited/samplecontexts.tsv +++ b/tab-delimited/samplecontexts.tsv @@ -65,4 +65,3 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological -i dont know From 29c4f927584f2774419fba93af08e7f4ff57115d Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:12:52 +0100 Subject: [PATCH 089/119] Revert "Added postgres dump for test.tsv" This reverts commit 00fd0ff36a435b9a31bdb380241612ea87e8026c. --- postgres/EMPD2.sql | 114 ++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/postgres/EMPD2.sql b/postgres/EMPD2.sql index 8c6961531..359334330 100644 --- a/postgres/EMPD2.sql +++ b/postgres/EMPD2.sql @@ -2,12 +2,11 @@ -- PostgreSQL database dump -- --- Dumped from database version 11.2 (Debian 11.2-1.pgdg90+1) --- Dumped by pg_dump version 11.2 (Debian 11.2-1.pgdg90+1) +-- Dumped from database version 9.5.15 +-- Dumped by pg_dump version 9.5.15 SET statement_timeout = 0; SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); @@ -15,6 +14,20 @@ SET check_function_bodies = false; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + SET default_tablespace = ''; SET default_with_oids = false; @@ -309,9 +322,6 @@ C Within last 250 years 50-200BP -- COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; -test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 -test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 \. @@ -480,9 +490,6 @@ X Do not use! \N -- COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere Germany 10 50 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 -test_a2 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 \N \N \N \N \N \N \N f \N \N EMPD2 -test_a3 orig_test_a2 somewhere Germany 10 52 340 X 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 30 i dont know \N \N \N \N \N \N f \N \N EMPD2 \. @@ -507,12 +514,6 @@ COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -- COPY public.p_counts (samplename, var_, count) FROM stdin; -test_a2 1 45 -test_a2 2 6 -test_a2 3 55 -test_a3 1 45 -test_a3 2 6 -test_a3 3 55 \. @@ -521,9 +522,6 @@ test_a3 3 55 -- COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; -1 \N Pinus Pinus TRSH NULL -2 \N Exotic Exotics (counted) NOPO NULL -3 \N Umbelliferae Apiaceae HERB NULL \. @@ -606,7 +604,6 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological -i dont know \. @@ -679,7 +676,7 @@ COPY public.workers (workerid, firstname, lastname, initials, address1, email1, -- --- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ageuncertainties @@ -687,7 +684,7 @@ ALTER TABLE ONLY public.ageuncertainties -- --- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.countries @@ -695,7 +692,7 @@ ALTER TABLE ONLY public.countries -- --- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -703,7 +700,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.groupid @@ -711,7 +708,7 @@ ALTER TABLE ONLY public.groupid -- --- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.locationreliabilities @@ -719,7 +716,7 @@ ALTER TABLE ONLY public.locationreliabilities -- --- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -727,7 +724,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -735,7 +732,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -743,7 +740,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -751,7 +748,7 @@ ALTER TABLE ONLY public.climate -- --- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -759,7 +756,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -767,7 +764,7 @@ ALTER TABLE ONLY public.p_vars -- --- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -775,7 +772,7 @@ ALTER TABLE ONLY public.publications -- --- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.publications @@ -783,7 +780,7 @@ ALTER TABLE ONLY public.publications -- --- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplecontexts @@ -791,7 +788,7 @@ ALTER TABLE ONLY public.samplecontexts -- --- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.samplemethods @@ -799,7 +796,7 @@ ALTER TABLE ONLY public.samplemethods -- --- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.sampletypes @@ -807,7 +804,7 @@ ALTER TABLE ONLY public.sampletypes -- --- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workerroles @@ -815,7 +812,7 @@ ALTER TABLE ONLY public.workerroles -- --- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.workers @@ -823,7 +820,7 @@ ALTER TABLE ONLY public.workers -- --- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.climate @@ -831,7 +828,7 @@ ALTER TABLE ONLY public.climate -- --- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.ecosystems @@ -839,7 +836,7 @@ ALTER TABLE ONLY public.ecosystems -- --- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -847,7 +844,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -855,7 +852,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -863,7 +860,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -871,7 +868,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -879,7 +876,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metadata @@ -887,7 +884,7 @@ ALTER TABLE ONLY public.metadata -- --- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -895,7 +892,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metapubli @@ -903,7 +900,7 @@ ALTER TABLE ONLY public.metapubli -- --- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -911,7 +908,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -919,7 +916,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.metaworker @@ -927,7 +924,7 @@ ALTER TABLE ONLY public.metaworker -- --- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -935,7 +932,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_counts @@ -943,7 +940,7 @@ ALTER TABLE ONLY public.p_counts -- --- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- Name: p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- ALTER TABLE ONLY public.p_vars @@ -951,6 +948,15 @@ ALTER TABLE ONLY public.p_vars -- --- PostgreSQL database dump complete +-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres -- +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM postgres; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- From a65d0ceb387b2797be4eb5fcef71e745820daaca Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:12:55 +0100 Subject: [PATCH 090/119] Revert "Updated fixed tables" This reverts commit b8eeb67b4590da73c16b285b5d34a2027e6808ad. --- postgres/scripts/tables/SampleContext.tsv | 1 - 1 file changed, 1 deletion(-) diff --git a/postgres/scripts/tables/SampleContext.tsv b/postgres/scripts/tables/SampleContext.tsv index 2c321e858..9d8aead53 100644 --- a/postgres/scripts/tables/SampleContext.tsv +++ b/postgres/scripts/tables/SampleContext.tsv @@ -65,4 +65,3 @@ wetland bog wetland bog/scattered trees/shrubs wetland bog/treeless vegetation archaeological -i dont know From 951baea3a50a5a3a4c5697b9887c5e5b04af2d75 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:13:12 +0100 Subject: [PATCH 091/119] Revert "Merged test.tsv into meta.tsv [skip ci]" This reverts commit a7976b54f260770ff91abc98722ed7210bfd9c13. --- meta.tsv | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta.tsv b/meta.tsv index dbc448a22..9c9cfeb6f 100644 --- a/meta.tsv +++ b/meta.tsv @@ -1,4 +1 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From b3cac3cb214c9a598c1b9dede8ac69c0482f8baa Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:13:30 +0100 Subject: [PATCH 092/119] Revert "[test_a1]: Changed country from France to Germany" This reverts commit 9128681cc04d13c442d11a424cbcedccd1b047df. --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index dbc448a22..2650801dc 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere Germany 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 73ffad20f20aea6ba6ae85a18fbdc25eaa92efdb Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 1 Apr 2019 06:59:47 +0200 Subject: [PATCH 093/119] Updated 1 samples in test.tsv as requested by Philipp Sommer --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 2650801dc..25b512de8 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 79768dffc14c2a84017431d56764a520798ffcd8 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 1 Apr 2019 07:40:29 +0200 Subject: [PATCH 094/119] Updated 1 samples in test.tsv as requested by {submitter} --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 25b512de8..b2be9869f 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a12 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 279f8ff137275dd08cd330f43db3e7e47029ef30 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 1 Apr 2019 16:06:06 +0200 Subject: [PATCH 095/119] Updated 1 sample in test.tsv as requested by {submitter} --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index b2be9869f..25b512de8 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a12 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From a902077dde74ea5f8140f0ea45109d20f85c04f4 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 1 Apr 2019 16:01:39 +0000 Subject: [PATCH 096/119] Updated 1 sample in test.tsv as requested by {submitter} --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 25b512de8..b2be9869f 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a12 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From a0b6efc98fa3b62b3eed276425b42a4cfa062d86 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 1 Apr 2019 19:09:26 +0000 Subject: [PATCH 097/119] Updated 1 sample in test.tsv as requested by {submitter} --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index b2be9869f..25b512de8 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a12 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 512a50c8a1d0986e034f7fe4ddb26c97f7578244 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Mon, 1 Apr 2019 19:09:47 +0000 Subject: [PATCH 098/119] Updated 1 sample in test.tsv as requested by {submitter} --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 25b512de8..b2be9869f 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a12 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 43ab5e1c67edb7a2bc02125259f0d039b447bcb0 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Tue, 2 Apr 2019 13:36:56 +0000 Subject: [PATCH 099/119] Updated 1 sample in test.tsv as requested by {submitter} --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index b2be9869f..2ef76d35f 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a12 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 False test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From b81ae5b3906d00ab5b02cbdffa353cf9d609ab55 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Wed, 3 Apr 2019 10:45:04 +0000 Subject: [PATCH 100/119] Updated 1 sample in test.tsv as requested by {submitter} --- test.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.tsv b/test.tsv index 2ef76d35f..f359f1bf3 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept -test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 False +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 True test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From f3dc5d1c9a0b1fcbfec3d1a1aa8a32868b14f6e2 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Mon, 25 Mar 2019 14:12:24 +0100 Subject: [PATCH 101/119] set initial publi_id and worker_id from existing tables --- postgres/scripts/import_into_empd2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index bfd3ce071..51729319d 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -71,8 +71,11 @@ def clean_doi(doi): return DOI -PUBLI_ID = 1 # TODO: Has to be set from EMPD2 -WORKER_ID = 1 # TODO: Has to be set from EMPD2 +cursor.execute('SELECT MAX(publiid) FROM publications') +PUBLI_ID = (cursor.fetchall()[0][0] or 0) + 1 + +cursor.execute('SELECT MAX(workerid) FROM workers') +WORKER_ID = (cursor.fetchall()[0][0] or 0) + 1 err = 0 list_of_errors = [] From bfe1321410b313d6cf0c5c472ff9d41187b9513e Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 27 Mar 2019 12:06:25 +0100 Subject: [PATCH 102/119] added extract-failed argument --- tests/conftest.py | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 75db3f2c0..5e96fc11c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ # configuration module for pytest from collections import OrderedDict import pathlib +import os import os.path as osp import pytest from functools import partial @@ -10,6 +11,8 @@ report_path = '' +extract_failed = False + _meta_file = pathlib.Path( osp.join(osp.dirname(osp.abspath(__file__)), '..', 'meta.tsv')).resolve() @@ -199,13 +202,23 @@ def get_priority(report): return 2 -def pytest_runtest_logreport(report): - if report_path: - all_reports.setdefault(report.nodeid, []).append(report) +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # we only look at actual failing test calls, not setup/teardown + if rep.when == "call" and rep.failed: + all_reports.setdefault(rep.nodeid, []).append(rep) def pytest_sessionfinish(session): md = '' + meta = _meta() + samples = meta.index.values + failed_samples = set() + for key, reports in all_reports.items(): reports.sort(key=get_priority) @@ -233,10 +246,26 @@ def pytest_sessionfinish(session): key, '```\n%s\n```' % s if key == 'error' else s) md += '\n' + details + '\n' md += '\n' + if report.outcome == 'failed': + failed_samples.update(set(report.keywords).intersection(samples)) if report_path: report_path.write_text(md, encoding='utf-8') + if extract_failed: + target_dir = osp.join(osp.dirname(_meta_file), 'failures') + target = osp.join(target_dir, extract_failed) + os.makedirs(target_dir, exist_ok=True) + meta.loc[list(failed_samples)].to_csv( + target, sep='\t', float_format='%1.8g') + + if _commit_fixes: + from git import Repo + local_repo = Repo(osp.dirname(_meta_file)) + local_repo.index.add([target]) + local_repo.index.commit( + f"Extracted failures to {target} [skip ci]") + def pytest_addoption(parser): group = parser.getgroup("terminal reporting") @@ -257,15 +286,24 @@ def pytest_addoption(parser): "well.")) group.addoption('--empd-meta', default=str(_meta_file), help="The path to the meta file") + group.addoption( + '--extract-failed', metavar='filename.tsv', nargs='?', + const='failed.tsv', default=False, + help=("Extract the meta data of failed samples into a separate file " + "in the `failures` directory. Without argument, failed samples " + "will be extracted to ``%(const)s``.")) def pytest_configure(config): global report_path, _meta_file, _base_meta_file, _commit_fixes + global extract_failed if config.option.markdown_report: report_path = pathlib.Path( config.option.markdown_report).expanduser().resolve() if config.option.commit: _commit_fixes = True + if config.option.extract_failed: + extract_failed = config.option.extract_failed _meta_file = pathlib.Path(config.option.empd_meta).expanduser().resolve() _base_meta_file = pathlib.Path( osp.join(osp.dirname(_meta_file), 'meta.tsv')) From 7d6dbf9c7b19dcef4107dd4fa8193fdd1be86a62 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 27 Mar 2019 12:07:38 +0100 Subject: [PATCH 103/119] added fix_trailing_spaces and fix_newline_chars --- tests/fixes.py | 56 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/fixes.py b/tests/fixes.py index 32c479f70..bdc882ca3 100644 --- a/tests/fixes.py +++ b/tests/fixes.py @@ -16,16 +16,59 @@ def get_elevation(points): return elevation +@pytest.mark.formatfix +@pytest.mark.dbfix +def fix_trailing_spaces(meta, meta_row, meta_file, commit_fixes, local_repo, + skip_ci): + meta_row = meta_row.astype(str) + stripped = meta_row.str.strip() + cols = meta_row[stripped != meta_row].index + if len(cols): + message = "Removed trailing spaces for %s" % (cols, ) + print(message) + meta.loc[meta_row.name, cols] = stripped[cols] + meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') + if commit_fixes: + local_repo.index.add([meta_file.name]) + local_repo.index.commit( + ("[%s]: %s" % (meta_row.name, message)) + + ('\n\n[skip ci]' if skip_ci else '')) + + +@pytest.mark.formatfix +@pytest.mark.dbfix +def fix_newline_chars(meta, meta_row, meta_file, commit_fixes, local_repo, + skip_ci): + meta_row = meta_row.astype(str) + stripped = meta_row.str.replace('\n', ' ').str.replace('\r', ' ') + cols = meta_row[stripped != meta_row].index + if len(cols): + message = "Removed newline characters for %s" % (cols, ) + print(message) + meta.loc[meta_row.name, cols] = stripped[cols] + meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') + if commit_fixes: + local_repo.index.add([meta_file.name]) + local_repo.index.commit( + ("[%s]: %s" % (meta_row.name, message)) + + ('\n\n[skip ci]' if skip_ci else '')) + + +@pytest.mark.metafix @pytest.mark.dbfix def fix_country(meta, meta_row, meta_file, commit_fixes, local_repo, - skip_ci): + skip_ci, countries): country = get_country(meta_row.Latitude, meta_row.Longitude) + try: + country = countries.loc[country] + except KeyError: + pass current = getattr(meta_row, 'Country', '') if country != current: message = "Changed country from %s to %s" % (current, country) print(message) meta.loc[meta_row.name, 'Country'] = country - meta.to_csv(str(meta_file), sep='\t') + meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( @@ -33,6 +76,7 @@ def fix_country(meta, meta_row, meta_file, commit_fixes, local_repo, ('\n\n[skip ci]' if skip_ci else '')) +@pytest.mark.metafix @pytest.mark.dbfix def fix_elevation(meta, meta_file, commit_fixes, local_repo, skip_ci): try: @@ -45,7 +89,7 @@ def fix_elevation(meta, meta_file, commit_fixes, local_repo, skip_ci): points = meta.loc[samples, ['Latitude', 'Longitude']].values elev = get_elevation(points) meta.loc[samples, 'Elevation'] = elev - meta.to_csv(str(meta_file), sep='\t') + meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( @@ -54,6 +98,7 @@ def fix_elevation(meta, meta_file, commit_fixes, local_repo, skip_ci): pytest.skip("All elevations already specified") +@pytest.mark.metafix @pytest.mark.dbfix def fix_temperature(meta, meta_file, commit_fixes, local_repo, skip_ci): try: @@ -68,13 +113,14 @@ def fix_temperature(meta, meta_file, commit_fixes, local_repo, skip_ci): variables=['tavg']) meta.loc[samples, 'Temperature'] = list(map( ','.join, temperature.values.astype(str))) - meta.to_csv(str(meta_file), sep='\t') + meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( message + ('\n\n[skip ci]' if skip_ci else '')) +@pytest.mark.metafix @pytest.mark.dbfix def fix_precipitation(meta, meta_file, commit_fixes, local_repo, skip_ci): try: @@ -89,7 +135,7 @@ def fix_precipitation(meta, meta_file, commit_fixes, local_repo, skip_ci): variables=['prec']) meta.loc[samples, 'Precipitation'] = list(map( ','.join, precip.values.astype(str))) - meta.to_csv(str(meta_file), sep='\t') + meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( From 77d52ac41e871b74e68ca8f172e13c7cf1239f46 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 27 Mar 2019 12:08:00 +0100 Subject: [PATCH 104/119] save the original metadata after having removed okexcept fields --- postgres/scripts/import_into_empd2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index 51729319d..1605b0748 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -83,6 +83,7 @@ def clean_doi(doi): METADATA = pd.read_csv(meta, sep='\t') +orig_METADATA = METADATA.copy(True) base_cols = pd.read_csv(base_meta, nrows=1, sep='\t').columns for col in set(base_cols) - set(METADATA.columns): @@ -94,6 +95,7 @@ def clean_doi(doi): # check okexcept column and update fixed tables okexcept = defaultdict(set) +save_orig = False for key, row in METADATA[METADATA.okexcept.astype(bool)].iterrows(): if row.okexcept and str(row.okexcept) != 'nan': row_okexcept = row.okexcept.split(',') @@ -104,6 +106,11 @@ def clean_doi(doi): okexcept[col].add(row[col]) row_okexcept.remove(col) METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) + orig_METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) + save_orig = True + +if save_orig: + orig_METADATA.to_csv(meta, sep='\t', index=False, float_format='%1.8g') table_map = { 'Country': 'countries', From 7ce6c4b6eca3c34a5745a900164e5b805f71fa5c Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 27 Mar 2019 22:16:04 +0100 Subject: [PATCH 105/119] Always add dbfix mark when fix-db option is passed --- tests/conftest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5e96fc11c..200d0e225 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -307,8 +307,8 @@ def pytest_configure(config): _meta_file = pathlib.Path(config.option.empd_meta).expanduser().resolve() _base_meta_file = pathlib.Path( osp.join(osp.dirname(_meta_file), 'meta.tsv')) - if not config.option.fix_db: - if getattr(config.option, 'markexpr', None): - config.option.markexpr += ' and not dbfix' - else: - config.option.markexpr = 'not dbfix' + fix_db = config.option.fix_db + if getattr(config.option, 'markexpr', None): + config.option.markexpr += ' and %sdbfix' % ('' if fix_db else 'not ') + else: + config.option.markexpr = '%sdbfix' % ('' if fix_db else 'not ') From 6f07b3e58971fd28f8f8c0cdef142852b8bb7c75 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 27 Mar 2019 22:16:43 +0100 Subject: [PATCH 106/119] Fixed fixing of temperatures and precipitation --- tests/fixes.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/fixes.py b/tests/fixes.py index bdc882ca3..7918d600f 100644 --- a/tests/fixes.py +++ b/tests/fixes.py @@ -102,11 +102,14 @@ def fix_elevation(meta, meta_file, commit_fixes, local_repo, skip_ci): @pytest.mark.dbfix def fix_temperature(meta, meta_file, commit_fixes, local_repo, skip_ci): try: - samples = meta[meta.Temperature.isnull()].index.values + mask = meta.Temperature.isnull() + mask |= meta.Temperature.astype(str).str.contains('nan') except AttributeError: - samples = meta.index.values + mask = meta.index.astype(bool) + mask &= meta.Longitude.notnull() & meta.Latitude.notnull() + samples = meta[mask].index.values if len(samples): - message = "Set temperature from WorldClim v2 for %s" % samples + message = "Set temperature from WorldClim v2 for %i" % len(samples) print(message) temperature = get_climate( *meta.loc[samples, ['Latitude', 'Longitude']].values.T, @@ -117,18 +120,22 @@ def fix_temperature(meta, meta_file, commit_fixes, local_repo, skip_ci): if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( - message + ('\n\n[skip ci]' if skip_ci else '')) + message + ('\n\n[skip ci]\n' if skip_ci else '\n\n') + samples) @pytest.mark.metafix @pytest.mark.dbfix def fix_precipitation(meta, meta_file, commit_fixes, local_repo, skip_ci): try: - samples = meta[meta.Precipitation.isnull()].index.values + mask = meta.Precipitation.isnull() + mask |= meta.Precipitation.astype(str).str.contains('nan') except AttributeError: - samples = meta.index.values + mask = meta.index.astype(bool) + mask &= meta.Longitude.notnull() & meta.Latitude.notnull() + samples = meta[mask].index.values if len(samples): - message = "Set temperature from WorldClim v2 for %s" % samples + + message = "Set precipitation from WorldClim v2 for %i" % len(samples) print(message) precip = get_climate( *meta.loc[samples, ['Latitude', 'Longitude']].values.T, @@ -139,4 +146,4 @@ def fix_precipitation(meta, meta_file, commit_fixes, local_repo, skip_ci): if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( - message + ('\n\n[skip ci]' if skip_ci else '')) + message + ('\n\n[skip ci]\n' if skip_ci else '\n\n') + samples) From e8d93bf12abcd1621fd2d4de0a953cc623bff0b5 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 27 Mar 2019 22:22:25 +0100 Subject: [PATCH 107/119] minor --- tests/fixes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/fixes.py b/tests/fixes.py index 7918d600f..151a1d0f2 100644 --- a/tests/fixes.py +++ b/tests/fixes.py @@ -120,7 +120,8 @@ def fix_temperature(meta, meta_file, commit_fixes, local_repo, skip_ci): if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( - message + ('\n\n[skip ci]\n' if skip_ci else '\n\n') + samples) + message + ('\n\n[skip ci]\n' if skip_ci else '\n\n') + + ', '.join(samples)) @pytest.mark.metafix @@ -146,4 +147,5 @@ def fix_precipitation(meta, meta_file, commit_fixes, local_repo, skip_ci): if commit_fixes: local_repo.index.add([meta_file.name]) local_repo.index.commit( - message + ('\n\n[skip ci]\n' if skip_ci else '\n\n') + samples) + message + ('\n\n[skip ci]\n' if skip_ci else '\n\n') + + ', '.join(samples)) From ba030e505aa3a32d0b04074b97ab7eecff0745f8 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 27 Mar 2019 22:33:50 +0100 Subject: [PATCH 108/119] minor change in precip and temperature fix --- tests/fixes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fixes.py b/tests/fixes.py index 151a1d0f2..0afb36542 100644 --- a/tests/fixes.py +++ b/tests/fixes.py @@ -114,8 +114,8 @@ def fix_temperature(meta, meta_file, commit_fixes, local_repo, skip_ci): temperature = get_climate( *meta.loc[samples, ['Latitude', 'Longitude']].values.T, variables=['tavg']) - meta.loc[samples, 'Temperature'] = list(map( - ','.join, temperature.values.astype(str))) + meta.loc[samples, 'Temperature'] = temperature.to_csv( + header=False, index=False, float_format='%1.8g').splitlines() meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') if commit_fixes: local_repo.index.add([meta_file.name]) @@ -141,8 +141,8 @@ def fix_precipitation(meta, meta_file, commit_fixes, local_repo, skip_ci): precip = get_climate( *meta.loc[samples, ['Latitude', 'Longitude']].values.T, variables=['prec']) - meta.loc[samples, 'Precipitation'] = list(map( - ','.join, precip.values.astype(str))) + meta.loc[samples, 'Precipitation'] = precip.to_csv( + header=False, index=False, float_format='%1.8g').splitlines() meta.to_csv(str(meta_file), sep='\t', float_format='%1.8g') if commit_fixes: local_repo.index.add([meta_file.name]) From 2cf8ace93daad4c6d0e68ca7048d0971cf23a5b3 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Thu, 28 Mar 2019 16:04:55 +0100 Subject: [PATCH 109/119] do not run tests per meta_row --- postgres/scripts/import_into_empd2.py | 7 +- postgres/scripts/tables/Country.tsv | 16 +- tests/conftest.py | 91 +++++- tests/fixes.py | 8 +- tests/test_critical.py | 31 +- tests/test_formatting.py | 46 ++- tests/test_meta.py | 442 +++++++++++++++++++------- 7 files changed, 495 insertions(+), 146 deletions(-) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index 1605b0748..bf1a2ed12 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -101,10 +101,15 @@ def clean_doi(doi): row_okexcept = row.okexcept.split(',') for col in row_okexcept[:]: col = col.strip() - if col in ['Country', 'GroupID', 'SampleContext', 'SampleMethod', + if col in ['Country', 'SampleContext', 'SampleMethod', 'SampleType'] and notnull(row[col]): okexcept[col].add(row[col]) row_okexcept.remove(col) + elif col == 'GroupID': + COUNTS = pd.read_csv( + os.path.join(samples_dir, row.SampleName + '.tsv'), '\t') + okexcept[col].update(COUNTS['groupid']) + row_okexcept.remove(col) METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) orig_METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) save_orig = True diff --git a/postgres/scripts/tables/Country.tsv b/postgres/scripts/tables/Country.tsv index f4572c6e6..b7aa5a8b0 100644 --- a/postgres/scripts/tables/Country.tsv +++ b/postgres/scripts/tables/Country.tsv @@ -10,10 +10,11 @@ Belarus Belarus Belgium Belgium Bosnia And Herzegovina Bosnia and Herzegovina Bulgaria Bulgaria -Cape Verde Cape Verde +Cape Verde Cabo Verde +China, People s Republic Of China Croatia Croatia Cyprus Cyprus -Czech Republic Czech Republic +Czech Republic Czechia Denmark Denmark Egypt Egypt Eritrea Eritrea @@ -30,15 +31,16 @@ Greenland Greenland Guernsey Guernsey Hungary Hungary Iceland Iceland +India India Iran, Islamic Republic Of Iran Iraq Iraq Ireland Ireland Isle Of Man Isle of Man Israel Israel Italy Italy -India India Japan Japan Jersey Jersey +Jordan Jordan Kazakhstan Kazakhstan Kuwait Kuwait Kyrgyzstan Kyrgyzstan @@ -52,9 +54,11 @@ Macedonia, The Former Yugoslav Republic Of Macedonia Malta Malta Moldova, Republic Of Moldova Monaco Monaco +Serbia And Montenegro Montenegro Morocco Morocco Netherlands Netherlands Norway Norway +Svalbard and Jan Mayen Norway Oman Oman Palestinian Territory, Occupied Palestine Poland Poland @@ -64,7 +68,6 @@ Romania Romania Russian Federation Russia San Marino San Marino Saudi Arabia Saudi Arabia -Serbia And Montenegro Montenegro Slovakia Slovakia Slovenia Slovenia Spain Spain @@ -73,14 +76,11 @@ Switzerland Switzerland Syrian Arab Republic Syria Tunisia Tunisia Turkey Turkey +Turkmenistan, Republic Of Turkmenistan Ukraine Ukraine United Arab Emirates United Arab Emirates United Kingdom United Kingdom Yemen Yemen -Jordan Jordan Black Sea Dead Sea -Turkmenistan, Republic Of Turkmenistan Adriatic Sea -Svalbard and Jan Mayen Norway -China, People s Republic Of China diff --git a/tests/conftest.py b/tests/conftest.py index 200d0e225..00eeb02db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,8 @@ import os.path as osp import pytest from functools import partial +from itertools import starmap +import textwrap all_reports = OrderedDict() @@ -24,7 +26,10 @@ def _meta(fname=None): import pandas as pd fname = fname or _meta_file - return pd.read_csv(str(fname), sep='\t', index_col='SampleName') + ret = pd.read_csv(str(fname), sep='\t', index_col='SampleName') + if 'okexcept' not in ret.columns: + ret['okexcept'] = '' + return ret def _data_files(): @@ -93,6 +98,24 @@ def countries(meta_file): 'natural_earth').iloc[:, 0] +@pytest.fixture(scope='session') +def nat_earth_countries(meta, countries): + import pandas as pd + import numpy as np + lat = meta.Latitude.astype(float).values + lon = meta.Longitude.astype(float).values + mask = ~(np.isnan(lat) | np.isnan(lon)) + try: + import geopandas as gpd # Using geopandas is much faster! + except ImportError: + from latlon_utils import get_country + countries = list(starmap(get_country, zip(lat[mask], lon[mask]))) + else: + from latlon_utils import get_country_gpd + countries = get_country_gpd(lat[mask], lon[mask]) + return pd.Series(countries, index=meta.index[mask]) + + @pytest.fixture(scope='session') def samplecontexts(meta_file): import pandas as pd @@ -135,6 +158,15 @@ def locationreliabilities(meta_file): sep='\t').iloc[:, 0].values +@pytest.fixture(scope='session') +def groupids(meta_file): + import pandas as pd + return pd.read_csv( + osp.join(osp.dirname(meta_file), 'tab-delimited', + 'groupid.tsv'), + sep='\t').iloc[:, 0].values + + @pytest.fixture(scope='session') def ageuncertainties(meta_file): import pandas as pd @@ -149,6 +181,12 @@ def samples_dir(meta_file): return osp.join(osp.dirname(meta_file), 'samples') +@pytest.fixture(scope='session') +def data_files(meta_file, meta): + base = osp.join(osp.dirname(meta_file), 'samples') + return [osp.join(base, sample + '.tsv') for sample in meta.index.values] + + @pytest.fixture def data_file(request): """The path to a data file (set with metafunc.parameterize down below)""" @@ -167,6 +205,18 @@ def meta_row(request): return request.param +@pytest.fixture(scope='session') +def okexcept(meta): + meta_okexcept = meta.okexcept.astype(str).fillna('') + + def okexcept(column): + ret = meta_okexcept.str.contains(column) + ret.name = f'okexcept({column})' + return ret + + return okexcept + + @pytest.fixture(scope='session') def local_repo(): try: @@ -214,6 +264,7 @@ def pytest_runtest_makereport(item, call): def pytest_sessionfinish(session): + import pandas as pd md = '' meta = _meta() samples = meta.index.values @@ -229,10 +280,11 @@ def pytest_sessionfinish(session): for attr in ['stderr', 'stdout', 'log']: if getattr(report, 'cap' + attr): d[attr] = getattr(report, 'cap' + attr) + user_props = dict(report.user_properties) if d and not any(report.skipped for report in reports): md += '
%s..%s\n' % ( report.nodeid, report.outcome.upper()) - if len(d) == 1: + if len(d) == 1 and not 'failed_samples' in user_props: message, val = next(iter(d.items())) if message == 'error': val = '\n\n```\n%s\n```' % val @@ -245,9 +297,40 @@ def pytest_sessionfinish(session): '%s\n
') % ( key, '```\n%s\n```' % s if key == 'error' else s) md += '\n' + details + '\n' + if 'failed_samples' in user_props or 'failed_data': + try: + df = user_props['failed_samples'] + summary = '%i failed samples' % len(df) + except KeyError: + df = user_props['failed_data'] + summary = '%i failed data rows' % len(df) + n = len(df) + df = df.head(session.config.getoption('--max-table-len')) + df = pd.concat([ + pd.DataFrame([('---', ) * len(df.columns)], + index=['---'], columns=df.columns), + df]) + details = ( + '
\n' + '%s\n\n' + '%s\n\nDisplaying %i of %i samples\n
') % ( + summary, + textwrap.indent( + df.to_csv(sep='|', float_format='%1.8g'), + '| '), len(df) - 1, n) + md += '\n' + details + '\n' + md += '\n' if report.outcome == 'failed': failed_samples.update(set(report.keywords).intersection(samples)) + if 'failed_samples' in user_props: + df = user_props['failed_samples'] + failed_samples.update(df.index) + meta = meta.join( + df[[col for col in df.columns if col not in meta.columns]]) + elif 'failed_data' in user_props: + df = user_props['failed_data'] + failed_samples.update(df.samplename) if report_path: report_path.write_text(md, encoding='utf-8') @@ -272,6 +355,10 @@ def pytest_addoption(parser): group.addoption( "--markdown-report", action="store", metavar="path", default=None, help="create markdown report file at given path.") + group.addoption( + '--max-table-len', metavar='NUM', type=int, default=100, + help=("The maximal length of tables in the markdown-report. " + "Default: %(default)s")) group = parser.getgroup('EMPD') group.addoption( "--fix-db", action='store_true', default=False, diff --git a/tests/fixes.py b/tests/fixes.py index 0afb36542..6db2c5afb 100644 --- a/tests/fixes.py +++ b/tests/fixes.py @@ -1,10 +1,16 @@ # Module for fixing errors in the meta data -from latlon_utils import get_country, get_climate import requests from itertools import starmap import pandas as pd import pytest +try: + import geopandas +except ImportError: + from latlon_utils import get_country, get_climate +else: + from latlon_utils import get_country_gpd as get_country, get_climate + def get_elevation(points): query = ('https://api.open-elevation.com/api/v1/lookup?locations=' + diff --git a/tests/test_critical.py b/tests/test_critical.py index dcf765edc..6b7d1d45f 100644 --- a/tests/test_critical.py +++ b/tests/test_critical.py @@ -1,5 +1,8 @@ # Critical tests for the EMPD data. If they do not run, you can abort the tests import pytest +import os.path as osp +import pandas as pd +import textwrap @pytest.mark.critical @@ -20,10 +23,24 @@ def test_samplename_duplicates(meta): @pytest.mark.critical -def test_data_columns(data_frame): - ref_cols = {'var_', 'samplename', 'count', 'acc_var_', - 'original_varname', 'acc_varname', 'groupid', 'groupname', - 'higher_groupid', 'included_in_percent_sum', 'make_percent', - 'higher_groupname', 'percentage'} - assert ref_cols <= set(data_frame.columns) - assert data_frame.columns.all() # make sure every column has a title +def test_data_columns(meta, data_files, record_property): + + def test_data(fname): + if osp.exists(fname): + data_frame = pd.read_csv(fname, sep='\t') + return (ref_cols <= set(data_frame.columns) and + data_frame.columns.fillna('').astype(bool).all()) + + ref_cols = {'count', 'original_varname', 'acc_varname', 'groupid'} + meta = meta.copy() + meta['files'] = data_files + meta['valid'] = meta['files'].apply(test_data) + + failed = meta[~meta['valid']] + + record_property('failed_samples', failed[['files', 'valid']]) + + msg = "Found %i invalid data columns: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg diff --git a/tests/test_formatting.py b/tests/test_formatting.py index 69dbfa6a3..eb21b392e 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -1,26 +1,46 @@ # Test the formatting of the data import pytest +import numpy as np -def test_trailing_spaces(meta_row): - meta_row = meta_row.astype(str) - assert (meta_row.str.strip() == meta_row).all() +def test_trailing_spaces(meta, record_property): + has_failed = np.zeros_like(meta, dtype=bool) + for i, (col, s) in enumerate(meta.astype(str).fillna('').items()): + has_failed[:, i] = s.str.strip() != s + failed = meta.where(has_failed).dropna(how='all').dropna(axis=1, how='all') + record_property('failed_samples', + failed.astype(str).fillna('').applymap(repr)) -def test_newline_chars(meta_row): - meta_row = meta_row.astype(str) - assert (meta_row.str.replace('\n', ' ').str.replace('\r', ' ') == - meta_row).all() + msg = "Found %i cells with trailing spaces" % has_failed.sum() + assert not len(failed), msg -@pytest.mark.parametrize('doi_field', ['DOI1', 'DOI2', 'DOI3', 'DOI4', 'DOI5']) -def test_doi(meta_row, doi_field): - if meta_row.notnull()[doi_field]: - doi = meta_row.copy(True)[doi_field] +def test_newline_chars(meta, record_property): + has_failed = np.zeros_like(meta, dtype=bool) + for i, (col, s) in enumerate(meta.astype(str).fillna('').items()): + has_failed[:, i] = s.str.replace('\n', ' ').str.replace( + '\r', ' ') != s + failed = meta.where(has_failed).dropna(how='all').dropna(axis=1, how='all') + record_property('failed_samples', + failed.astype(str).fillna('').applymap(repr)) + + msg = "Found %i cells with newline characters" % has_failed.sum() + assert not len(failed), msg + +@pytest.mark.parametrize('doi_field', ['DOI1', 'DOI2', 'DOI3', 'DOI4', 'DOI5']) +def test_doi(meta, doi_field, record_property): + if doi_field in meta.columns and meta[doi_field].fillna('').any(): + col = meta[doi_field].astype(str).fillna('') + orig = col for patt in ['https://doi.org/', 'http://dx.doi.org/', 'doi: ', 'doi.org/', 'DOI: ', 'DOI ', 'doi:', ' ', 'https://link.springer.com/article/']: - doi = doi.replace(patt, '') - assert doi == meta_row.loc[doi_field] + col = col.replace(patt, '') + failed = meta[col != orig] + record_property('failed_samples', failed[[doi_field]]) + + msg = "Found %i cells with invalid DOI" % len(failed) + assert not len(failed[doi_field]), msg else: pytest.skip("%s field empty specified" % doi_field) diff --git a/tests/test_meta.py b/tests/test_meta.py index 75c24aabd..1e9bec595 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -2,10 +2,10 @@ import os.path as osp import glob import pytest -from latlon_utils import get_country from itertools import starmap import numpy as np -import pandas as pd +from difflib import get_close_matches +import textwrap def okexcept(meta_row, col): @@ -35,128 +35,342 @@ def test_meta_exists(samples_dir, meta, base_meta): assert not missing, f"Missing meta information for samples {missing}" -def test_lat_lon(meta_row): - assert okexcept(meta_row, 'Latitude') or ( - meta_row.Latitude >= -90 and meta_row.Latitude <= 90) - assert okexcept(meta_row, 'Longitude') or ( - meta_row.Longitude >= -180 and meta_row.Longitude <= 360) - - -def test_country(meta_row, countries): - assert okexcept(meta_row, 'Country') or meta_row.Country in countries.loc[ - [get_country(*meta_row[['Latitude', 'Longitude']])]].values - - -def test_samplecontext(meta_row, samplecontexts): - samplecontext = str(getattr(meta_row, 'SampleContext', '')) - if isnull(samplecontext): +def test_lat_lon(meta, okexcept, record_property): + meta = meta.copy() + meta = meta.join(okexcept('Latitude')) + meta['Latitude_ok'] = (meta.Latitude >= -90) & (meta.Latitude <= 90) + meta = meta.join(okexcept('Longitude')) + meta['Longitude_ok'] = (meta.Longitude >= -180) & (meta.Longitude <= 360) + failed = meta[~(meta.iloc[:, -4] | meta.iloc[:, -3]) | + ~(meta.iloc[:, -2] | meta.iloc[:, -1])] + record_property('failed_samples', + failed[['Latitude', 'Longitude'] + list(meta.columns[-4:])] + ) + msg = "Found %i invalid Latitude-Longitude pairs: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_country(meta, countries, nat_earth_countries, okexcept, + record_property): + ref = countries + meta = meta.copy() + s = meta['Country'].fillna('') + + s_ok = okexcept(s.name) + s_ok |= (okexcept("Latitude") & okexcept("Longitude")).values + meta = meta.join(s_ok) + meta['nat_earth'] = nat_earth_countries + meta['empd_countries'] = [ + '; '.join(ref.loc[[n]]) if n in ref.index else c + for c, n in meta[[s.name, 'nat_earth']].values] + meta[s.name + '_ok'] = [n in ref.index and c in ref.loc[[n]].values + for c, n in meta[[s.name, 'nat_earth']].values] + failed = meta[~(s_ok | meta.Country_ok)] + record_property('failed_samples', + failed[[s.name, 'nat_earth', 'empd_countries', s_ok.name, + s.name + '_ok']] + ) + msg = "Found %i invalid %s: %s" % ( + len(failed), s.name if len(failed) == 1 else 'Countries', + textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_samplecontext(meta, samplecontexts, okexcept, record_property): + if 'SampleContext' not in meta.columns: return pytest.skip("No SampleContext specified.") - assert (samplecontext.lower() == samplecontext and - (okexcept(meta_row, 'SampleContext') or - samplecontext in samplecontexts)) - - -def test_sampletype(meta_row, sampletypes): - sampletype = str(getattr(meta_row, 'SampleType', '')) - if isnull(sampletype): + ref = samplecontexts + meta = meta.copy() + s = meta['SampleContext'].fillna('') + + s_ok = okexcept(s.name) + meta = meta.join(s_ok) + meta[s.name + '_ok'] = (~s.astype(bool)) | np.isin(s, ref) + meta[s.name + '_lower'] = s.str.lower() == s + + failed = meta[ + ~(s_ok | (meta[s.name + '_lower'] & meta[s.name + '_ok']))] + failed[s.name + '_suggestions'] = failed[s.name].apply( + lambda s: '; '.join(get_close_matches(s, ref))) + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_lower', + s.name + '_ok', s.name + '_suggestions']]) + msg = "Found %i invalid %ss: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_sampletype(meta, sampletypes, okexcept, record_property): + if 'SampleType' not in meta.columns: return pytest.skip("No SampleType specified.") - assert (okexcept(meta_row, 'SampleType') or sampletype in sampletypes) - - -def test_samplemethod(meta_row, samplemethods): - samplemethod = str(getattr(meta_row, 'SampleMethod', '')) - if isnull(samplemethod): - return pytest.skip("No samplemethod specified.") - assert (okexcept(meta_row, 'SampleMethod') or - samplemethod in samplemethods) + ref = sampletypes + meta = meta.copy() + s = meta['SampleType'].fillna('') + + s_ok = okexcept(s.name) + meta = meta.join(s_ok) + meta[s.name + '_ok'] = (~s.astype(bool)) | np.isin(s, ref) + + failed = meta[~(s_ok | meta[s.name + '_ok'])] + failed[s.name + '_suggestions'] = failed[s.name].apply( + lambda s: '; '.join(get_close_matches(s, ref))) + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_ok', + s.name + '_suggestions']]) + + msg = "Found %i invalid SampleTypes: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_samplemethod(meta, samplemethods, okexcept, record_property): + if 'SampleMethod' not in meta.columns: + return pytest.skip("No SampleMethod specified.") + ref = samplemethods + meta = meta.copy() + s = meta['SampleMethod'].fillna('') + + s_ok = okexcept(s.name) + meta = meta.join(s_ok) + meta[s.name + '_ok'] = (~s.astype(bool)) | np.isin(s, ref) + + failed = meta[~(s_ok | meta[s.name + '_ok'])] + failed[s.name + '_suggestions'] = failed[s.name].apply( + lambda s: '; '.join(get_close_matches(s, ref))) + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_ok', + s.name + '_suggestions']]) + + msg = "Found %i invalid %ss: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg @pytest.mark.parametrize('worker', ['Worker1', 'Worker2', 'Worker3', 'Worker4']) -def test_workerrole(meta_row, worker, workerroles): - if (isnull(getattr(meta_row, worker + '_LastName', '')) or - isnull(getattr(meta_row, worker + '_Role', ''))): - return pytest.skip(worker + "_Role not specified.") - assert meta_row[worker + '_Role'] in workerroles - - -def test_locationreliability(meta_row, locationreliabilities): - locationreliability = str(getattr(meta_row, 'LocationReliability', '')) - if isnull(locationreliability): - return pytest.skip("No LocationReliability specified.") - assert (okexcept(meta_row, 'LocationReliability') or - locationreliability in locationreliabilities) - - -def test_ageuncertainty(meta_row, ageuncertainties): - ageuncertainty = str(getattr(meta_row, 'AgeUncertainty', '')) - if isnull(ageuncertainty): - return pytest.skip("No AgeUncertainty specified.") - assert (okexcept(meta_row, 'AgeUncertainty') or - ageuncertainty in ageuncertainties) - - -def test_elevation(meta_row): - assert okexcept(meta_row, 'Elevation') or ~np.isnan(meta_row.Elevation) - - -def test_temperature(meta_row): - if okexcept(meta_row, 'Temperature'): - return - assert meta_row.Temperature - temperature = np.array(meta_row.Temperature.split(',')).astype(float) - assert len(temperature) == 17 - assert ((temperature > -100) & (temperature < 100)).all() - - -def test_precip(meta_row): - if okexcept(meta_row, 'Precipitation'): - return - assert meta_row.Precipitation - precip = np.array(meta_row.Precipitation.split(',')).astype(float) - assert len(precip) == 17 - assert ((precip > 0) & (precip < 10000)).all() +def test_workerrole(meta, worker, workerroles, record_property): + if worker + '_Role' not in meta.columns: + return pytest.skip(f"No {worker}_Role specified.") + ref = workerroles + meta = meta.copy() + s = meta[worker + '_Role'].fillna('') + + meta[s.name + '_ok'] = (~s.astype(bool)) | np.isin(s, ref) + + failed = meta[~meta[s.name + '_ok']] + failed[s.name + '_suggestions'] = failed[s.name].apply( + lambda s: '; '.join(get_close_matches(s, ref))) + record_property( + 'failed_samples', + failed[[s.name, s.name + '_ok', + s.name + '_suggestions']]) + + msg = "Found %i invalid values for %s: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_locationreliability(meta, locationreliabilities, record_property): + if 'LocationReliability' not in meta.columns: + return pytest.skip(f"No LocationReliability specified.") + ref = locationreliabilities + meta = meta.copy() + s = meta['LocationReliability'].fillna('') + + meta[s.name + '_ok'] = ((~s.astype(bool)) | np.isin(s, ref)) + + failed = meta[~(meta[s.name + '_ok'].values)] + failed[s.name + '_suggestions'] = failed[s.name].apply( + lambda s: '; '.join(get_close_matches(s, ref))) + record_property( + 'failed_samples', + failed[[s.name, s.name + '_ok', + s.name + '_suggestions']]) + + msg = "Found %i invalid values for %s: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_ageuncertainty(meta, ageuncertainties, record_property): + ref = ageuncertainties + if 'AgeUncertainty' not in meta.columns: + return pytest.skip(f"No AgeUncertainty specified.") + meta = meta.copy() + s = meta['AgeUncertainty'].replace(np.nan, '') + meta[s.name + '_ok'] = ((~s.astype(bool)) | np.isin(s, ref)) + + failed = meta[~meta[s.name + '_ok']] + failed[s.name + '_suggestions'] = failed[s.name].apply( + lambda s: '; '.join(get_close_matches(s, ref))) + record_property( + 'failed_samples', + failed[[s.name, s.name + '_ok', + s.name + '_suggestions']]) + msg = "Found %i invalid values for %s: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_elevation(meta, okexcept, record_property): + meta = meta.copy() + s = meta.Elevation.astype(float) + s_ok = okexcept(s.name) + meta = meta.join(s_ok) + meta[s.name + '_ok'] = notnull = s.notnull() + + failed = meta[~(s_ok | notnull)] + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_ok']]) + msg = "Found %i invalid values for %s: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_temperature(meta, okexcept, record_property): + + def test_range(l): + try: + temperature = np.array(l, dtype=float) + except Exception: + return False + else: + return ((temperature > -100) & (temperature < 100)).all() + + meta = meta.copy() + s = meta.Temperature.astype(str).fillna('') + s_ok = okexcept(s.name) + meta = meta.join(s_ok) + meta[s.name + '_nvals'] = nvals_ok = s.str.split(',').apply(len) == 17 + meta[s.name + '_ok'] = range_ok = s.str.split(',').apply(test_range) + + failed = meta[~(s_ok | (nvals_ok & range_ok))] + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_nvals', s.name + '_ok']]) + msg = "Found %i invalid values for %s: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg + + +def test_precip(meta, okexcept, record_property): + + def test_range(l): + try: + precip = np.array(l, dtype=float) + except Exception: + return False + else: + return ((precip > 0) & (precip < 10000)).all() + + meta = meta.copy() + s = meta.Precipitation.astype(str).fillna('') + s_ok = okexcept(s.name) + meta = meta.join(s_ok) + meta[s.name + '_nvals'] = nvals_ok = s.str.split(',').apply(len) == 17 + meta[s.name + '_ok'] = range_ok = s.str.split(',').apply(test_range) + + failed = meta[~(s_ok | (nvals_ok & range_ok))] + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_nvals', s.name + '_ok']]) + msg = "Found %i invalid values for %s: %s" % ( + len(failed), s.name, textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg # --- sample file tests -def test_orig_varnames(data_frame): - assert data_frame.original_varname.all() - duplicated = data_frame.duplicated('original_varname') - assert not duplicated.any(), ( - 'Duplicated acc_varnames: ' + str( - data_frame[duplicated].original_varname)) - - -def test_acc_varnames(data_frame): - assert data_frame.acc_varname.all() - duplicated = data_frame.duplicated('acc_varname', keep=False) +def test_orig_varnames(counts, record_property): + counts = counts.copy() + counts.original_varname.fillna('') + counts['original_varname_valid'] = names = counts.original_varname.astype( + bool) + counts['duplicated'] = duplicated = counts.duplicated( + ['samplename', 'original_varname'], keep=False) + + failed = counts[(~names) | duplicated] + record_property('failed_data', failed) + msg = "Found %i invalid original varnames: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.samplename.unique()), 80, placeholder='...')) + assert not len(failed), msg + + +def test_acc_varnames(counts, record_property): + counts = counts.copy() + counts.acc_varname.fillna('') + counts['acc_varname_valid'] = names = counts.acc_varname.astype(bool) + duplicated = counts.duplicated( + ['samplename', 'acc_varname'], keep=False) if duplicated.any(): - names = data_frame[duplicated][['original_varname', 'acc_varname']] + rows = counts[duplicated][ + ['samplename', 'original_varname', 'acc_varname']].sort_values( + ['samplename', 'acc_varname']) print('Found several duplicated acc_varnames:\n\n- ' + - '\n - '.join(starmap('{}: {}'.format, names.values))) - - -def test_counts(data_frame): - assert data_frame['count'].notnull().all() - - -def test_groupid(data_frame): - assert data_frame.groupid.all() - - -def test_percentage(data_frame): - assert data_frame[data_frame.make_percent].percentage.notnull().all() - - -def test_samplename(data_file): - samplename = osp.splitext(osp.basename(data_file))[0] - df = pd.read_csv(data_file, sep='\t') - samplenames = df.samplename - assert set(samplenames) == {samplename} - - -def test_data_exists(meta_row, samples_dir): - fname = osp.join(samples_dir, meta_row.name) + '.tsv' - assert osp.exists(fname), f"Missing {fname}" + '\n - '.join(starmap('{}: {} --> {}'.format, rows.values))) + + failed = counts[~names] + record_property('failed_data', failed) + msg = "Found %i invalid accepted varnames: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.samplename.unique()), 80, placeholder='...')) + assert not len(failed), msg + + +def test_counts(counts, record_property): + failed = counts[counts['count'].isnull() | counts['count'] < 0] + record_property('failed_data', failed) + msg = "Found %i rows with invalid count data: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.samplename.unique()), 80, placeholder='...')) + assert not len(failed), msg + + +def test_groupid(counts, okexcept, record_property, groupids): + counts = counts.copy() + counts['groupid'] = counts['groupid'].astype(str).fillna('') + s_ok = okexcept('GroupID') + counts = counts.merge( + s_ok.to_frame().reset_index().rename(columns={ + 'SampleName': 'samplename'}), + on='samplename', how='left') + + counts['valid_groupid'] = valid = counts.groupid.str.strip().astype(bool) + counts['existing_groupid'] = exists = np.isin(counts.groupid, groupids) + + failed = counts[~(valid & (counts[s_ok.name] | exists))] + record_property('failed_data', failed) + msg = "Found %i invalid groupids: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.samplename.unique()), 80, placeholder='...')) + assert not len(failed), msg + + +def test_data_exists(meta, data_files, record_property): + meta['data_file'] = data_files + meta['data_exists'] = meta.data_file.apply(osp.exists) + + failed = meta[~meta.data_exists] + record_property('failed_samples', failed[['data_file', 'data_exists']]) + msg = "Missing %i datafiles: %s" % ( + len(failed), textwrap.shorten( + ', '.join(failed.index), 80, placeholder='...')) + assert not len(failed), msg From 9ced84fd91c0af7f92503957523cc7e9c2e6d763 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 3 Apr 2019 08:39:17 +0200 Subject: [PATCH 110/119] Fixes for no sample submission --- tests/conftest.py | 7 +++- tests/test_critical.py | 3 +- tests/test_meta.py | 95 +++++++++++++++++++++++------------------- 3 files changed, 61 insertions(+), 44 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 00eeb02db..725a392bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,7 +48,10 @@ def skip_ci(request): def counts(): import pandas as pd read_tsv = partial(pd.read_csv, sep='\t') - return pd.concat(list(map(read_tsv, _data_files()))) + files = _data_files() + if not len(files): + pytest.skip("No sample data provided") + return pd.concat(list(map(read_tsv, files))) @pytest.fixture(scope='session') @@ -102,6 +105,8 @@ def countries(meta_file): def nat_earth_countries(meta, countries): import pandas as pd import numpy as np + if not len(meta): + return pytest.skip("No samples provided") lat = meta.Latitude.astype(float).values lon = meta.Longitude.astype(float).values mask = ~(np.isnan(lat) | np.isnan(lon)) diff --git a/tests/test_critical.py b/tests/test_critical.py index 6b7d1d45f..69e1ebcf4 100644 --- a/tests/test_critical.py +++ b/tests/test_critical.py @@ -38,7 +38,8 @@ def test_data(fname): failed = meta[~meta['valid']] - record_property('failed_samples', failed[['files', 'valid']]) + if len(failed): + record_property('failed_samples', failed[['files', 'valid']]) msg = "Found %i invalid data columns: %s" % ( len(failed), textwrap.shorten( diff --git a/tests/test_meta.py b/tests/test_meta.py index 1e9bec595..8e90b2467 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -68,10 +68,11 @@ def test_country(meta, countries, nat_earth_countries, okexcept, meta[s.name + '_ok'] = [n in ref.index and c in ref.loc[[n]].values for c, n in meta[[s.name, 'nat_earth']].values] failed = meta[~(s_ok | meta.Country_ok)] - record_property('failed_samples', - failed[[s.name, 'nat_earth', 'empd_countries', s_ok.name, - s.name + '_ok']] - ) + if len(failed): + record_property('failed_samples', + failed[[s.name, 'nat_earth', 'empd_countries', s_ok.name, + s.name + '_ok']] + ) msg = "Found %i invalid %s: %s" % ( len(failed), s.name if len(failed) == 1 else 'Countries', textwrap.shorten( @@ -95,10 +96,11 @@ def test_samplecontext(meta, samplecontexts, okexcept, record_property): ~(s_ok | (meta[s.name + '_lower'] & meta[s.name + '_ok']))] failed[s.name + '_suggestions'] = failed[s.name].apply( lambda s: '; '.join(get_close_matches(s, ref))) - record_property( - 'failed_samples', - failed[[s.name, s_ok.name, s.name + '_lower', - s.name + '_ok', s.name + '_suggestions']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_lower', + s.name + '_ok', s.name + '_suggestions']]) msg = "Found %i invalid %ss: %s" % ( len(failed), s.name, textwrap.shorten( ', '.join(failed.index), 80, placeholder='...')) @@ -119,10 +121,11 @@ def test_sampletype(meta, sampletypes, okexcept, record_property): failed = meta[~(s_ok | meta[s.name + '_ok'])] failed[s.name + '_suggestions'] = failed[s.name].apply( lambda s: '; '.join(get_close_matches(s, ref))) - record_property( - 'failed_samples', - failed[[s.name, s_ok.name, s.name + '_ok', - s.name + '_suggestions']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_ok', + s.name + '_suggestions']]) msg = "Found %i invalid SampleTypes: %s" % ( len(failed), textwrap.shorten( @@ -144,10 +147,11 @@ def test_samplemethod(meta, samplemethods, okexcept, record_property): failed = meta[~(s_ok | meta[s.name + '_ok'])] failed[s.name + '_suggestions'] = failed[s.name].apply( lambda s: '; '.join(get_close_matches(s, ref))) - record_property( - 'failed_samples', - failed[[s.name, s_ok.name, s.name + '_ok', - s.name + '_suggestions']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_ok', + s.name + '_suggestions']]) msg = "Found %i invalid %ss: %s" % ( len(failed), s.name, textwrap.shorten( @@ -169,10 +173,10 @@ def test_workerrole(meta, worker, workerroles, record_property): failed = meta[~meta[s.name + '_ok']] failed[s.name + '_suggestions'] = failed[s.name].apply( lambda s: '; '.join(get_close_matches(s, ref))) - record_property( - 'failed_samples', - failed[[s.name, s.name + '_ok', - s.name + '_suggestions']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s.name + '_ok', s.name + '_suggestions']]) msg = "Found %i invalid values for %s: %s" % ( len(failed), s.name, textwrap.shorten( @@ -192,10 +196,10 @@ def test_locationreliability(meta, locationreliabilities, record_property): failed = meta[~(meta[s.name + '_ok'].values)] failed[s.name + '_suggestions'] = failed[s.name].apply( lambda s: '; '.join(get_close_matches(s, ref))) - record_property( - 'failed_samples', - failed[[s.name, s.name + '_ok', - s.name + '_suggestions']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s.name + '_ok', s.name + '_suggestions']]) msg = "Found %i invalid values for %s: %s" % ( len(failed), s.name, textwrap.shorten( @@ -214,10 +218,10 @@ def test_ageuncertainty(meta, ageuncertainties, record_property): failed = meta[~meta[s.name + '_ok']] failed[s.name + '_suggestions'] = failed[s.name].apply( lambda s: '; '.join(get_close_matches(s, ref))) - record_property( - 'failed_samples', - failed[[s.name, s.name + '_ok', - s.name + '_suggestions']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s.name + '_ok', s.name + '_suggestions']]) msg = "Found %i invalid values for %s: %s" % ( len(failed), s.name, textwrap.shorten( ', '.join(failed.index), 80, placeholder='...')) @@ -232,9 +236,9 @@ def test_elevation(meta, okexcept, record_property): meta[s.name + '_ok'] = notnull = s.notnull() failed = meta[~(s_ok | notnull)] - record_property( - 'failed_samples', - failed[[s.name, s_ok.name, s.name + '_ok']]) + if len(failed): + record_property( + 'failed_samples', failed[[s.name, s_ok.name, s.name + '_ok']]) msg = "Found %i invalid values for %s: %s" % ( len(failed), s.name, textwrap.shorten( ', '.join(failed.index), 80, placeholder='...')) @@ -259,9 +263,10 @@ def test_range(l): meta[s.name + '_ok'] = range_ok = s.str.split(',').apply(test_range) failed = meta[~(s_ok | (nvals_ok & range_ok))] - record_property( - 'failed_samples', - failed[[s.name, s_ok.name, s.name + '_nvals', s.name + '_ok']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_nvals', s.name + '_ok']]) msg = "Found %i invalid values for %s: %s" % ( len(failed), s.name, textwrap.shorten( ', '.join(failed.index), 80, placeholder='...')) @@ -286,9 +291,10 @@ def test_range(l): meta[s.name + '_ok'] = range_ok = s.str.split(',').apply(test_range) failed = meta[~(s_ok | (nvals_ok & range_ok))] - record_property( - 'failed_samples', - failed[[s.name, s_ok.name, s.name + '_nvals', s.name + '_ok']]) + if len(failed): + record_property( + 'failed_samples', + failed[[s.name, s_ok.name, s.name + '_nvals', s.name + '_ok']]) msg = "Found %i invalid values for %s: %s" % ( len(failed), s.name, textwrap.shorten( ', '.join(failed.index), 80, placeholder='...')) @@ -307,7 +313,8 @@ def test_orig_varnames(counts, record_property): ['samplename', 'original_varname'], keep=False) failed = counts[(~names) | duplicated] - record_property('failed_data', failed) + if len(failed): + record_property('failed_data', failed) msg = "Found %i invalid original varnames: %s" % ( len(failed), textwrap.shorten( ', '.join(failed.samplename.unique()), 80, placeholder='...')) @@ -328,7 +335,8 @@ def test_acc_varnames(counts, record_property): '\n - '.join(starmap('{}: {} --> {}'.format, rows.values))) failed = counts[~names] - record_property('failed_data', failed) + if len(failed): + record_property('failed_data', failed) msg = "Found %i invalid accepted varnames: %s" % ( len(failed), textwrap.shorten( ', '.join(failed.samplename.unique()), 80, placeholder='...')) @@ -337,7 +345,8 @@ def test_acc_varnames(counts, record_property): def test_counts(counts, record_property): failed = counts[counts['count'].isnull() | counts['count'] < 0] - record_property('failed_data', failed) + if len(failed): + record_property('failed_data', failed) msg = "Found %i rows with invalid count data: %s" % ( len(failed), textwrap.shorten( ', '.join(failed.samplename.unique()), 80, placeholder='...')) @@ -357,7 +366,8 @@ def test_groupid(counts, okexcept, record_property, groupids): counts['existing_groupid'] = exists = np.isin(counts.groupid, groupids) failed = counts[~(valid & (counts[s_ok.name] | exists))] - record_property('failed_data', failed) + if len(failed): + record_property('failed_data', failed) msg = "Found %i invalid groupids: %s" % ( len(failed), textwrap.shorten( ', '.join(failed.samplename.unique()), 80, placeholder='...')) @@ -369,7 +379,8 @@ def test_data_exists(meta, data_files, record_property): meta['data_exists'] = meta.data_file.apply(osp.exists) failed = meta[~meta.data_exists] - record_property('failed_samples', failed[['data_file', 'data_exists']]) + if len(failed): + record_property('failed_samples', failed[['data_file', 'data_exists']]) msg = "Missing %i datafiles: %s" % ( len(failed), textwrap.shorten( ', '.join(failed.index), 80, placeholder='...')) From e53a01d9fec66fd666b0cc03ac9401a8083dc395 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 3 Apr 2019 21:32:11 +0200 Subject: [PATCH 111/119] disable updates of groupids --- postgres/scripts/import_into_empd2.py | 6 ------ postgres/scripts/updateFixedTables.py | 16 ---------------- tests/test_meta.py | 9 ++------- 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/postgres/scripts/import_into_empd2.py b/postgres/scripts/import_into_empd2.py index bf1a2ed12..e80160847 100644 --- a/postgres/scripts/import_into_empd2.py +++ b/postgres/scripts/import_into_empd2.py @@ -105,11 +105,6 @@ def clean_doi(doi): 'SampleType'] and notnull(row[col]): okexcept[col].add(row[col]) row_okexcept.remove(col) - elif col == 'GroupID': - COUNTS = pd.read_csv( - os.path.join(samples_dir, row.SampleName + '.tsv'), '\t') - okexcept[col].update(COUNTS['groupid']) - row_okexcept.remove(col) METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) orig_METADATA.loc[key, 'okexcept'] = ','.join(row_okexcept) save_orig = True @@ -119,7 +114,6 @@ def clean_doi(doi): table_map = { 'Country': 'countries', - 'GroupID': 'groupID', 'SampleContext': 'sampleContexts', 'SampleType': 'sampleTypes', 'SampleMethod': 'sampleMethods', diff --git a/postgres/scripts/updateFixedTables.py b/postgres/scripts/updateFixedTables.py index fbb32e4f5..9caad119d 100644 --- a/postgres/scripts/updateFixedTables.py +++ b/postgres/scripts/updateFixedTables.py @@ -46,20 +46,4 @@ def is_null_str(x): conn.commit() -# --- -if 'GroupID' in to_update or 'all' in to_update: - groupIDs = read_tsv(tables + "GroupID.tsv") - groupIDs.replace(np.nan, '', inplace=True) - - for x in range(groupIDs.shape[0]): - cursor.execute( - "UPDATE groupID SET groupname = %s, higher_groupid = %s " - "WHERE groupID = %s" % ( - is_null_str(groupIDs.iloc[x][1]), - is_null_str(groupIDs.iloc[x][2]), - is_null_str(groupIDs.iloc[x][0]))) - - conn.commit() - - conn.close() diff --git a/tests/test_meta.py b/tests/test_meta.py index 8e90b2467..e8e86a162 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -353,19 +353,14 @@ def test_counts(counts, record_property): assert not len(failed), msg -def test_groupid(counts, okexcept, record_property, groupids): +def test_groupid(counts, record_property, groupids): counts = counts.copy() counts['groupid'] = counts['groupid'].astype(str).fillna('') - s_ok = okexcept('GroupID') - counts = counts.merge( - s_ok.to_frame().reset_index().rename(columns={ - 'SampleName': 'samplename'}), - on='samplename', how='left') counts['valid_groupid'] = valid = counts.groupid.str.strip().astype(bool) counts['existing_groupid'] = exists = np.isin(counts.groupid, groupids) - failed = counts[~(valid & (counts[s_ok.name] | exists))] + failed = counts[~(valid & exists)] if len(failed): record_property('failed_data', failed) msg = "Found %i invalid groupids: %s" % ( From 29ed8055aee639a3cb3d6ca343d5d00ef1689b9b Mon Sep 17 00:00:00 2001 From: Chilipp Date: Wed, 3 Apr 2019 22:47:04 +0200 Subject: [PATCH 112/119] Merge branch 'upstream/master' into test-data From 8897b82f6000e03d1245c1ea5f07317b71ecfcba Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Wed, 3 Apr 2019 21:57:18 +0000 Subject: [PATCH 113/119] Extracted failures to failures/failed.tsv [skip ci] --- failures/failed.tsv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 failures/failed.tsv diff --git a/failures/failed.tsv b/failures/failed.tsv new file mode 100644 index 000000000..a01324407 --- /dev/null +++ b/failures/failed.tsv @@ -0,0 +1,2 @@ +SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept nat_earth empd_countries okexcept(Country) Country_ok +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 True Germany Germany False False From edeb8efe17c28a6f0583bef8835c276b0fe72f1e Mon Sep 17 00:00:00 2001 From: Chilipp Date: Thu, 4 Apr 2019 00:17:26 +0200 Subject: [PATCH 114/119] Minor From e9dccef35556bbc70f0e8a0415929c8e164f8e5c Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Sun, 20 Oct 2019 17:41:10 +0000 Subject: [PATCH 115/119] Updated fixed tables --- postgres/scripts/tables/SampleContext.tsv | 1 + test.tsv | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/postgres/scripts/tables/SampleContext.tsv b/postgres/scripts/tables/SampleContext.tsv index 4958e8baa..24357555a 100644 --- a/postgres/scripts/tables/SampleContext.tsv +++ b/postgres/scripts/tables/SampleContext.tsv @@ -68,3 +68,4 @@ archaeological moss section sediments +i dont know diff --git a/test.tsv b/test.tsv index f359f1bf3..9112660ae 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 True -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 True Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know True From 426ff360ce1543fea156ec057e873d645647dd87 Mon Sep 17 00:00:00 2001 From: EMPD-admin Date: Sun, 20 Oct 2019 17:41:11 +0000 Subject: [PATCH 116/119] Added postgres dump for test.tsv --- postgres/test.sql | 960 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 960 insertions(+) create mode 100644 postgres/test.sql diff --git a/postgres/test.sql b/postgres/test.sql new file mode 100644 index 000000000..cd2db90ec --- /dev/null +++ b/postgres/test.sql @@ -0,0 +1,960 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 11.2 +-- Dumped by pg_dump version 11.2 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: ageuncertainties; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ageuncertainties ( + ageuncertainty character varying(1) NOT NULL, + description character varying(21) NOT NULL, + age character varying(14) +); + + +ALTER TABLE public.ageuncertainties OWNER TO postgres; + +-- +-- Name: climate; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.climate ( + samplename character varying(25) NOT NULL, + t_jan double precision, + t_feb double precision, + t_mar double precision, + t_apr double precision, + t_may double precision, + t_jun double precision, + t_jul double precision, + t_aug double precision, + t_sep double precision, + t_oct double precision, + t_nov double precision, + t_dec double precision, + t_djf double precision, + t_mam double precision, + t_jja double precision, + t_son double precision, + t_ann double precision, + p_jan double precision, + p_feb double precision, + p_mar double precision, + p_apr double precision, + p_may double precision, + p_jun double precision, + p_jul double precision, + p_aug double precision, + p_sep double precision, + p_oct double precision, + p_nov double precision, + p_dec double precision, + p_djf double precision, + p_mam double precision, + p_jja double precision, + p_son double precision, + p_ann double precision +); + + +ALTER TABLE public.climate OWNER TO postgres; + +-- +-- Name: countries; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.countries ( + country character varying(42) NOT NULL, + natural_earth character varying(42) +); + + +ALTER TABLE public.countries OWNER TO postgres; + +-- +-- Name: ecosystems; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.ecosystems ( + samplename character varying(25) NOT NULL, + realm character(16), + biome character(62), + ecoregion character(65) +); + + +ALTER TABLE public.ecosystems OWNER TO postgres; + +-- +-- Name: groupid; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.groupid ( + groupid character varying(9) NOT NULL, + groupname character varying(60) NOT NULL, + higher_groupid character varying(4) NOT NULL, + used_in_sum boolean DEFAULT false, + percent_values boolean DEFAULT false +); + + +ALTER TABLE public.groupid OWNER TO postgres; + +-- +-- Name: locationreliabilities; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.locationreliabilities ( + locationreliability character varying(1) NOT NULL, + description character varying(55) NOT NULL, + error character varying(5) +); + + +ALTER TABLE public.locationreliabilities OWNER TO postgres; + +-- +-- Name: metadata; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metadata ( + samplename character varying(25) NOT NULL, + originalsamplename character varying(70), + sitename character varying(70), + country character varying(42), + longitude double precision, + latitude double precision, + elevation double precision, + locationreliability character varying(1), + locationnotes text, + areaofsite double precision, + samplecontext character varying(40), + sitedescription text, + vegdescription text, + sampletype character varying(14), + samplemethod character varying(31), + agebp double precision, + ageuncertainty character varying(1), + ispercent boolean DEFAULT false, + notes text, + okexcept text, + empd_version character varying(5) +); + + +ALTER TABLE public.metadata OWNER TO postgres; + +-- +-- Name: metapubli; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metapubli ( + samplename character varying(25) NOT NULL, + publiid integer NOT NULL +); + + +ALTER TABLE public.metapubli OWNER TO postgres; + +-- +-- Name: metaworker; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.metaworker ( + samplename character varying(25) NOT NULL, + workerid integer NOT NULL, + workerrole character varying(3) +); + + +ALTER TABLE public.metaworker OWNER TO postgres; + +-- +-- Name: p_counts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_counts ( + samplename character varying(25) NOT NULL, + var_ integer NOT NULL, + count double precision NOT NULL, + percentage double precision +); + + +ALTER TABLE public.p_counts OWNER TO postgres; + +-- +-- Name: p_vars; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.p_vars ( + var_ integer NOT NULL, + acc_var_ integer, + original_varname character varying(100) NOT NULL, + acc_varname character varying(100), + groupid character varying(9), + notes text +); + + +ALTER TABLE public.p_vars OWNER TO postgres; + +-- +-- Name: publications; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.publications ( + publiid integer NOT NULL, + doi character varying(50), + reference text NOT NULL +); + + +ALTER TABLE public.publications OWNER TO postgres; + +-- +-- Name: samplecontexts; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplecontexts ( + samplecontext character varying(40) NOT NULL +); + + +ALTER TABLE public.samplecontexts OWNER TO postgres; + +-- +-- Name: samplemethods; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.samplemethods ( + samplemethod character varying(31) NOT NULL +); + + +ALTER TABLE public.samplemethods OWNER TO postgres; + +-- +-- Name: sampletypes; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.sampletypes ( + sampletype character varying(14) NOT NULL, + notes character varying(13) +); + + +ALTER TABLE public.sampletypes OWNER TO postgres; + +-- +-- Name: workerroles; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workerroles ( + workerrole character varying(3) NOT NULL, + description character varying(35) NOT NULL +); + + +ALTER TABLE public.workerroles OWNER TO postgres; + +-- +-- Name: workers; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workers ( + workerid integer NOT NULL, + firstname character varying(25), + lastname character varying(25), + initials character varying(10), + address1 text, + email1 character varying(50), + phone1 character varying(25), + address2 text, + email2 character varying(50), + phone2 character varying(25) +); + + +ALTER TABLE public.workers OWNER TO postgres; + +-- +-- Data for Name: ageuncertainties; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ageuncertainties (ageuncertainty, description, age) FROM stdin; +A Modern sample 0BP to Present +B Within last 100 years 0-50BP +C Within last 250 years 50-200BP +\. + + +-- +-- Data for Name: climate; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.climate (samplename, t_jan, t_feb, t_mar, t_apr, t_may, t_jun, t_jul, t_aug, t_sep, t_oct, t_nov, t_dec, t_djf, t_mam, t_jja, t_son, t_ann, p_jan, p_feb, p_mar, p_apr, p_may, p_jun, p_jul, p_aug, p_sep, p_oct, p_nov, p_dec, p_djf, p_mam, p_jja, p_son, p_ann) FROM stdin; +test_a1 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a2 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +test_a3 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 +\. + + +-- +-- Data for Name: countries; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.countries (country, natural_earth) FROM stdin; +Albania Albania +Algeria Algeria +Andorra Andorra +Armenia Armenia +Austria Austria +Azerbaijan Azerbaijan +Bahrain Bahrain +Belarus Belarus +Belgium Belgium +Bosnia And Herzegovina Bosnia and Herzegovina +Bulgaria Bulgaria +Cape Verde Cabo Verde +China, People s Republic Of China +Croatia Croatia +Cyprus Cyprus +Czech Republic Czechia +Denmark Denmark +Egypt Egypt +Eritrea Eritrea +Estonia Estonia +Ethiopia Ethiopia +Faroe Islands Faroe Islands +Finland Finland +France France +Georgia Georgia +Germany Germany +Gibraltar Gibraltar +Greece Greece +Greenland Greenland +Guernsey Guernsey +Hungary Hungary +Iceland Iceland +India India +Iran, Islamic Republic Of Iran +Iraq Iraq +Ireland Ireland +Isle Of Man Isle of Man +Israel Israel +Italy Italy +Japan Japan +Jersey Jersey +Jordan Jordan +Kazakhstan Kazakhstan +Kuwait Kuwait +Kyrgyzstan Kyrgyzstan +Latvia Latvia +Lebanon Lebanon +Libyan Arab Jamahiriya Libya +Liechtenstein Liechtenstein +Lithuania Lithuania +Luxembourg Luxembourg +Macedonia, The Former Yugoslav Republic Of Macedonia +Malta Malta +Moldova, Republic Of Moldova +Monaco Monaco +Serbia And Montenegro Montenegro +Morocco Morocco +Netherlands Netherlands +Norway Norway +Svalbard and Jan Mayen Norway +Oman Oman +Palestinian Territory, Occupied Palestine +Poland Poland +Portugal Portugal +Qatar Qatar +Romania Romania +Russian Federation Russia +San Marino San Marino +Saudi Arabia Saudi Arabia +Slovakia Slovakia +Slovenia Slovenia +Spain Spain +Sweden Sweden +Switzerland Switzerland +Syrian Arab Republic Syria +Tunisia Tunisia +Turkey Turkey +Turkmenistan, Republic Of Turkmenistan +Ukraine Ukraine +United Arab Emirates United Arab Emirates +United Kingdom United Kingdom +Yemen Yemen +Black Sea \N +Dead Sea \N +Adriatic Sea \N +\. + + +-- +-- Data for Name: ecosystems; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.ecosystems (samplename, realm, biome, ecoregion) FROM stdin; +\. + + +-- +-- Data for Name: groupid; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.groupid (groupid, groupname, higher_groupid, used_in_sum, percent_values) FROM stdin; +ACRI Acritarchs NOPO f f +ALGA Algae AQUA f t +AQBR Aquatic Bryophytes AQUA f t +AQPT Aquatic Pteridophyta AQUA f t +AQUA Aquatics AQUA f t +AQVP Aquatic Vascular Plants AQUA f t +BRYO Bryophytes VACR t t +CHAR Charcoal CHAR f f +CHLO Chlorophytes NOPO f f +CYAN Cyanobacteria NOPO f f +DINO Dinoflagellates NOPO f f +DWAR Dwarf shrubs TRSH t t +FUNG Fungi NOPO f f +HELO Helophytes AQUA f t +HEMI Hemi-parasitic NOPO f f +HERB Herbs HERB t t +INUN Indeterminables and unknowns INUN f f +INVE Invertebrates NOPO f f +LIAN Liana LIAN t t +MACR Macrofossils NOPO f f +MICR Microcrustaceans NOPO f f +NEMA Nematoda (roundworms) NOPO f f +NOPO Nonpollen NOPO f f +PLAT Platyhelminthes (flatworms) NOPO f f +PREQ Pre-Quaternary type NOPO f f +RHIZ Rhizopods NOPO f f +ROTI Rotifera (Rotifers) NOPO f f +SPOR Taxonomically undifferentiated spores INUN f f +TEAM Testate amoebae NOPO f f +TRSH Trees & Shrubs TRSH t t +UNPP Undifferentiated NPP INUN f f +UPBR Upland Bryophytes VACR t t +UPHE Upland herbs HERB t t +VACR Vascular cryptogams (Pteridophytes) VACR t t +VEMI Vegetative microfossils NOPO f f +MOSS Moss VACR t t +\. + + +-- +-- Data for Name: locationreliabilities; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.locationreliabilities (locationreliability, description, error) FROM stdin; +A Good for high resolution remote sensing (<100m) <100m +B Good for lower resolution remote sensing (<1km) <1km +C Ok for climate reconstruction or regional scale mapping <5km +D Ok for mapping at continental scale <20km +X Do not use! \N +\. + + +-- +-- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; +test_a1 orig_test_a1 somewhere France 10 50 340 X Actually in Germany 30 \N \N \N \N \N \N \N f \N \N \N +test_a2 orig_test_a2 somewhere Germany 10 52 340 X Actually in Germany 30 \N \N \N \N \N \N \N f \N Country \N +test_a3 orig_test_a2 somewhere Germany 10 52 340 X Actually in Germany 30 i dont know \N \N \N \N \N \N f \N \N \N +\. + + +-- +-- Data for Name: metapubli; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metapubli (samplename, publiid) FROM stdin; +\. + + +-- +-- Data for Name: metaworker; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; +\. + + +-- +-- Data for Name: p_counts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_counts (samplename, var_, count, percentage) FROM stdin; +test_a2 1 45 \N +test_a2 2 6 \N +test_a2 3 55 \N +test_a3 1 45 \N +test_a3 2 6 \N +test_a3 3 55 \N +\. + + +-- +-- Data for Name: p_vars; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.p_vars (var_, acc_var_, original_varname, acc_varname, groupid, notes) FROM stdin; +1 \N Pinus Pinus TRSH NULL +2 \N Exotic Exotics (counted) NOPO NULL +3 \N Umbelliferae Apiaceae HERB NULL +\. + + +-- +-- Data for Name: publications; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.publications (publiid, doi, reference) FROM stdin; +\. + + +-- +-- Data for Name: samplecontexts; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplecontexts (samplecontext) FROM stdin; +arable +blanket bog +bog +cave +cirque lake +closed forest +coastal +coastal lake +coastal wetland +drained_lake +ephemeral lake +ephemeral lake/pond +estuarine +fallow +fallow - scattered trees/shrubs +fen +floodplain mire +fluvial +forest +forest undefined +heath +ice from glacier +kettle lake +lagoon +lake +maquis +marine +marsh +marshland +mire +mor humus +moss-shrubby tundra +natural grassland +natural grasslands +natural open water +open forest +open forest/orchard +open forest/scattered trees/shrubs +orchard +orchard/scattered trees/shrubs +pasture +peatbog +pond +riverine +salt lake +salt marsh +scattered trees +scattered trees/shrubs +shrubs +shrubs/scrubland +soil +soligenous (mineraltrophic) Mire +sparse/no vegetation +swamp +tectonic lake +terrestrial +treeless vegetation +treeless vegetation/natural grassland +treeless vegetation/sparse vegetation +urban +valley mire +wetland +wetland bog +wetland bog/scattered trees/shrubs +wetland bog/treeless vegetation +archaeological +moss +section +sediments +i dont know +\. + + +-- +-- Data for Name: samplemethods; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.samplemethods (samplemethod) FROM stdin; +hand picking +hand picking - multiple samples +unspecified +auger corer +freeze corer +gravity corer +piston corer +russian corer +box +spade +hiller corer +gouge auger +core +corer unspecified +eckman-grab +box corer +\. + + +-- +-- Data for Name: sampletypes; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.sampletypes (sampletype, notes) FROM stdin; +dung \N +litter \N +moss \N +pollen trap To be removed +sediment submerged +spider web \N +soil terrestrial +sphagnum peat \N +peat \N +lichen \N +submerged \N +core_top \N +tussock \N +epiphytic moss \N +\. + + +-- +-- Data for Name: workerroles; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workerroles (workerrole, description) FROM stdin; +R Responsible Person +R/A Both Responsible Person and Analyst +A Analyst +A/D Analyst (Deceased) +A/I Analysis (Inactive) +A/U Analyst (Unknown) +\. + + +-- +-- Data for Name: workers; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.workers (workerid, firstname, lastname, initials, address1, email1, phone1, address2, email2, phone2) FROM stdin; +\. + + +-- +-- Name: ageuncertainties ageuncertainties_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ageuncertainties + ADD CONSTRAINT ageuncertainties_pkey PRIMARY KEY (ageuncertainty); + + +-- +-- Name: countries countries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.countries + ADD CONSTRAINT countries_pkey PRIMARY KEY (country); + + +-- +-- Name: ecosystems ecosystems_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_pkey PRIMARY KEY (samplename); + + +-- +-- Name: groupid groupid_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.groupid + ADD CONSTRAINT groupid_pkey PRIMARY KEY (groupid); + + +-- +-- Name: locationreliabilities locationreliabilities_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.locationreliabilities + ADD CONSTRAINT locationreliabilities_pkey PRIMARY KEY (locationreliability); + + +-- +-- Name: metadata metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_pkey PRIMARY KEY (samplename); + + +-- +-- Name: metapubli metapubli_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_pkey PRIMARY KEY (samplename, publiid); + + +-- +-- Name: metaworker metaworker_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_pkey PRIMARY KEY (samplename, workerid); + + +-- +-- Name: climate metaworkerclimate_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT metaworkerclimate_pkey PRIMARY KEY (samplename); + + +-- +-- Name: p_counts p_counts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_pkey PRIMARY KEY (samplename, var_); + + +-- +-- Name: p_vars p_vars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_pkey PRIMARY KEY (var_); + + +-- +-- Name: publications publications_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_doi_key UNIQUE (doi); + + +-- +-- Name: publications publications_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.publications + ADD CONSTRAINT publications_pkey PRIMARY KEY (publiid); + + +-- +-- Name: samplecontexts samplecontexts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplecontexts + ADD CONSTRAINT samplecontexts_pkey PRIMARY KEY (samplecontext); + + +-- +-- Name: samplemethods samplemethods_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.samplemethods + ADD CONSTRAINT samplemethods_pkey PRIMARY KEY (samplemethod); + + +-- +-- Name: sampletypes sampletypes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.sampletypes + ADD CONSTRAINT sampletypes_pkey PRIMARY KEY (sampletype); + + +-- +-- Name: workerroles workerroles_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workerroles + ADD CONSTRAINT workerroles_pkey PRIMARY KEY (workerrole); + + +-- +-- Name: workers workers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workers + ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); + + +-- +-- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.climate + ADD CONSTRAINT climates_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: ecosystems ecosystems_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.ecosystems + ADD CONSTRAINT ecosystems_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metadata metadata_ageuncertainty_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_ageuncertainty_fk FOREIGN KEY (ageuncertainty) REFERENCES public.ageuncertainties(ageuncertainty); + + +-- +-- Name: metadata metadata_country_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_country_fk FOREIGN KEY (country) REFERENCES public.countries(country); + + +-- +-- Name: metadata metadata_locationreliability_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_locationreliability_fk FOREIGN KEY (locationreliability) REFERENCES public.locationreliabilities(locationreliability); + + +-- +-- Name: metadata metadata_samplecontext_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplecontext_fk FOREIGN KEY (samplecontext) REFERENCES public.samplecontexts(samplecontext); + + +-- +-- Name: metadata metadata_samplemethod_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_samplemethod_fk FOREIGN KEY (samplemethod) REFERENCES public.samplemethods(samplemethod); + + +-- +-- Name: metadata metadata_sampletype_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metadata + ADD CONSTRAINT metadata_sampletype_fk FOREIGN KEY (sampletype) REFERENCES public.sampletypes(sampletype); + + +-- +-- Name: metapubli metapubli_publiid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_publiid_fk FOREIGN KEY (publiid) REFERENCES public.publications(publiid); + + +-- +-- Name: metapubli metapubli_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metapubli + ADD CONSTRAINT metapubli_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: metaworker metaworker_workerid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerid_fk FOREIGN KEY (workerid) REFERENCES public.workers(workerid); + + +-- +-- Name: metaworker metaworker_workerrole_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.metaworker + ADD CONSTRAINT metaworker_workerrole_fk FOREIGN KEY (workerrole) REFERENCES public.workerroles(workerrole); + + +-- +-- Name: p_counts p_counts_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_samplename_fk FOREIGN KEY (samplename) REFERENCES public.metadata(samplename); + + +-- +-- Name: p_counts p_counts_var__fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_counts + ADD CONSTRAINT p_counts_var__fk FOREIGN KEY (var_) REFERENCES public.p_vars(var_); + + +-- +-- Name: p_vars p_vars_groupid_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.p_vars + ADD CONSTRAINT p_vars_groupid_fk FOREIGN KEY (groupid) REFERENCES public.groupid(groupid); + + +-- +-- PostgreSQL database dump complete +-- + From e888c471caeef18ae2153408fcf5e9e4535f4b67 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 20 Oct 2019 20:16:56 +0200 Subject: [PATCH 117/119] Revert "Updated fixed tables" This reverts commit e9dccef35556bbc70f0e8a0415929c8e164f8e5c. --- postgres/scripts/tables/SampleContext.tsv | 1 - test.tsv | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/postgres/scripts/tables/SampleContext.tsv b/postgres/scripts/tables/SampleContext.tsv index 24357555a..4958e8baa 100644 --- a/postgres/scripts/tables/SampleContext.tsv +++ b/postgres/scripts/tables/SampleContext.tsv @@ -68,4 +68,3 @@ archaeological moss section sediments -i dont know diff --git a/test.tsv b/test.tsv index 9112660ae..f359f1bf3 100644 --- a/test.tsv +++ b/test.tsv @@ -1,4 +1,4 @@ SampleName OriginalSampleName SiteName Country Longitude Latitude Elevation Temperature Precipitation LocationReliability LocationNotes AreaOfSite SampleContext SiteDescription VegDescription SampleType SampleMethod AgeBP AgeUncertainty ispercent Notes EMPD_version Publication1 DOI1 Publication2 DOI2 Publication3 DOI3 Publication4 DOI4 Publication5 DOI5 Worker1_Role Worker1_LastName Worker1_Initials Worker1_FirstName Worker1_Address1 Worker1_Address2 Worker1_Email1 Worker1_Email2 Worker1_Phone1 Worker1_Phone2 Worker2_Role Worker2_LastName Worker2_Initials Worker2_FirstName Worker2_Address1 Worker2_Address2 Worker2_Email1 Worker2_Email2 Worker2_Phone1 Worker2_Phone2 Worker3_Role Worker3_LastName Worker3_Initials Worker3_FirstName Worker3_Address1 Worker3_Address2 Worker3_Email1 Worker3_Email2 Worker3_Phone1 Worker3_Phone2 Worker4_Role Worker4_LastName Worker4_Initials Worker4_FirstName Worker4_Address1 Worker4_Address2 Worker4_Email1 Worker4_Email2 Worker4_Phone1 Worker4_Phone2 okexcept test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 True -test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 True Country -test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know True +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know SampleContext From 17504e790e12720248f2197d61403e74e26f58b8 Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 20 Oct 2019 20:54:51 +0200 Subject: [PATCH 118/119] Added metaViewer table to postgres dump --- postgres/test.sql | 101 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/postgres/test.sql b/postgres/test.sql index cd2db90ec..3116a132c 100644 --- a/postgres/test.sql +++ b/postgres/test.sql @@ -131,6 +131,89 @@ CREATE TABLE public.locationreliabilities ( ALTER TABLE public.locationreliabilities OWNER TO postgres; +-- +-- Name: metaViewer; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public."metaViewer" ( + "SampleName" text, + "OriginalSampleName" text, + "SiteName" text, + "Country" text, + "Longitude" double precision, + "Latitude" double precision, + "Elevation" double precision, + "Temperature" text, + "Precipitation" text, + "LocationReliability" text, + "LocationNotes" text, + "AreaOfSite" double precision, + "SampleContext" text, + "SiteDescription" text, + "VegDescription" text, + "SampleType" text, + "SampleMethod" text, + "AgeBP" double precision, + "AgeUncertainty" text, + ispercent boolean, + "Notes" text, + "EMPD_version" text, + "Publication1" text, + "DOI1" text, + "Publication2" text, + "DOI2" text, + "Publication3" text, + "DOI3" text, + "Publication4" text, + "DOI4" text, + "Publication5" text, + "DOI5" text, + "Worker1_Role" text, + "Worker1_LastName" text, + "Worker1_Initials" text, + "Worker1_FirstName" text, + "Worker1_Address1" text, + "Worker1_Address2" text, + "Worker1_Email1" text, + "Worker1_Email2" text, + "Worker1_Phone1" text, + "Worker1_Phone2" text, + "Worker2_Role" text, + "Worker2_LastName" text, + "Worker2_Initials" text, + "Worker2_FirstName" text, + "Worker2_Address1" text, + "Worker2_Address2" text, + "Worker2_Email1" text, + "Worker2_Email2" text, + "Worker2_Phone1" text, + "Worker2_Phone2" text, + "Worker3_Role" text, + "Worker3_LastName" text, + "Worker3_Initials" text, + "Worker3_FirstName" text, + "Worker3_Address1" text, + "Worker3_Address2" text, + "Worker3_Email1" text, + "Worker3_Email2" text, + "Worker3_Phone1" text, + "Worker3_Phone2" text, + "Worker4_Role" text, + "Worker4_LastName" text, + "Worker4_Initials" text, + "Worker4_FirstName" text, + "Worker4_Address1" text, + "Worker4_Address2" text, + "Worker4_Email1" text, + "Worker4_Email2" text, + "Worker4_Phone1" text, + "Worker4_Phone2" text, + okexcept text +); + + +ALTER TABLE public."metaViewer" OWNER TO postgres; + -- -- Name: metadata; Type: TABLE; Schema: public; Owner: postgres -- @@ -476,6 +559,17 @@ X Do not use! \N \. +-- +-- Data for Name: metaViewer; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public."metaViewer" ("SampleName", "OriginalSampleName", "SiteName", "Country", "Longitude", "Latitude", "Elevation", "Temperature", "Precipitation", "LocationReliability", "LocationNotes", "AreaOfSite", "SampleContext", "SiteDescription", "VegDescription", "SampleType", "SampleMethod", "AgeBP", "AgeUncertainty", ispercent, "Notes", "EMPD_version", "Publication1", "DOI1", "Publication2", "DOI2", "Publication3", "DOI3", "Publication4", "DOI4", "Publication5", "DOI5", "Worker1_Role", "Worker1_LastName", "Worker1_Initials", "Worker1_FirstName", "Worker1_Address1", "Worker1_Address2", "Worker1_Email1", "Worker1_Email2", "Worker1_Phone1", "Worker1_Phone2", "Worker2_Role", "Worker2_LastName", "Worker2_Initials", "Worker2_FirstName", "Worker2_Address1", "Worker2_Address2", "Worker2_Email1", "Worker2_Email2", "Worker2_Phone1", "Worker2_Phone2", "Worker3_Role", "Worker3_LastName", "Worker3_Initials", "Worker3_FirstName", "Worker3_Address1", "Worker3_Address2", "Worker3_Email1", "Worker3_Email2", "Worker3_Phone1", "Worker3_Phone2", "Worker4_Role", "Worker4_LastName", "Worker4_Initials", "Worker4_FirstName", "Worker4_Address1", "Worker4_Address2", "Worker4_Email1", "Worker4_Email2", "Worker4_Phone1", "Worker4_Phone2", okexcept) FROM stdin; +test_a1 orig_test_a1 somewhere France 10 50 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 \N \N \N \N \N \N \N t \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +test_a2 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 \N \N \N \N \N \N \N t \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N Country +test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 X Actually in Germany 30 i dont know \N \N \N \N \N \N t \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N SampleContext +\. + + -- -- Data for Name: metadata; Type: TABLE DATA; Schema: public; Owner: postgres -- @@ -826,6 +920,13 @@ ALTER TABLE ONLY public.workers ADD CONSTRAINT workers_pkey PRIMARY KEY (workerid); +-- +-- Name: ix_metaViewer_SampleName; Type: INDEX; Schema: public; Owner: postgres +-- + +CREATE INDEX "ix_metaViewer_SampleName" ON public."metaViewer" USING btree ("SampleName"); + + -- -- Name: climate climates_samplename_fk; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- From c013d567cf44e07d5c71c8d7746fdbf411c57b0f Mon Sep 17 00:00:00 2001 From: Chilipp Date: Sun, 20 Oct 2019 21:40:04 +0200 Subject: [PATCH 119/119] added counts for test_a1 --- postgres/test.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/postgres/test.sql b/postgres/test.sql index 3116a132c..aa627fde8 100644 --- a/postgres/test.sql +++ b/postgres/test.sql @@ -575,9 +575,9 @@ test_a3 orig_test_a2 somewhere Germany 10 52 340 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 -- COPY public.metadata (samplename, originalsamplename, sitename, country, longitude, latitude, elevation, locationreliability, locationnotes, areaofsite, samplecontext, sitedescription, vegdescription, sampletype, samplemethod, agebp, ageuncertainty, ispercent, notes, okexcept, empd_version) FROM stdin; -test_a1 orig_test_a1 somewhere France 10 50 340 X Actually in Germany 30 \N \N \N \N \N \N \N f \N \N \N -test_a2 orig_test_a2 somewhere Germany 10 52 340 X Actually in Germany 30 \N \N \N \N \N \N \N f \N Country \N -test_a3 orig_test_a2 somewhere Germany 10 52 340 X Actually in Germany 30 i dont know \N \N \N \N \N \N f \N \N \N +test_a1 orig_test_a1 somewhere France 10 50 340 X Actually in Germany 30 \N \N \N \N \N \N \N f \N \N EMPD2 +test_a2 orig_test_a2 somewhere Germany 10 52 340 X Actually in Germany 30 \N \N \N \N \N \N \N f \N Country EMPD2 +test_a3 orig_test_a2 somewhere Germany 10 52 340 X Actually in Germany 30 i dont know \N \N \N \N \N \N f \N \N EMPD2 \. @@ -602,6 +602,9 @@ COPY public.metaworker (samplename, workerid, workerrole) FROM stdin; -- COPY public.p_counts (samplename, var_, count, percentage) FROM stdin; +test_a1 1 45 \N +test_a1 2 6 \N +test_a1 3 55 \N test_a2 1 45 \N test_a2 2 6 \N test_a2 3 55 \N