Revision 10618
Added by Aaron Marcuse-Kubitza over 11 years ago
| inputs/UNCC/Specimen/postprocess.sql | ||
|---|---|---|
| 17 | 17 |
$$_map('UNCCD=>UNCC, *=>*', "herbarium")$$)
|
| 18 | 18 |
; -- runtime: 1 s ("Time: 1057.222 ms") @starscream
|
| 19 | 19 |
|
| 20 |
SELECT mk_derived_col((:table_str, 'recordedBy'), |
|
| 21 |
$$_join("collector", "collector1", "collector2", "collector3")$$)
|
|
| 22 |
; -- runtime: 1 s ("Time: 1001.616 ms") @starscream
|
|
| 23 |
|
|
| 24 |
SELECT mk_derived_col((:table_str, 'occurrenceRemarks'), |
|
| 25 |
$$_join("comment1", "comment2", "comment3", "comment4")$$)
|
|
| 26 |
; -- runtime: 1 s ("Time: 996.729 ms") @starscream
|
|
| 27 |
|
|
| 28 |
|
|
| 20 | 29 |
SELECT mk_derived_col((:table_str, 'cultivated__campus'), |
| 21 | 30 |
/* "this refers to occurrence on the UNCC campus, but I do not know the meaning |
| 22 | 31 |
of the codes. Presumably C = cultivated" |
| ... | ... | |
| 30 | 39 |
$$_map('*=>true', "*cultivated")$$)
|
| 31 | 40 |
; -- runtime: 0.5 s ("Time: 403.916 ms") @starscream
|
| 32 | 41 |
|
| 42 |
SELECT mk_derived_col((:table_str, 'cultivated'), |
|
| 43 |
$$_alt("cultivated__main", "cultivated__campus")$$)
|
|
| 44 |
; -- runtime: 0.5 s ("Time: 394.182 ms") @starscream
|
|
| 33 | 45 |
|
| 46 |
|
|
| 34 | 47 |
CREATE OR REPLACE FUNCTION map_reproductive_condition(value text) |
| 35 | 48 |
RETURNS text AS |
| 36 | 49 |
$BODY$ |
| ... | ... | |
| 52 | 65 |
|
| 53 | 66 |
SELECT mk_derived_col((:table_str, 'flower'), $$map_reproductive_condition("*flower")$$); -- runtime: 0.5 s ("Time: 635.054 ms") @starscream
|
| 54 | 67 |
SELECT mk_derived_col((:table_str, 'fruit' ), $$map_reproductive_condition("*fruit" )$$); -- runtime: 0.5 s ("Time: 591.854 ms") @starscream
|
| 68 |
|
|
| 69 |
SELECT mk_derived_col((:table_str, 'reproductiveCondition'), |
|
| 70 |
-- using the suggested values at DwC.vegpath.org?reproductiveCondition |
|
| 71 |
$$_join( |
|
| 72 |
_map(('{immature,mature}'::text[] => 'in bloom' ) || '*=>NULL', "flower")
|
|
| 73 |
, _map(('{immature,mature}'::text[] => 'fruit-bearing') || '*=>NULL', "fruit" )
|
|
| 74 |
)$$) |
|
| 75 |
; -- runtime: 1.5 s ("Time: 1433.204 ms") @starscream
|
|
| inputs/UNCC/Specimen/map.csv | ||
|---|---|---|
| 18 | 18 |
infraname,infraspecificEpithet,, |
| 19 | 19 |
SciName,taxonName,, |
| 20 | 20 |
authors,scientificNameAuthorship,, |
| 21 |
collector,recordedBy,/_join/1,
|
|
| 22 |
collector1,recordedBy,/_join/2,
|
|
| 23 |
collector2,recordedBy,/_join/3,
|
|
| 24 |
collector3,recordedBy,/_join/4,
|
|
| 21 |
collector,recordedBy__primary,,
|
|
| 22 |
collector1,recordedBy__additional__1,,
|
|
| 23 |
collector2,recordedBy__additional__2,,
|
|
| 24 |
collector3,recordedBy__additional__3,,
|
|
| 25 | 25 |
collectno,collectorNumber,, |
| 26 | 26 |
collmonth,monthCollected,, |
| 27 | 27 |
collday,dayCollected,, |
| ... | ... | |
| 36 | 36 |
root,*root,, |
| 37 | 37 |
locality,locality,, |
| 38 | 38 |
habitat,habitat,, |
| 39 |
comment1,occurrenceRemarks,/_join/1,
|
|
| 40 |
comment2,occurrenceRemarks,/_join/2,
|
|
| 41 |
comment3,occurrenceRemarks,/_join/3,
|
|
| 42 |
comment4,occurrenceRemarks,/_join/4,
|
|
| 39 |
comment1,occurrenceRemarks__1,,
|
|
| 40 |
comment2,occurrenceRemarks__2,,
|
|
| 41 |
comment3,occurrenceRemarks__3,,
|
|
| 42 |
comment4,occurrenceRemarks__4,,
|
|
| 43 | 43 |
loanto,*loanto,,only populated in one row |
| 44 | 44 |
inorout,*inorout,,"values: I, A. only populated in 19 rows." |
| 45 | 45 |
sheetno,associatedMedia,,? |
| 46 | 46 |
cultivated,cultivated__main__verbatim,, |
| 47 | 47 |
filler,*filler,,"Where used, same as family" |
| 48 | 48 |
:UNCC,specimenHolderInstitutions,, |
| 49 |
flower,reproductiveCondition,/_join/1, |
|
| 50 |
fruit,reproductiveCondition,/_join/2, |
|
| 51 |
cultivated__campus,cultivated,/_alt/2, |
|
| 52 |
cultivated__main,cultivated,/_alt/1, |
|
| 49 |
recordedBy,recordedBy,, |
|
| 50 |
cultivated,cultivated,, |
|
| 51 |
reproductiveCondition,reproductiveCondition,, |
|
| 52 |
occurrenceRemarks,occurrenceRemarks,, |
|
| inputs/UNCC/Specimen/VegBIEN.csv | ||
|---|---|---|
| 38 | 38 |
HAVING count(*) > 1 |
| 39 | 39 |
-----" |
| 40 | 40 |
collectno,"/location/iscultivated/_or/1/_if[@name=""if TaxonOccurrence""]/cond/_exists", |
| 41 |
cultivated__main,"/location/iscultivated/_or/1/_if[@name=""if TaxonOccurrence""]/else/_first/1/_alt/1", |
|
| 42 |
cultivated__campus,"/location/iscultivated/_or/1/_if[@name=""if TaxonOccurrence""]/else/_first/1/_alt/2", |
|
| 41 |
cultivated,"/location/iscultivated/_or/1/_if[@name=""if TaxonOccurrence""]/else/_first/1", |
|
| 43 | 42 |
locality,/location/iscultivated/_or/2/_locationnarrative_is_cultivated/locationnarrative/_join/1, |
| 44 | 43 |
habitat,/location/iscultivated/_or/2/_locationnarrative_is_cultivated/locationnarrative/_join/3, |
| 45 | 44 |
collday,"/location/locationevent/taxonoccurrence/aggregateoccurrence/collectiondate/_alt/2/_date/day/_nullIf:[null=0,type=float]/value", |
| 46 | 45 |
collmonth,"/location/locationevent/taxonoccurrence/aggregateoccurrence/collectiondate/_alt/2/_date/month/_nullIf:[null=0,type=float]/value", |
| 47 | 46 |
collyear,"/location/locationevent/taxonoccurrence/aggregateoccurrence/collectiondate/_alt/2/_date/year/_nullIf:[null=0,type=float]/value", |
| 48 |
comment1,/location/locationevent/taxonoccurrence/aggregateoccurrence/notes/_join/1/_join/1, |
|
| 49 |
comment2,/location/locationevent/taxonoccurrence/aggregateoccurrence/notes/_join/1/_join/2, |
|
| 50 |
comment3,/location/locationevent/taxonoccurrence/aggregateoccurrence/notes/_join/1/_join/3, |
|
| 51 |
comment4,/location/locationevent/taxonoccurrence/aggregateoccurrence/notes/_join/1/_join/4, |
|
| 47 |
occurrenceRemarks,/location/locationevent/taxonoccurrence/aggregateoccurrence/notes/_join/1, |
|
| 52 | 48 |
collectno,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/authorplantcode/_first/2, |
| 53 |
flower,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/reproductivecondition/_join/1, |
|
| 54 |
fruit,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/reproductivecondition/_join/2, |
|
| 49 |
reproductiveCondition,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/reproductivecondition, |
|
| 55 | 50 |
accession,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/catalognumber_dwc/_if[@name=""if indirect voucher""]/else","Globally unique, although occasionally duplicated. [1] |
| 56 | 51 |
|
| 57 | 52 |
[1] Using the following query: |
| ... | ... | |
| 75 | 70 |
:UNCC,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/institution_id/sourcelist/_if[@name=""if specimen""]/then/sourcename/name/_first/2/_split:[separator=""[,;] *""]/value", |
| 76 | 71 |
collectno,"/location/locationevent/taxonoccurrence/authortaxoncode/_if[@name=""if in plot""]/then/_first/2/_first/3", |
| 77 | 72 |
SciName,"/location/locationevent/taxonoccurrence/authortaxoncode/_if[@name=""if in plot""]/then/_first/3/_first/3", |
| 78 |
collector,/location/locationevent/taxonoccurrence/collector_id/party/fullname/_join/1, |
|
| 79 |
collector1,/location/locationevent/taxonoccurrence/collector_id/party/fullname/_join/2, |
|
| 80 |
collector2,/location/locationevent/taxonoccurrence/collector_id/party/fullname/_join/3, |
|
| 81 |
collector3,/location/locationevent/taxonoccurrence/collector_id/party/fullname/_join/4, |
|
| 73 |
recordedBy,/location/locationevent/taxonoccurrence/collector_id/party/fullname, |
|
| 82 | 74 |
collectno,"/location/locationevent/taxonoccurrence/iscultivated/_alt/1/_if[@name=""if TaxonOccurrence""]/cond/_exists", |
| 83 |
cultivated__main,"/location/locationevent/taxonoccurrence/iscultivated/_alt/1/_if[@name=""if TaxonOccurrence""]/then/_first/1/_alt/1", |
|
| 84 |
cultivated__campus,"/location/locationevent/taxonoccurrence/iscultivated/_alt/1/_if[@name=""if TaxonOccurrence""]/then/_first/1/_alt/2", |
|
| 75 |
cultivated,"/location/locationevent/taxonoccurrence/iscultivated/_alt/1/_if[@name=""if TaxonOccurrence""]/then/_first/1", |
|
| 85 | 76 |
usdaRank,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/_if[@name=""if has explicit parent""]/else/rank",is infraspecificRank__abbr |
| 86 | 77 |
species,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/parent_id/_forEach:[in:[cultivar,forma,variety,subspecies,],do=""taxonlabel[rank=_val]/parent_id""]/taxonlabel[rank=species]/taxonepithet", |
| 87 | 78 |
genus,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/parent_id/_forEach:[in:[cultivar,forma,variety,subspecies,species,],do=""taxonlabel[rank=_val]/parent_id""]/taxonlabel[rank=genus]/taxonepithet", |
| ... | ... | |
| 122 | 113 |
state,/location/locationplace/*_id/place/matched_place_id/place:[source_id/source/shortname=geoscrub]/stateprovince/_first/2, |
| 123 | 114 |
state,/location/locationplace/*_id/place/stateprovince, |
| 124 | 115 |
campus,,"** No join mapping for cultivated__campus__verbatim ** ""this refers to occurrence on the UNCC campus, but I do not know the meaning of the codes. Presumably C = cultivated"" (https://projects.nceas.ucsb.edu/nceas/attachments/download/723/UNCC-validate_rkp_bb_rkp.xlsx#Campus)" |
| 116 |
collector,,** No join mapping for recordedBy__primary ** |
|
| 117 |
collector1,,** No join mapping for recordedBy__additional__1 ** |
|
| 118 |
collector2,,** No join mapping for recordedBy__additional__2 ** |
|
| 119 |
collector3,,** No join mapping for recordedBy__additional__3 ** |
|
| 120 |
comment1,,** No join mapping for occurrenceRemarks__1 ** |
|
| 121 |
comment2,,** No join mapping for occurrenceRemarks__2 ** |
|
| 122 |
comment3,,** No join mapping for occurrenceRemarks__3 ** |
|
| 123 |
comment4,,** No join mapping for occurrenceRemarks__4 ** |
|
| 125 | 124 |
cultivated,,** No join mapping for cultivated__main__verbatim ** |
| 126 | 125 |
filler,,"** No join mapping for *filler ** Where used, same as family" |
| 127 | 126 |
flower,,** No join mapping for flower_verbatim ** |
| inputs/UNCC/Specimen/test.xml.ref | ||
|---|---|---|
| 51 | 51 |
</year> |
| 52 | 52 |
</_date> |
| 53 | 53 |
</collectiondate> |
| 54 |
<notes> |
|
| 55 |
<_join> |
|
| 56 |
<1>$comment1</1> |
|
| 57 |
<2>$comment2</2> |
|
| 58 |
<3>$comment3</3> |
|
| 59 |
<4>$comment4</4> |
|
| 60 |
</_join> |
|
| 61 |
</notes> |
|
| 54 |
<notes>$occurrenceRemarks</notes> |
|
| 62 | 55 |
<plantobservation> |
| 63 | 56 |
<authorplantcode>$collectno</authorplantcode> |
| 64 |
<reproductivecondition> |
|
| 65 |
<_join> |
|
| 66 |
<1>$flower</1> |
|
| 67 |
<2>$fruit</2> |
|
| 68 |
</_join> |
|
| 69 |
</reproductivecondition> |
|
| 57 |
<reproductivecondition>$reproductiveCondition</reproductivecondition> |
|
| 70 | 58 |
<specimenreplicate> |
| 71 | 59 |
<catalognumber_dwc>$accession</catalognumber_dwc> |
| 72 | 60 |
<collectionnumber>$collectno</collectionnumber> |
| ... | ... | |
| 86 | 74 |
</specimenreplicate> |
| 87 | 75 |
</plantobservation> |
| 88 | 76 |
</aggregateoccurrence> |
| 89 |
<collector_id> |
|
| 90 |
<party> |
|
| 91 |
<fullname> |
|
| 92 |
<_join> |
|
| 93 |
<1>$collector</1> |
|
| 94 |
<2>$collector1</2> |
|
| 95 |
<3>$collector2</3> |
|
| 96 |
<4>$collector3</4> |
|
| 97 |
</_join> |
|
| 98 |
</fullname> |
|
| 99 |
</party> |
|
| 100 |
</collector_id> |
|
| 101 |
<iscultivated> |
|
| 102 |
<_alt> |
|
| 103 |
<1>$cultivated__main</1> |
|
| 104 |
<2>$cultivated__campus</2> |
|
| 105 |
</_alt> |
|
| 106 |
</iscultivated> |
|
| 77 |
<collector_id><party><fullname>$recordedBy</fullname></party></collector_id> |
|
| 78 |
<iscultivated>$cultivated</iscultivated> |
|
| 107 | 79 |
<taxondetermination> |
| 108 | 80 |
<taxonverbatim_id> |
| 109 | 81 |
<taxonverbatim> |
| inputs/UNCC/Specimen/new_terms.csv | ||
|---|---|---|
| 4 | 4 |
infraname,infraspecificEpithet,, |
| 5 | 5 |
SciName,taxonName,, |
| 6 | 6 |
authors,scientificNameAuthorship,, |
| 7 |
collector1,recordedBy,/_join/2,
|
|
| 8 |
collector2,recordedBy,/_join/3,
|
|
| 9 |
collector3,recordedBy,/_join/4,
|
|
| 7 |
collector1,recordedBy__additional__1,,
|
|
| 8 |
collector2,recordedBy__additional__2,,
|
|
| 9 |
collector3,recordedBy__additional__3,,
|
|
| 10 | 10 |
collectno,collectorNumber,, |
| 11 | 11 |
collmonth,monthCollected,, |
| 12 | 12 |
collday,dayCollected,, |
| 13 | 13 |
collyear,yearCollected,, |
| 14 | 14 |
campus,cultivated__campus__verbatim,,"""this refers to occurrence on the UNCC campus, but I do not know the meaning of the codes. Presumably C = cultivated"" (https://projects.nceas.ucsb.edu/nceas/attachments/download/723/UNCC-validate_rkp_bb_rkp.xlsx#Campus)" |
| 15 |
comment1,occurrenceRemarks,/_join/1,
|
|
| 16 |
comment2,occurrenceRemarks,/_join/2,
|
|
| 17 |
comment3,occurrenceRemarks,/_join/3,
|
|
| 18 |
comment4,occurrenceRemarks,/_join/4,
|
|
| 15 |
comment1,occurrenceRemarks__1,,
|
|
| 16 |
comment2,occurrenceRemarks__2,,
|
|
| 17 |
comment3,occurrenceRemarks__3,,
|
|
| 18 |
comment4,occurrenceRemarks__4,,
|
|
| 19 | 19 |
sheetno,associatedMedia,,? |
| 20 |
cultivated__campus,cultivated,/_alt/2, |
|
| 21 |
cultivated__main,cultivated,/_alt/1, |
|
| inputs/UNCC/Specimen/unmapped_terms.csv | ||
|---|---|---|
| 1 | 1 |
*row_num |
| 2 | 2 |
specimenDonorInstitution_verbatim |
| 3 |
recordedBy__primary |
|
| 4 |
recordedBy__additional__1 |
|
| 5 |
recordedBy__additional__2 |
|
| 6 |
recordedBy__additional__3 |
|
| 3 | 7 |
cultivated__campus__verbatim |
| 4 | 8 |
*leaves |
| 5 | 9 |
flower_verbatim |
| 6 | 10 |
fruit_verbatim |
| 7 | 11 |
*root |
| 12 |
occurrenceRemarks__1 |
|
| 13 |
occurrenceRemarks__2 |
|
| 14 |
occurrenceRemarks__3 |
|
| 15 |
occurrenceRemarks__4 |
|
| 8 | 16 |
*loanto |
| 9 | 17 |
*inorout |
| 10 | 18 |
associatedMedia |
Also available in: Unified diff
inputs/UNCC/Specimen/: translated multi-column filters to postprocessing derived columns, using the steps at http://wiki.vegpath.org/Adding_new-style_import_to_a_datasource#Translating-filters-to-postprocessing-derived-columns. reproductiveCondition: use the suggested values at DwC.vegpath.org?reproductiveCondition instead of just joining together the values in flower and fruit.