Project

General

Profile

« Previous | Next » 

Revision 10535

inputs/HVAA/: switched to new-style import, using the steps at wiki.vegpath.org/Adding_new-style_import_to_a_datasource

View differences:

inputs/HVAA/Specimen/postprocess.sql
1 1
SELECT util.search_path_append('util');
2 2

  
3
SELECT util.rename_cols(:table_str, $$
4
occurrenceRemarks => *occurrenceRemarks,
5
$$::hstore);
6 3

  
7

  
8 4
SELECT mk_derived_col((:table_str, 'occurrenceRemarks'),
9
$$_merge("*occurrenceRemarks", "dynamicProperties")$$)
5
$$_merge("occurrenceRemarks_from_occurrenceRemarks", "occurrenceRemarks_from_dynamicProperties")$$)
10 6
; -- runtime: 1 s ("Time: 590.875 ms") @starscream
inputs/HVAA/Specimen/run
1
#!/bin/bash -e
2
. "$(dirname "${BASH_SOURCE[0]}")"/../table.run
0 3

  
inputs/HVAA/Specimen/map.csv
71 71
rightsHolder,dataOwners,,
72 72
accessRights,accessRights,,
73 73
symbiotaId,occurrenceID,,
74
occurrenceRemarks,occurrenceRemarks,,
inputs/HVAA/Specimen/test.xml.ref
9 9
                    <location>
10 10
                        <authorlocationcode>
11 11
                            <_alt>
12
                                <1>$symbiotaId</1>
12
                                <1>$occurrenceID</1>
13 13
                                <2>
14 14
                                    <_join>
15
                                        <1>$institutionCode</1>
16
                                        <2>$collectionCode</2>
17
                                        <3>$catalogNumber</3>
15
                                        <1>$specimenHolderInstitutions</1>
16
                                        <2>$collection</2>
17
                                        <3>$accessionNumber</3>
18 18
                                    </_join>
19 19
                                </2>
20 20
                            </_alt>
......
54 54
                                <aggregateoccurrence>
55 55
                                    <collectiondate>
56 56
                                        <_alt>
57
                                            <1>$eventDate</1>
57
                                            <1>$dateCollected</1>
58 58
                                            <2>
59 59
                                                <_date>
60 60
                                                    <day>
61 61
                                                        <_nullIf>
62 62
                                                            <null>0</null>
63 63
                                                            <type>float</type>
64
                                                            <value>$day</value>
64
                                                            <value>$dayCollected</value>
65 65
                                                        </_nullIf>
66 66
                                                    </day>
67 67
                                                    <month>
68 68
                                                        <_nullIf>
69 69
                                                            <null>0</null>
70 70
                                                            <type>float</type>
71
                                                            <value>$month</value>
71
                                                            <value>$monthCollected</value>
72 72
                                                        </_nullIf>
73 73
                                                    </month>
74 74
                                                    <year>
75 75
                                                        <_nullIf>
76 76
                                                            <null>0</null>
77 77
                                                            <type>float</type>
78
                                                            <value>$year</value>
78
                                                            <value>$yearCollected</value>
79 79
                                                        </_nullIf>
80 80
                                                    </year>
81 81
                                                </_date>
......
85 85
                                    <count>$individualCount</count>
86 86
                                    <notes>$occurrenceRemarks</notes>
87 87
                                    <plantobservation>
88
                                        <authorplantcode>$recordNumber</authorplantcode>
88
                                        <authorplantcode>$collectorNumber</authorplantcode>
89 89
                                        <definedvalue fkey="tablerecord_id">
90 90
                                            <userdefined_id>
91 91
                                                <userdefined>
......
97 97
                                        </definedvalue>
98 98
                                        <reproductivecondition>$reproductiveCondition</reproductivecondition>
99 99
                                        <specimenreplicate>
100
                                            <catalognumber_dwc>$catalogNumber</catalognumber_dwc>
101
                                            <collectioncode_dwc>$collectionCode</collectioncode_dwc>
102
                                            <collectionnumber>$recordNumber</collectionnumber>
100
                                            <catalognumber_dwc>$accessionNumber</catalognumber_dwc>
101
                                            <collectioncode_dwc>$collection</collectioncode_dwc>
102
                                            <collectionnumber>$collectorNumber</collectionnumber>
103 103
                                            <institution_id>
104 104
                                                <sourcelist>
105
                                                    <name>$institutionCode</name>
105
                                                    <name>$specimenHolderInstitutions</name>
106 106
                                                    <sourcename>
107 107
                                                        <name>
108 108
                                                            <_split>
109 109
                                                                <separator>[,;] *</separator>
110
                                                                <value>$institutionCode</value>
110
                                                                <value>$specimenHolderInstitutions</value>
111 111
                                                            </_split>
112 112
                                                        </name>
113 113
                                                    </sourcename>
114 114
                                                </sourcelist>
115 115
                                            </institution_id>
116
                                            <sourceaccessioncode>$symbiotaId</sourceaccessioncode>
116
                                            <sourceaccessioncode>$occurrenceID</sourceaccessioncode>
117 117
                                        </specimenreplicate>
118 118
                                    </plantobservation>
119 119
                                </aggregateoccurrence>
......
134 134
                                        <value>$establishmentMeans</value>
135 135
                                    </_map>
136 136
                                </isnative>
137
                                <sourceaccessioncode>$symbiotaId</sourceaccessioncode>
137
                                <sourceaccessioncode>$occurrenceID</sourceaccessioncode>
138 138
                                <taxondetermination>
139 139
                                    <party_id><party><fullname>$identifiedBy</fullname></party></party_id>
140 140
                                    <taxonverbatim_id>
inputs/HVAA/Specimen/VegBIEN.csv
1
HVAA,VegBIEN:/_setDefault:[source_id/source/shortname/_env:[name=source]]/path/_simplifyPath:[next=parent_id]/path,Comments
2
catalogNumber,"/_if[@name=""if specimen""]/cond/_exists",
3
symbiotaId,"/_if[@name=""if specimen""]/cond/_exists",
4
institutionCode,"/_if[@name=""if specimen""]/else/source/shortname/_first/1",
5
symbiotaId,"/location/_if[@name=""if subplot""]/else/authorlocationcode/_first/3/_if[@name=""if plot""]/else/_alt/1",
6
catalogNumber,"/location/_if[@name=""if subplot""]/else/authorlocationcode/_first/3/_if[@name=""if plot""]/else/_alt/2/_if[@name=""if catalogNumber""]/cond/_exists",
7
catalogNumber,"/location/_if[@name=""if subplot""]/else/authorlocationcode/_first/3/_if[@name=""if plot""]/else/_alt/2/_if[@name=""if catalogNumber""]/then/_join/1/_if[@name=""if specimen""]/cond/_exists",
8
symbiotaId,"/location/_if[@name=""if subplot""]/else/authorlocationcode/_first/3/_if[@name=""if plot""]/else/_alt/2/_if[@name=""if catalogNumber""]/then/_join/1/_if[@name=""if specimen""]/cond/_exists",
9
institutionCode,"/location/_if[@name=""if subplot""]/else/authorlocationcode/_first/3/_if[@name=""if plot""]/else/_alt/2/_if[@name=""if catalogNumber""]/then/_join/1/_if[@name=""if specimen""]/then/_first/2",
10
collectionCode,"/location/_if[@name=""if subplot""]/else/authorlocationcode/_first/3/_if[@name=""if plot""]/else/_alt/2/_if[@name=""if catalogNumber""]/then/_join/2/_join/1",
11
catalogNumber,"/location/_if[@name=""if subplot""]/else/authorlocationcode/_first/3/_if[@name=""if plot""]/else/_alt/2/_if[@name=""if catalogNumber""]/then/_join/3/_if[@name=""if indirect voucher""]/else",
12
rights,/location/accessconditions,
13
accessRights,/location/accesslevel,
14
verbatimElevation,"/location/elevation_m/_alt/1/_units:[to=m,to=]/value",
15
minimumElevationInMeters,/location/elevation_m/_alt/2/_avg/1,
16
maximumElevationInMeters,/location/elevation_m/_alt/2/_avg/2,
17
recordNumber,"/location/iscultivated/_or/1/_if[@name=""if TaxonOccurrence""]/cond/_exists",
18
symbiotaId,"/location/iscultivated/_or/1/_if[@name=""if TaxonOccurrence""]/cond/_exists",
19
locality,/location/iscultivated/_or/2/_locationnarrative_is_cultivated/locationnarrative/_join/1,
20
habitat,/location/iscultivated/_or/2/_locationnarrative_is_cultivated/locationnarrative/_join/3,
21
samplingProtocol,/location/locationevent/*_id/method/name,
22
dateIdentified,"/location/locationevent/taxonoccurrence/_if[@name=""if has accepted name""]/then/taxondetermination:[determinationtype=accepted]/determinationdate/_alt/1/_dateRangeStart/value",
23
taxonRemarks,"/location/locationevent/taxonoccurrence/_if[@name=""if has accepted name""]/then/taxondetermination:[determinationtype=accepted]/notes",
24
identificationQualifier,"/location/locationevent/taxonoccurrence/_if[@name=""if has accepted name""]/then/taxondetermination:[determinationtype=accepted]/taxonfit",
25
eventDate,/location/locationevent/taxonoccurrence/aggregateoccurrence/collectiondate/_alt/1,
26
day,"/location/locationevent/taxonoccurrence/aggregateoccurrence/collectiondate/_alt/2/_date/day/_nullIf:[null=0,type=float]/value",
27
month,"/location/locationevent/taxonoccurrence/aggregateoccurrence/collectiondate/_alt/2/_date/month/_nullIf:[null=0,type=float]/value",
28
year,"/location/locationevent/taxonoccurrence/aggregateoccurrence/collectiondate/_alt/2/_date/year/_nullIf:[null=0,type=float]/value",
29
individualCount,/location/locationevent/taxonoccurrence/aggregateoccurrence/count,
30
occurrenceRemarks,/location/locationevent/taxonoccurrence/aggregateoccurrence/notes/_join/1,
31
recordNumber,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/authorplantcode/_first/2,
32
sex,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/definedvalue[*_id/userdefined[tablename=plantobservation,userdefinedname=sex]]:[@fkey=tablerecord_id]/definedvalue","Brad: OMIT. Apparently not used by this institution and of uncertain applicability to plants. With other collections, *sometimes* used for phenological information (if specimen in flower, in fruit, sterile, etc.). If so, this is important information we should keep. Bob, Nick, anywhere for phenological information in VB and VX?; Aaron: Keeping it because it's useful for some collections"
33
reproductiveCondition,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/reproductivecondition,
34
catalogNumber,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/catalognumber_dwc/_if[@name=""if indirect voucher""]/else",
35
collectionCode,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/collectioncode_dwc/_alt/1,Brad: Not sure if mapping correct. DwC element; code for entire collection; often same as InstitutionCode; Aaron: Combining with InstitutionCode to create collection name
36
recordNumber,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/collectionnumber/_if[@name=""if indirect voucher""]/else",
37
catalogNumber,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/institution_id/sourcelist/_if[@name=""if specimen""]/cond/_exists",
38
symbiotaId,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/institution_id/sourcelist/_if[@name=""if specimen""]/cond/_exists",
39
institutionCode,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/institution_id/sourcelist/_if[@name=""if specimen""]/then/name/_first/2",
40
institutionCode,"/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/institution_id/sourcelist/_if[@name=""if specimen""]/then/sourcename/name/_first/2/_split:[separator=""[,;] *""]/value",
41
symbiotaId,/location/locationevent/taxonoccurrence/aggregateoccurrence/plantobservation/specimenreplicate/sourceaccessioncode,
42
recordNumber,"/location/locationevent/taxonoccurrence/authortaxoncode/_if[@name=""if in plot""]/then/_first/2/_first/3",
43
scientificName,"/location/locationevent/taxonoccurrence/authortaxoncode/_if[@name=""if in plot""]/then/_first/3/_first/2",
44
recordedBy,/location/locationevent/taxonoccurrence/collector_id/party/fullname,
45
recordNumber,"/location/locationevent/taxonoccurrence/iscultivated/_alt/1/_if[@name=""if TaxonOccurrence""]/cond/_exists",
46
symbiotaId,"/location/locationevent/taxonoccurrence/iscultivated/_alt/1/_if[@name=""if TaxonOccurrence""]/cond/_exists",
47
establishmentMeans,"/location/locationevent/taxonoccurrence/iscultivated/_alt/2/_map:[cultivated=t,wild=f,*=]/value",
48
establishmentMeans,"/location/locationevent/taxonoccurrence/isnative/_map:[native=t,exotic=f,*=]/value",
49
symbiotaId,/location/locationevent/taxonoccurrence/sourceaccessioncode/_first/3,
50
identifiedBy,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/party/fullname,
51
taxonRank,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/_if[@name=""if has explicit parent""]/else/rank",
52
specificEpithet,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/parent_id/_forEach:[in:[cultivar,forma,variety,subspecies,],do=""taxonlabel[rank=_val]/parent_id""]/taxonlabel[rank=species]/taxonepithet",
53
genus,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/parent_id/_forEach:[in:[cultivar,forma,variety,subspecies,species,],do=""taxonlabel[rank=_val]/parent_id""]/taxonlabel[rank=genus]/taxonepithet",
54
family,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/parent_id/_forEach:[in:[cultivar,forma,variety,subspecies,species,genus,],do=""taxonlabel[rank=_val]/parent_id""]/taxonlabel[rank=family]/taxonepithet",
55
infraspecificEpithet,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/parent_id/taxonlabel/taxonepithet,
56
taxonRank,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/rank,
57
taxonRank,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonepithet/_if[@name=""if taxonomic name is epithet""]/cond/_alt:[2=true]/1/_taxonomic_name_is_epithet/rank",
58
family,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/prefix/_taxon_family_require_std/family,
59
scientificName,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/value/_alt/1,
60
genus,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/value/_alt/2/_join_words/1/_alt/2/_join_words/2/_filter_genus/value,
61
specificEpithet,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/value/_alt/2/_join_words/1/_alt/2/_join_words/3/_join_words/1,
62
infraspecificEpithet,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/value/_alt/2/_join_words/1/_alt/2/_join_words/5/_join_words/1/_if[@name=""if has infraspecificEpithet""]/cond",
63
taxonRank,"/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/value/_alt/2/_join_words/1/_alt/2/_join_words/5/_join_words/1/_if[@name=""if has infraspecificEpithet""]/then",
64
infraspecificEpithet,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/value/_alt/2/_join_words/1/_alt/2/_join_words/5/_join_words/2,
65
scientificNameAuthorship,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/*_id/taxonlabel/taxonomicname/_merge_prefix/value/_alt/2/_join_words/2,
66
scientificNameAuthorship,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/author,
67
family,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/family,
68
genus,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/genus,
69
specificEpithet,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/specific_epithet,
70
scientificName,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/taxonomicname,
71
taxonRank,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/*_id/taxonverbatim/verbatimrank,
72
dateIdentified,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/determinationdate/_alt/1/_dateRangeStart/value,
73
taxonRemarks,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/notes,
74
identificationQualifier,/location/locationevent/taxonoccurrence/taxondetermination[!isoriginal]/taxonfit,
75
catalogNumber,"/location/locationevent/taxonoccurrence/voucher/*_id/specimenreplicate/catalognumber_dwc/_if[@name=""if indirect voucher""]/then",
76
recordNumber,"/location/locationevent/taxonoccurrence/voucher/*_id/specimenreplicate/collectionnumber/_if[@name=""if indirect voucher""]/then",
77
locality,/location/locationnarrative/_merge/1,
78
habitat,"/location/locationnarrative/_merge/3/_label[label=""habitat""]/value","Brad: Free-text description of vegetation community where collected, frequently redundane wrt 'Vegetation'. Bob, Nick: keep as user defined or create special element?"
79
locationRemarks,/location/locationnarrative/_merge/4,
80
coordinateUncertaintyInMeters,/location/locationplace/*_id/place/*_id/coordinates/coordsaccuracy_m/_noCV/value,
81
footprintWKT,/location/locationplace/*_id/place/*_id/coordinates/footprintgeometry_dwc,
82
decimalLatitude,"/location/locationplace/*_id/place/*_id/coordinates/latitude_deg/_nullIf:[null=0,type=float]/value",
83
decimalLongitude,"/location/locationplace/*_id/place/*_id/coordinates/longitude_deg/_nullIf:[null=0,type=float]/value",
84
verbatimCoordinates,/location/locationplace/*_id/place/*_id/coordinates/verbatimcoordinates,
85
country,/location/locationplace/*_id/place/country,
86
county,/location/locationplace/*_id/place/county,
87
decimalLatitude,"/location/locationplace/*_id/place/matched_place_id/place:[.,source_id/source/shortname=geoscrub]/*_id/coordinates:[source_id/source/shortname=geoscrub]/_first/2/latitude_deg/_nullIf:[null=0,type=float]/value",""".,"" sorts it with other coordinates mappings"
88
decimalLongitude,"/location/locationplace/*_id/place/matched_place_id/place:[.,source_id/source/shortname=geoscrub]/*_id/coordinates:[source_id/source/shortname=geoscrub]/_first/2/longitude_deg/_nullIf:[null=0,type=float]/value",""".,"" sorts it with other coordinates mappings"
89
country,/location/locationplace/*_id/place/matched_place_id/place:[source_id/source/shortname=geoscrub]/country/_first/2,
90
county,/location/locationplace/*_id/place/matched_place_id/place:[source_id/source/shortname=geoscrub]/county/_first/2,
91
stateProvince,/location/locationplace/*_id/place/matched_place_id/place:[source_id/source/shortname=geoscrub]/stateprovince/_first/2,
92
stateProvince,/location/locationplace/*_id/place/stateprovince,
93
verbatimElevation,/location/verbatimelevation,
94
associatedTaxa,,** No join mapping for associatedTaxa ** 
95
basisOfRecord,,** No join mapping for basisOfRecord ** 
96
coordinatePrecision,,** No non-empty join mapping for UNUSED#coordinatePrecision ** 
97
disposition,,** No join mapping for disposition ** 
98
dynamicProperties,,** No join mapping for occurrenceRemarks_from_dynamicProperties ** Contains occurrenceRemarks data
99
endDayOfYear,,** No join mapping for endDayOfYear ** 
100
fieldNotes,,** No non-empty join mapping for UNUSED#fieldNotes ** 
101
fieldNumber,,** No non-empty join mapping for UNUSED#fieldNumber ** 
102
geodeticDatum,,** No join mapping for geodeticDatum ** 
103
georeferenceProtocol,,** No join mapping for georeferenceProtocol ** 
104
georeferenceRemarks,,** No join mapping for georeferenceRemarks ** 
105
georeferenceSources,,** No join mapping for georeferenceSources ** 
106
georeferenceVerificationStatus,,** No join mapping for georeferenceVerificationStatus ** 
107
georeferencedBy,,** No join mapping for georeferencedBy ** 
108
identificationReferences,,** No join mapping for identificationReferences ** 
109
identificationRemarks,,** No join mapping for identificationRemarks ** 
110
language,,** No join mapping for language ** 
111
lifeStage,,** No join mapping for lifeStage ** 
112
modified,,** No join mapping for modified ** 
113
municipality,,** No join mapping for municipality ** 
114
occurrenceRemarks,,** No join mapping for occurrenceRemarks_from_occurrenceRemarks ** 
115
otherCatalogNumbers,,** No join mapping for otherCatalogNumbers ** 
116
ownerInstitutionCode,,** No join mapping for specimenOwner ** 
117
preparations,,** No join mapping for preparations ** 
118
rightsHolder,,** No join mapping for dataOwners ** 
119
row_num,,** No join mapping for *row_num ** 
120
startDayOfYear,,** No join mapping for startDayOfYear ** 
121
typeStatus,,** No join mapping for typeStatus ** 
122
verbatimCoordinateSystem,,** No join mapping for coordinateUnits ** 
123
verbatimEventDate,,** No join mapping for verbatimEventDate ** 
1
link ../../../mappings/VegCore-VegBIEN.csv
124 2

  
inputs/HVAA/run
1
#!/bin/bash -e
2
. "$(dirname "${BASH_SOURCE[0]}")"/../../lib/runscripts/datasrc_dir.run
0 3

  

Also available in: Unified diff