Project

General

Profile

« Previous | Next » 

Revision 1537

Added by Aaron Marcuse-Kubitza almost 13 years ago

xml_func.py: _map: Instead of _closed special entry, make all maps closed by default and open them if special entry "*=*" is present. Support using a _map to filter values by interpreting special entry "*=" as removing all values not explicitly specified, and by interpreting special value "*" as keeping input value the same.

View differences:

inputs/SALVIAS/maps/VegX.plots.csv
34 34 
"sand_percent","->/*s/abioticObservation[*ID]/simpleUserdefined[name=sandPercent]/value/_units:[units=""%""]/value",
35 35 
"pH","->/*s/abioticObservation[*ID]/ph","Brad: For all these soil variables, how do we store information on method, units. Seems like an ontological structure would be more appropriate (measurementName, measurementValue, measurementUnits, measurementMethod). Same as BIEN traits table. Something to discuss."
36 36 
"soil_N","->/*s/abioticObservation[*ID]/nitrogen/_units:[units=""%""]/value",
37 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0]/value/_units:[units=""%""]/value",
37 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0,*=*]/value/_units:[units=""%""]/value",
38 38 
"soil_C","->/*s/abioticObservation[*ID]/simpleUserdefined[name=carbon]/value/_units:[units=""%""]/value",
39 39 
"soil_K","->/*s/abioticObservation[*ID]/pottasium/_units:[units=""%""]/value",
40 40 
"soil_Mg","->/*s/abioticObservation[*ID]/magnesium/_units:[units=""%""]/value",
41 41 
"soil_Ca","->/*s/abioticObservation[*ID]/simpleUserdefined[name=calcium]/value/_units:[units=""%""]/value",
42 42 
"soil_Na","->/*s/abioticObservation[*ID]/simpleUserdefined[name=sodium]/value/_units:[units=""%""]/value",
43 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0]/value/_units:[units=""%""]/value",
43 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0,*=*]/value/_units:[units=""%""]/value",
44 44 
"soil_base","->/*s/abioticObservation[*ID]/simpleUserdefined[name=base]/value/_units:[units=""%""]/value",
45 45 
"soil_cation_cap","->/*s/abioticObservation[*ID]/simpleUserdefined[name=cationCap]/value",
46 46 
"soil_conductivity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=conductivity]/value",
inputs/SALVIAS/maps/VegX.plots.full.csv
34 34 
"sand_percent","->/*s/abioticObservation[*ID]/simpleUserdefined[name=sandPercent]/value/_units:[units=""%""]/value",
35 35 
"pH","->/*s/abioticObservation[*ID]/ph","Brad: For all these soil variables, how do we store information on method, units. Seems like an ontological structure would be more appropriate (measurementName, measurementValue, measurementUnits, measurementMethod). Same as BIEN traits table. Something to discuss."
36 36 
"soil_N","->/*s/abioticObservation[*ID]/nitrogen/_units:[units=""%""]/value",
37 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0]/value/_units:[units=""%""]/value",
37 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0,*=*]/value/_units:[units=""%""]/value",
38 38 
"soil_C","->/*s/abioticObservation[*ID]/simpleUserdefined[name=carbon]/value/_units:[units=""%""]/value",
39 39 
"soil_K","->/*s/abioticObservation[*ID]/pottasium/_units:[units=""%""]/value",
40 40 
"soil_Mg","->/*s/abioticObservation[*ID]/magnesium/_units:[units=""%""]/value",
41 41 
"soil_Ca","->/*s/abioticObservation[*ID]/simpleUserdefined[name=calcium]/value/_units:[units=""%""]/value",
42 42 
"soil_Na","->/*s/abioticObservation[*ID]/simpleUserdefined[name=sodium]/value/_units:[units=""%""]/value",
43 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0]/value/_units:[units=""%""]/value",
43 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0,*=*]/value/_units:[units=""%""]/value",
44 44 
"soil_base","->/*s/abioticObservation[*ID]/simpleUserdefined[name=base]/value/_units:[units=""%""]/value",
45 45 
"soil_cation_cap","->/*s/abioticObservation[*ID]/simpleUserdefined[name=cationCap]/value",
46 46 
"soil_conductivity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=conductivity]/value",
inputs/SALVIAS/maps/VegBIEN.plots.csv
19 19 
date_finish,/obsenddate/_*/date,
20 20 
date_start,/obsstartdate/_*/date,
21 21 
Precip,/precipitation,
22 
soil_acidity,"/soilobs/acidity/_map:[""<0.05""=0]/value/_units:[units=""%""]/value",
22 
soil_acidity,"/soilobs/acidity/_map:[""<0.05""=0,*=*]/value/_units:[units=""%""]/value",
23 23 
soil_base,"/soilobs/basesaturation/_units:[units=""%""]/value",
24 24 
soil_Ca,"/soilobs/calcium/_units:[units=""%""]/value",
25 25 
soil_C,"/soilobs/carbon/_units:[units=""%""]/value",
......
30 30 
soil_N,"/soilobs/nitrogen/_units:[units=""%""]/value",
31 31 
organic_percent,"/soilobs/organic/_units:[units=""%""]/value",
32 32 
pH,/soilobs/ph,"Brad: For all these soil variables, how do we store information on method, units. Seems like an ontological structure would be more appropriate (measurementName, measurementValue, measurementUnits, measurementMethod). Same as BIEN traits table. Something to discuss."
33 
soil_P,"/soilobs/phosphorus/_map:[""<1,5""=0]/value/_units:[units=""%""]/value",
33 
soil_P,"/soilobs/phosphorus/_map:[""<1,5""=0,*=*]/value/_units:[units=""%""]/value",
34 34 
soil_K,"/soilobs/potassium/_units:[units=""%""]/value",
35 35 
sand_percent,"/soilobs/sand/_units:[units=""%""]/value",
36 36 
silt_percent,"/soilobs/silt/_units:[units=""%""]/value",
inputs/UNCC/maps/DwC.specimens.csv
15 15 
collday,dayCollected,
16 16 
collyear,yearCollected,
17 17 
campus,"fieldNotes/_merge/5/_label[label=""campus""]/value",
18 
leaves,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
19 
flower,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
20 
fruit,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
21 
root,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
18 
leaves,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
19 
flower,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
20 
fruit,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
21 
root,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
22 22 
comment1,fieldNotes/_merge/1,
23 23 
comment2,fieldNotes/_merge/2,
24 24 
comment3,fieldNotes/_merge/3,
......
26 26 
loanto,"fieldNotes/_merge/6/_label[label=""loaned to""]/value",
27 27 
inorout,"fieldNotes/_merge/7/_label[label=""in or out""]/value",
28 28 
sheetno,associatedMedia,?
29 
cultivated,"establishmentMeans/_map:[Y=cultivated,N=wild]/value",Ignore other values
29 
cultivated,"establishmentMeans/_map:[Y=cultivated,N=wild,*=]/value",Ignore other values
30 30 
filler,,"Where used, same as family"
inputs/UNCC/maps/DwC.specimens.full.csv
9 9 
country,country
10 10 
county,county
11 11 
collday,dayCollected,
12 
cultivated,"establishmentMeans/_map:[Y=cultivated,N=wild]/value",Ignore other values
12 
cultivated,"establishmentMeans/_map:[Y=cultivated,N=wild,*=]/value",Ignore other values
13 13 
family,family
14 14 
comment1,fieldNotes/_merge/1,
15 15 
comment2,fieldNotes/_merge/2,
......
24 24 
locality,locality
25 25 
collmonth,monthCollected,
26 26 
accession,occurrenceID,
27 
flower,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
28 
fruit,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
29 
leaves,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
30 
root,reproductiveCondition/_map:[]/value,"Main values are A, I, and M. What do they mean?"
27 
flower,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
28 
fruit,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
29 
leaves,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
30 
root,"reproductiveCondition/_map:[A=*,I=*,M=*]/value","Main values are A, I, and M. What do they mean?"
31 31 
SciName,scientificName,
32 32 
authors,scientificNameAuthorship,
33 33 
species,species
inputs/UNCC/maps/VegBIEN.specimens.csv
27 27 
inorout,"/description/_merge/1/_merge/7/_label[label=""in or out""]/value",,
28 28 
accession,":[datasource_id/party/organizationname/_alt/2=$/_ignore/inLabel,*_id/taxonoccurrence]/sourceaccessioncode","Brad: OMIT; Aaron: We need this field because it is the only ID column with an entry in every row; Brad: OK, use this as primary key",19
29 29 
collectno,:[datasource_id/party/organizationname/_alt/2=$/_ignore/inLabel]/collectionnumber/_alt/1,,22
30 
cultivated,,"** No join mapping for establishmentMeans/_map:[Y=cultivated,N=wild]/value ** Ignore other values"
30 
cultivated,,"** No join mapping for establishmentMeans/_map:[Y=cultivated,N=wild,*=]/value ** Ignore other values"
31 31 
filler,,"Where used, same as family"
32 
flower,,"** No join mapping for reproductiveCondition/_map:[]/value ** Main values are A, I, and M. What do they mean?"
33 
fruit,,"** No join mapping for reproductiveCondition/_map:[]/value ** Main values are A, I, and M. What do they mean?"
32 
flower,,"** No join mapping for reproductiveCondition/_map:[A=*,I=*,M=*]/value ** Main values are A, I, and M. What do they mean?"
33 
fruit,,"** No join mapping for reproductiveCondition/_map:[A=*,I=*,M=*]/value ** Main values are A, I, and M. What do they mean?"
34 34 
infrarank,,** No join mapping for taxonRank/_alt/2 **
35 
leaves,,"** No join mapping for reproductiveCondition/_map:[]/value ** Main values are A, I, and M. What do they mean?"
36 
root,,"** No join mapping for reproductiveCondition/_map:[]/value ** Main values are A, I, and M. What do they mean?"
35 
leaves,,"** No join mapping for reproductiveCondition/_map:[A=*,I=*,M=*]/value ** Main values are A, I, and M. What do they mean?"
36 
root,,"** No join mapping for reproductiveCondition/_map:[A=*,I=*,M=*]/value ** Main values are A, I, and M. What do they mean?"
37 37 
sheetno,,** No join mapping for associatedMedia ** ?
38 38 
usdaRank,,** No join mapping for taxonRank/_alt/1 **
inputs/NCU-NCSC/maps/DwC.specimens.csv
4 4 
Barcode,CatalogNumber,
5 5 
Full Taxon Name,ScientificName,
6 6 
Collect Date,CollectedDate,
7 
Cultivated?,"establishmentMeans/_map:[_closed=1,Yes=cultivated,No=wild]/value",
7 
Cultivated?,"establishmentMeans/_map:[Yes=cultivated,No=wild]/value",
8 8 
Elevation,VerbatimElevation,
9 9 
Lat (-=S),"Latitude/_replace:[""\?$""=]/value",
10 10 
Long (-=W),"Longitude/_replace:[""\?$""=]/value",
inputs/NCU-NCSC/maps/DwC.specimens.full.csv
11 11 
Full Taxon Name,ScientificName,
12 12 
State,State
13 13 
Elevation,VerbatimElevation,
14 
Cultivated?,"establishmentMeans/_map:[_closed=1,Yes=cultivated,No=wild]/value",
14 
Cultivated?,"establishmentMeans/_map:[Yes=cultivated,No=wild]/value",
15 15 
Accession Number,occurrenceID,
16 16 
Alt E/W,,
17 17 
Alt N/S,,
inputs/NCU-NCSC/maps/VegBIEN.specimens.csv
12 12 
Accession Number,":[datasource_id/party/organizationname/_alt/2=$/_ignore/inLabel,*_id/taxonoccurrence]/sourceaccessioncode","Brad: OMIT; Aaron: We need this field because it is the only ID column with an entry in every row; Brad: OK, use this as primary key",19
13 13 
Alt E/W,,
14 14 
Alt N/S,,
15 
Cultivated?,,"** No join mapping for establishmentMeans/_map:[_closed=1,Yes=cultivated,No=wild]/value ** "
15 
Cultivated?,,"** No join mapping for establishmentMeans/_map:[Yes=cultivated,No=wild]/value ** "
16 16 
Elevation,,** No join mapping for VerbatimElevation **
17 17 
FIPS,,FIPS county code
18 18 
Format,,** No join mapping for PreparationType **
inputs/SALVIAS-CSV/maps/VegX.plots.csv
22 22 
"organic_percent","->/*s/abioticObservation[*ID]/simpleUserdefined[name=organicPercent]/value/_units:[units=""%""]/value",
23 23 
"pH","->/*s/abioticObservation[*ID]/ph","Brad: For all these soil variables, how do we store information on method, units. Seems like an ontological structure would be more appropriate (measurementName, measurementValue, measurementUnits, measurementMethod). Same as BIEN traits table. Something to discuss."
24 24 
"soil_N","->/*s/abioticObservation[*ID]/nitrogen/_units:[units=""%""]/value",
25 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0]/value/_units:[units=""%""]/value",
25 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0,*=*]/value/_units:[units=""%""]/value",
26 26 
"soil_C","->/*s/abioticObservation[*ID]/simpleUserdefined[name=carbon]/value/_units:[units=""%""]/value",
27 27 
"soil_K","->/*s/abioticObservation[*ID]/pottasium/_units:[units=""%""]/value",
28 28 
"soil_Mg","->/*s/abioticObservation[*ID]/magnesium/_units:[units=""%""]/value",
29 29 
"soil_Ca","->/*s/abioticObservation[*ID]/simpleUserdefined[name=calcium]/value/_units:[units=""%""]/value",
30 30 
"soil_Na","->/*s/abioticObservation[*ID]/simpleUserdefined[name=sodium]/value/_units:[units=""%""]/value",
31 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0]/value/_units:[units=""%""]/value",
31 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0,*=*]/value/_units:[units=""%""]/value",
32 32 
"soil_base","->/*s/abioticObservation[*ID]/simpleUserdefined[name=base]/value/_units:[units=""%""]/value",
33 33 
"soil_cation_cap","->/*s/abioticObservation[*ID]/simpleUserdefined[name=cationCap]/value",
34 34 
"soil_conductivity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=conductivity]/value",
inputs/SALVIAS-CSV/maps/VegX.plots.full.csv
22 22 
"organic_percent","->/*s/abioticObservation[*ID]/simpleUserdefined[name=organicPercent]/value/_units:[units=""%""]/value",
23 23 
"pH","->/*s/abioticObservation[*ID]/ph","Brad: For all these soil variables, how do we store information on method, units. Seems like an ontological structure would be more appropriate (measurementName, measurementValue, measurementUnits, measurementMethod). Same as BIEN traits table. Something to discuss."
24 24 
"soil_N","->/*s/abioticObservation[*ID]/nitrogen/_units:[units=""%""]/value",
25 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0]/value/_units:[units=""%""]/value",
25 
"soil_P","->/*s/abioticObservation[*ID]/phosphorus/_map:[""<1,5""=0,*=*]/value/_units:[units=""%""]/value",
26 26 
"soil_C","->/*s/abioticObservation[*ID]/simpleUserdefined[name=carbon]/value/_units:[units=""%""]/value",
27 27 
"soil_K","->/*s/abioticObservation[*ID]/pottasium/_units:[units=""%""]/value",
28 28 
"soil_Mg","->/*s/abioticObservation[*ID]/magnesium/_units:[units=""%""]/value",
29 29 
"soil_Ca","->/*s/abioticObservation[*ID]/simpleUserdefined[name=calcium]/value/_units:[units=""%""]/value",
30 30 
"soil_Na","->/*s/abioticObservation[*ID]/simpleUserdefined[name=sodium]/value/_units:[units=""%""]/value",
31 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0]/value/_units:[units=""%""]/value",
31 
"soil_acidity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=acidity]/value/_map:[""<0.05""=0,*=*]/value/_units:[units=""%""]/value",
32 32 
"soil_base","->/*s/abioticObservation[*ID]/simpleUserdefined[name=base]/value/_units:[units=""%""]/value",
33 33 
"soil_cation_cap","->/*s/abioticObservation[*ID]/simpleUserdefined[name=cationCap]/value",
34 34 
"soil_conductivity","->/*s/abioticObservation[*ID]/simpleUserdefined[name=conductivity]/value",
inputs/SALVIAS-CSV/maps/VegBIEN.plots.csv
19 19 
date_finish,/obsenddate/_*/date,
20 20 
date_start,/obsstartdate/_*/date,
21 21 
precip_mm,/precipitation,
22 
soil_acidity,"/soilobs/acidity/_map:[""<0.05""=0]/value/_units:[units=""%""]/value",
22 
soil_acidity,"/soilobs/acidity/_map:[""<0.05""=0,*=*]/value/_units:[units=""%""]/value",
23 23 
soil_base,"/soilobs/basesaturation/_units:[units=""%""]/value",
24 24 
soil_Ca,"/soilobs/calcium/_units:[units=""%""]/value",
25 25 
soil_C,"/soilobs/carbon/_units:[units=""%""]/value",
......
30 30 
soil_N,"/soilobs/nitrogen/_units:[units=""%""]/value",
31 31 
organic_percent,"/soilobs/organic/_units:[units=""%""]/value",
32 32 
pH,/soilobs/ph,"Brad: For all these soil variables, how do we store information on method, units. Seems like an ontological structure would be more appropriate (measurementName, measurementValue, measurementUnits, measurementMethod). Same as BIEN traits table. Something to discuss."
33 
soil_P,"/soilobs/phosphorus/_map:[""<1,5""=0]/value/_units:[units=""%""]/value",
33 
soil_P,"/soilobs/phosphorus/_map:[""<1,5""=0,*=*]/value/_units:[units=""%""]/value",
34 34 
soil_K,"/soilobs/potassium/_units:[units=""%""]/value",
35 35 
sand_percent,"/soilobs/sand/_units:[units=""%""]/value",
36 36 
silt_percent,"/soilobs/silt/_units:[units=""%""]/value",
lib/xml_func.py
137 137 
funcs['_nullIf'] = _nullIf
138 138 

  		




  139
  139
  
    
def _map(items):


  		




  
  140
  
    
    '''Raises error if value not in map and no special '*' entry


  		




  
  141
  
    
    @param items


  		




  
  142
  
    
        <last_entry> Value


  		




  
  143
  
    
        <other_entries> name=value Mappings


  		




  
  144
  
    
            name "*" means all other input values


  		




  
  145
  
    
            value "*" means keep input value the same


  		




  
  146
  
    
            value "" means ignore input value


  		




  
  147
  
    
    '''


  		




  140
  148
  
    
    items = conv_items(str, items) # get *once* from iter and check types


  		




  141
  149
  
    
    try: value = items.pop()[1] # last entry contains value


  		




  142
  150
  
    
    except IndexError, e: raise SyntaxException(e)


  		




  143
  151
  
    
    map_ = dict(items)


  		




  144
  
  
    
    closed = bool(map_.pop('_closed', False))


  		




  145
  
  
    
    try: value = map_[value]


  		




  
  152
  
    
    


  		




  
  153
  
    
    try: new_value = map_[value]


  		




  146
  154
  
    
    except KeyError, e:


  		




  147
  
  
    
        if closed: raise SyntaxException(e)


  		




  148
  
  
    
        else: return value


  		




  149
  
  
    
    return util.none_if(value, u'') # empty map entry means None


  		




  
  155
  
    
        # Save traceback right away in case another exception raised


  		




  
  156
  
    
        se = SyntaxException(e) 


  		




  
  157
  
    
        try: new_value = map_['*']


  		




  
  158
  
    
        except KeyError: raise se


  		




  
  159
  
    
    if new_value == '*': new_value = value # '*' means keep input value the same


  		




  
  160
  
    
    return util.none_if(new_value, u'') # empty map entry means None


  		




  150
  161
  
    
funcs['_map'] = _map


  		




  151
  162
  
    

  		




  152
  163
  
    
def _replace(items):


  		












Also available in:  Unified diff