Project

General

Profile

« Previous | Next » 

Revision 131

Added by Aaron Marcuse-Kubitza over 12 years ago

Added support for multiple database engines. Changed SALVIAS_db input to use user-entered password.

View differences:

scripts/test/input/SALVIAS_db.organisms.sh
1 1 
#!/bin/sh
2 
export in_host=localhost in_user=aaronmk in_database=salvias_plots
2 
export in_engine=MySQL in_host=localhost in_user="$USER" \
3 
in_database=salvias_plots
scripts/test/map
8 8 
shopt -s nullglob
9 9 

  		




  10
  10
  
    
test -n "$n" || n=2


  		




  11
  
  
    
let n++ # include header row


  		




  
  11
  
    
let n++ # account for header row


  		




  12
  12
  
    

  		




  
  13
  
    
. ../util/env_password in_password


  		




  
  14
  
    

  		




  
  15
  
    

  		




  13
  16
  
    
# Update generated mappings


  		




  14
  17
  
    
../../mappings/extract_plot_map


  		




  15
  18
  
    
../../mappings/join_all_vegbank


  		












  

    
      scripts/lib/util.py
    
  





  1
  1
  
    
# Useful functions and classes


  		




  2
  2
  
    

  		




  3
  
  
    
class Obj:


  		




  4
  
  
    
    def __init__(self, **attrs): self.__dict__ = attrs


  		




  5
  
  
    
    


  		




  6
  
  
    
    def __repr__(self): return repr(self.__dict__)


  		




  
  3
  
    
def rename_key(dict_, orig, new):


  		




  
  4
  
    
    try: dict_[new] = dict_.pop(orig)


  		




  
  5
  
    
    except KeyError: pass


  		












  

    
      scripts/lib/sql.py
    
  





  5
  5
  
    
import sys


  		




  6
  6
  
    

  		




  7
  7
  
    
import ex


  		




  
  8
  
    
import util


  		




  8
  9
  
    

  		




  9
  10
  
    
def _add_cursor_info(e, cur): ex.add_msg(e, 'query: '+cur.query)


  		




  10
  11
  
    

  		




  ......




  114
  115
  
    
            return last_insert_id(db)


  		




  115
  116
  
    
        except DuplicateKeyException, e:


  		




  116
  117
  
    
            return value(select(db, table, [pkey], {e.col: row[e.col]}))


  		




  
  118
  
    

  		




  
  119
  
    
db_engines = {


  		




  
  120
  
    
    'MySQL': ('MySQLdb', {'password': 'passwd', 'database': 'db'}),


  		




  
  121
  
    
    'PostgreSQL': ('psycopg2', {}),


  		




  
  122
  
    
}


  		




  
  123
  
    

  		




  
  124
  
    
def connect(db_config):


  		




  
  125
  
    
    db_config = db_config.copy() # don't modify input!


  		




  
  126
  
    
    module, mappings = db_engines[db_config.pop('engine')]


  		




  
  127
  
    
    for orig, new in mappings.iteritems(): util.rename_key(db_config, orig, new)


  		




  
  128
  
    
    return __import__(module).connect(**db_config)


  		












  

    
      scripts/map
    
  





  11
  11
  
    
sys.path.append(os.path.dirname(__file__)+"/lib")


  		




  12
  12
  
    

  		




  13
  13
  
    
import opts


  		




  
  14
  
    
import sql


  		




  14
  15
  
    
import xml_func


  		




  15
  16
  
    

  		




  16
  17
  
    
def metadata_value(name):


  		




  ......




  18
  19
  
    
    else: return None


  		




  19
  20
  
    

  		




  20
  21
  
    
def main():


  		




  
  22
  
    
    env_names = []


  		




  
  23
  
    
    def usage_err():


  		




  
  24
  
    
        raise SystemExit('Usage: '+opts.env_usage(env_names, True)


  		




  
  25
  
    
            +' [commit=1] '+sys.argv[0]+' [map_path] [<input] [>output]')


  		




  
  26
  
    
    


  		




  21
  27
  
    
    # Get db config from env vars


  		




  22
  
  
    
    db_config_names = ['host', 'user', 'password', 'database']


  		




  23
  
  
    
    env_names = []


  		




  
  28
  
    
    db_config_names = ['engine', 'host', 'user', 'password', 'database']


  		




  24
  29
  
    
    def get_db_config(prefix):


  		




  25
  30
  
    
        return opts.get_env_vars(db_config_names, prefix, env_names)


  		




  26
  31
  
    
    in_db_config = get_db_config('in')


  		




  27
  32
  
    
    out_db_config = get_db_config('out')


  		




  28
  
  
    
    in_is_db = in_db_config != {}


  		




  29
  
  
    
    out_is_db = out_db_config != {}


  		




  
  33
  
    
    in_is_db = 'engine' in in_db_config


  		




  
  34
  
    
    out_is_db = 'engine' in out_db_config


  		




  30
  35
  
    
    


  		




  31
  36
  
    
    # Parse args


  		




  32
  37
  
    
    map_path = None


  		




  33
  38
  
    
    try: _prog_name, map_path = sys.argv


  		




  34
  39
  
    
    except ValueError:


  		




  35
  
  
    
        if in_is_db or not out_is_db: raise SystemExit('Usage: '


  		




  36
  
  
    
            +opts.env_usage(env_names, True)+' [commit=1] '+sys.argv[0]


  		




  37
  
  
    
            +' [map_path] [<input] [>output]')


  		




  
  40
  
    
        if in_is_db or not out_is_db: usage_err()


  		




  38
  41
  
    
    commit = opts.env_flag('commit')


  		




  39
  42
  
    
    


  		




  40
  43
  
    
    # Load map header


  		




  ......




  68
  71
  
    
    # Input datasource to XML tree, mapping if needed


  		




  69
  72
  
    
    if in_is_xml: doc0 = xml.dom.minidom.parse(sys.stdin)


  		




  70
  73
  
    
    if map_path != None:


  		




  71
  
  
    
        doc1 = xml.dom.minidom.getDOMImplementation().createDocument(None,


  		




  72
  
  
    
            dest, None)


  		




  
  74
  
    
        doc1 = xml.dom.minidom.getDOMImplementation().createDocument(None, dest,


  		




  
  75
  
    
            None)


  		




  73
  76
  
    
        if in_is_db:


  		




  74
  77
  
    
            assert in_is_xpaths


  		




  75
  78
  
    
            


  		




  76
  
  
    
            import psycopg2


  		




  77
  79
  
    
            import db_xml


  		




  78
  80
  
    
            


  		




  79
  81
  
    
            try: src_root = xpath.parse(src_root)


  		




  80
  82
  
    
            except SyntaxException, ex: raise SystemExit(str(ex))


  		




  81
  83
  
    
            


  		




  82
  
  
    
            in_db = psycopg2.connect(**in_db_config)


  		




  
  84
  
    
            in_db = sql.connect(in_db_config)


  		




  83
  85
  
    
            for in_, out in mappings:


  		




  84
  86
  
    
                value = metadata_value(in_)


  		




  85
  87
  
    
                if value == None:


  		




  ......




  108
  110
  
    
    


  		




  109
  111
  
    
    # Output XML tree


  		




  110
  112
  
    
    if out_is_db:


  		




  111
  
  
    
        import psycopg2


  		




  112
  113
  
    
        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE


  		




  113
  114
  
    
        import db_xml


  		




  114
  115
  
    
        


  		




  115
  
  
    
        db = psycopg2.connect(**out_db_config)


  		




  
  116
  
    
        db = sql.connect(out_db_config)


  		




  116
  117
  
    
        db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)


  		




  117
  118
  
    
        try:


  		




  118
  119
  
    
            row_ct_ref = [0]


  		












  

    
      scripts/map2vegbank
    
  





  3
  3
  
    

  		




  4
  4
  
    
selfDir="$(dirname -- "$0")"


  		




  5
  5
  
    

  		




  6
  
  
    
export out_host=localhost out_user=vegbank out_password=vegbank \


  		




  7
  
  
    
out_database=vegbank


  		




  
  6
  
    
export out_engine=PostgreSQL out_host=localhost out_user=vegbank \


  		




  
  7
  
    
out_password=vegbank out_database=vegbank


  		




  8
  8
  
    

  		




  9
  9
  
    
"$selfDir/map" "$@"


  		












  

    
      scripts/util/env_password
    
  





  
  1
  
    
#!/bin/sh


  		




  
  2
  
    
# Sets a password environment variable


  		




  
  3
  
    

  		




  
  4
  
    
if test "$#" -lt 1; then


  		




  
  5
  
    
    echo "Usage: . $0 env_var_name (note the \".\" at the beginning)"


  		




  
  6
  
    
    exit 2


  		




  
  7
  
    
fi


  		




  
  8
  
    

  		




  
  9
  
    
if test -z "${!1+t}"; then # env var with name $1 is unset


  		




  
  10
  
    
    read -s -p "Enter $1: "; echo


  		




  
  11
  
    
    export "$1"="$REPLY"


  		




  
  12
  
    
fi


  		




  0
  13
  
    

  		












Also available in:  Unified diff