Project

General

Profile

« Previous | Next » 

Revision 1646

Added by Aaron Marcuse-Kubitza almost 13 years ago

inputs/REMIB/src/nodes.make: Support custom starting node ID and # nodes processed via env vars

View differences:

nodes.make
1 1 
#!/usr/bin/env python
2 2 
# Downloads REMIB data for all nodes
3 
# Usage: self
3 
# Usage: env [start=...] [n=...] self 2>>log
4 4 

  		




  5
  5
  
    
import csv


  		




  6
  6
  
    
import itertools


  		




  ......




  12
  12
  
    
sys.path.append(os.path.dirname(__file__)+"/../../../lib")


  		




  13
  13
  
    

  		




  14
  14
  
    
import exc


  		




  
  15
  
    
import opts


  		




  15
  16
  
    
import profiling


  		




  16
  17
  
    
import streams


  		




  17
  18
  
    
import strings


  		




  ......




  29
  30
  
    
    return line == '' or line.startswith('\t') or line.find(',') < 0


  		




  30
  31
  
    

  		




  31
  32
  
    
def main():


  		




  
  33
  
    
    # Get config from env vars


  		




  
  34
  
    
    start = util.cast(int, opts.get_env_var('start', 1))


  		




  
  35
  
    
    end = util.cast(int, util.none_if(opts.get_env_var('n', None), u''))


  		




  
  36
  
    
    if end != None: end += start


  		




  
  37
  
    
    


  		




  32
  38
  
    
    def clear_line(): sys.stderr.write('\n')


  		




  33
  39
  
    
    log_indent = 0


  		




  34
  40
  
    
    def log(msg, line_ending='\n'):


  		




  ......




  42
  48
  
    
        'remib_distribucion.cgi?lengua=EN&niveltax=familia&taxon=[prefix]%25&'


  		




  43
  49
  
    
        'pais=Todos&pais_otro=&estado=100&formato=csv&mapa=no&mapabase=estados'


  		




  44
  50
  
    
        '&coleccion=id%3D[node_id]')


  		




  
  51
  
    
    


  		




  
  52
  
    
    if end == None: node_ids = itertools.count(start)


  		




  
  53
  
    
    else: node_ids = xrange(start, end)


  		




  45
  54
  
    
    done = False


  		




  46
  
  
    
    for node_id in itertools.count(1):


  		




  
  55
  
    
    for node_id in node_ids:


  		




  47
  56
  
    
        if done: break


  		




  48
  57
  
    
        log('Processing node #'+str(node_id)+'...')


  		




  49
  58
  
    
        log_indent += 1


  		












Also available in:  Unified diff