Project

General

Profile

« Previous | Next » 

Revision 1669

Added by Aaron Marcuse-Kubitza almost 13 years ago

inputs/REMIB/src/nodes.make: Stop after a configurable # of empty responses (indicating no more nodes), instead of at a preset node ID, because there seem to be many more nodes than are listed on the web form

View differences:

nodes.make
19 19 
import timeout
20 20 
import util
21 21 

  		




  
  22
  
    
# Config


  		




  22
  23
  
    
timeout_ = 20 # sec


  		




  
  24
  
    
max_consec_empty_responses = 10


  		




  23
  25
  
    

  		




  24
  26
  
    
alphabet = map(chr, xrange(ord('A'), ord('Z')+1))


  		




  25
  27
  
    

  		




  ......




  35
  37
  
    
    # Get config from env vars


  		




  36
  38
  
    
    start = util.cast(int, opts.get_env_var('start', 1))


  		




  37
  39
  
    
    end = util.cast(int, util.none_if(opts.get_env_var('n', None), u''))


  		




  38
  
  
    
    if end == None: end = 150 # about 120 nodes listed on the web form


  		




  39
  
  
    
    else: end += start


  		




  
  40
  
    
    if end != None: end += start


  		




  40
  41
  
    
    


  		




  41
  42
  
    
    def clear_line(): sys.stderr.write('\n')


  		




  42
  43
  
    
    log_indent = 0


  		




  ......




  52
  53
  
    
        'pais=Todos&pais_otro=&estado=100&formato=csv&mapa=no&mapabase=estados'


  		




  53
  54
  
    
        '&coleccion=id%3D[node_id]')


  		




  54
  55
  
    
    


  		




  55
  
  
    
    for node_id in xrange(start, end):


  		




  
  56
  
    
    if end != None: node_ids = xrange(start, end)


  		




  
  57
  
    
    else: node_ids = itertools.count(start)


  		




  
  58
  
    
    consec_empty_responses = 0


  		




  
  59
  
    
    for node_id in node_ids:


  		




  
  60
  
    
        if consec_empty_responses > max_consec_empty_responses: break


  		




  
  61
  
    
        


  		




  56
  62
  
    
        log('Processing node #'+str(node_id)+'...')


  		




  57
  63
  
    
        log_indent += 1


  		




  58
  64
  
    
        profiler = profiling.ItersProfiler(start_now=True, iter_text='row')


  		




  ......




  77
  83
  
    
            stream = streams.StreamIter(streams.TimeoutInputStream(


  		




  78
  84
  
    
                urllib2.urlopen(url), timeout_))


  		




  79
  85
  
    
            


  		




  
  86
  
    
            is_empty_response = False


  		




  80
  87
  
    
            try:


  		




  81
  88
  
    
                util.skip(stream, is_ignore) # skip header


  		




  82
  89
  
    
                try: metadata_row = csv.reader(stream).next()


  		




  ......




  96
  103
  
    
                    if row_ct % 100 == 0: print_status('\r')


  		




  97
  104
  
    
                        # CR at end so next print overwrites msg


  		




  98
  105
  
    
            except EmptyResponseException, e: # must come before InputException


  		




  
  106
  
    
                is_empty_response = True


  		




  99
  107
  
    
                log_ex(e)


  		




  100
  108
  
    
                break # assume node doesn't exist, so abort node


  		




  101
  109
  
    
            except InputException, e: log_ex(e)


  		




  ......




  108
  116
  
    
                profiler.add_iters(row_ct)


  		




  109
  117
  
    
                print_status()


  		




  110
  118
  
    
                log_indent -= 1


  		




  
  119
  
    
                


  		




  
  120
  
    
                if is_empty_response: consec_empty_responses += 1


  		




  
  121
  
    
                else: consec_empty_responses = 0 # reset count


  		




  111
  122
  
    
            


  		




  112
  123
  
    
        profiler.stop()


  		




  113
  124
  
    
        log(profiler.msg())


  		












Also available in:  Unified diff