Project

General

Profile

« Previous | Next » 

Revision 1859

Added by Aaron Marcuse-Kubitza over 12 years ago

bin/map: Use new parallel.py for parallel processing

View differences:

map
17 17 
import iters
18 18 
import maps
19 19 
import opts
20 
import parallel
20 21 
import Parser
21 22 
import profiling
22 23 
import sql
......
29 30 
import xml_func
30 31 
import xml_parse
31 32 

  		




  32
  
  
    
class Pool:


  		




  33
  
  
    
    def apply_async(func, args=None, kw_args=None, callback=None):


  		




  34
  
  
    
        if args == None: args = ()


  		




  35
  
  
    
        if kwds == None: kwds = {}


  		




  36
  
  
    
        if callback == None: callback = lambda v: None


  		




  37
  
  
    
        


  		




  38
  
  
    
        callback(func(*args, **kw_args))


  		




  39
  
  
    

  		




  40
  33
  
    
def get_with_prefix(map_, prefixes, key):


  		




  41
  34
  
    
    '''Gets all entries for the given key with any of the given prefixes'''


  		




  42
  35
  
    
    values = []


  		




  ......




  123
  116
  
    
    profiler = profiling.ItersProfiler(start_now=True, iter_text='row')


  		




  124
  117
  
    
    


  		




  125
  118
  
    
    # Parallel processing


  		




  126
  
  
    
    try:


  		




  127
  
  
    
        if cpus == 0: raise ImportError('Parallel processing turned off')


  		




  128
  
  
    
        import multiprocessing


  		




  129
  
  
    
        import multiprocessing.pool


  		




  130
  
  
    
    except ImportError, e:


  		




  131
  
  
    
        log_start('Not using parallel processing: '+str(e))


  		




  132
  
  
    
        job_server = Pool()


  		




  133
  
  
    
    else:


  		




  134
  
  
    
        if cpus == None: cpus = multiprocessing.cpu_count()


  		




  135
  
  
    
        log_start('Using '+str(cpus)+' CPUs')


  		




  136
  
  
    
        job_server = multiprocessing.pool.Pool(processes=cpus)


  		




  
  119
  
    
    pool, cpu_ct = parallel.mk_pool(cpus)


  		




  
  120
  
    
    log_start('Using '+str(cpu_ct)+' parallel CPUs')


  		




  137
  121
  
    
    


  		




  138
  122
  
    
    doc = xml_dom.create_doc()


  		




  139
  123
  
    
    root = doc.documentElement


  		












Also available in:  Unified diff