Revision 1859
Added by Aaron Marcuse-Kubitza over 12 years ago
| bin/map | ||
|---|---|---|
| 17 | 17 |
import iters |
| 18 | 18 |
import maps |
| 19 | 19 |
import opts |
| 20 |
import parallel |
|
| 20 | 21 |
import Parser |
| 21 | 22 |
import profiling |
| 22 | 23 |
import sql |
| ... | ... | |
| 29 | 30 |
import xml_func |
| 30 | 31 |
import xml_parse |
| 31 | 32 |
|
| 32 |
class Pool: |
|
| 33 |
def apply_async(func, args=None, kw_args=None, callback=None): |
|
| 34 |
if args == None: args = () |
|
| 35 |
if kwds == None: kwds = {}
|
|
| 36 |
if callback == None: callback = lambda v: None |
|
| 37 |
|
|
| 38 |
callback(func(*args, **kw_args)) |
|
| 39 |
|
|
| 40 | 33 |
def get_with_prefix(map_, prefixes, key): |
| 41 | 34 |
'''Gets all entries for the given key with any of the given prefixes''' |
| 42 | 35 |
values = [] |
| ... | ... | |
| 123 | 116 |
profiler = profiling.ItersProfiler(start_now=True, iter_text='row') |
| 124 | 117 |
|
| 125 | 118 |
# Parallel processing |
| 126 |
try: |
|
| 127 |
if cpus == 0: raise ImportError('Parallel processing turned off')
|
|
| 128 |
import multiprocessing |
|
| 129 |
import multiprocessing.pool |
|
| 130 |
except ImportError, e: |
|
| 131 |
log_start('Not using parallel processing: '+str(e))
|
|
| 132 |
job_server = Pool() |
|
| 133 |
else: |
|
| 134 |
if cpus == None: cpus = multiprocessing.cpu_count() |
|
| 135 |
log_start('Using '+str(cpus)+' CPUs')
|
|
| 136 |
job_server = multiprocessing.pool.Pool(processes=cpus) |
|
| 119 |
pool, cpu_ct = parallel.mk_pool(cpus) |
|
| 120 |
log_start('Using '+str(cpu_ct)+' parallel CPUs')
|
|
| 137 | 121 |
|
| 138 | 122 |
doc = xml_dom.create_doc() |
| 139 | 123 |
root = doc.documentElement |
Also available in: Unified diff
bin/map: Use new parallel.py for parallel processing