Revision 1856
Added by Aaron Marcuse-Kubitza over 12 years ago
| map | ||
|---|---|---|
| 72 | 72 |
# Optimization |
| 73 | 73 |
if test: cpus_default = 0 |
| 74 | 74 |
else: cpus_default = None |
| 75 |
cpus = util.cast(int, opts.get_env_var('cpus', cpus_default, env_names))
|
|
| 75 |
cpus = util.cast(int, util.none_if(opts.get_env_var('cpus', cpus_default,
|
|
| 76 |
env_names), u'')) |
|
| 76 | 77 |
|
| 77 | 78 |
# Debugging |
| 78 | 79 |
debug = opts.env_flag('debug', False, env_names)
|
| ... | ... | |
| 113 | 114 |
ex_tracker = exc.ExPercentTracker(iter_text='row') |
| 114 | 115 |
profiler = profiling.ItersProfiler(start_now=True, iter_text='row') |
| 115 | 116 |
|
| 117 |
# Parallel processing |
|
| 116 | 118 |
if cpus != 0: |
| 117 |
try: import pp |
|
| 119 |
try: |
|
| 120 |
import multiprocessing |
|
| 121 |
import multiprocessing.pool |
|
| 118 | 122 |
except ImportError: pass |
| 119 | 123 |
else: |
| 120 |
if cpus != None: args = [cpus] |
|
| 121 |
else: args = [] |
|
| 122 |
job_server = pp.Server(*args) |
|
| 123 |
log_start('Using '+str(job_server.get_ncpus())+' CPUs')
|
|
| 124 |
if cpus == None: cpus = multiprocessing.cpu_count() |
|
| 125 |
job_server = multiprocessing.pool.Pool(processes=cpus) |
|
| 126 |
log_start('Using '+str(cpus)+' CPUs')
|
|
| 124 | 127 |
|
| 125 | 128 |
doc = xml_dom.create_doc() |
| 126 | 129 |
root = doc.documentElement |
Also available in: Unified diff
bin/map: Use multiprocessing instead of pp for parallel processing because it's easier to use (it uses the Python threading API and doesn't require providing all the functions a task calls). Allow the user to set the cpus option to to use all system CPUs (needed because in test mode, the default is 0 CPUs to turn off parallel processing).