From 0663c432764117c42e226d8cac623a9fcf3e8daf Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Wed, 13 Feb 2013 17:04:37 -0500 Subject: Parallelized plotting and parsing. --- parse_exps.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'parse_exps.py') diff --git a/parse_exps.py b/parse_exps.py index c2376de..f27021a 100755 --- a/parse_exps.py +++ b/parse_exps.py @@ -16,7 +16,7 @@ from parse.dir_map import DirMap from parse.point import ExpPoint from parse.tuple_table import TupleTable,ReducedTupleTable from parse.col_map import ColMapBuilder - +from multiprocessing import Pool, cpu_count def parse_args(): # TODO: convert data-dir to proper option, clean 'dest' options @@ -106,7 +106,7 @@ def parse_exp(exp, force): pickle.dump(result, f) - return result + return (exp, result) def main(): opts, args = parse_args() @@ -126,8 +126,11 @@ def main(): result_table = TupleTable(col_map) sys.stderr.write("Parsing data...\n") - for i,exp in enumerate(exps): - result = parse_exp(exp, opts.force) + + procs = min(len(exps), cpu_count()/2) + pool = Pool(processes=procs) + enum = pool.imap_unordered(parse_exp, exps, [opts.force]*len(exps)) + for i, (exp, result) in enumerate(enum): if opts.verbose: print(result) else: -- cgit v1.2.2