Allow batchrun to run more than one job at a time

diff --git a/batchrun.py b/batchrun.py
index ab64380e09f21218fac72b4245a1b370021b279c..ae6eef00ceffc84eb2bf57459f5bf96ebcc3187f 100755 (executable)
--- a/batchrun.py
+++ b/batchrun.py
@@ -1,14 +1,22 @@
 #!/usr/bin/env python
 
-import optparse, shlex, subprocess, sys, itertools
+import optparse, shlex, subprocess, os, sys, itertools
 
 def parseArgs():
        parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
+       parser.add_option("-j", "--jobs",
+               action="store", dest="jobs", default=1, type="int",
+               help="The number of concurrent jobs to run")
        opts, args = parser.parse_args(sys.argv[1:])
-       return args
+       opts.runningJobs = []
+       return opts, args
 
-def run(args):
-       subprocess.Popen(args).wait()
+def run(batchOpts, args):
+       batchOpts.runningJobs.append(subprocess.Popen(args, stdin=file(os.devnull, 'r')))
+       if len(batchOpts.runningJobs) >= batchOpts.jobs:
+               for job in batchOpts.runningJobs:
+                       job.wait()
+               batchOpts.runningJobs = []
 
 def getblocks(fd):
        def _countIndentationLevel(s):
@@ -27,13 +35,13 @@ def getblocks(fd):
                line = line[level:] # Slice off the indentation
                yield level, line
                        
-def batchProcess(fd):
+def batchProcess(batchOpts, fd):
        opts = []
 
        for level, line in getblocks(fd):
                oldLevel = len(opts) - 1
                if level <= oldLevel:
-                       run(itertools.chain(*opts))
+                       run(batchOpts, itertools.chain(*opts))
 
                opts = opts[:level] # Delete all options that belong to groups that are indented more than this one
                assert len(opts) == level, 'Seems we missed some options somewhere'
@@ -41,13 +49,13 @@ def batchProcess(fd):
                opts.append(shlex.split(line))
 
        if len(opts) > 0:
-               run(itertools.chain(*opts))
+               run(batchOpts, itertools.chain(*opts))
 
 
 def main():
-       args = parseArgs()
+       opts, args = parseArgs()
        for name in args:
-               batchProcess(open(name))
+               batchProcess(opts, open(name))
 
 if __name__ == "__main__":
        main()