batchrun.py --jobs now starts a new jobs immediately after one finishes

diff --git a/batchrun.py b/batchrun.py
index ae6eef00ceffc84eb2bf57459f5bf96ebcc3187f..fa799dffc359b7364d78bcab63c014709b14b8f7 100755 (executable)
--- a/batchrun.py
+++ b/batchrun.py
@@ -1,25 +1,28 @@
 #!/usr/bin/env python
 
-import optparse, shlex, subprocess, os, sys, itertools
+import itertools
+import optparse
+import os
+import shlex
+import subprocess
+import sys
+import time
 
-def parseArgs():
-       parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
-       parser.add_option("-j", "--jobs",
-               action="store", dest="jobs", default=1, type="int",
-               help="The number of concurrent jobs to run")
-       opts, args = parser.parse_args(sys.argv[1:])
-       opts.runningJobs = []
-       return opts, args
 
-def run(batchOpts, args):
-       batchOpts.runningJobs.append(subprocess.Popen(args, stdin=file(os.devnull, 'r')))
-       if len(batchOpts.runningJobs) >= batchOpts.jobs:
-               for job in batchOpts.runningJobs:
-                       job.wait()
-               batchOpts.runningJobs = []
+def run(running_jobs, cmd):
+       p = subprocess.Popen(cmd, stdin=file(os.devnull, 'r'))
+       running_jobs.append(p)
+
+def wait_for_completion(max_jobs, running_jobs):
+       while len(running_jobs) >= max_jobs:
+               time.sleep(1)
+               for job in running_jobs:
+                       if job.poll() is not None:
+                               running_jobs.remove(job)
+                               job.wait()
 
-def getblocks(fd):
-       def _countIndentationLevel(s):
+def parse_file(fd):
+       def count_indent(s):
                level = 0
                for ch in s:
                        if ch == "\t":
@@ -31,31 +34,45 @@ def getblocks(fd):
        for line in fd:
                if not line.strip():
                        continue # Ignore blank lines
-               level = _countIndentationLevel(line)
+               level = count_indent(line)
                line = line[level:] # Slice off the indentation
                yield level, line
-                       
-def batchProcess(batchOpts, fd):
-       opts = []
+       yield 0, None
 
-       for level, line in getblocks(fd):
-               oldLevel = len(opts) - 1
-               if level <= oldLevel:
-                       run(batchOpts, itertools.chain(*opts))
+def batch_process(opts, lines):
+       running_jobs = []
+       cmd = []
 
-               opts = opts[:level] # Delete all options that belong to groups that are indented more than this one
-               assert len(opts) == level, 'Seems we missed some options somewhere'
+       for level, line in lines:
+               old_level = len(cmd) - 1
+               if level <= old_level:
+                       run(running_jobs, itertools.chain(*cmd))
+                       wait_for_completion(opts.jobs, running_jobs)
 
-               opts.append(shlex.split(line))
+               # Delete all options that belong to groups that are indented more than
+               # this one
+               cmd = cmd[:level]
+               assert len(cmd) == level
 
-       if len(opts) > 0:
-               run(batchOpts, itertools.chain(*opts))
+               if line:
+                       cmd.append(shlex.split(line))
 
+       # Wait for remaining jobs to finish
+       wait_for_completion(1, running_jobs)
+
+def parse_args():
+       parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
+       parser.add_option("-j", "--jobs",
+               action="store", dest="jobs", default=1, type="int",
+               help="The number of concurrent jobs to run")
+       opts, args = parser.parse_args(sys.argv[1:])
+       opts.running_jobs = []
+       return opts, args
 
 def main():
-       opts, args = parseArgs()
+       opts, args = parse_args()
        for name in args:
-               batchProcess(opts, open(name))
+               batch_process(opts, parse_file(open(name)))
 
 if __name__ == "__main__":
        main()