From d395206f2969ec7835372e1553f90bcb1b882098 Mon Sep 17 00:00:00 2001
From: James Bunton <jamesbunton@fastmail.fm>
Date: Sat, 2 Oct 2010 13:28:39 +1000
Subject: [PATCH] batchrun.py --jobs now starts a new jobs immediately after
 one finishes

---
 batchrun.py | 81 ++++++++++++++++++++++++++++++++---------------------
 1 file changed, 49 insertions(+), 32 deletions(-)

diff --git a/batchrun.py b/batchrun.py
index ae6eef0..fa799df 100755
--- a/batchrun.py
+++ b/batchrun.py
@@ -1,25 +1,28 @@
 #!/usr/bin/env python
 
-import optparse, shlex, subprocess, os, sys, itertools
+import itertools
+import optparse
+import os
+import shlex
+import subprocess
+import sys
+import time
 
-def parseArgs():
-	parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
-	parser.add_option("-j", "--jobs",
-		action="store", dest="jobs", default=1, type="int",
-		help="The number of concurrent jobs to run")
-	opts, args = parser.parse_args(sys.argv[1:])
-	opts.runningJobs = []
-	return opts, args
 
-def run(batchOpts, args):
-	batchOpts.runningJobs.append(subprocess.Popen(args, stdin=file(os.devnull, 'r')))
-	if len(batchOpts.runningJobs) >= batchOpts.jobs:
-		for job in batchOpts.runningJobs:
-			job.wait()
-		batchOpts.runningJobs = []
+def run(running_jobs, cmd):
+	p = subprocess.Popen(cmd, stdin=file(os.devnull, 'r'))
+	running_jobs.append(p)
+
+def wait_for_completion(max_jobs, running_jobs):
+	while len(running_jobs) >= max_jobs:
+		time.sleep(1)
+		for job in running_jobs:
+			if job.poll() is not None:
+				running_jobs.remove(job)
+				job.wait()
 
-def getblocks(fd):
-	def _countIndentationLevel(s):
+def parse_file(fd):
+	def count_indent(s):
 		level = 0
 		for ch in s:
 			if ch == "\t":
@@ -31,31 +34,45 @@ def getblocks(fd):
 	for line in fd:
 		if not line.strip():
 			continue # Ignore blank lines
-		level = _countIndentationLevel(line)
+		level = count_indent(line)
 		line = line[level:] # Slice off the indentation
 		yield level, line
-			
-def batchProcess(batchOpts, fd):
-	opts = []
+	yield 0, None
 
-	for level, line in getblocks(fd):
-		oldLevel = len(opts) - 1
-		if level <= oldLevel:
-			run(batchOpts, itertools.chain(*opts))
+def batch_process(opts, lines):
+	running_jobs = []
+	cmd = []
 
-		opts = opts[:level] # Delete all options that belong to groups that are indented more than this one
-		assert len(opts) == level, 'Seems we missed some options somewhere'
+	for level, line in lines:
+		old_level = len(cmd) - 1
+		if level <= old_level:
+			run(running_jobs, itertools.chain(*cmd))
+			wait_for_completion(opts.jobs, running_jobs)
 
-		opts.append(shlex.split(line))
+		# Delete all options that belong to groups that are indented more than
+		# this one
+		cmd = cmd[:level]
+		assert len(cmd) == level
 
-	if len(opts) > 0:
-		run(batchOpts, itertools.chain(*opts))
+		if line:
+			cmd.append(shlex.split(line))
 
+	# Wait for remaining jobs to finish
+	wait_for_completion(1, running_jobs)
+
+def parse_args():
+	parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
+	parser.add_option("-j", "--jobs",
+		action="store", dest="jobs", default=1, type="int",
+		help="The number of concurrent jobs to run")
+	opts, args = parser.parse_args(sys.argv[1:])
+	opts.running_jobs = []
+	return opts, args
 
 def main():
-	opts, args = parseArgs()
+	opts, args = parse_args()
 	for name in args:
-		batchProcess(opts, open(name))
+		batch_process(opts, parse_file(open(name)))
 
 if __name__ == "__main__":
 	main()
-- 
2.39.2