X-Git-Url: https://code.delx.au/transcoding/blobdiff_plain/2a0d436d209917e0b0ac8eefd15e2e8313f33fe9..d395206f2969ec7835372e1553f90bcb1b882098:/batchrun.py

diff --git a/batchrun.py b/batchrun.py
index b8596ac..fa799df 100755
--- a/batchrun.py
+++ b/batchrun.py
@@ -1,30 +1,78 @@
 #!/usr/bin/env python
 
-import optparse, shlex, subprocess, sys
+import itertools
+import optparse
+import os
+import shlex
+import subprocess
+import sys
+import time
 
-def parseArgs():
-	parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
-	opts, args = parser.parse_args(sys.argv[1:])
-	return args
 
-def run(args):
-	subprocess.Popen(args).wait()
+def run(running_jobs, cmd):
+	p = subprocess.Popen(cmd, stdin=file(os.devnull, 'r'))
+	running_jobs.append(p)
+
+def wait_for_completion(max_jobs, running_jobs):
+	while len(running_jobs) >= max_jobs:
+		time.sleep(1)
+		for job in running_jobs:
+			if job.poll() is not None:
+				running_jobs.remove(job)
+				job.wait()
+
+def parse_file(fd):
+	def count_indent(s):
+		level = 0
+		for ch in s:
+			if ch == "\t":
+				level += 1
+			else:
+				break
+		return level
 
-def batchProcess(fd):
-	opts = [[], []]
 	for line in fd:
-		args = shlex.split(line)
-		if line.startswith("\t\t"):
-			run(opts[0] + opts[1] + args)
-		elif line.startswith("\t"):
-			opts[1] = args
-		else:
-			opts[0] = args
+		if not line.strip():
+			continue # Ignore blank lines
+		level = count_indent(line)
+		line = line[level:] # Slice off the indentation
+		yield level, line
+	yield 0, None
+
+def batch_process(opts, lines):
+	running_jobs = []
+	cmd = []
+
+	for level, line in lines:
+		old_level = len(cmd) - 1
+		if level <= old_level:
+			run(running_jobs, itertools.chain(*cmd))
+			wait_for_completion(opts.jobs, running_jobs)
+
+		# Delete all options that belong to groups that are indented more than
+		# this one
+		cmd = cmd[:level]
+		assert len(cmd) == level
+
+		if line:
+			cmd.append(shlex.split(line))
+
+	# Wait for remaining jobs to finish
+	wait_for_completion(1, running_jobs)
+
+def parse_args():
+	parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
+	parser.add_option("-j", "--jobs",
+		action="store", dest="jobs", default=1, type="int",
+		help="The number of concurrent jobs to run")
+	opts, args = parser.parse_args(sys.argv[1:])
+	opts.running_jobs = []
+	return opts, args
 
 def main():
-	args = parseArgs()
+	opts, args = parse_args()
 	for name in args:
-		batchProcess(open(name))
+		batch_process(opts, parse_file(open(name)))
 
 if __name__ == "__main__":
 	main()