code.delx.au - transcoding/blob - batch-run

   1 #!/usr/bin/env python
   2
   3 import itertools
   4 import optparse
   5 import os
   6 import shlex
   7 import subprocess
   8 import sys
   9 import time
  10
  11
  12 def run(running_jobs, cmd):
  13         p = subprocess.Popen(cmd, stdin=file(os.devnull, 'r'))
  14         running_jobs.append(p)
  15
  16 def wait_for_completion(max_jobs, running_jobs):
  17         while len(running_jobs) >= max_jobs:
  18                 time.sleep(1)
  19                 for job in running_jobs:
  20                         if job.poll() is not None:
  21                                 running_jobs.remove(job)
  22                                 job.wait()
  23
  24 def parse_file(fd):
  25         def count_indent(s):
  26                 level = 0
  27                 for ch in s:
  28                         if ch == "\t":
  29                                 level += 1
  30                         else:
  31                                 break
  32                 return level
  33
  34         for line in fd:
  35                 if not line.strip():
  36                         continue # Ignore blank lines
  37                 level = count_indent(line)
  38                 line = line[level:] # Slice off the indentation
  39                 yield level, line
  40         yield 0, None
  41
  42 def batch_process(opts, lines):
  43         running_jobs = []
  44         cmd = []
  45
  46         for level, line in lines:
  47                 old_level = len(cmd) - 1
  48                 if level <= old_level:
  49                         run(running_jobs, itertools.chain(*cmd))
  50                         wait_for_completion(opts.jobs, running_jobs)
  51
  52                 # Delete all options that belong to groups that are indented more than
  53                 # this one
  54                 cmd = cmd[:level]
  55                 assert len(cmd) == level
  56
  57                 if line:
  58                         cmd.append(shlex.split(line))
  59
  60         # Wait for remaining jobs to finish
  61         wait_for_completion(1, running_jobs)
  62
  63 def parse_args():
  64         parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
  65         parser.add_option("-j", "--jobs",
  66                 action="store", dest="jobs", default=1, type="int",
  67                 help="The number of concurrent jobs to run")
  68         opts, args = parser.parse_args(sys.argv[1:])
  69         opts.running_jobs = []
  70         return opts, args
  71
  72 def main():
  73         opts, args = parse_args()
  74         for name in args:
  75                 batch_process(opts, parse_file(open(name)))
  76
  77 if __name__ == "__main__":
  78         main()
  79