code.delx.au - transcoding/blob - batch-run

   1 #!/usr/bin/python
   2
   3 import itertools
   4 import optparse
   5 import os
   6 import shlex
   7 import subprocess
   8 import sys
   9 import time
  10
  11
  12 def run(running_jobs, cmd):
  13     p = subprocess.Popen(cmd, stdin=subprocess.DEVNULL)
  14     running_jobs.append(p)
  15
  16 def wait_for_completion(max_jobs, running_jobs):
  17     while len(running_jobs) >= max_jobs:
  18         time.sleep(1)
  19         for job in running_jobs:
  20             if job.poll() is not None:
  21                 running_jobs.remove(job)
  22                 job.wait()
  23
  24 def parse_file(fd):
  25     def count_indent(s):
  26         level = 0
  27         for ch in s:
  28             if ch == "\t":
  29                 level += 1
  30             else:
  31                 break
  32         return level
  33
  34     for line in fd:
  35         if not line.strip():
  36             continue # Ignore blank lines
  37         level = count_indent(line)
  38         line = line[level:] # Slice off the indentation
  39         yield level, line
  40     yield 0, None
  41
  42 def batch_process(opts, lines):
  43     running_jobs = []
  44     cmd = []
  45
  46     for level, line in lines:
  47         old_level = len(cmd) - 1
  48         if level <= old_level:
  49             run(running_jobs, itertools.chain(*cmd))
  50             wait_for_completion(opts.jobs, running_jobs)
  51
  52         # Delete all options that belong to groups that are indented more than
  53         # this one
  54         cmd = cmd[:level]
  55         assert len(cmd) == level
  56
  57         if line:
  58             cmd.append(shlex.split(line))
  59
  60     # Wait for remaining jobs to finish
  61     wait_for_completion(1, running_jobs)
  62
  63 def make_and_chdir(filename):
  64     dirname = os.path.splitext(filename)[0] + ".out"
  65     try:
  66         os.makedirs(dirname)
  67     except FileExistsError:
  68         pass
  69     os.chdir(dirname)
  70
  71 def parse_args():
  72     parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
  73     parser.add_option("-j", "--jobs",
  74         action="store", dest="jobs", default=1, type="int",
  75         help="The number of concurrent jobs to run")
  76     opts, args = parser.parse_args(sys.argv[1:])
  77     opts.running_jobs = []
  78     args = map(os.path.abspath, args)
  79     return opts, args
  80
  81 def main():
  82     opts, args = parse_args()
  83     filenames = list(map(os.path.abspath, args))
  84     for filename in filenames:
  85         print("Processing", filename)
  86         make_and_chdir(filename)
  87         with open(filename) as fd:
  88             batch_process(opts, parse_file(fd))
  89
  90 if __name__ == "__main__":
  91     main()
  92