Commit 0a03d70a authored by Peter-Bernd Otte's avatar Peter-Bernd Otte

Added extended support for large execname lines

parent e73ba8d7
......@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
#
# Workload Distributor
# Version 0.3
# Version 0.4
#
# Minimal python version 3.6
# on Mogon 2: run first
......@@ -30,22 +30,29 @@ logger = logging.getLogger('rank' + str(rank))
now = datetime.datetime.now()
parser = argparse.ArgumentParser(description='Workload distributor for trivial parallelism.')
parser.add_argument('execname', help='name of executable to call')
parser.add_argument("-v", "--verbosity", help="increase output verbosity", default=0, action="count")
parser.add_argument("-V", "--version", help="Returns the actual program version", action="version",
version='%(prog)s 0.3')
version='%(prog)s 0.4')
parser.add_argument("-d", "--delay", help="time delay in ms between starts of consecutive jobs to help "
"distributing load", default=50, type=int)
parser.add_argument("-i", "--input-dir", help="specifies the directory with files to process", default=".")
parser.add_argument("-o", "--output-dir", help="output directory, default = output[datetime]", default=os.getcwd() +
"/output_{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}".format(now.year, now.month, now.day,
now.hour, now.minute, now.second) )
parser.add_argument("-a", "--argument", help="line called for each job. Default = "
"'{execname} {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt'",
default='{execname} {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt')
parser.add_argument("-s", "--single-argument", help="do not add the default placeholders to the execname", action="store_true")
parser.add_argument('execname', help='name of executable to call OR complete shell command line. If no'
'placeholders are provided, " {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt" '
'is added.')
parser.add_argument('remainderargs', nargs=argparse.REMAINDER) #the remainder shall be added to the args.execname
clients = {}
def join_l(l, sep):
out_str = ''
for i, el in enumerate(l):
out_str += '{}{}'.format(sep, el)
return out_str #[:-len(sep)]
if rank == 0:
if size < 2:
logger.warning(
......@@ -62,9 +69,19 @@ if rank == 0:
if not os.path.isdir(inputdir):
logger.error("Input directory does not exist. Exiting.")
exit(1)
# prepare execname
execname = args.execname+join_l(args.remainderargs," ")
if execname.find('{') == -1:
if args.single_argument:
logger.info("Single-Argument option: no placeholders are added to the execname.")
else:
logger.info("No placeholders are provided with the execname. Default will be added.")
execname += " {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt"
logger.info("Input dir: " + inputdir)
logger.info("Delay between files: " + str(args.delay) + " ms")
logger.info("Executing: " + str(args.execname))
logger.info("Executing: \"" + execname +"\"")
# prepare list of files to process
inputfilelist = [f for f in os.listdir(inputdir) if os.path.isfile(os.path.join(inputdir, f))]
......@@ -86,7 +103,7 @@ if rank == 0:
data = {'execname':args.execname, 'inputdir':inputdir, 'jobid': i, 'inputfilename':v, 'outputdir':outputdir}
# more hints: see https://pyformat.info
# worklist.append( ('{execname} {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt').format(**data) )
worklist.append( args.argument.format(**data) )
worklist.append( execname.format(**data) )
logger.info("List if jobs:")
logger.info(worklist)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment