Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Open sidebar
Peter-Bernd Otte
Workload Manager
Commits
0a03d70a
Commit
0a03d70a
authored
May 22, 2019
by
Peter-Bernd Otte
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Added extended support for large execname lines
parent
e73ba8d7
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
25 additions
and
8 deletions
+25
-8
wkmgr.py
wkmgr.py
+25
-8
No files found.
wkmgr.py
View file @
0a03d70a
...
...
@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
#
# Workload Distributor
# Version 0.
3
# Version 0.
4
#
# Minimal python version 3.6
# on Mogon 2: run first
...
...
@@ -30,22 +30,29 @@ logger = logging.getLogger('rank' + str(rank))
now
=
datetime
.
datetime
.
now
()
parser
=
argparse
.
ArgumentParser
(
description
=
'Workload distributor for trivial parallelism.'
)
parser
.
add_argument
(
'execname'
,
help
=
'name of executable to call'
)
parser
.
add_argument
(
"-v"
,
"--verbosity"
,
help
=
"increase output verbosity"
,
default
=
0
,
action
=
"count"
)
parser
.
add_argument
(
"-V"
,
"--version"
,
help
=
"Returns the actual program version"
,
action
=
"version"
,
version
=
'%(prog)s 0.
3
'
)
version
=
'%(prog)s 0.
4
'
)
parser
.
add_argument
(
"-d"
,
"--delay"
,
help
=
"time delay in ms between starts of consecutive jobs to help "
"distributing load"
,
default
=
50
,
type
=
int
)
parser
.
add_argument
(
"-i"
,
"--input-dir"
,
help
=
"specifies the directory with files to process"
,
default
=
"."
)
parser
.
add_argument
(
"-o"
,
"--output-dir"
,
help
=
"output directory, default = output[datetime]"
,
default
=
os
.
getcwd
()
+
"/output_{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}"
.
format
(
now
.
year
,
now
.
month
,
now
.
day
,
now
.
hour
,
now
.
minute
,
now
.
second
)
)
parser
.
add_argument
(
"-a"
,
"--argument"
,
help
=
"line called for each job. Default = "
"'{execname} {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt'"
,
default
=
'{execname} {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt'
)
parser
.
add_argument
(
"-s"
,
"--single-argument"
,
help
=
"do not add the default placeholders to the execname"
,
action
=
"store_true"
)
parser
.
add_argument
(
'execname'
,
help
=
'name of executable to call OR complete shell command line. If no'
'placeholders are provided, " {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt" '
'is added.'
)
parser
.
add_argument
(
'remainderargs'
,
nargs
=
argparse
.
REMAINDER
)
#the remainder shall be added to the args.execname
clients
=
{}
def
join_l
(
l
,
sep
):
out_str
=
''
for
i
,
el
in
enumerate
(
l
):
out_str
+=
'{}{}'
.
format
(
sep
,
el
)
return
out_str
#[:-len(sep)]
if
rank
==
0
:
if
size
<
2
:
logger
.
warning
(
...
...
@@ -62,9 +69,19 @@ if rank == 0:
if
not
os
.
path
.
isdir
(
inputdir
):
logger
.
error
(
"Input directory does not exist. Exiting."
)
exit
(
1
)
# prepare execname
execname
=
args
.
execname
+
join_l
(
args
.
remainderargs
,
" "
)
if
execname
.
find
(
'{'
)
==
-
1
:
if
args
.
single_argument
:
logger
.
info
(
"Single-Argument option: no placeholders are added to the execname."
)
else
:
logger
.
info
(
"No placeholders are provided with the execname. Default will be added."
)
execname
+=
" {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt"
logger
.
info
(
"Input dir: "
+
inputdir
)
logger
.
info
(
"Delay between files: "
+
str
(
args
.
delay
)
+
" ms"
)
logger
.
info
(
"Executing: "
+
str
(
args
.
execname
)
)
logger
.
info
(
"Executing:
\"
"
+
execname
+
"
\"
"
)
# prepare list of files to process
inputfilelist
=
[
f
for
f
in
os
.
listdir
(
inputdir
)
if
os
.
path
.
isfile
(
os
.
path
.
join
(
inputdir
,
f
))]
...
...
@@ -86,7 +103,7 @@ if rank == 0:
data
=
{
'execname'
:
args
.
execname
,
'inputdir'
:
inputdir
,
'jobid'
:
i
,
'inputfilename'
:
v
,
'outputdir'
:
outputdir
}
# more hints: see https://pyformat.info
# worklist.append( ('{execname} {inputdir}{inputfilename} {outputdir}{jobid}/outfile.txt').format(**data) )
worklist
.
append
(
args
.
argument
.
format
(
**
data
)
)
worklist
.
append
(
execname
.
format
(
**
data
)
)
logger
.
info
(
"List if jobs:"
)
logger
.
info
(
worklist
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment