QCWrapper 28.3 KB
Newer Older
1 2 3
#!/bin/bash

#set -x
4 5
set -e
PS4='Line ${LINENO}: '
6 7 8 9 10 11 12 13 14

# constantly set:
#set -u pipefail

module purge

# load the most current version of GNU parallel
module load tools/parallel

15
# which cluster are we ony
16 17 18 19
cluster=$(sacctmgr show cluster -p| tail -n1| cut -f1 -d '|')

_arg_inputdir=""
_arg_outputdir=""
20
_arg_executable="flexbar"
21
declare -i _arg_runlimit=300
22
declare -i _arg_paired=1
23 24 25 26 27 28 29 30
_arg_assoc=$(sacct -nu "$USER" -o Account%20 | tail -n1 | tr -d '[:space:]')
if [ "$cluster" = "mogon" ]; then
    _arg_queue="nodeshort"
else
    _arg_queue="parallel"
    _arg_constraint="skylake"
fi
_arg_dependency=""
31
_arg_tag=""
32 33
_arg_adapter=""
_arg_args=""
34 35
#-----
# flexbar specific defaults
36
_arg_qtrim="WIN"
37
_arg_qtrim_format="i1.8"
38 39 40

SCRIPT="$0"

Christian Meesters's avatar
Christian Meesters committed
41
SCRIPT_VERSION="0.3.1"
Christian Meesters's avatar
Christian Meesters committed
42

43 44 45 46 47 48 49 50 51 52 53 54
die()
{
        local _ret=$2
        test -n "$_ret" || _ret=1
        test "$_PRINT_HELP" = yes && print_help >&2
        echo "$1" >&2
        exit ${_ret}
}

print_help ()
{
    echo "This script's help msg"
55
    echo 'Usage: QCWrapper [options] <readdir>'
56
    echo
57
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "--executable" "mandatory argument to designate the executable"
58
    printf "\\t\\t\\t%s\\n"                 "- possible arguments: cutadapt, flexbar, trimmomatic"
59
    printf "\\t\\t\\t%s\\n"                 "- check is case insensitive"
60
    printf "\\t\\t\\t%s\\n"                 "- defaults to 'flexbar'"
61 62 63 64 65 66
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-l,--runlimit" "runlimit default is 300 min"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-p,--partition" "SLURM partition"
    printf "\\t\\t\\t%s\\n"                                "- default is 'parallel' on Mogon II and 'nodeshort' on Mogon I"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-A,--account" "SLURM account"
    printf "\\t\\t\\t%s\\n"                              "- default is the last submit account"
    printf "\\t\\t\\t%s\\n"                              "- an error is triggered if none specified nor can be deduced"
67 68
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-t,--threads" "number of threads the executable should use"
    printf "\\t\\t\\t%s\\n"                " defaults are application dependend"
69 70
    printf "\\t\\033[1m%s\\033[0m\\t\\t%s\\n" "--args" "arguments otherwise not set by the wrapper"
    printf "\\t\\t\\t%s\\n"                          "- the defaults of the choosen executable apply for unset arguments"
71 72
    printf "\\t\\t\\t%s\\n"                          "- will superseed the defaults,"
    printf "\\t\\t\\t%s\\n"                          "e.g. 'LEADING:3 TRAILING:3 SLIDINGWINDOW:4:15 MINLEN:36' for trimmomatic"
73 74 75 76
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-d,--dependency" "list of comma separated jobids, the job will wait for to finish"
    printf "\\t\\t\\t%s\\n"                                 "- default: None"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-o,--outdir" "output directory path (default is the current working directory)"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-a,--adapter" "if executable is trimmomatic:"
77 78
    printf "\\t\\t\\t%s\\n"                               "- a selection of one of Trimmomatics pre-defined adapters,"
    printf "\\t\\t\\t%s\\n"                                  "defaults to 'TruSeq3-PE.fa'"
79 80
    printf "\\t\\t\\t%s\\n"                               "else"
    printf "\\t\\t\\t%s\\n"                               "- an adaptor string specification according to the selected software."
81 82
    printf "\\t\\t\\t%s\\n"                               "  defaults to 'TACACTCTTTCCCTACACGACGCTCTTCCGATCT' (adaptor 1 of TruSeq3-PE.fa)"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "--adapterp"   "adaptor for the mate pair"
83 84
    printf "\\t\\t\\t%s\\n"                               "- if the excecutable is trimmomatic, this argument is not necessary"
    printf "\\t\\t\\t%s\\n"                               " (it is contained in the global adaptor selection)"
85 86
    printf "\\t\\t\\t%s\\n"                               "- if the excecutable is cutadapt, this arguments defaults to"
    printf "\\t\\t\\t%s\\n"                               "  'GTGACTGGAGTTCAGACGTGTGCTCTTCCGATCT' (the mate of TruSeq3-PE.fa)"
87 88 89 90
    printf "\\t\\t\\t%s\\n"                               "- if the excecutable is flexbar, this argument is not necessary"
    printf "\\t\\t\\t%s\\n"                               " (it is contained in the global adaptor selection)"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "--single"     "if given, single end data will be assumed"
    printf "\\t\\t\\t%s\\n"                               "otherwise: paired-end data are default"
91 92 93 94 95 96 97
    echo 
    printf "\\t%s\\n"                                     "========== flexbar specific =========="
    printf "\\t\\033[1m%s\\033[0m\\t\\t%s\\n" "--qtrim"      "see '--qtrim' option of flexbar, default to 'WiN'"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "--qtrim-format" "see '-qf/--qtrim-format' option of flexbar, default is 'i1.5'"
    printf "\\t%s\\n"                                     "======================================"
    echo 
    
98
    if [ "$cluster" = "mogon2" ]; then
99
        printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "--constraint" "defaults to 'broadwell'"
100
    fi
101
    printf "\\t\\033[1m%s\\033[0m\\t\\t%s\\n"     "--tag" "a jobtag (default is decuced by naming scheme)"
102 103 104 105
    
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "--credits" "shows credits and version history"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "--version" "shows the version number"
    printf "\\t\\033[1m%s\\033[0m\\t%s\\n" "-h,--help" "Prints help"
106 107 108
    echo
    echo "Report issues and create feature requests at: https://gitlab.rlp.net/hpc-jgu-lifescience/seq-analysis/issues"
    echo
109 110 111 112 113 114 115 116 117
    echo "This script will submit itself. DO NOT run it within a job!"

    echo 
    echo -e "\e[3m Limitations:\e[0m"
    echo "   - 'ILLUMINACLIP' is hard coded, as it is the only option for Trimmomatic"
}

credits()
{
Christian Meesters's avatar
Christian Meesters committed
118 119
   echo "Version is: $SCRIPT_VERSION"
   echo
120 121 122 123 124 125
   echo "History of the implementation:"
   echo "- v0.1   --  4. Sep. 2018 -- release of the implementation for SLURM, supporting the ability"
   echo "                             compute accross nodes."
   echo "- v0.1.1 -- 10. Sep. 2018 -- new features / fixes:"
   echo "                             - new dependency switch, enabling job dependency"
   echo "                             - made trimming options variable"
126
   echo "- v0.2.1 -- 26. Oct. 2019 -- minor improvements and fixes"
Christian Meesters's avatar
Christian Meesters committed
127
   echo "- v0.3   -- 28. Jan. 2019 -- update including:"
128
   echo "                             - support for multiple tools, new: cutadapt"
Christian Meesters's avatar
Christian Meesters committed
129
   echo "                             - support for multiple tools, new: flexbar"
130
   echo "                             - resource release is build-in"
Christian Meesters's avatar
Christian Meesters committed
131
   echo "                             - bug fix: file name handling"
132 133 134
   echo "- v0.3.1 -- 22. Feb. 2019 -- bug fixes:"
   echo "                             - striping only, when creating unstriped output dir on MII"
   echo "                             - preventing to write to HOME"
135 136 137 138 139 140 141
   echo 
   echo "This script was written by Christian Meesters 2018-2019 (ZDV, HPC group)".
   echo "See https://gitlab.rlp.net/hpc-jgu-lifescience/seq-analysis/blob/master/LICENSE"
   echo "for licensing information."
   echo "A (personal) acknowledgement is welcomed. Please refer to:"
   echo "https://hpc.uni-mainz.de/high-performance-computing/publikationen/"
   exit
142
}
Christian Meesters's avatar
Christian Meesters committed
143

144 145 146 147 148 149 150 151 152 153 154 155 156 157
# function to redirect simple error messages to stderr
error() {
    (>&2 echo "ERROR: $1")
}

warning() {
    (>&2 echo "WARNING: $1")
}

# command line parsing
while test $# -gt 0
do
        _key="$1"
        case "$_key" in
158 159 160 161 162 163 164 165 166 167
                --debug)
                    set -x
                    ;;
                --test)
                    _arg_test=on
                    ;;
                --executable|--executable=*)
                        _val_from_long="${_key##--executable=}"
                        _arg_executable=$_val_from_long
                        ;;
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
                -l*|--runlimit|--runlimit=*)
                        _val_from_long="${_key##--runlimit=}"
                        _val_from_short="${_key##-l}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_runlimit="$_val"
                        ;;
                -p*|--partition|--partition=*)
                        _val_from_long="${_key##--partition=}"
                        _val_from_short="${_key##-p}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_queue="$_val"
                        ;;
                -A*|--account|--account=*)
                        _val_from_long="${_key##--account=}"
                        _val_from_short="${_key##-A}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_assoc="$_val"
                        ;;
                -N*|--nodes|--nodes=*)
                        _val_from_long="${_key##--nodes=}"
                        _val_from_short="${_key##-N}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_nodes="$_val"
                        ;;
                -o*|--outdir|--outdir=*)
                        _val_from_long="${_key##--outdir=}"
                        _val_from_short="${_key##-o}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_outdir="$_val"
                        ;;
                -a*|--adapter|--adapter=*)
                        _val_from_long="${_key##--adapter=}"
                        _val_from_short="${_key##-a}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_adapter="$_val"
                        ;;
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
                --adapterp|--adapterp=*)
                        _val_from_long="${_key##--adapter=}"
                        _val_from_short="${_key##-a}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_adapterp="$_val"
                        ;;
280
                --args|--args=*)
281 282 283 284 285 286 287 288 289 290 291 292 293 294
                        _val_from_long="${_key##--args=}"
                        _val_from_short="${_key##-a}"
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_args="$_val"
295
                        ;;
296
                --qtrim|--qtrim=*)
297
                        _val_from_long="${_key##--qtrim=}"
298
                        _arg_qtrim=$_val_from_long
299
                        echo $_arg_qtrim
300 301
                        ;;
                --qtrim-format|--qtrim-format=*)
302
                        _val_from_long="${_key##--qtrim-format=}"
303 304
                        _arg_qtrim_format="$_val_from_long"
                        ;;
305 306 307 308 309 310 311 312
                --tag|--tag=*)
                        _val_from_long="${_key##--tag=}"
                        _arg_tag=$_val_from_long
                        ;;
                --dependency|--dependency=*)
                        _val_from_long="${_key##--dependency=}"
                        _arg_dependency=$_val_from_long
                        ;;
313 314 315 316 317 318 319 320 321 322
                --reservation|--reservation=*)
                        _val="${_key##--reservation=}"
                        if test "$_val" = "$_key"
                        then
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_reservation="$_val"
                        ;;
323 324
                -t|--threads|--threads=*)
                        _val_from_long="${_key##--threads=}"
325
                        _val_from_short="${_key##-t}"
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
                        if test "$_val_from_long" != "$_key"
                        then
                                _val="$_val_from_long"
                        elif test "$_val_from_short" != "$_key" -a -n "$_val_from_short"
                        then
                                _val="$_val_from_short"
                        else
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _val="$2"
                                shift
                        fi
                        _arg_threads="$_val"
                        ;;
                --constraint|--constraint=*)
                        _val_from_long="${_key##--constraint=}"
                        _arg_constraint=$_val_from_long
                        ;;
343 344 345
                --single)
                        _arg_paired=0
                        ;;
Christian Meesters's avatar
Christian Meesters committed
346
                --credits)
347 348 349
                        credits
                        exit 0
                        ;;
Christian Meesters's avatar
Christian Meesters committed
350 351 352 353
                --version)
                        echo "$SCRIPT_VERSION"
                        exit 0
                        ;;
354 355 356 357 358 359 360 361 362 363 364
                -h*|--help)
                        print_help
                        exit 0
                        ;;
                *)
                        _positionals+=("$1")
                        ;;
        esac
        shift
done

365

366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
_positional_names=('_arg_inputdir' )
_required_args_string="'inputdir'"

for (( ii = 0; ii < 1; ii++)); do
    eval "${_positional_names[ii]}=\${_positionals[ii]}" || die "Error during argument parsing, possibly an Argbash bug." 1
done
_our_args=$((${#_positionals[@]} - ${#_positional_names[@]}))
for (( ii = 0; ii < _our_args; ii++)); do 
    _positional_names+=("_arg_leftovers[(($ii + 0))]")
done

#check whether the account is given or could be deduced
if [[ -z "${_arg_assoc// }" ]]; then
   error "Unable to deduce the user association and it is not defined.\n\tUse -A/--account to define your association."
   print_help
   exit 1
fi

### check if input directory exist
if [ ! -d "$_arg_inputdir" ]; then
    error "inputdir directory not found: '$_arg_inputdir'!"
    echo
    print_help
    exit 1
fi

392
if [ "trimmomatic" = "${_arg_executable,,}" ]; then
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
    ### set our default adapter:
    adapter="TruSeq3-PE.fa"
    ### overwrite with wanted adaptor - check subsequently
    
    if [ ! -z "${_arg_adapter// }" ]; then
       adapter="${_arg_adapter}"
    fi
    ### check whether the selected adapter is in the list of allowed ones
    allowed="NexteraPE-PE.fa TruSeq2-PE.fa TruSeq3-PE-2.fa TruSeq3-PE.fa"
    if [[ ! $allowed =~ (^| )"$adapter"($| ) ]]; then
        error "allowed adapters for trimmomatic include '$allowed'"
        exit 1
    fi
 
    ### likewise for our default arguments
    args="LEADING:3 TRAILING:3 SLIDINGWINDOW:4:15 MINLEN:36"
    ### overwrite with the wanted ones - no check is applicable
    if [ ! -z "${_arg_args}" ]; then
       args="${_arg_args}"
    fi
    
    ### overwrite the number of threads, if really desired by the user
    if [ -n "${_arg_threads}" ]; then
       threads=${_arg_threads}
    else
       threads=8
    fi
    if [ $_arg_paired -eq 0 ]; then
       error "single end data are currently not supported withing this wrapper for trimmomatic"
       exit 3
    fi
    if [ -z "$SLURM_JOB_ID" ]; then
425
       source $(dirname $SCRIPT)/trimmomatic_wrap.sh
426
    else
427
       source ${SCRIPT_PATH}/trimmomatic_wrap.sh
428
    fi
429
elif [ "cutadapt" = "${_arg_executable,,}" ]; then
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
    if [ "$cluster" = "mogon1" ]; then
       error "Python scripts (as 'cutadapt' are not suitable to crunch large amounts of data and are henceforth not supported on Mogon II"
    fi
    ### let the cutadapt default adapter be the same as for trimmomatic:
    adapter="TACACTCTTTCCCTACACGACGCTCTTCCGATCT"
    ### overwrite with wanted - as the check is passed
    if [ ! -z "${_arg_adapter// }" ]; then
       adapter="${_arg_adapter}"
    fi
    ### hence the second argument is 
    adapterp="GTGACTGGAGTTCAGACGTGTGCTCTTCCGATCT"
    if [ ! -z "${_arg_adapterp// }" ]; then
       adapterp="${_arg_adapterp}"
    fi
    ### overwrite the number of threads, if really desired by the user
    if [ -n "${_arg_threads}" ]; then
       threads=${_arg_threads}
    else
       threads=1
    fi
    if [ -z "$SLURM_JOB_ID" ]; then
        source $(dirname $SCRIPT)/cutadapt_wrap.sh
    else
        source ${SCRIPT_PATH}/cutadapt_wrap.sh
    fi
455
elif [ "flexbar" = "${_arg_executable,,}" ]; then
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
    ### set our default adapter:
    adapter="TruSeq"
       ### overwrite with wanted - as the check is passed
    if [ ! -z "${_arg_adapter// }" ]; then
       adapter="${_arg_adapter}"
    fi
    ### check whether the selected adapter is in the list of allowed ones
    allowed="TruSeq SmallRNA Methyl Ribo Nextera NexteraMP"
    if [[ ! $allowed =~ (^| )"$adapter"($| ) ]]; then
        error "allowed adapters for flexbar include '$allowed'"
        exit 1
    fi
 
    ### overwrite the number of threads, if really desired by the user
    if [ -n "${_arg_threads}" ]; then
       threads=${_arg_threads}
    else
       #TODO: check sensible defaults
474
       threads=8
475 476 477 478 479 480
    fi
    if [ -z "$SLURM_JOB_ID" ]; then
        source $(dirname $SCRIPT)/flexbar_wrap.sh
    else
        source ${SCRIPT_PATH}/flexbar_wrap.sh
    fi
481 482 483 484
else 
    error "executable '$_arg_executable' not recognized."
    print_help
    exit 2
485 486
fi

487 488 489 490 491 492
### what shall be our job tag?
JOBTAG="QC_on_$(basename $_arg_inputdir)"
if [[ ! -z "${_arg_tag/ /}" ]]; then
    JOBTAG=${_arg_tag}
fi

493 494
### check if output directory already exists
if [[  -z "${_arg_outdir// }" ]]; then
495
    _arg_outdir=$(dirname $_arg_inputdir)/${JOBTAG}
Christian Meesters's avatar
Christian Meesters committed
496 497 498 499 500
else
   if [[ $(realpath "$_arg_outdir") == *"home"* ]]; then
      error "The reference, input and output may not reside in HOME"
      exit 1
   fi
501 502 503 504
fi
### create this directory unconditionally
mkdir -p $_arg_outdir

505
if [ "$cluster" = "mogon2" ] && [ -z "$SLURM_JOB_ID" ]; then
506
    ### set stripecount - if unset
507
    stripe_count=$(lfs getstripe $_arg_outdir | grep 'stripe_count:' | head -1 | tr -s ' ' | cut -d ' ' -f2)
508 509 510
    if [ $stripe_count -eq 1 ]; then
        lfs setstripe -c 6 $_arg_outdir
    fi
511
    ### check whether constraint is allowed
512
    echo "skylake broadwell anyarch" | grep $_arg_constraint &> /dev/null
513 514 515 516 517 518 519 520 521 522
    ret=$?
    if [ $ret -ne 0 ]; then
        error "given constraint not allowed: '$_arg_constraint'!"
        echo
        print_help
        exit 1
    fi
fi

# first: get some samples - any!
523
samples=$(find -L $(realpath "$_arg_inputdir") -type f \( -iname \*.fastq\* -o -iname \*.fq\* \) )
524
# transform into an array
525 526
read -r -a samples <<< $samples

527 528 529 530 531
if [ ${#samples[@]} -eq 0 ]; then
    error "no input found (was looking for '*gz', '*fastq' and '*fq')."
    exit 1
fi

532 533 534 535 536
if [ $_arg_paired -eq 1 ]; then
    # test for paired end scheme
    if [[ ${samples[0]} == *"_R1"* || ${samples[0]} == *"_R2"* ]]; then
        first='_R1'
        second='_R2'
Christian Meesters's avatar
Christian Meesters committed
537
    elif [[ ${samples[0]} == *"_1"* ]] || [[ ${samples[0]} == *"_2"* ]]; then
538 539 540 541 542 543
        first='_1'
        second='_2'
    else
        error "paired end scheme has to contain tuples of either '*_R1*'/'*_R2*' or '*_1*'/'*_2*' mates"
        exit 1
    fi
544 545
fi

546
# assume to be true by default
Christian Meesters's avatar
Christian Meesters committed
547
compression=".gz"
548 549 550 551
# determine the actual suffix, to determine the compression / no compression type
# just look on the first file
ffile=${samples[0]}
if [ ${ffile: -4} = ".bz2" ]; then
Christian Meesters's avatar
Christian Meesters committed
552
    compression=".bz2"
553 554 555 556
elif [ ${ffile: -3} = ".gz" ]; then
    compression=".gz"
else
    compression=""
557
fi 
558 559 560 561 562 563
if [ ${#samples[@]} -eq 0 ]; then
    samples=$(find $(realpath $_arg_inputdir) -name '*bz2' | grep -v $second)
    # transform into array
    read -r -a samples <<< $samples
    compression='.bz2'
fi
564 565 566 567
if [ ${#samples[@]} -eq 0 ]; then
    samples=$(find $(realpath $_arg_inputdir) -name '*fastq' | grep -v $second)
    # transform into array
    read -r -a samples <<< $samples
568
    compression=''
569 570
fi
if [ ${#samples[@]} -eq 0 ]; then
571 572 573
    samples=$(find $(realpath $_arg_inputdir) -name '*fq' | grep -v $second)
    # transform into array
    read -r -a samples <<< $samples
574
    compression=''
575 576
fi
if [ ${#samples[@]} -eq 0 ]; then
577 578 579 580
    error "no input found (was looking for '*gz', '*fastq' and '*fq')."
    exit 1
fi

Christian Meesters's avatar
Christian Meesters committed
581 582 583 584 585
### special case for jgu-cbdm
if [[ "$cluster" = "mogon" && "$_arg_assoc" = "jgu-cbdm" ]]; then
    _arg_queue="andrade,${_arg_queue}"
fi

586 587
### what shall be our job tag?
JOBTAG="QC_on_$(basename $_arg_inputdir)"
588 589
if [[ ! -z "${_arg_tag/ /}" ]]; then
    JOBTAG=${_arg_tag}
590
>>>>>>> update_mapper
591 592 593 594 595 596
fi

INPUTDIR=$_arg_inputdir

#check if this script is on node by checking env-variable $SLURM_JOB_ID, else send it to SLURM with given parameters and exit
if [ -z "$SLURM_JOB_ID" ]; then
597 598
    # depending on the number of samples, we need to submit to different partitions:
    #   (we are disregarding samples containing "unpaired" as they may be a lot smaller and account for short runtimes)
599
    nsamples=$(( $(wc -w <<< $(echo "${samples[@]}" |egrep -v 'unpaired') ) ))
600
    if [ "$cluster" = "mogon1" ]; then
601 602
        if [ $nsamples -lt 128 ]; then
           _arg_queue="short"
603
           ntasks_per_node=$((nsamples / threads))
604 605 606
        else
           _arg_queue="nodeshort"
           ntasks_per_node=64
Christian Meesters's avatar
Christian Meesters committed
607 608
           # how many nodes do we need?
           nnodes=$(printf %.0f $(echo "$nsamples / $ntasks_per_node" | bc -l))
609
        fi
610
    else
611
        # set a maximum for the number of tasks, if single threaded
Christian Meesters's avatar
Christian Meesters committed
612 613 614 615
        if [ "flexbar" = "${_arg_executable,,}" ]; then
            _arg_constraint="skylake"
        fi

616 617 618 619 620 621
        if [ ${_arg_constraint} == "broadwell" ]; then
            ntasks_per_node=40
        else
            ntasks_per_node=64
        fi
        # re-calculate with the actual number of task, when assuming a threaded application
622
        ntasks_per_node=$((ntasks_per_node / threads))
Christian Meesters's avatar
gnarf  
Christian Meesters committed
623
        # how many nodes do we need?
624
        nnodes=$(printf %.0f $(echo "$nsamples / $ntasks_per_node" | bc -l))
625 626

        # if there are (tasks * threads * samples) < partition dependent core number, submit to smp
627
        ncount=$((ntasks_per_node * threads * nsamples))
628 629 630 631 632 633 634 635 636
        # overwrite constraint, if too few samples
        if [ $ncount -lt 40 ]; then
            _arg_constraint="broadwell"
        fi
        if [ ${_arg_constraint} == "broadwell" ] && [ $ncount -lt 40 ]; then
            _arg_queue="smp"
        else
            _arg_queue="parallel"
        fi
637
    fi
638

639 640
    ### overwrite our calculation, if really desired by the user
    if [ -n "${_arg_nodes}" ]; then
641
       nnodes=$_arg_nodes
642 643
    fi
    # safe guards
644
    if [ $nnodes -eq 0 ]; then
645
       nnodes=1
646 647
    fi

648
    #submit_statement="sbatch --distribution=block:cyclic --no-requeue -o "${_arg_outdir}/${JOBTAG}_%j.out" -J $JOBTAG -p $_arg_queue -A $_arg_assoc -c $threads -t $_arg_runlimit -N $nnodes --ntasks-per-node=$((64/threads))" 
649
    submit_statement="sbatch  --no-requeue -o "${_arg_outdir}/${JOBTAG}_%j.out" -J $JOBTAG -p $_arg_queue -A $_arg_assoc -c $threads -t $_arg_runlimit -N $nnodes --ntasks-per-node=${ntasks_per_node}" 
650

651
    if [ "$cluster" = "mogon2" ]; then
Christian Meesters's avatar
Christian Meesters committed
652 653 654 655
        submit_statement="${submit_statement} -C $_arg_constraint" 
        if [ "trimmomatic" = "${_arg_executable,,}" ]; then
            submit_statement="${submit_statement} --mem=177000"
        fi
656
    fi
Christian Meesters's avatar
Christian Meesters committed
657

658 659 660
    if [ -n "$_arg_dependency" ]; then
        submit_statement="${submit_statement} --dependency=afterany:${_arg_dependency}" 
    fi
661 662 663 664
    script_statement="$SCRIPT --executable=$_arg_executable --tag=$JOBTAG" 
    if [ ! -z "${_arg_args}" ]; then
        script_statement="${script_statement} --args='$_arg_args'"
    fi
665 666 667 668 669 670 671
    if [ ! -z "${_arg_qtrim}" ]; then
        script_statement="${script_statement} --qtrim='$_arg_qtrim'"
    fi
    if [ ! -z "${_arg_qtrim_format}" ]; then
        script_statement="${script_statement} --qtrim-format='$_arg_qtrim_format'"
    fi
    
672 673
    # amend script statement with mandatory arguments
    script_statement="${script_statement} $INPUTDIR" 
674 675 676 677 678 679
    # paste them together
    submit_statement="${submit_statement} ${script_statement}"
    echo $submit_statement
    if [[ $_arg_test == "on" ]]; then
        exit
    fi
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
    ### AMMEND THE SCRIPT PATH - FOR INTERNAL PURPOSES, ONLY ###
    export SCRIPT_PATH=$(dirname $0)

    ### check wether this is an interactive shell ###
    #   if so - be a verbose submitter
    #   else  - only report the jobid, to work with dependencies
    parent=$(ps -o stat= -p $PPID)
    if [[ $parent == *s* ]]; then
        echo "sending job:"
        echo $submit_statement
        eval $submit_statement
        exit
    else
        jobid=$(eval $submit_statement)
        echo ${jobid##* } # just report the numerical ID
        exit
    fi
697
fi
698

699 700
cmdfile=/localscratch/$SLURM_JOB_ID/dummy.sh

701 702 703 704
cmdfilewriter

chmod +x $cmdfile

705
newcmd=/localscratch/$SLURM_JOBID/trimfilter_wrapper.sh
706 707 708 709 710
sbcast $cmdfile $newcmd

rm $cmdfile
cmdfile=$newcmd

Christian Meesters's avatar
Christian Meesters committed
711 712 713
### append a finishing token to the samples
samples+=('done')

714
srun="srun --cpu-bind=q -n 1 -N1 --exclusive --ntasks-per-node=$SLURM_NTASKS_PER_NODE -c $SLURM_CPUS_PER_TASK --jobid $SLURM_JOBID --mem-per-cpu=$((SLURM_MEM_PER_NODE / SLURM_CPUS_ON_NODE))"
715
parallel="parallel --no-notice -j $SLURM_NTASKS -P $SLURM_NTASKS "
716 717

$parallel $srun $cmdfile {} ::: ${samples[@]}