script_that_sometimes_fails.sh

#!/bin/bash

seed=$SLURM_JOB_ID
ID=$1

prng () {
    # Use the linear conguential generator algorithm:
    # https://en.wikipedia.org/wiki/Random_number_generation#Computational_methods
    #
    # We seed b with the SLURM_JOB_ID so that we independently have
    # the same seed for all tasks for a given job.

    x_n=0
    a=1
    b=$seed
    m=$SLURM_NTASKS
    # Workaround bug# 4
    if (( b % m == 0 ))
    then
	b+=1
    fi
    # Recur as many times as the task id to generate different numbers
    # for each SLURM task.
    for i in $(seq 1 $ID)
    do
	x_n=$(( $(( a * $((x_n + b)) )) % m))
    done
    echo $x_n
}

main () {
    # Randomly fail half of the tasks.
    random_int=$(prng)
    echo -n "Task $ID started (seed $seed, random number $random_int) ... "
    sleep "$random_int"
    if (( $random_int % 4 == 0 ))
    then
	echo "succeeded!"
	exit 0
    fi
    echo "failed!"
    exit 1
}

[ "$0" != "${BASH_SOURCE[0]}" ] || main "$@"