script_that_sometimes_fails.sh

#!/bin/bash

prng () {
    # Use the linear conguential generator algorithm:
    # https://en.wikipedia.org/wiki/Random_number_generation#Computational_methods
    #
    # Set it with:
    # a = 1
    # b = SLURM_JOB_ID
    # m = SLURM_NTASKS
    #
    # We seed b with the SLURM_JOB_ID so that we independently have
    # the same seed for all tasks for a given job.

    x_n=0
    a=1
    b=$SLURM_JOB_ID
    m=$SLURM_NTASKS
    # Recur as many times as the task id to generate different numbers
    # for each SLURM task.
    for _ in 1..$SLURM_PROCID
    do
	x_n=$(( $(( a * $((x_n + b)) )) % m))
    done
}

main () {
    # Randomly fail half of the tasks.
    "Task $SLURM_PROC_ID started..."
    random_int=$(prng)
    sleep "$random_int"
    if _=$(( random_int % 2 ))
    then
	"Task $SLURM_PROC_ID failed!"
	exit 1
    fi
    "Task $SLURM_PROC_ID succeeded!"
}

[ "$0" != "${BASH_SOURCE[0]}" ] || main "$@"