02-submit-resumable.slurm

#!/bin/bash -x
#SBATCH --nodes 2
#SBATCH --ntasks 5
#SBATCH --output submit.out

#SBATCH --dependency singleton
#SBATCH --job-name unambiguous-name-for-resumable-job
# Kill job after 15 seconds to show resuming feature.
#SBATCH --time 0:15

# Overwrite instead of appending to output file.
echo -n > submit.out

parallel_opts=$(~/parallel-slurm/parallel_opts.sh)
module load parallel

# Run a failure prone program.
parallel $parallel_opts \
	 --joblog joblog \
	 --resume \
	 --line-buffer \
	 ./script_that_sometimes_fails.sh \
	 ::: $(seq $SLURM_NTASKS)