Home
lawn-hark: Parallelization using xargs(1). - gopher-lawn - The gopher lawn gopher directory project. HTML git clone git://bitreich.org/gopher-lawn/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/gopher-lawn/ DIR Log DIR Files DIR Refs DIR Tags --- DIR commit e3cc5279072ddbee7f2d58152732c759a9ede582 DIR parent b9a49018389fa0126149e3cbef3ecdef4defe6ef HTML Author: Julian Schweinsberg <pazz0@0xfa.de> Date: Tue, 19 Dec 2023 20:25:02 +0100 lawn-hark: Parallelization using xargs(1). Signed-off-by: Christoph Lohmann <20h@r-36.net> Diffstat: M lawn-hark/lawn-hark.sh | 73 +++++++++++++++++-------------- 1 file changed, 41 insertions(+), 32 deletions(-) --- DIR diff --git a/lawn-hark/lawn-hark.sh b/lawn-hark/lawn-hark.sh @@ -5,6 +5,7 @@ errorthreshold=3 timeout=15 +maxworkers=8 onionsocksproxy="127.0.0.1:9050" function tcpdial() { @@ -95,26 +96,14 @@ function checkraw() { return 0 } -checktime="$(date +%s)" -statedir="$1" +program="$(readlink -f "$0")" -if [ -z "${statedir}" ]; +if [ "${LAWNHARK_WORKER}" = "1" ]; then - printf "You need to specify a state dir.\n" >&2 - exit 1 -fi - -mkdir -p "${statedir}" -if [ ! -d "${statedir}" ]; -then - printf "%s is not a directory! Aborting.\n" "${statedir}" >&2 - exit 1 -fi + statedir="$1" + checktime="$2" + f="$3" -shift - -for f; -do type="" selector="" host="" @@ -145,12 +134,10 @@ do || [ -z "${host}" ] \ || [ -z "${port}" ]; then - printf "ERROR\t%s\tInvalid entry!\n" "${f}" >&2 - continue + flock -x "${program}" printf "ERROR\t%s\tInvalid entry!\n" "${f}" >&2 + exit fi - #printf "DEBUG\t%s\tchecking\t%s\t%s\t%s\t%s\n" "${f}" "${type}" "${selector}" "${host}" "${port}" - case "${type}" in cso ) error="$(checkcso "${host}" "${port}")" @@ -182,8 +169,8 @@ do error="$(checkraw "${sshhost}" "${sshport}")" ;; * ) - printf "TODO\t%s\tCan't handle %s\n" "${f}" "${url}" - continue + flock -x "${program}" printf "TODO\t%s\tCan't handle %s\n" "${f}" "${url}" >&2 + exit ;; esac else @@ -215,20 +202,42 @@ do if [ ${errorcount} -ge ${errorthreshold} ]; then - printf "ERROR\t%s\t%s\n" "${f}" "${error}" >&2 + flock -x "${program}" printf "ERROR\t%s\t%s\n" "${f}" "${error}" >&2 fi printf "%s\t%s\n" "${checktime}" "${errorcount}" > "${statefile}" -done +else + checktime="$(date +%s)" + statedir="$1" -# garbage collection -find "${statedir}" -type f | while read -r f; -do - IFS=" " read -r lastcheck errorcount < "${f}" + if [ -z "${statedir}" ]; + then + printf "You need to specify a state dir.\n" >&2 + exit 1 + fi - if [ ${lastcheck} -ne ${checktime} ]; + mkdir -p "${statedir}" + if [ ! -d "${statedir}" ]; then - rm -f "${f}" + printf "%s is not a directory! Aborting.\n" "${statedir}" >&2 + exit 1 fi -done + shift + + for f; + do + printf "%s\0" "${f}" + done | LAWNHARK_WORKER=1 xargs -r -0 -P "${maxworkers}" -L1 "${program}" "${statedir}" "${checktime}" + + # garbage collection + find "${statedir}" -type f | while read -r f; + do + IFS=" " read -r lastcheck errorcount < "${f}" + + if [ ${lastcheck} -ne ${checktime} ]; + then + rm -f "${f}" + fi + done +fi