checkdir

#!/bin/bash
# Author: Christopher Lovejoy <c@lovejoy.io>

SCRIPT_NAME=checkdir
SCRIPT_VERSION=0.3.2
SCRIPT_MODIFIED=2026-09-10

## PLUMBING:

yell() { echo "$@" >&2; }
die() { yell "$@"; exit 111; }
must() { "$@" || die "cannot $*"; }
loglevel() { [[ $opt_verbose -ge $1 ]]; }
feedback() { loglevel $1 || return; shift; yell "$@"; }

toLowerCase() {
  #perl -C -lne 'print uc';
  tr '[:upper:]' '[:lower:]';
}
toUpperCase() {
  #perl -C -lne 'print uc';
  tr '[:lower:]' '[:upper:]';
}


## CONST/CONFIGURATION:

if [[ -z "$TMPDIR" || ! -d "$TMPDIR" ]]; then
  if [[ -d '/dev/shm' ]]; then
    export TMPDIR='/dev/shm';
  else
    export TMPDIR="$PWD";
  fi
fi


## DOCUMENTATION:


SCRIPT_USAGE="${0##*/} [OPTIONS] [SUMFILE…]"
SCRIPT_OPTIONS=(
  "--help, help, -h\tShow this help."
  "--version\tShow the script version."
  "-a ALG\tSet encryption algorithm: 'sha256', 'b2', 'sha256e' (alias 'annex')."
  "-e\t\tOnly check if files in SUMFILE exist, and output missing filenames."
  "-l\t\tOnly output the SUMFILE path found/selected."
  "-I END\t(Only with -a sha256e) Ignore END and anything after it on each line. If omitted, this option obeys a header describing such a separator."
  "-v\t\tShow more verbose output."
)
SCRIPT_DESCRIPTION_LINES=(
  "If no SUMFILE is specified, .*SHA256SUM* and *SHA256SUM* is globbed in the current directory, and the newest one is used."
)


## METHODS:

showHelp() {
  echo "USAGE: $SCRIPT_USAGE";
  echo "";
  echo "OPTIONS:";
  local opt;
  for opt in "${SCRIPT_OPTIONS[@]}"; do
    echo -e "  ${opt}"
  done
  echo "";
  echo "DESCRIPTION:";
  for line in "${SCRIPT_DESCRIPTION_LINES[@]}"; do
    echo "  ${line}"
  done
  echo
  echo "${SCRIPT_NAME} version ${SCRIPT_VERSION} (${SCRIPT_MODIFIED})"
}

confirm() {
  local prompt="$1";
  local answer="$2";
  local REPLY;
  local defaultYes=true
  if [[ $answer =~ [Nn] ]]; then
    defaultYes=false
    options='y/N'
  else
    options='Y/n'
  fi
  read -n1 -p "${prompt} (${options}) ";
  if [[ $defaultYes == true ]]; then
    [[ -z $REPLY || $REPLY =~ [Yy] ]];
    return
  else
    [[ $REPLY =~ [Yy] ]];
    return
  fi
}

if ! command -v sha256sum &>/dev/null; then
  if command -v shasum &>/dev/null; then
    sha256sum() { shasum -a 256 "$@"; }
  fi
fi

initializeSumCommand() {
  # NOTE: Each command MUST support the '--check SUMFILE' option.
  opt_algorithm="$(echo "$opt_algorithm" | toLowerCase)";
  case $opt_algorithm in
    sha256e|*annex) opt_algorithm=sha256e;
      if ! command -v sha256sum &>/dev/null; then
        die "Missing checksum software: sha256sum or shasum.";
      fi
      sumCommand=(checksumSHA256E);
      ;;
    sha*) sumCommand=(sha256sum);
      if ! command -v sha256sum &>/dev/null; then
        die "Missing checksum software: sha256sum or shasum.";
      fi
      ;;
    b2*|blake*) sumCommand=(b2sum);;
    *) die "Unknown sum algorithm '${opt_algorithm}'.";;
  esac
  if command -v "${sumCommand[0]}" &>/dev/null; then
    feedback 1 "Using $(command -v "${sumCommand[0]}")";
  else
    die "Missing checksum software: ${sumCommand[0]}.";
  fi
}

getSumFile() {
  local match;
  match="$(ls -1t .*SUM* 2>/dev/null)" \
    || match="$(ls -t *SUM* 2>/dev/null)" \
    || return $?;

  echo -en "$match" | head -1
}

countFilesInSumFile() {
  grep -Ecv '^\s*#' "$1";
}

describeIgnoreAfter() {
  local sumFile="$1"
  local pattern="^\\s*#\\s*[^:]*(separat|metadata|ignore)[^:]*: ?'?(.+)'?\$"
  local header
  header="$(head -10 "$sumFile" | grep -i -P "$pattern")" || return
  echo "$header" | perl -pe "s/$pattern/\$2/i";
}

describeSumFile() {
  local sumFile="$1";
  local fileCount="$2";
  [[ -z $fileCount ]] && fileCount="$(countFilesInSumFile "$sumFile")";
  local dirCount="$(grep -Ec '^[a-f0-9]+  .*/' "$sumFile")";

  local description="${fileCount} files";

  if [[ $dirCount -gt 0 ]]; then
    description+="; descends into directories";
  fi;

  local metadataSeparator="$3"
  if [[ -n $metadataSeparator ]]; then
    description+="; metadata after ${metadataSeparator}";
  fi;

  echo -n "$description";
}

checksumSHA256E() {
  local sumfile path basename size sum ext baseKey
  local status # <enum:'OK','FAILED','REF'>
  local MAX_KEY_LENGTH=300
  while [[ $# -gt 0 ]]; do
    case $1 in
      --|--check) shift; continue;;
      *) sumfile="$1";
    esac
    shift
  done
  local readOpts=(-r)
  while read "${readOpts[@]}" key path; do
    [[ $key =~ ^\s*# ]] && continue; # comment
    if [[ ${key:0:1} == '*' ]]; then
      status=REF
      key="${key#\*}"
    fi
    if [[ -n $opt_ignoreAfter ]]; then
      path="${path%%${opt_ignoreAfter}*}"
    fi
    if ! [[ -e "$path" ]]; then
      status='FAILED open or read';
    else
      basename="$(basename "$path")"
      size=($(du -L -b "$path")) || continue;
      sum=($(sha256sum "$path")) || continue;
      baseKey="SHA256E-s${size[0]}--${sum[0]}"
      if [[ ${key:0:${#baseKey}} == $baseKey ]]; then
        status=OK;
      elif [[ $status == REF && $size -lt $MAX_KEY_LENGTH ]] \
        && grep -qF "$key" "$path"
      then
        status='REFerence to file not present'
      else
        status=FAILED;
      fi
    fi
    echo "${path}: ${status}";
  done <"$sumfile"
}
export -f checksumSHA256E

## Giving it a descriptive name, because (with option -e) it shows up in the "full" output to an
## interactive shell.
checkdir_doPathsInSumFileExist() {
  local sumFile="$1"; shift;
  local numTotal=0 numPresent=0 numMissing=0;
  local exitStatus=0;

  feedback 2 "About to read sumfile(${sumFile}) line by line in Bash.";
  local hash path;
  while read hash path; do
    if [[ -z $hash || -z "$path" || $hash =~ ^[[:space:]]*# ]]; then
      feedback 2 "Skipping commented line: ${hash} ${path}";
      continue;
    fi;

    let numTotal++;
    if [[ -e "$path" ]]; then
      let numPresent++;
      echo "EXISTS: ${path}";
    else
      let numMissing++;
      exitStatus=1;
      echo "MISSING: ${path}";
    fi
  done <"$sumFile"

  if [[ $numMissing -gt 0 ]]; then
    yell "Missing ${numMissing} files.";
  fi

  return $exitStatus;
}

main() {
  local sumFile="$1";
  if [[ -z "$sumFile" ]]; then
    sumFile="$(getSumFile)" || die "No hash file to check against. Run sumdir to make one.";
  fi

  if [[ $sumFile =~ [bB]2 ]]; then
    opt_algorithm=b2;
  elif [[ $sumFile =~ SHA256E ]]; then
    opt_algorithm=sha256e;
  else
    opt_algorithm=sha256;
  fi

  checkOutputCacheFile="${TMPDIR}/.checking.${sumFile##*/}.${RANDOM}.log";
  local fileCount="$(countFilesInSumFile "$sumFile")";

  local metadataSeparatorDescription
  if [[ -z $opt_ignoreAfter ]]; then
    metadataSeparatorDescription="$(describeIgnoreAfter "$sumFile")"
    metadataSeparatorStatus=$?
    if [[ $metadataSeparatorStatus -eq 0 ]]; then
      case $metadataSeparatorDescription in
        *\\t*|*Tab*|*tab*) opt_ignoreAfter=$'\t';;
        *) opt_ignoreAfter="$metadataSeparatorDescription";;
      esac
    else
      metadataSeparatorDescription=
    fi
  fi
  #loglevel 1 && >&2 declare -p opt_ignoreAfter metadataSeparatorDescription metadataSeparatorStatus;

  yell "Checking files against $sumFile ($(describeSumFile "$sumFile" "$fileCount" "$metadataSeparatorDescription"))";

  ## PREPARING EXECUTION:
  checkCommand=();
  if [[ $opt_check_exists_only != 'true' ]]; then
    initializeSumCommand;
    checkCommand+=("${sumCommand[@]}" --check);
  else
    checkCommand+=(checkdir_doPathsInSumFileExist);
  fi
  checkCommand+=("$sumFile")

  ## EXECUTION:
  # WISH (B) 2024-07-07 parallelize file hash check
  # DONE x 2024-07-07 (A) 2024-07-07 Show progress
  local checkOutput="$("${checkCommand[@]}" | {
    outputLineCount=0;
    while read outputLine; do
      let 'outputLineCount++';
      echo "$outputLine";
      >&2 echo -n $'\r'"  $((100 * outputLineCount / fileCount))% (${outputLineCount} files checked)       "; # space at the end to blank out long numbers
    done;
    >&2 echo;
  };)";
  
  checkStatus=$?; # FIXME Status of checkCommand obscured by reading outputLine and subsequent echos.  uuid:14601836-19ee-4744-8e0c-cd40cea53393

  ## HANDLE OUTPUT:
  local failedLines
  failedLines="$(echo "$checkOutput" \
    | tee "${checkOutputCacheFile}" \
    | grep ': \(FAILED\|MISSING\)')"
  failedMatchStatus=$?
  local failedLineCount=$(echo "$failedLines" | wc -l)
  if [[ $failedMatchStatus -eq 0 ]]; then
    yell "$failedLines";
    yell "";
    yell "${failedLineCount} of ${fileCount} files failed checks ($((100 * (fileCount - failedLineCount) / fileCount))% OK)";
  elif [[ $checkStatus != 0 ]]; then
    yell "✗ Finished with problems.";
  else
    yell "✓ All OK ($fileCount files)";
  fi

  [[ -t 0 ]] \
    && confirm "See full output?" 'n' \
    && cat <(echo "### output from '${checkCommand[*]}':") "$checkOutputCacheFile" <(echo -e "\n### sum file data being checked:") "$sumFile" | less

  [[ -e "$checkOutputCacheFile" ]] && rm "$checkOutputCacheFile"

  return $checkStatus;
}



## EXECUTION:

opt_algorithm=sha256;
opt_check_exists_only=false;
opt_verbose=0;
opt_ignoreAfter=
sumCommand=();
sumFileArgs=();

while [[ $# -gt 0 ]]; do
  case $1 in
    --help|help|-h) showHelp; exit;;
    -a) opt_algorithm="$2"; shift;;
    -e) opt_check_exists_only=true;;
    -I) opt_ignoreAfter+="$2"; shift;;
    -l) getSumFile; exit;;
    -v) let opt_verbose++;;
    --version) 
      if loglevel 1; then
        echo "$(basename "$0") version ${SCRIPT_VERSION} (${SCRIPT_MODIFIED})";
      else
        echo "${SCRIPT_VERSION}";
      fi
      exit;;
    *) sumFileArgs+=("$1");;
  esac
  shift;
done

feedback 1 "Log level: $opt_verbose"

main "${sumFileArgs[@]}";

exit;