tootpaste/tootpaste.sh

#!/usr/bin/env bash
set -euo pipefail

# Steps to run
DO_ACCOUNTS_CULL=${DO_ACCOUNTS_CULL:-true}
DO_MEDIA_REMOVE=${DO_MEDIA_REMOVE:-true}
DO_STATUSES_REMOVE=${DO_STATUSES_REMOVE:-true}
DO_CACHE_RECOUNT=${DO_CACHE_RECOUNT:-true}
# Environment variables for main settings
TOOTCTL=${TOOTCTL:-/home/mastodon/live/bin/tootctl}
export DB_POOL=${DB_POOL:-5}
DRY_RUN=${DRY_RUN:-true}
# Date difference for expired certificate, default 2w
TLS_EXPIRED_MAX_SEC=${TLS_EXPIRED_MAX_SEC:-1210000}
# How many seconds to wait to connect to an instance already in error in the
# past
INSTANCE_LAST_CHANCE_TIMEOUT=${INSTANCE_LAST_CHANCE_TIMEOUT:-30}
# How old media attachments have to be before getting removed
MEDIA_REMOVE_DAYS=${MEDIA_REMOVE_DAYS:-7}
CARDS_REMOVE_DAYS=${CARDS_REMOVE_DAYS:-15}
# How old unreferenced statuses have to be before getting removed
STATUSES_REMOVE_DAYS=${STATUSES_REMOVE_DAYS:-30}
# Path to logs files
CULL_LOG=$(mktemp /tmp/tootpaste_XXX)
TLS_EXPIRED_LOG=$(mktemp /tmp/tootpaste_XXX)
OTHER_ERRORS_LOG=$(mktemp /tmp/tootpaste_XXX)
PREV_ERRORS_LOG=/tmp/tootpaste_prev_errors

accounts_cull() {
  
  $DRY_RUN \
    && $TOOTCTL accounts cull \
    --dry-run \
    --concurrency "$DB_POOL" \
    > "$CULL_LOG"
  $DRY_RUN \
    || $TOOTCTL accounts cull \
    --concurrency "$DB_POOL" \
    > "$CULL_LOG"

  # Remove instances that have an expired certificate from more than
  # TLS_EXPIRED_MAX_SEC
  if grep -q 'certificate has expired' "$CULL_LOG"; then
    grep 'certificate has expired' "$CULL_LOG" \
      | awk '{print $NF}' \
      | cut -d'/' -f3 \
      | sort -u \
      > "$TLS_EXPIRED_LOG"
  
    while read -r instance; do
      TLS_EXPIRED_TS=$(
        date -d "$(
          echo Q \
            | openssl s_client \
              -servername "$instance" \
              -connect "${instance}":443 \
              2>/dev/null \
            | openssl x509 -noout -dates \
            | grep 'notAfter' \
            | cut -d'=' -f2
        )" +%s
      )
      DATE_DIFF=$(($(date +%s) - TLS_EXPIRED_TS))
      if [[ $DATE_DIFF -gt $TLS_EXPIRED_MAX_SEC ]]; then
        echo "${instance} has a certificate expired for more than TLS_EXPIRED_MAX_SEC, purging..."
        $DRY_RUN \
          && $TOOTCTL domains purge \
          --concurrency "$DB_POOL" \
          --dry-run \
          "$instance"
        $DRY_RUN \
          || $TOOTCTL domains purge \
          --concurrency "$DB_POOL" \
          "$instance"
      fi
    done < "$TLS_EXPIRED_LOG"
  fi

  # Log other instances errors, then if they were already in the log, purge them
  if grep -q 'https' "$CULL_LOG"; then
    grep \
      -e 'certificate verify failed' \
      -e 'timed out' \
      -e 'sslv3 alert handshake failure' \
      -e 'TooManyRedirectsError' \
      -e 'EndlessRedirectError' \
      -e 'HostValidationError' \
      "$CULL_LOG" \
      | awk '{print $NF}' \
      | cut -d'/' -f3 \
      | sort -u \
      > "$OTHER_ERRORS_LOG"
  fi

  # Log unjoinable instances, then if they were already in the log, purge them
  if grep -q 'not available during the check:' "$CULL_LOG"; then
    grep \
      -A 9999 \
      'not available during the check:' \
      "$CULL_LOG" \
      | tail -n +2 \
      | sed -E 's/\s+//' \
      > "$OTHER_ERRORS_LOG"
  fi
  
  test -f $PREV_ERRORS_LOG || touch $PREV_ERRORS_LOG
  while read -r instance; do
    if grep -q "$instance" $PREV_ERRORS_LOG; then
      error=false
      echo "${instance} was already in error last time your ran tootpaste, trying access..."
      curl \
        --output /dev/null \
        --silent \
        --show-error \
        --max-time "$INSTANCE_LAST_CHANCE_TIMEOUT" \
        https://"${instance}" \
        || error=true
      if $error; then
        echo "${instance} still cannot be accessed, purging..."
        $DRY_RUN \
          && $TOOTCTL domains purge \
          --concurrency "$DB_POOL" \
          --dry-run \
          "$instance"
        $DRY_RUN \
          || $TOOTCTL domains purge \
          --concurrency "$DB_POOL" \
          "$instance"
        else
          echo "${instance} can now be accessed, not purging!"
      fi
    fi
  done < "$OTHER_ERRORS_LOG"
  cat "$OTHER_ERRORS_LOG" >> $PREV_ERRORS_LOG
}

cache_recount(){

  $DRY_RUN && echo 'Not running cache recount in dry run.'
  $DRY_RUN || $TOOTCTL cache recount accounts --concurrency "$DB_POOL"
  $DRY_RUN || $TOOTCTL cache recount statuses --concurrency "$DB_POOL"
}

media_remove(){

  $DRY_RUN \
    && $TOOTCTL media remove \
    --days "$MEDIA_REMOVE_DAYS" \
    --concurrency "$DB_POOL" \
    --dry-run
  $DRY_RUN \
    || $TOOTCTL media remove \
    --days "$MEDIA_REMOVE_DAYS" \
    --concurrency "$DB_POOL"
  
  $DRY_RUN && $TOOTCTL media remove-orphans --dry-run
  $DRY_RUN || $TOOTCTL media remove-orphans
  
  $DRY_RUN \
    && $TOOTCTL preview_cards remove \
    --days "$MEDIA_REMOVE_DAYS" \
    --concurrency "$DB_POOL" \
    --link \
    --dry-run
  $DRY_RUN \
    || $TOOTCTL preview_cards remove \
    --days "$CARDS_REMOVE_DAYS" \
    --concurrency "$DB_POOL" \
    --link
}

statuses_remove(){
 
  $DRY_RUN && echo 'Not removing old statuses in dry-run.'
  $DRY_RUN || $TOOTCTL statuses remove --days "$STATUSES_REMOVE_DAYS"
}

check_command(){
  command -v "$1" > /dev/null
}

for command in $TOOTCTL curl grep awk cut sort openssl; do
  check_command "$command" || (echo "$command not found, exiting..."; exit 1)
done

$DO_ACCOUNTS_CULL && accounts_cull
$DO_MEDIA_REMOVE && media_remove
$DO_STATUSES_REMOVE && statuses_remove
$DO_CACHE_RECOUNT && cache_recount