Init first version

tootpaste.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# vim: set sw=2 sts=2 ts=2 expandtab smarttab
+set -euo pipefail
+
+# Steps to run
+DO_ACCOUNTS_CULL=${DO_ACCOUNTS_CULL:-true}
+DO_MEDIA_REMOVE=${DO_MEDIA_REMOVE:-true}
+DO_STATUSES_REMOVE=${DO_STATUSES_REMOVE:-true}
+DO_CACHE_RECOUNT=${DO_CACHE_RECOUNT:-true}
+# Environment variables for main settings
+TOOTCTL=${TOOTCTL:-/home/mastodon/mastodon/bin/tootctl}
+export DB_POOL=${DB_POOL:-90}
+DRY_RUN=${DRY_RUN:-true}
+# Date difference for expired certificate, default 2w
+TLS_EXPIRED_MAX_SEC=${TLS_EXPIRED_MAX_SEC:-1210000}
+# How many seconds to wait to connect to an instance already in error in the
+# past
+INSTANCE_LAST_CHANCE_TIMEOUT=30
+# How old media attachments have to be before getting removed
+MEDIA_REMOVE_DAYS=7
+CARDS_REMOVE_DAYS=15
+# How old unreferenced statuses have to be before getting removed
+STATUSES_REMOVE_DAYS=30
+# Path to logs files
+CULL_LOG=/tmp/$(mktemp tootpaste_XXX)
+TLS_EXPIRED_LOG=/tmp/$(mktemp tootpaste_XXX)
+OTHER_ERRORS_LOG=/tmp/$(mktemp tootpaste_XXX)
+PREV_ERRORS_LOG=/tmp/tootpaste_prev_errors
+
+accounts_cull() {
+  
+  $DRY_RUN && $TOOTCTL accounts cull --dry-run --concurrency $DB_POOL > $CULL_LOG
+  $DRY_RUN || $TOOTCTL accounts cull --concurrency $DB_POOL > $CULL_LOG
+
+  # Remove instances that have an expired certificate from more than
+  # TLS_EXPIRED_MAX_SEC
+  grep 'certificate has expired' $CULL_LOG \
+    | awk '{print $NF}' \
+    | cut -d'/' -f3 \
+    | sort -u \
+    > $TLS_EXPIRED_LOG
+  
+  while read instance; do
+    TLS_EXPIRED_TS=$(
+      date -d "$(
+        echo Q \
+          | openssl s_client \
+            -servername $instance \
+            -connect ${instance}:443 \
+            2>/dev/null \
+          | openssl x509 -noout -dates \
+          | grep 'notAfter' \
+          | cut -d'=' -f2
+      )" +%s
+    )
+    DATE_DIFF=$(($(date +%s) - $TLS_EXPIRED_TS))
+    if [[ $DATE_DIFF -gt $TLS_EXPIRED_MAX_SEC ]]; then
+      echo "${instance} has a certificate expired for more than TLS_EXPIRED_MAX_SEC, purging..."
+      $DRY_RUN && $TOOTCTL domains purge --concurrency $DB_POOL --dry-run $instance
+      $DRY_RUN || $TOOTCTL domains purge --concurrency $DB_POOL $instance
+    fi
+  done < $TLS_EXPIRED_LOG
+
+  # Log other instances errors, then if they were already in the log, purge
+  # them
+  grep \
+    -e 'certificate verify failed' \
+    -e 'timed out' \
+    -e 'sslv3 alert handshake failure' \
+    -e 'TooManyRedirectsError' \
+    $CULL_LOG \
+    | awk '{print $NF}' \
+    | cut -d'/' -f3 \
+    | sort -u \
+    > $OTHER_ERRORS_LOG
+  
+  while read instance; do
+    if grep -q $instance $PREV_ERRORS_LOG; then
+      error=false
+      echo "${instance} was already in error last time your ran tootpaste, trying access..."
+      curl \
+        --silent \
+        --show-error \
+        --max-time $INSTANCE_LAST_CHANCE_TIMEOUT \
+        https://${instance} \
+        || error=true
+      if $error; then
+        echo "${instance} still cannot be accessed, purging..."
+        $DRY_RUN && $TOOTCTL domains purge --concurrency $DB_POOL --dry-run $instance
+        $DRY_RUN || $TOOTCTL domains purge --concurrency $DB_POOL $instance
+      fi
+    fi
+  done < $OTHER_ERRORS_LOG
+  cat $OTHER_ERRORS_LOG >> $PREV_ERRORS_LOG
+}
+
+cache_recount(){
+
+  $DRY_RUN && echo 'Not running cache recount in dry run.'
+  $DRY_RUN || $TOOTCTL cache recount accounts --concurrency $DB_POOL
+  $DRY_RUN || $TOOTCTL cache recount statuses --concurrency $DB_POOL
+}
+
+media_remove(){
+
+  $DRY_RUN && $TOOTCTL media remove --days $MEDIA_REMOVE_DAYS --concurrency $DB_POOL --dry-run
+  $DRY_RUN || $TOOTCTL media remove --days $MEDIA_REMOVE_DAYS --concurrency $DB_POOL
+  
+  $DRY_RUN && $TOOTCTL media remove-orphans --dry-run
+  $DRY_RUN || $TOOTCTL media remove-orphans
+  $DRY_RUN && $TOOTCTL preview_cards remove --days $MEDIA_REMOVE_DAYS --concurrency $DB_POOL --dry-run
+  $DRY_RUN || $TOOTCTL preview_cards remove --days $CARDS_REMOVE_DAYS --concurrency $DB_POOL --link
+}
+
+statuses_remove(){
+ 
+  $DRY_RUN && echo 'Not removing old statuses in dry-run.'
+  $DRY_RUN || $TOOTCTL statuses remove --days $STATUSES_REMOVE_DAYS
+}
+
+$DO_ACCOUNTS_CULL && accounts_cull
+$DO_MEDIA_REMOVE && media_remove
+$DO_STATUSES_REMOVE && statuses_remove
+$DO_CACHE_RECOUNT && cache_recount