tootpaste/tootpaste.sh
2022-04-22 17:22:02 +09:00

189 lines
5.4 KiB
Bash

#!/usr/bin/env bash
set -euo pipefail
# Steps to run
DO_ACCOUNTS_CULL=${DO_ACCOUNTS_CULL:-true}
DO_MEDIA_REMOVE=${DO_MEDIA_REMOVE:-true}
DO_STATUSES_REMOVE=${DO_STATUSES_REMOVE:-true}
DO_CACHE_RECOUNT=${DO_CACHE_RECOUNT:-true}
# Environment variables for main settings
TOOTCTL=${TOOTCTL:-/home/mastodon/live/bin/tootctl}
export DB_POOL=${DB_POOL:-5}
DRY_RUN=${DRY_RUN:-true}
# Date difference for expired certificate, default 2w
TLS_EXPIRED_MAX_SEC=${TLS_EXPIRED_MAX_SEC:-1210000}
# How many seconds to wait to connect to an instance already in error in the
# past
INSTANCE_LAST_CHANCE_TIMEOUT=${INSTANCE_LAST_CHANCE_TIMEOUT:-30}
# How old media attachments have to be before getting removed
MEDIA_REMOVE_DAYS=${MEDIA_REMOVE_DAYS:-7}
CARDS_REMOVE_DAYS=${CARDS_REMOVE_DAYS:-15}
# How old unreferenced statuses have to be before getting removed
STATUSES_REMOVE_DAYS=${STATUSES_REMOVE_DAYS:-30}
# Path to logs files
CULL_LOG=$(mktemp /tmp/tootpaste_XXX)
TLS_EXPIRED_LOG=$(mktemp /tmp/tootpaste_XXX)
OTHER_ERRORS_LOG=$(mktemp /tmp/tootpaste_XXX)
PREV_ERRORS_LOG=/tmp/tootpaste_prev_errors
accounts_cull() {
$DRY_RUN \
&& $TOOTCTL accounts cull \
--dry-run \
--concurrency "$DB_POOL" \
> "$CULL_LOG"
$DRY_RUN \
|| $TOOTCTL accounts cull \
--concurrency "$DB_POOL" \
> "$CULL_LOG"
# Remove instances that have an expired certificate from more than
# TLS_EXPIRED_MAX_SEC
if grep -q 'certificate has expired' "$CULL_LOG"; then
grep 'certificate has expired' "$CULL_LOG" \
| awk '{print $NF}' \
| cut -d'/' -f3 \
| sort -u \
> "$TLS_EXPIRED_LOG"
while read -r instance; do
TLS_EXPIRED_TS=$(
date -d "$(
echo Q \
| openssl s_client \
-servername "$instance" \
-connect "${instance}":443 \
2>/dev/null \
| openssl x509 -noout -dates \
| grep 'notAfter' \
| cut -d'=' -f2
)" +%s
)
DATE_DIFF=$(($(date +%s) - TLS_EXPIRED_TS))
if [[ $DATE_DIFF -gt $TLS_EXPIRED_MAX_SEC ]]; then
echo "${instance} has a certificate expired for more than TLS_EXPIRED_MAX_SEC, purging..."
$DRY_RUN \
&& $TOOTCTL domains purge \
--concurrency "$DB_POOL" \
--dry-run \
"$instance"
$DRY_RUN \
|| $TOOTCTL domains purge \
--concurrency "$DB_POOL" \
"$instance"
fi
done < "$TLS_EXPIRED_LOG"
fi
# Log other instances errors, then if they were already in the log, purge them
if grep -q 'https' "$CULL_LOG"; then
grep \
-e 'certificate verify failed' \
-e 'timed out' \
-e 'sslv3 alert handshake failure' \
-e 'TooManyRedirectsError' \
-e 'EndlessRedirectError' \
-e 'HostValidationError' \
"$CULL_LOG" \
| awk '{print $NF}' \
| cut -d'/' -f3 \
| sort -u \
> "$OTHER_ERRORS_LOG"
fi
# Log unjoinable instances, then if they were already in the log, purge them
if grep -q 'not available during the check:' "$CULL_LOG"; then
grep \
-A 9999 \
'not available during the check:' \
"$CULL_LOG" \
| tail -n +2 \
| sed -E 's/\s+//' \
> "$OTHER_ERRORS_LOG"
fi
test -f $PREV_ERRORS_LOG || touch $PREV_ERRORS_LOG
while read -r instance; do
if grep -q "$instance" $PREV_ERRORS_LOG; then
error=false
echo "${instance} was already in error last time your ran tootpaste, trying access..."
curl \
--output /dev/null \
--silent \
--show-error \
--max-time "$INSTANCE_LAST_CHANCE_TIMEOUT" \
https://"${instance}" \
|| error=true
if $error; then
echo "${instance} still cannot be accessed, purging..."
$DRY_RUN \
&& $TOOTCTL domains purge \
--concurrency "$DB_POOL" \
--dry-run \
"$instance"
$DRY_RUN \
|| $TOOTCTL domains purge \
--concurrency "$DB_POOL" \
"$instance"
else
echo "${instance} can now be accessed, not purging!"
fi
fi
done < "$OTHER_ERRORS_LOG"
cat "$OTHER_ERRORS_LOG" >> $PREV_ERRORS_LOG
}
cache_recount(){
$DRY_RUN && echo 'Not running cache recount in dry run.'
$DRY_RUN || $TOOTCTL cache recount accounts --concurrency "$DB_POOL"
$DRY_RUN || $TOOTCTL cache recount statuses --concurrency "$DB_POOL"
}
media_remove(){
$DRY_RUN \
&& $TOOTCTL media remove \
--days "$MEDIA_REMOVE_DAYS" \
--concurrency "$DB_POOL" \
--dry-run
$DRY_RUN \
|| $TOOTCTL media remove \
--days "$MEDIA_REMOVE_DAYS" \
--concurrency "$DB_POOL"
$DRY_RUN && $TOOTCTL media remove-orphans --dry-run
$DRY_RUN || $TOOTCTL media remove-orphans
$DRY_RUN \
&& $TOOTCTL preview_cards remove \
--days "$MEDIA_REMOVE_DAYS" \
--concurrency "$DB_POOL" \
--link \
--dry-run
$DRY_RUN \
|| $TOOTCTL preview_cards remove \
--days "$CARDS_REMOVE_DAYS" \
--concurrency "$DB_POOL" \
--link
}
statuses_remove(){
$DRY_RUN && echo 'Not removing old statuses in dry-run.'
$DRY_RUN || $TOOTCTL statuses remove --days "$STATUSES_REMOVE_DAYS"
}
check_command(){
command -v "$1" > /dev/null
}
for command in $TOOTCTL curl grep awk cut sort openssl; do
check_command "$command" || (echo "$command not found, exiting..."; exit 1)
done
$DO_ACCOUNTS_CULL && accounts_cull
$DO_MEDIA_REMOVE && media_remove
$DO_STATUSES_REMOVE && statuses_remove
$DO_CACHE_RECOUNT && cache_recount