#!/usr/bin/env bash
DOMAIN=LE FQDN DE VOTRE INSTANCE ICI
API_URL="http://localhost:8008"
DBNAME="NOM DE LA BDD"
DBUSER="USER DE LA BDD"
export PGPASSWORD="MOT DE PASSE SECRET"
# Définition du delai de rétention:
TIME1='6 months ago'
# Création de la date from the given time string:
UNIX_TIMESTAMP1=$(date +%s%3N --date='TZ="UTC+2" '"$TIME1")
# Localisation des fichiers temporaires
ROOMLIST=/tmp/roomlist.json
EMPTYROOMLIST_PURGE=/tmp/emptyrooms_to_purge.txt
ROOMLIST_PURGE=/tmp/rooms_to_purge.txt
ROOMLIST_COMPRESS=/tmp/rooms_to_compress.txt
ROOMLIST_COMPRESS2=/tmp/rooms_to_compress2.txt
###############################################################################
# set an access token
###############################################################################
TOKEN=VOTRE TOCKEN MATRIX USER ADMIN ICI
###############################################################################
# Début
###############################################################################
echo ""
echo " - Espace disque avant"
date
df -h
###############################################################################
# Supprimer les rooms vides
###############################################################################
echo ""
echo " - Création de la liste des rooms vides"
curl --no-progress-meter -s -H "Authorization: Bearer $TOKEN" "$API_URL/_synapse/admin/v1/rooms?limit=999999999" > $ROOMLIST
jq -r '[ .rooms[] | select(.joined_local_members == 0) ] | .[].room_id' < $ROOMLIST > $EMPTYROOMLIST_PURGE
readarray -t ROOMS_ARRAY < $EMPTYROOMLIST_PURGE
for ROOM_NAME in "${ROOMS_ARRAY[@]}"
do
:
echo ""
echo "--- Suppression de la room $ROOM_NAME "
curl --no-progress-meter -s -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/$ROOM_NAME" -d '{"block": false, "purge": true}'
done
###############################################################################
# Supprimer les TRES GROSSES ROOMS (LISTE STATIQUE)
###############################################################################
echo ""
echo "--- Suppression DES TRES GROSSES ROOMS (LISTE STATIQUE) "
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!OGEhHVWSdvArJzumhm:matrix.org" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!yUHxXrvmurdnsPyhiH:sakura.ci" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!mKkDdfCLCtkTyqcxEc:nerdsin.space" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!fzfHhoTplYBEXfWOaI:matrix.org" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!AGeUOyHpLMMrLYAkXW:matrix.org" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!PpSxdCZfUIFXMZXivf:waifuhunter.club" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!wOlkWNmgkAZFxbTaqj:matrix.org" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!GtIfdsfQtQIgbQSxwJ:archlinux.org" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!UPsqJAKJwKaqqYkxNZ:cutefunny.art" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!UxQFGskJBlUowxdIxQ:tapenet.org" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!wOlkWNmgkAZFxbTaqj:matrix.org" -d '{"block": false, "purge": true}'
curl -H "Authorization: Bearer $TOKEN" -X DELETE -H "Content-Type: application/json" "$API_URL/_synapse/admin/v2/rooms/!zAeWUNBoXONgkVtgqn:mozilla.org" -d '{"block": false, "purge": true}'
###############################################################################
# Purge de l'historique des rooms restantes
###############################################################################
### NE PAS ACTIVER TANT QU'ON NE SAIT PAS EXACTEMENT CE QUE CA FAIT ###
### FAIT DOUBLON AVEC https://github.com/matrix-org/synapse/blob/develop/docs/message_retention_policies.md MAIS LE FAIT PEUT ETRE MIEUX ###
### NE PURGE PAS LES STATES EVENTS => ON NE PEUT QUE LES COMPRESSER avec COMPRESSOR) ###
echo ""
echo " - Création de la liste des rooms dont l'historique de plus de $TIME1 va être purgé "
curl --no-progress-meter -s -H "Authorization: Bearer $TOKEN" "$API_URL/_synapse/admin/v1/rooms?limit=999999999" > $ROOMLIST
jq '.rooms[] | .room_id' < $ROOMLIST > $ROOMLIST_PURGE
readarray -t ROOMS_ARRAY < $ROOMLIST_PURGE
for ROOM_NAME in "${ROOMS_ARRAY[@]}"
do
:
echo ""
# Yeah, pretty ugly but y'know, json stuff...
ROOM_NAME_TRIM=$(echo $ROOM_NAME | sed 's/"//g')
echo "--- Suppression de l'historique de la room $ROOM_NAME_TRIM "
curl --no-progress-meter -s -H "Authorization: Bearer $TOKEN" -XPOST -H "Content-Type: application/json" -d "{ \"delete_local_events\": false, \"purge_up_to_ts\": $UNIX_TIMESTAMP1 }" "$API_URL/_synapse/admin/v1/purge_history/$ROOM_NAME_TRIM"
done
###############################################################################
# Purge des medias:
###############################################################################
echo ""
echo " - Purge des médias distants datant d'avant $TIME1"
curl --no-progress-meter -s -H "Authorization: Bearer $TOKEN" -X POST "$API_URL/_synapse/admin/v1/purge_media_cache?before_ts=$UNIX_TIMESTAMP1"
echo ""
echo ""
echo " - Purge des médias locaux datant d'avant $TIME1"
curl --no-progress-meter -s -H "Authorization: Bearer $TOKEN" -X POST "$API_URL/_synapse/admin/v1/media/$DOMAIN/delete?before_ts=$UNIX_TIMESTAMP1"
echo ""
###############################################################################
# Compression ciblée des public.state_groups_state avec synapse_state_compressor
# Opération effectée ponctuellement - le récurrent est assuré par synapse_auto_compressor
###############################################################################
echo ""
date
echo " - DEBUT COMPRESSION"
# Durée d'éxécution : 30min
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER --command="SELECT room_id, count(*) AS count FROM state_groups_state GROUP BY room_id HAVING count(*) > 99999 ORDER BY count DESC;" | sed -r 's/\s//g' | cut -d'|' -f1 | egrep -v '^$' > $ROOMLIST_COMPRESS
#/root/synapse_compress_state -t -o /mnt/STK4/synapse_compress_state_OGEhHVWSdvArJzumhm.sql -p "host=localhost user=${DBUSER} password=${PGPASSWORD} dbname=${DBNAME}" -r "!OGEhHVWSdvArJzumhm:matrix.org"
#/root/synapse_compress_state -t -o /mnt/STK4/synapse_compress_state_zAeWUNBoXONgkVtgqn.sql -p "host=localhost user=${DBUSER} password=${PGPASSWORD} dbname=${DBNAME}" -r "!zAeWUNBoXONgkVtgqn:mozilla.org"
#/root/synapse_compress_state -t -o /mnt/STK4/synapse_compress_state_wOlkWNmgkAZFxbTaqj.sql -p "host=localhost user=${DBUSER} password=${PGPASSWORD} dbname=${DBNAME}" -r "!wOlkWNmgkAZFxbTaqj:matrix.org"
#/root/synapse_compress_state -t -o /mnt/STK4/synapse_compress_state_hYNwBrDyxpZUxyHcHA.sql -p "host=localhost user=${DBUSER} password=${PGPASSWORD} dbname=${DBNAME}" -r "!hYNwBrDyxpZUxyHcHA:matrix.underworld.fr"
#/root/synapse_compress_state -t -o /mnt/STK4/synapse_compress_state_HxxSmgxPkysXjCBLeB.sql -p "host=localhost user=${DBUSER} password=${PGPASSWORD} dbname=${DBNAME}" -r "!HxxSmgxPkysXjCBLeB:matrix.org"
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER < /mnt/STK4/synapse_compress_state_OGEhHVWSdvArJzumhm.sql
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER < /mnt/STK4/synapse_compress_state_zAeWUNBoXONgkVtgqn.sql
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER < /mnt/STK4/synapse_compress_state_wOlkWNmgkAZFxbTaqj.sql
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER < /mnt/STK4/synapse_compress_state_hYNwBrDyxpZUxyHcHA.sql
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER < /mnt/STK4/synapse_compress_state_HxxSmgxPkysXjCBLeB.sql
echo " - FIN COMPRESSION"
###############################################################################
# Compression des public.state_groups_state avec synapse_auto_compressor
###############################################################################
echo ""
date
echo " - DEBUT COMPRESSION AUTO"
# Purge de la table d'état du compressor avant éxécution en cas de plantage
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER --command="
#DELETE
#FROM state_compressor_state AS scs
#WHERE NOT EXISTS
# (SELECT *
# FROM rooms AS r
# WHERE r.room_id = scs.room_id);"
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER --command="
#DELETE
#FROM state_compressor_state AS scs
#WHERE scs.room_id in
# (SELECT DISTINCT room_id
# FROM state_compressor_state AS scs2
# WHERE scs2.current_head IS NOT NULL
# AND NOT EXISTS
# (SELECT *
# FROM state_groups AS sg
# WHERE sg.id = scs2.current_head));"
#psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER --command="
#DELETE
#FROM state_compressor_progress AS scp
#WHERE NOT EXISTS
# (SELECT *
# FROM state_compressor_state AS scs
# WHERE scs.room_id = scp.room_id);"
/root/synapse_auto_compressor -p postgresql://$DBUSER:$PGPASSWORD@localhost/$DBNAME -c 100000 -n 10000000
echo " - FIN COMPRESSION AUTO"
###############################################################################
# Nettoyage de la base de données
# !!! NECESSITE D'AVOIR 50% D'ESPACE DISQUE DE LIBRE !!!
# !!! IMPLIQUE UN ARRETE DE SYNAPSE !!!
###############################################################################
### NE PAS ACTIVER TANT QU'ON NE SAIT PAS EXACTEMENT CE QUE CA FAIT ###
echo ""
date
echo " - Cleaning database"
echo " --- Stopping server and wait 2 minutes"
systemctl stop matrix-synapse
sleep 120
echo " --- VACUUM"
psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER --command="VACUUM FULL VERBOSE;"
sleep 120
echo " --- Restarting server"
systemctl start matrix-synapse
###############################################################################
# Finish
###############################################################################
###############################################################################
# STATISTIQUES
###############################################################################
psql -t -A -h localhost --dbname=$DBNAME --username=$DBUSER --command="SELECT nspname || '.' || relname AS "relation",
pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size"
FROM pg_class C
LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
WHERE nspname NOT IN ('pg_catalog', 'information_schema')
AND C.relkind <> 'i'
AND nspname !~ '^pg_toast'
ORDER BY pg_total_relation_size(C.oid) DESC
LIMIT 20;"
#
#
###############################################################################
echo ""
/sbin/fstrim -av
date
echo " - Espace disque après"
df -h
![UNDERWORLD [UWD]](/user/themes/quark/images/logo/Fedivers2.png){kind=logo}