diff --git a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh index 702de62ba..5fa477621 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh @@ -6,40 +6,70 @@ ## extensions, containing regtypes referencing system OIDs. # Extensions to be reenabled after pg_upgrade. -# Running an upgrade with these extensions enabled will result in errors due to +# Running an upgrade with these extensions enabled will result in errors due to # them depending on regtypes referencing system OIDs. Thus they have been disabled # beforehand. EXTENSIONS_TO_REENABLE=( "pg_graphql" ) +set -eEuo pipefail run_sql() { - STATEMENT=$1 - psql -h localhost -U supabase_admin -d postgres -c "$STATEMENT" + psql -h localhost -U supabase_admin -d postgres "$@" +} + +cleanup() { + UPGRADE_STATUS=${1:-"failed"} + EXIT_CODE=${?:-0} + + echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status + + exit $EXIT_CODE } function complete_pg_upgrade { + if [ -f /tmp/pg-upgrade-status ]; then + echo "Upgrade job already started. Bailing." + exit 0 + fi + + echo "running" > /tmp/pg-upgrade-status + mount -a -v # copying custom configurations cp -R /data/conf/* /etc/postgresql-custom/ + chown -R postgres:postgres /var/lib/postgresql/data + chown -R postgres:postgres /data/pgdata service postgresql start - su -c 'vacuumdb --all --analyze-in-stages' -s $SHELL postgres for EXTENSION in "${EXTENSIONS_TO_REENABLE[@]}"; do - run_sql "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;" + run_sql -c "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;" done - - sleep 5 - service postgresql restart + + if [ -d /data/sql ]; then + for FILE in /data/sql/*.sql; do + if [ -f "$FILE" ]; then + run_sql -f $FILE + fi + done + fi sleep 5 service postgresql restart + + start_vacuum_analyze + + echo "Upgrade job completed" +} + +function start_vacuum_analyze { + su -c 'vacuumdb --all --analyze-in-stages' -s $SHELL postgres + cleanup "complete" } -set -euo pipefail +trap cleanup ERR -complete_pg_upgrade >> /var/log/pg-upgrade-complete.log 2>&1 -echo "Upgrade job completed" +complete_pg_upgrade >>/var/log/pg-upgrade-complete.log 2>&1 & diff --git a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh index 9614a845b..aeb4f7f70 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh @@ -28,6 +28,10 @@ cleanup() { UPGRADE_STATUS=${1:-"failed"} EXIT_CODE=${?:-0} + if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then + cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ + fi + if [ -L /var/lib/postgresql ]; then rm /var/lib/postgresql mv /var/lib/postgresql.bak /var/lib/postgresql @@ -42,9 +46,6 @@ cleanup() { done run_sql "ALTER USER postgres WITH NOSUPERUSER;" - if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then - cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ - fi umount $MOUNT_POINT echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status @@ -53,11 +54,19 @@ cleanup() { } function initiate_upgrade { - BLOCK_DEVICE=$(lsblk -dpno name | grep -v "/dev/nvme[0-1]") echo "running" > /tmp/pg-upgrade-status + # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere + # excluding nvme0 since it is the root disk + BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }') + + if [ -x "$(command -v blockdev)" ]; then + blockdev --rereadpt "$BLOCK_DEVICE" + fi + mkdir -p "$MOUNT_POINT" mount "$BLOCK_DEVICE" "$MOUNT_POINT" + resize2fs "$BLOCK_DEVICE" SHARED_PRELOAD_LIBRARIES=$(cat /etc/postgresql/postgresql.conf | grep shared_preload_libraries | sed "s/shared_preload_libraries = '\(.*\)'.*/\1/") PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") @@ -73,6 +82,12 @@ function initiate_upgrade { cp /root/pg_upgrade_pgsodium_getkey.sh "$PGSHARENEW/extension/pgsodium_getkey" chmod +x "$PGSHARENEW/extension/pgsodium_getkey" + if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then + cp "$MOUNT_POINT/pgsodium_root.key" /etc/postgresql-custom/pgsodium_root.key + chown postgres:postgres /etc/postgresql-custom/pgsodium_root.key + chmod 600 /etc/postgresql-custom/pgsodium_root.key + fi + chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION" for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do @@ -90,7 +105,9 @@ function initiate_upgrade { WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }') # upgrade job outputs a log in the cwd; needs write permissions - cd /tmp + mkdir -p /tmp/pg_upgrade + chown -R postgres:postgres /tmp/pg_upgrade + cd /tmp/pg_upgrade UPGRADE_COMMAND=$(cat <> /var/log/pg-upgrade-initiate.log 2>&1 -echo "Upgrade initiate job completed " +initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1 & +echo "Upgrade initiate job completed" diff --git a/ansible/files/admin_api_scripts/pg_upgrade_prepare.sh b/ansible/files/admin_api_scripts/pg_upgrade_prepare.sh index 97547264d..7d7eb9890 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_prepare.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_prepare.sh @@ -1,14 +1,15 @@ #! /usr/bin/env bash -## This script is runs in advance of the database version upgrade, on the newly -## launched instance which will eventually be promoted to become the primary +## This script is runs in advance of the database version upgrade, on the newly +## launched instance which will eventually be promoted to become the primary ## database instance once the upgrade successfully completes, terminating the ## previous (source) instance. -## The following commands safely stop the Postgres service and unmount +## The following commands safely stop the Postgres service and unmount ## the data disk off the newly launched instance, to be re-attached to the ## source instance and run the upgrade there. set -euo pipefail systemctl stop postgresql -umount /data +cp /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key +umount /data diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 055d46dc4..02bf1ca67 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.0.33" +postgres-version = "15.1.0.35"