chore: update pg upgrade scripts

ansible/files/admin_api_scripts/pg_upgrade_complete.sh

@@ -6,40 +6,70 @@
 ## extensions, containing regtypes referencing system OIDs.
 
 # Extensions to be reenabled after pg_upgrade.
-# Running an upgrade with these extensions enabled will result in errors due to 
+# Running an upgrade with these extensions enabled will result in errors due to
 # them depending on regtypes referencing system OIDs. Thus they have been disabled
 # beforehand.
 EXTENSIONS_TO_REENABLE=(
     "pg_graphql"
 )
 
+set -eEuo pipefail
 
 run_sql() {
-    STATEMENT=$1
-    psql -h localhost -U supabase_admin -d postgres -c "$STATEMENT"
+    psql -h localhost -U supabase_admin -d postgres "$@"
+}
+
+cleanup() {
+    UPGRADE_STATUS=${1:-"failed"}
+    EXIT_CODE=${?:-0}
+
+    echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status
+
+    exit $EXIT_CODE
 }
 
 function complete_pg_upgrade {
+    if [ -f /tmp/pg-upgrade-status ]; then
+        echo "Upgrade job already started. Bailing."
+        exit 0
+    fi
+
+    echo "running" > /tmp/pg-upgrade-status
+
     mount -a -v
 
     # copying custom configurations
     cp -R /data/conf/* /etc/postgresql-custom/
+    chown -R postgres:postgres /var/lib/postgresql/data
+    chown -R postgres:postgres /data/pgdata
 
     service postgresql start
-    su -c 'vacuumdb --all --analyze-in-stages' -s $SHELL postgres
 
     for EXTENSION in "${EXTENSIONS_TO_REENABLE[@]}"; do
-        run_sql "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;"
+        run_sql -c "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;"
     done
-
-    sleep 5
-    service postgresql restart
+
+    if [ -d /data/sql ]; then
+        for FILE in /data/sql/*.sql; do
+            if [ -f "$FILE" ]; then
+                run_sql -f $FILE
+            fi
+        done
+    fi
 
     sleep 5
     service postgresql restart
+
+    start_vacuum_analyze
+
+    echo "Upgrade job completed"
+}
+
+function start_vacuum_analyze {
+    su -c 'vacuumdb --all --analyze-in-stages' -s $SHELL postgres
+    cleanup "complete"
 }
 
-set -euo pipefail
+trap cleanup ERR
 
-complete_pg_upgrade >> /var/log/pg-upgrade-complete.log 2>&1
-echo "Upgrade job completed"
+complete_pg_upgrade >>/var/log/pg-upgrade-complete.log 2>&1 &

ansible/files/admin_api_scripts/pg_upgrade_initiate.sh

@@ -28,6 +28,10 @@ cleanup() {
     UPGRADE_STATUS=${1:-"failed"}
     EXIT_CODE=${?:-0}
 
+    if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then
+        cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/
+    fi
+
     if [ -L /var/lib/postgresql ]; then
         rm /var/lib/postgresql
         mv /var/lib/postgresql.bak /var/lib/postgresql
@@ -42,9 +46,6 @@ cleanup() {
     done
 
     run_sql "ALTER USER postgres WITH NOSUPERUSER;"
-    if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then
-        cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/
-    fi
 
     umount $MOUNT_POINT
     echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status
@@ -53,11 +54,19 @@ cleanup() {
 }
 
 function initiate_upgrade {
-    BLOCK_DEVICE=$(lsblk -dpno name | grep -v "/dev/nvme[0-1]")
     echo "running" > /tmp/pg-upgrade-status
 
+    # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere
+    # excluding nvme0 since it is the root disk
+    BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }')
+
+    if [ -x "$(command -v blockdev)" ]; then 
+        blockdev --rereadpt "$BLOCK_DEVICE"
+    fi
+
     mkdir -p "$MOUNT_POINT"
     mount "$BLOCK_DEVICE" "$MOUNT_POINT"
+    resize2fs "$BLOCK_DEVICE"
 
     SHARED_PRELOAD_LIBRARIES=$(cat /etc/postgresql/postgresql.conf | grep shared_preload_libraries |  sed "s/shared_preload_libraries = '\(.*\)'.*/\1/")
     PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/")    
@@ -73,6 +82,12 @@ function initiate_upgrade {
     cp /root/pg_upgrade_pgsodium_getkey.sh "$PGSHARENEW/extension/pgsodium_getkey"
     chmod +x "$PGSHARENEW/extension/pgsodium_getkey"
 
+    if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then
+        cp "$MOUNT_POINT/pgsodium_root.key" /etc/postgresql-custom/pgsodium_root.key
+        chown postgres:postgres /etc/postgresql-custom/pgsodium_root.key
+        chmod 600 /etc/postgresql-custom/pgsodium_root.key
+    fi
+
     chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION"
 
     for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do
@@ -90,7 +105,9 @@ function initiate_upgrade {
     WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }')
 
     # upgrade job outputs a log in the cwd; needs write permissions
-    cd /tmp
+    mkdir -p /tmp/pg_upgrade
+    chown -R postgres:postgres /tmp/pg_upgrade
+    cd /tmp/pg_upgrade
 
     UPGRADE_COMMAND=$(cat <<EOF
     time ${PGBINNEW}/pg_upgrade \
@@ -108,17 +125,28 @@ EOF
     mv /var/lib/postgresql /var/lib/postgresql.bak
     ln -s /tmp/pg_upgrade_bin/15/share /var/lib/postgresql
 
+    if [ ! -L /var/lib/postgresql.bak/data ]; then
+        if [ -L /var/lib/postgresql/data ]; then
+            rm /var/lib/postgresql/data
+        fi
+        ln -s /var/lib/postgresql.bak/data /var/lib/postgresql/data
+    fi
+
     systemctl stop postgresql
     su -c "$UPGRADE_COMMAND" -s $SHELL postgres
 
     # copying custom configurations
-    mkdir -p $MOUNT_POINT/conf
-    cp -R /etc/postgresql-custom/* $MOUNT_POINT/conf/
+    mkdir -p "$MOUNT_POINT/conf"
+    cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/"
+
+    # copy sql files generated by pg_upgrade
+    mkdir -p "$MOUNT_POINT/sql"
+    cp /tmp/pg_upgrade/*.sql "$MOUNT_POINT/sql/"  || true
 
     cleanup "complete"
 }
 
 trap cleanup ERR
 
-initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1
-echo "Upgrade initiate job completed "
+initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1 &
+echo "Upgrade initiate job completed"

ansible/files/admin_api_scripts/pg_upgrade_prepare.sh

@@ -1,14 +1,15 @@
 #! /usr/bin/env bash
-## This script is runs in advance of the database version upgrade, on the newly 
-## launched instance which will eventually be promoted to become the primary 
+## This script is runs in advance of the database version upgrade, on the newly
+## launched instance which will eventually be promoted to become the primary
 ## database instance once the upgrade successfully completes, terminating the
 ## previous (source) instance.
-## The following commands safely stop the Postgres service and unmount 
+## The following commands safely stop the Postgres service and unmount
 ## the data disk off the newly launched instance, to be re-attached to the
 ## source instance and run the upgrade there.
 
 set -euo pipefail
 
 systemctl stop postgresql
-umount /data
 
+cp /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key
+umount /data

common.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.1.0.33"
+postgres-version = "15.1.0.35"