diff --git a/.github/workflows/check-shellscripts.yml b/.github/workflows/check-shellscripts.yml new file mode 100644 index 000000000..e32066677 --- /dev/null +++ b/.github/workflows/check-shellscripts.yml @@ -0,0 +1,20 @@ +name: Check shell scripts + +on: + push: + branches: + - develop + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + env: + SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 + with: + scandir: './ansible/files/admin_api_scripts' diff --git a/ansible/files/admin_api_scripts/manage_readonly_mode.sh b/ansible/files/admin_api_scripts/manage_readonly_mode.sh index b8bbb8230..41c9f5a1e 100644 --- a/ansible/files/admin_api_scripts/manage_readonly_mode.sh +++ b/ansible/files/admin_api_scripts/manage_readonly_mode.sh @@ -27,7 +27,7 @@ FROM role_comment; EOF ) RESULT=$(psql -h localhost -U supabase_admin -d postgres -At -c "$COMMAND") - echo -n $RESULT + echo -n "$RESULT" } case $SUBCOMMAND in @@ -36,7 +36,7 @@ case $SUBCOMMAND in ;; "set") shift - set_mode $@ + set_mode "$@" ;; *) echo "Error: '$SUBCOMMAND' is not a known subcommand." diff --git a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh index 5fa477621..5e0330c2f 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh @@ -5,14 +5,6 @@ ## The following commands copy custom PG configs and enable previously disabled ## extensions, containing regtypes referencing system OIDs. -# Extensions to be reenabled after pg_upgrade. -# Running an upgrade with these extensions enabled will result in errors due to -# them depending on regtypes referencing system OIDs. Thus they have been disabled -# beforehand. -EXTENSIONS_TO_REENABLE=( - "pg_graphql" -) - set -eEuo pipefail run_sql() { @@ -23,9 +15,9 @@ cleanup() { UPGRADE_STATUS=${1:-"failed"} EXIT_CODE=${?:-0} - echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status + echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status - exit $EXIT_CODE + exit "$EXIT_CODE" } function complete_pg_upgrade { @@ -36,37 +28,39 @@ function complete_pg_upgrade { echo "running" > /tmp/pg-upgrade-status + echo "1. Mounting data disk" mount -a -v # copying custom configurations + echo "2. Copying custom configurations" cp -R /data/conf/* /etc/postgresql-custom/ chown -R postgres:postgres /var/lib/postgresql/data chown -R postgres:postgres /data/pgdata + echo "3. Starting postgresql" service postgresql start - for EXTENSION in "${EXTENSIONS_TO_REENABLE[@]}"; do - run_sql -c "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;" - done - + echo "4. Running generated SQL files" if [ -d /data/sql ]; then for FILE in /data/sql/*.sql; do if [ -f "$FILE" ]; then - run_sql -f $FILE + run_sql -f "$FILE" fi done fi sleep 5 + + echo "5. Restarting postgresql" service postgresql restart + echo "6. Starting vacuum analyze" start_vacuum_analyze - - echo "Upgrade job completed" } function start_vacuum_analyze { - su -c 'vacuumdb --all --analyze-in-stages' -s $SHELL postgres + su -c 'vacuumdb --all --analyze-in-stages' -s "$SHELL" postgres + echo "Upgrade job completed" cleanup "complete" } diff --git a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh index aeb4f7f70..8c1b15fa3 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh @@ -8,27 +8,58 @@ # Extensions to disable before running pg_upgrade. # Running an upgrade with these extensions enabled will result in errors due to -# them depending on regtypes referencing system OIDs. +# them depending on regtypes referencing system OIDs or outdated library files. EXTENSIONS_TO_DISABLE=( "pg_graphql" + "plv8" + "plcoffee" + "plls" +) + +PG14_EXTENSIONS_TO_DISABLE=( + "wrappers" + "pgrouting" +) + +PG13_EXTENSIONS_TO_DISABLE=( + "pgrouting" ) set -eEuo pipefail PGVERSION=$1 +IS_DRY_RUN=${2:-false} +if [ "$IS_DRY_RUN" != false ]; then + IS_DRY_RUN=true +fi MOUNT_POINT="/data_migration" run_sql() { - STATEMENT=$1 - psql -h localhost -U supabase_admin -d postgres -c "$STATEMENT" + psql -h localhost -U supabase_admin -d postgres "$@" } +POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" +OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") +# If upgrading from older major PG versions, disable specific extensions +if [[ "$OLD_PGVERSION" =~ 14* ]]; then + EXTENSIONS_TO_DISABLE+=("${PG14_EXTENSIONS_TO_DISABLE[@]}") +fi +if [[ "$OLD_PGVERSION" =~ 13* ]]; then + EXTENSIONS_TO_DISABLE+=("${PG13_EXTENSIONS_TO_DISABLE[@]}") +fi + + cleanup() { UPGRADE_STATUS=${1:-"failed"} EXIT_CODE=${?:-0} + if [ "$UPGRADE_STATUS" = "failed" ]; then + echo "Upgrade job failed. Cleaning up and exiting." + fi + if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then + echo "Copying pg_upgrade output to /var/log" cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ fi @@ -37,77 +68,142 @@ cleanup() { mv /var/lib/postgresql.bak /var/lib/postgresql fi - systemctl restart postgresql - sleep 10 - systemctl restart postgresql + if [ -L /usr/lib/postgresql/lib/aarch64/libpq.so.5 ]; then + rm /usr/lib/postgresql/lib/aarch64/libpq.so.5 + fi - for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do - run_sql "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;" - done + if [ "$IS_DRY_RUN" = false ]; then + echo "Restarting postgresql" + systemctl restart postgresql + fi - run_sql "ALTER USER postgres WITH NOSUPERUSER;" + echo "Re-enabling extensions" + if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then + run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT + fi - umount $MOUNT_POINT - echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status + echo "Removing SUPERUSER grant from postgres" + run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" - exit $EXIT_CODE -} + if [ "$IS_DRY_RUN" = false ]; then + echo "Unmounting data disk from ${MOUNT_POINT}" + umount $MOUNT_POINT + fi + echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status -function initiate_upgrade { - echo "running" > /tmp/pg-upgrade-status + exit "$EXIT_CODE" +} - # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere - # excluding nvme0 since it is the root disk - BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }') +function handle_extensions { + rm -f $POST_UPGRADE_EXTENSION_SCRIPT + touch $POST_UPGRADE_EXTENSION_SCRIPT - if [ -x "$(command -v blockdev)" ]; then - blockdev --rereadpt "$BLOCK_DEVICE" - fi + # Disable extensions if they're enabled + # Generate SQL script to re-enable them after upgrade + for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do + EXTENSION_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = '${EXTENSION}');") + if [ "$EXTENSION_ENABLED" = "t" ]; then + echo "Disabling extension ${EXTENSION}" + run_sql -c "DROP EXTENSION IF EXISTS ${EXTENSION} CASCADE;" + cat << EOF >> $POST_UPGRADE_EXTENSION_SCRIPT +DO \$\$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = '${EXTENSION}') THEN + CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE; + END IF; +END; +\$\$; +EOF + fi + done +} +function initiate_upgrade { mkdir -p "$MOUNT_POINT" - mount "$BLOCK_DEVICE" "$MOUNT_POINT" - resize2fs "$BLOCK_DEVICE" + SHARED_PRELOAD_LIBRARIES=$(cat /etc/postgresql/postgresql.conf | grep shared_preload_libraries | sed "s/shared_preload_libraries = '\(.*\)'.*/\1/") - SHARED_PRELOAD_LIBRARIES=$(cat /etc/postgresql/postgresql.conf | grep shared_preload_libraries | sed "s/shared_preload_libraries = '\(.*\)'.*/\1/") - PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") + # Wrappers officially launched in PG15; PG14 version is incompatible + if [[ "$OLD_PGVERSION" =~ 14* ]]; then + SHARED_PRELOAD_LIBRARIES=$(echo "$SHARED_PRELOAD_LIBRARIES" | sed "s/wrappers, //") + fi + + PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") PGDATANEW="$MOUNT_POINT/pgdata" - PGBINNEW="/tmp/pg_upgrade_bin/$PGVERSION/bin" - PGSHARENEW="/tmp/pg_upgrade_bin/$PGVERSION/share" + PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" + PGBINNEW="$PG_UPGRADE_BIN_DIR/bin" + PGLIBNEW="$PG_UPGRADE_BIN_DIR/lib" + PGSHARENEW="$PG_UPGRADE_BIN_DIR/share" + # running upgrade using at least 1 cpu core + WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }') + + echo "1. Extracting pg_upgrade binaries" mkdir -p "/tmp/pg_upgrade_bin" - tar zxvf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" + tar zxf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" # copy upgrade-specific pgsodium_getkey script into the share dir + chmod +x "/root/pg_upgrade_pgsodium_getkey.sh" cp /root/pg_upgrade_pgsodium_getkey.sh "$PGSHARENEW/extension/pgsodium_getkey" - chmod +x "$PGSHARENEW/extension/pgsodium_getkey" - - if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then - cp "$MOUNT_POINT/pgsodium_root.key" /etc/postgresql-custom/pgsodium_root.key - chown postgres:postgres /etc/postgresql-custom/pgsodium_root.key - chmod 600 /etc/postgresql-custom/pgsodium_root.key + if [ -d "/var/lib/postgresql/extension/" ]; then + cp /root/pg_upgrade_pgsodium_getkey.sh "/var/lib/postgresql/extension/pgsodium_getkey" + chown postgres:postgres "/var/lib/postgresql/extension/pgsodium_getkey" fi chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION" - for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do - run_sql "DROP EXTENSION IF EXISTS ${EXTENSION} CASCADE;" - done + # Make latest libpq available to pg_upgrade + mkdir -p /usr/lib/postgresql/lib/aarch64 + if [ ! -L /usr/lib/postgresql/lib/aarch64/libpq.so.5 ]; then + ln -s "$PGLIBNEW/libpq.so.5" /usr/lib/postgresql/lib/aarch64/libpq.so.5 + fi - run_sql "ALTER USER postgres WITH SUPERUSER;" + # upgrade job outputs a log in the cwd; needs write permissions + mkdir -p /tmp/pg_upgrade/ + chown -R postgres:postgres /tmp/pg_upgrade/ + cd /tmp/pg_upgrade/ + echo "running" > /tmp/pg-upgrade-status - chown -R postgres:postgres "$MOUNT_POINT/" - rm -rf "$PGDATANEW/" - su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/" -s $SHELL postgres + # Fixing erros generated by previous dpkg executions (package upgrades et co) + echo "2. Fixing potential errors generated by dpkg" + DEBIAN_FRONTEND=noninteractive dpkg --configure -a --force-confold || true # handle errors generated by dpkg - # running upgrade using at least 1 cpu core - WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }') + # Needed for PostGIS, since it's compiled with Protobuf-C support now + echo "3. Installing libprotobuf-c1 if missing" + if [[ ! "$(apt list --installed libprotobuf-c1 | grep "installed")" ]]; then + apt-get update && apt --fix-broken install -y libprotobuf-c1 + fi - # upgrade job outputs a log in the cwd; needs write permissions - mkdir -p /tmp/pg_upgrade - chown -R postgres:postgres /tmp/pg_upgrade - cd /tmp/pg_upgrade + if [ "$IS_DRY_RUN" = false ]; then + # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere + # excluding nvme0 since it is the root disk + echo "4. Determining block device to mount" + BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }') + echo "Block device found: $BLOCK_DEVICE" + + mkdir -p "$MOUNT_POINT" + echo "5. Mounting block device" + mount "$BLOCK_DEVICE" "$MOUNT_POINT" + resize2fs "$BLOCK_DEVICE" + fi + + if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then + cp "$MOUNT_POINT/pgsodium_root.key" /etc/postgresql-custom/pgsodium_root.key + chown postgres:postgres /etc/postgresql-custom/pgsodium_root.key + chmod 600 /etc/postgresql-custom/pgsodium_root.key + fi + + echo "6. Disabling extensions and generating post-upgrade script" + handle_extensions + + echo "7. Granting SUPERUSER to postgres user" + run_sql -c "ALTER USER postgres WITH SUPERUSER;" + + echo "8. Creating new data directory, initializing database" + chown -R postgres:postgres "$MOUNT_POINT/" + rm -rf "${PGDATANEW:?}/" + su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/" -s "$SHELL" postgres UPGRADE_COMMAND=$(cat <> /var/log/pg-upgrade-initiate.log 2>&1 & -echo "Upgrade initiate job completed" +if [ "$IS_DRY_RUN" = true ]; then + initiate_upgrade +else + initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1 & + echo "Upgrade initiate job completed" +fi diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 0d2436f88..87a6d5c28 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.0.40" +postgres-version = "15.1.0.41"