From 3d812b2e4e6635ddec1ce4af6ca8f5b9cfeca3ed Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Thu, 9 Feb 2023 11:08:30 +0200 Subject: [PATCH 1/8] chore: update upgrade scripts to be compatible with older instances; refactoring; fixes --- .../admin_api_scripts/pg_upgrade_complete.sh | 19 +- .../admin_api_scripts/pg_upgrade_initiate.sh | 219 +++++++++++++----- 2 files changed, 172 insertions(+), 66 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh index 5fa477621..9b51d5bcc 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh @@ -5,14 +5,6 @@ ## The following commands copy custom PG configs and enable previously disabled ## extensions, containing regtypes referencing system OIDs. -# Extensions to be reenabled after pg_upgrade. -# Running an upgrade with these extensions enabled will result in errors due to -# them depending on regtypes referencing system OIDs. Thus they have been disabled -# beforehand. -EXTENSIONS_TO_REENABLE=( - "pg_graphql" -) - set -eEuo pipefail run_sql() { @@ -36,19 +28,19 @@ function complete_pg_upgrade { echo "running" > /tmp/pg-upgrade-status + echo "1. Mounting data disk" mount -a -v # copying custom configurations + echo "2. Copying custom configurations" cp -R /data/conf/* /etc/postgresql-custom/ chown -R postgres:postgres /var/lib/postgresql/data chown -R postgres:postgres /data/pgdata + echo "3. Starting postgresql" service postgresql start - for EXTENSION in "${EXTENSIONS_TO_REENABLE[@]}"; do - run_sql -c "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;" - done - + echo "4. Running generated SQL files" if [ -d /data/sql ]; then for FILE in /data/sql/*.sql; do if [ -f "$FILE" ]; then @@ -58,8 +50,11 @@ function complete_pg_upgrade { fi sleep 5 + + echo "5. Restarting postgresql" service postgresql restart + echo "6. Starting vacuum analyze" start_vacuum_analyze echo "Upgrade job completed" diff --git a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh index aeb4f7f70..f11dc8b68 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh @@ -8,27 +8,59 @@ # Extensions to disable before running pg_upgrade. # Running an upgrade with these extensions enabled will result in errors due to -# them depending on regtypes referencing system OIDs. +# them depending on regtypes referencing system OIDs or outdated library files. EXTENSIONS_TO_DISABLE=( "pg_graphql" + "plv8" + "plcoffee" + "plls" +) + +PG14_EXTENSIONS_TO_DISABLE=( + "wrappers" + "pgrouting" +) + +PG13_EXTENSIONS_TO_DISABLE=( + "pgrouting" ) set -eEuo pipefail PGVERSION=$1 +IS_DRY_RUN=${2:-false} +if [ "$IS_DRY_RUN" != false ]; then + IS_DRY_RUN=true +fi MOUNT_POINT="/data_migration" run_sql() { STATEMENT=$1 - psql -h localhost -U supabase_admin -d postgres -c "$STATEMENT" + psql -h localhost -U supabase_admin -d postgres "$@" } +POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" +OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") +# If upgrading from older major PG versions, disable specific extensions +if [[ "$OLD_PGVERSION" =~ 14* ]]; then + EXTENSIONS_TO_DISABLE+=("${PG14_EXTENSIONS_TO_DISABLE[@]}") +fi +if [[ "$OLD_PGVERSION" =~ 13* ]]; then + EXTENSIONS_TO_DISABLE+=("${PG13_EXTENSIONS_TO_DISABLE[@]}") +fi + + cleanup() { UPGRADE_STATUS=${1:-"failed"} EXIT_CODE=${?:-0} + if [ "$UPGRADE_STATUS" = "failed" ]; then + echo "Upgrade job failed. Cleaning up and exiting." + fi + if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then + echo "Copying pg_upgrade output to /var/log" cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ fi @@ -37,50 +69,125 @@ cleanup() { mv /var/lib/postgresql.bak /var/lib/postgresql fi - systemctl restart postgresql - sleep 10 - systemctl restart postgresql + if [ -L /usr/lib/postgresql/lib/aarch64/libpq.so.5 ]; then + rm /usr/lib/postgresql/lib/aarch64/libpq.so.5 + fi - for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do - run_sql "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;" - done + if [ "$IS_DRY_RUN" = false ]; then + echo "Restarting postgresql" + systemctl restart postgresql + fi + + echo "Re-enabling extensions" + if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then + run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT + fi - run_sql "ALTER USER postgres WITH NOSUPERUSER;" + echo "Removing SUPERUSER grant from postgres" + run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" - umount $MOUNT_POINT + if [ "$IS_DRY_RUN" = false ]; then + echo "Unmounting data disk from ${MOUNT_POINT}" + umount $MOUNT_POINT + fi echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status exit $EXIT_CODE } -function initiate_upgrade { - echo "running" > /tmp/pg-upgrade-status - - # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere - # excluding nvme0 since it is the root disk - BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }') +function handle_extensions { + rm -f $POST_UPGRADE_EXTENSION_SCRIPT + touch $POST_UPGRADE_EXTENSION_SCRIPT - if [ -x "$(command -v blockdev)" ]; then - blockdev --rereadpt "$BLOCK_DEVICE" - fi + # Disable extensions if they're enabled + # Generate SQL script to re-enable them after upgrade + for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do + EXTENSION_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = '${EXTENSION}');") + if [ "$EXTENSION_ENABLED" = "t" ]; then + echo "Disabling extension ${EXTENSION}" + run_sql -c "DROP EXTENSION IF EXISTS ${EXTENSION} CASCADE;" + cat << EOF >> $POST_UPGRADE_EXTENSION_SCRIPT +DO \$\$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = '${EXTENSION}') THEN + CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE; + END IF; +END; +\$\$; +EOF + fi + done +} +function initiate_upgrade { mkdir -p "$MOUNT_POINT" - mount "$BLOCK_DEVICE" "$MOUNT_POINT" - resize2fs "$BLOCK_DEVICE" + SHARED_PRELOAD_LIBRARIES=$(cat /etc/postgresql/postgresql.conf | grep shared_preload_libraries | sed "s/shared_preload_libraries = '\(.*\)'.*/\1/") - SHARED_PRELOAD_LIBRARIES=$(cat /etc/postgresql/postgresql.conf | grep shared_preload_libraries | sed "s/shared_preload_libraries = '\(.*\)'.*/\1/") - PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") + # Wrappers officially launched in PG15; PG14 version is incompatible + if [[ "$OLD_PGVERSION" =~ 14* ]]; then + SHARED_PRELOAD_LIBRARIES=$(echo $SHARED_PRELOAD_LIBRARIES | sed "s/wrappers, //") + fi + + PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") PGDATANEW="$MOUNT_POINT/pgdata" - PGBINNEW="/tmp/pg_upgrade_bin/$PGVERSION/bin" - PGSHARENEW="/tmp/pg_upgrade_bin/$PGVERSION/share" + PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" + PGBINNEW="$PG_UPGRADE_BIN_DIR/bin" + PGLIBNEW="$PG_UPGRADE_BIN_DIR/lib" + PGSHARENEW="$PG_UPGRADE_BIN_DIR/share" + # running upgrade using at least 1 cpu core + WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }') + + echo "1. Extracting pg_upgrade binaries" mkdir -p "/tmp/pg_upgrade_bin" - tar zxvf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" + tar zxf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" # copy upgrade-specific pgsodium_getkey script into the share dir + chmod +x "/root/pg_upgrade_pgsodium_getkey.sh" cp /root/pg_upgrade_pgsodium_getkey.sh "$PGSHARENEW/extension/pgsodium_getkey" - chmod +x "$PGSHARENEW/extension/pgsodium_getkey" + if [ -d "/var/lib/postgresql/extension/" ]; then + cp /root/pg_upgrade_pgsodium_getkey.sh "/var/lib/postgresql/extension/pgsodium_getkey" + chown postgres:postgres "/var/lib/postgresql/extension/pgsodium_getkey" + fi + + chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION" + + # Make latest libpq available to pg_upgrade + mkdir -p /usr/lib/postgresql/lib/aarch64 + if [ ! -L /usr/lib/postgresql/lib/aarch64/libpq.so.5 ]; then + ln -s "$PGLIBNEW/libpq.so.5" /usr/lib/postgresql/lib/aarch64/libpq.so.5 + fi + + # upgrade job outputs a log in the cwd; needs write permissions + mkdir -p /tmp/pg_upgrade/ + chown -R postgres:postgres /tmp/pg_upgrade/ + cd /tmp/pg_upgrade/ + + echo "running" > /tmp/pg-upgrade-status + + # Fixing erros generated by previous dpkg executions (package upgrades et co) + echo "2. Fixing potential errors generated by dpkg" + DEBIAN_FRONTEND=noninteractive dpkg --configure -a --no-force overwrite || true # handle errors generated by dpkg + + # Needed for PostGIS, since it's compiled with Protobuf-C support now + echo "3. Installing libprotobuf-c1 if missing" + if [[ ! "$(apt list --installed libprotobuf-c1 | grep "installed")" ]]; then + apt-get update && apt --fix-broken install -y libprotobuf-c1 + fi + + if [ "$IS_DRY_RUN" = false ]; then + # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere + # excluding nvme0 since it is the root disk + echo "4. Determining block device to mount" + BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }') + echo "Block device found: $BLOCK_DEVICE" + + mkdir -p "$MOUNT_POINT" + echo "5. Mounting block device" + mount "$BLOCK_DEVICE" "$MOUNT_POINT" + resize2fs "$BLOCK_DEVICE" + fi if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then cp "$MOUNT_POINT/pgsodium_root.key" /etc/postgresql-custom/pgsodium_root.key @@ -88,27 +195,17 @@ function initiate_upgrade { chmod 600 /etc/postgresql-custom/pgsodium_root.key fi - chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION" - - for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do - run_sql "DROP EXTENSION IF EXISTS ${EXTENSION} CASCADE;" - done - - run_sql "ALTER USER postgres WITH SUPERUSER;" - + echo "6. Disabling extensions and generating post-upgrade script" + handle_extensions + + echo "7. Granting SUPERUSER to postgres user" + run_sql -c "ALTER USER postgres WITH SUPERUSER;" + echo "8. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "$PGDATANEW/" su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/" -s $SHELL postgres - # running upgrade using at least 1 cpu core - WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }') - - # upgrade job outputs a log in the cwd; needs write permissions - mkdir -p /tmp/pg_upgrade - chown -R postgres:postgres /tmp/pg_upgrade - cd /tmp/pg_upgrade - UPGRADE_COMMAND=$(cat <> /var/log/pg-upgrade-initiate.log 2>&1 & -echo "Upgrade initiate job completed" +if [ "$IS_DRY_RUN" = true ]; then + initiate_upgrade +else + initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1 & + echo "Upgrade initiate job completed" +fi From 1ce5cc6af723448036178780b0dc76010ccd038c Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 13 Feb 2023 09:52:27 +0200 Subject: [PATCH 2/8] chore: better handling of updated package default files --- ansible/files/admin_api_scripts/pg_upgrade_initiate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh index f11dc8b68..550d6a195 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh @@ -168,7 +168,7 @@ function initiate_upgrade { # Fixing erros generated by previous dpkg executions (package upgrades et co) echo "2. Fixing potential errors generated by dpkg" - DEBIAN_FRONTEND=noninteractive dpkg --configure -a --no-force overwrite || true # handle errors generated by dpkg + DEBIAN_FRONTEND=noninteractive dpkg --configure -a --force-confold || true # handle errors generated by dpkg # Needed for PostGIS, since it's compiled with Protobuf-C support now echo "3. Installing libprotobuf-c1 if missing" From 7923c8bf21b67334816b54053753147232225b49 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 13 Feb 2023 10:52:03 +0200 Subject: [PATCH 3/8] chore: bump pg version --- common.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index a1eee1cc9..7e46aee46 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.0.38" +postgres-version = "15.1.0.39" From 5435d7c8e515ae6a07006c952cd37e5dceaa9d84 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 13 Feb 2023 11:11:16 +0200 Subject: [PATCH 4/8] chore: add shellcheck --- .github/workflows/check-shellscripts.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/check-shellscripts.yml diff --git a/.github/workflows/check-shellscripts.yml b/.github/workflows/check-shellscripts.yml new file mode 100644 index 000000000..a2a281082 --- /dev/null +++ b/.github/workflows/check-shellscripts.yml @@ -0,0 +1,17 @@ +name: Check shell scripts + +on: + push: + branches: + - develop + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + scandir: './ansible/files/admin_scripts' \ No newline at end of file From a569572d5557ce9a3c5ecafba8e527ac7b33aceb Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 13 Feb 2023 11:51:21 +0200 Subject: [PATCH 5/8] chore: fix shellcheck path --- .github/workflows/check-shellscripts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-shellscripts.yml b/.github/workflows/check-shellscripts.yml index a2a281082..ca40f5117 100644 --- a/.github/workflows/check-shellscripts.yml +++ b/.github/workflows/check-shellscripts.yml @@ -14,4 +14,4 @@ jobs: - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: - scandir: './ansible/files/admin_scripts' \ No newline at end of file + scandir: './ansible/files/admin_api_scripts' From d7459fc4efca3dabd37dc71a56fe3a7b298fd4ce Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 13 Feb 2023 11:52:07 +0200 Subject: [PATCH 6/8] chore: add checkout step --- .github/workflows/check-shellscripts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check-shellscripts.yml b/.github/workflows/check-shellscripts.yml index ca40f5117..c31cefae1 100644 --- a/.github/workflows/check-shellscripts.yml +++ b/.github/workflows/check-shellscripts.yml @@ -11,6 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: From 13c8e6d13c74ed8723bc06bfff4b6131bd382c91 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 13 Feb 2023 16:46:46 +0200 Subject: [PATCH 7/8] chore: fix shellcheck tips --- .github/workflows/check-shellscripts.yml | 2 ++ .../files/admin_api_scripts/manage_readonly_mode.sh | 4 ++-- .../files/admin_api_scripts/pg_upgrade_initiate.sh | 13 ++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check-shellscripts.yml b/.github/workflows/check-shellscripts.yml index c31cefae1..e32066677 100644 --- a/.github/workflows/check-shellscripts.yml +++ b/.github/workflows/check-shellscripts.yml @@ -14,5 +14,7 @@ jobs: - uses: actions/checkout@v3 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master + env: + SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 with: scandir: './ansible/files/admin_api_scripts' diff --git a/ansible/files/admin_api_scripts/manage_readonly_mode.sh b/ansible/files/admin_api_scripts/manage_readonly_mode.sh index b8bbb8230..41c9f5a1e 100644 --- a/ansible/files/admin_api_scripts/manage_readonly_mode.sh +++ b/ansible/files/admin_api_scripts/manage_readonly_mode.sh @@ -27,7 +27,7 @@ FROM role_comment; EOF ) RESULT=$(psql -h localhost -U supabase_admin -d postgres -At -c "$COMMAND") - echo -n $RESULT + echo -n "$RESULT" } case $SUBCOMMAND in @@ -36,7 +36,7 @@ case $SUBCOMMAND in ;; "set") shift - set_mode $@ + set_mode "$@" ;; *) echo "Error: '$SUBCOMMAND' is not a known subcommand." diff --git a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh index 550d6a195..8c1b15fa3 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh @@ -36,7 +36,6 @@ fi MOUNT_POINT="/data_migration" run_sql() { - STATEMENT=$1 psql -h localhost -U supabase_admin -d postgres "$@" } @@ -90,9 +89,9 @@ cleanup() { echo "Unmounting data disk from ${MOUNT_POINT}" umount $MOUNT_POINT fi - echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status + echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status - exit $EXIT_CODE + exit "$EXIT_CODE" } function handle_extensions { @@ -125,7 +124,7 @@ function initiate_upgrade { # Wrappers officially launched in PG15; PG14 version is incompatible if [[ "$OLD_PGVERSION" =~ 14* ]]; then - SHARED_PRELOAD_LIBRARIES=$(echo $SHARED_PRELOAD_LIBRARIES | sed "s/wrappers, //") + SHARED_PRELOAD_LIBRARIES=$(echo "$SHARED_PRELOAD_LIBRARIES" | sed "s/wrappers, //") fi PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") @@ -203,8 +202,8 @@ function initiate_upgrade { echo "8. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" - rm -rf "$PGDATANEW/" - su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/" -s $SHELL postgres + rm -rf "${PGDATANEW:?}/" + su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/" -s "$SHELL" postgres UPGRADE_COMMAND=$(cat < Date: Mon, 13 Feb 2023 16:51:14 +0200 Subject: [PATCH 8/8] chore: more shellcheck fixes --- .../files/admin_api_scripts/pg_upgrade_complete.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh index 9b51d5bcc..5e0330c2f 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh @@ -15,9 +15,9 @@ cleanup() { UPGRADE_STATUS=${1:-"failed"} EXIT_CODE=${?:-0} - echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status + echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status - exit $EXIT_CODE + exit "$EXIT_CODE" } function complete_pg_upgrade { @@ -44,7 +44,7 @@ function complete_pg_upgrade { if [ -d /data/sql ]; then for FILE in /data/sql/*.sql; do if [ -f "$FILE" ]; then - run_sql -f $FILE + run_sql -f "$FILE" fi done fi @@ -56,12 +56,11 @@ function complete_pg_upgrade { echo "6. Starting vacuum analyze" start_vacuum_analyze - - echo "Upgrade job completed" } function start_vacuum_analyze { - su -c 'vacuumdb --all --analyze-in-stages' -s $SHELL postgres + su -c 'vacuumdb --all --analyze-in-stages' -s "$SHELL" postgres + echo "Upgrade job completed" cleanup "complete" }