diff --git a/.gitignore b/.gitignore index d32cc8f60..7b902d8aa 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ result* db/schema.sql common-nix.vars.pkr.hcl + +# CEP PostgreSQL source archives +*.tar.gz diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..89c3ae2d9 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,133 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## CRITICAL INSTRUCTION: PostgreSQL Version Restriction + +**ALWAYS USE POSTGRESQL 17 ONLY** + +- When starting servers, ONLY use: `nix run .#start-server 17` +- When building, ONLY target PostgreSQL 17 +- When testing, ONLY use: `nix build .#checks.psql_17 -L` +- When building AMIs, ONLY use: `nix run .#build-test-ami 17` +- NEVER use PostgreSQL 15 or OrioleDB 17 variants +- If asked to work with other versions, politely decline and explain you must only work with PostgreSQL 17 + +## Project Overview + +This is **Supabase's PostgreSQL distribution** - an enhanced PostgreSQL build with 40+ extensions across three major versions (PostgreSQL 15, 17, and OrioleDB 17). The project uses Nix as the primary build system for reproducible builds and provides Docker/Packer alternatives for different deployment scenarios. + +## Development Commands + +### Primary Development Environment (Nix) +```bash +# Enter development environment +nix develop + +# Start PostgreSQL server (ONLY use version 17) +nix run .#start-server 17 + +# Connect client with migrations applied +nix run .#start-client + +# Run database migrations +nix run .#dbmate-tool + +# Run all tests +nix flake check + +# Run tests for PostgreSQL 17 ONLY +nix build .#checks.psql_17 -L +``` + +### CI/CD and AMI Building +```bash +# Trigger Nix build +nix run .#trigger-nix-build + +# Build test AMI (PostgreSQL 17 ONLY) +nix run .#build-test-ami 17 + +# Run infrastructure tests +nix run .#run-testinfra + +# Cleanup AMI +nix run .#cleanup-ami +``` + +### Legacy Build System +```bash +# Packer-based image building +make init +make output-cloudimg/packer-cloudimg +make alpine-image +``` + +## Architecture Overview + +### PostgreSQL Version Support +- **PostgreSQL 17**: The ONLY version to use for all development and builds +- Other versions (15, OrioleDB 17) exist in the codebase but MUST NOT be used + +Extension configuration is in `nix/postgresql/` but ONLY work with PostgreSQL 17. + +### Build System Architecture +- **Primary**: Nix flakes (`flake.nix`) for reproducible builds +- **Fallback**: Docker containers for development +- **Production**: Packer + Ansible for AMI creation + +### Extension Management +Extensions are organized in `nix/ext/` by category: +- **Security**: pgsodium, vault, pgaudit +- **Analytics**: TimescaleDB, pg_stat_monitor +- **API**: pg_graphql, PostgREST integration +- **Vector Search**: pgvector +- **Geospatial**: PostGIS, pgRouting + +### Migration System +- Uses **dbmate** for schema migrations +- Located in `migrations/` directory +- Supports all PostgreSQL versions +- Append-only migration pattern + +### Testing Framework +- **pg_regress**: PostgreSQL regression testing +- **pgTAP**: Database unit testing in `migrations/tests/extensions/` +- **testinfra**: Infrastructure testing with Python pytest +- **PostgreSQL 17 testing**: All extensions tested against PostgreSQL 17 ONLY + +## Extension Development + +### Adding New Extensions +1. Create build stage in appropriate Dockerfile +2. Add version build args +3. Use `checkinstall` for source-built extensions +4. Copy package to extensions stage +5. Add pgTAP test in `migrations/tests/extensions/` + +### Extension Testing +Create test file in `migrations/tests/extensions/`: +```sql +BEGIN; +create extension if not exists your_extension with schema "extensions"; +ROLLBACK; +``` + +## Migration Guidelines +- **Never edit existing migrations** - always create new ones +- **Idempotent**: All migrations must be rerunnable +- **Role-based**: Use appropriate database roles for different migration phases +- **PostgreSQL 17 specific**: Target PostgreSQL 17 ONLY + +## Security Model +- **supabase_admin**: Superuser role for administration +- **authenticator**: Connection pooling role +- **Row Level Security**: Built-in RLS policies +- **Predefined roles**: For API access patterns + +## Key Configuration +- Default PostgreSQL port: `5435` +- Default host: `localhost` +- Superuser: `supabase_admin` +- WAL level: `logical` with 5 replication slots +- Large Systems Extensions enabled for ARM images \ No newline at end of file diff --git a/Dockerfile-17 b/Dockerfile-17 index 1f1309fc1..11ac383a1 100644 --- a/Dockerfile-17 +++ b/Dockerfile-17 @@ -186,7 +186,7 @@ RUN sed -i \ chown postgres:postgres /etc/postgresql-custom # Remove items from postgresql.conf -RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" +# RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf" RUN sed -i 's/ timescaledb,//g; s/ plv8,//g' "/etc/postgresql-custom/supautils.conf" diff --git a/README.md b/README.md index d7b106cf8..cba9d2ba7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,19 @@ # Postgres + goodies +This is a fork of the Supabase postgres project. It adds the full version of timescaledb for postgres17 and the timescaledb toolkit. + +To build: `docker build -f Dockerfile-17 -t ghcr.io/cepro/postgres:v .` + +To push your build to the Github docker registry: `docker push ghcr.io/cepro/postgres:v` + +Before pushing you will need to login to the Github docker registry with: `docker login --username ghcr.io` . You will need to use a personal access token as the password - see here for instructions: https://medium.com/devopsturkiye/pushing-docker-images-to-githubs-registry-manual-and-automated-methods-19cce3544eb1 + +NOTE: may need to create a 'classic' token and specifically give write:packages and read:packages permissions. In my case the 'fine grained token' didn't have any selectable permissions for packages. + +--- +--- +--- + Unmodified Postgres with some useful plugins. Our goal with this repo is not to modify Postgres, but to provide some of the most common extensions with a one-click install. ## Primary Features diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index d3209fc04..31178697d 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -18,20 +18,20 @@ set_fact: is_psql_15: "{{ psql_version in ['psql_15'] }}" - - name: Remove specified extensions from postgresql.conf if orioledb-17 or 17 build + - name: Remove specified extensions from postgresql.conf if orioledb-17 build ansible.builtin.command: cmd: > sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf - when: is_psql_oriole or is_psql_17 and stage2_nix + when: is_psql_oriole and stage2_nix become: yes - - name: Remove specified extensions from supautils.conf if orioledb-17 or 17 build + - name: Remove specified extensions from supautils.conf if orioledb-17 build ansible.builtin.command: cmd: > sed -i 's/ timescaledb,//g; s/ plv8,//g' /etc/postgresql-custom/supautils.conf - when: is_psql_oriole or is_psql_17 and stage2_nix + when: is_psql_oriole and stage2_nix become: yes - name: Remove db_user_namespace from postgresql.conf if orioledb-17 or 17 build diff --git a/ansible/vars.yml b/ansible/vars.yml index 2838d6f92..bb8c0e04a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -6,12 +6,14 @@ postgres_major: - "15" - "17" - "orioledb-17" + - "cep" # Full version strings for each major version postgres_release: postgresorioledb-17: "17.0.1.093-orioledb" postgres17: "17.4.1.043" postgres15: "15.8.1.100" + postgrescep: "17.4.1.043-cep" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/flake.nix b/flake.nix index f22415789..6fc47633d 100644 --- a/flake.nix +++ b/flake.nix @@ -131,6 +131,7 @@ ./nix/ext/rum.nix ./nix/ext/timescaledb.nix ./nix/ext/timescaledb-2.9.1.nix + ./nix/ext/timescaledb-toolkit.nix ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix @@ -163,17 +164,23 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option - #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions + #we're not using timescaledb or plv8 in the orioledb-17 version but timescaledb is now enabled for pg 17 orioleFilteredExtensions = builtins.filter ( x: x != ./nix/ext/timescaledb.nix && x != ./nix/ext/timescaledb-2.9.1.nix && + x != ./nix/ext/timescaledb-toolkit.nix && x != ./nix/ext/plv8.nix ) ourExtensions; orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; - dbExtensions17 = orioleFilteredExtensions; + dbExtensions17 = builtins.filter + ( + x: + x != ./nix/ext/plv8.nix && + x != ./nix/ext/timescaledb-2.9.1.nix + ) ourExtensions; getPostgresqlPackage = version: pkgs.postgresql."postgresql_${version}"; # Create a 'receipt' file for a given postgresql package. This is a way diff --git a/migrations/tests/extensions/timescaledb_toolkit.sql b/migrations/tests/extensions/timescaledb_toolkit.sql new file mode 100644 index 000000000..af3faee70 --- /dev/null +++ b/migrations/tests/extensions/timescaledb_toolkit.sql @@ -0,0 +1,3 @@ +BEGIN; +create extension if not exists timescaledb_toolkit with schema "extensions"; +ROLLBACK; \ No newline at end of file diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix index 92d5d73fe..cc75acaf2 100644 --- a/nix/ext/timescaledb-2.9.1.nix +++ b/nix/ext/timescaledb-2.9.1.nix @@ -14,7 +14,7 @@ stdenv.mkDerivation rec { hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; }; - cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=0" ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; # Fix the install phase which tries to install into the pgsql extension dir, diff --git a/nix/ext/timescaledb-toolkit.nix b/nix/ext/timescaledb-toolkit.nix new file mode 100644 index 000000000..b901a5348 --- /dev/null +++ b/nix/ext/timescaledb-toolkit.nix @@ -0,0 +1,65 @@ +{ lib, stdenv, fetchurl, postgresql, dpkg, patchelf, glibc }: + +stdenv.mkDerivation rec { + pname = "timescaledb-toolkit"; + version = "1.21.0"; + + # Use the official TimescaleDB toolkit package for PostgreSQL 17 + src = fetchurl { + url = "https://packagecloud.io/timescale/timescaledb/packages/ubuntu/jammy/timescaledb-toolkit-postgresql-17_${version}~ubuntu22.04_amd64.deb/download"; + sha256 = "12l8pngxa6vw7vkgngyyyx5gk151iay6js2smi8wkrwwscx80hjd"; + }; + + nativeBuildInputs = [ dpkg patchelf ]; + buildInputs = [ postgresql glibc ]; + + unpackPhase = '' + runHook preUnpack + dpkg-deb -x $src . + runHook postUnpack + ''; + + installPhase = '' + runHook preInstall + + # Create output directories + mkdir -p $out/lib + mkdir -p $out/share/postgresql/extension + + # Copy the shared library + if [ -f usr/lib/postgresql/17/lib/timescaledb_toolkit-*.so ]; then + cp usr/lib/postgresql/17/lib/timescaledb_toolkit-*.so $out/lib/ + fi + + # Copy extension files + if [ -d usr/share/postgresql/17/extension ]; then + cp -r usr/share/postgresql/17/extension/* $out/share/postgresql/extension/ + fi + + runHook postInstall + ''; + + postFixup = '' + # Fix the RPATH for the shared library + for lib in $out/lib/*.so; do + if [ -f "$lib" ]; then + patchelf --set-rpath "${lib.makeLibraryPath [ postgresql glibc ]}" "$lib" || true + fi + done + '' + lib.optionalString (!stdenv.isDarwin) '' + # Additional fixup for Linux + for lib in $out/lib/*.so; do + if [ -f "$lib" ]; then + patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$lib" 2>/dev/null || true + fi + done + ''; + + meta = with lib; { + description = "Extension for more hyperfunctions, fully compatible with TimescaleDB and PostgreSQL"; + homepage = "https://github.com/timescale/timescaledb-toolkit"; + license = licenses.asl20; + platforms = postgresql.meta.platforms; + maintainers = with maintainers; [ ]; + }; +} \ No newline at end of file diff --git a/nix/ext/timescaledb.nix b/nix/ext/timescaledb.nix index 1d7360762..80488a02e 100644 --- a/nix/ext/timescaledb.nix +++ b/nix/ext/timescaledb.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "timescaledb-apache"; - version = "2.16.1"; + version = "2.20.3"; nativeBuildInputs = [ cmake ]; buildInputs = [ postgresql openssl libkrb5 ]; @@ -11,10 +11,10 @@ stdenv.mkDerivation rec { owner = "timescale"; repo = "timescaledb"; rev = version; - hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w="; + hash = "sha256-Ma6h2ISMjBz14y5Pbx4T4QOMrrvUy5wkPyKawm9rpx0="; }; - cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=0" ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; # Fix the install phase which tries to install into the pgsql extension dir,