diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index 53d64cae8..a78756fa9 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -204,11 +204,17 @@ RUN sed -i \ # Remove items from postgresql.conf RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present -RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf" -RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf" -RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf" -RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf" - +RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf" +RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf" +RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf" + +# OrioleDB rewind configuration +# Enables time-based rewind capability for up to 20 minutes (1200 seconds) +# Buffer size: 1280 buffers * 8KB = 10MB for transaction retention +RUN echo "orioledb.enable_rewind = true" >> "/etc/postgresql/postgresql.conf" && \ + echo "orioledb.rewind_max_time = 1200" >> "/etc/postgresql/postgresql.conf" && \ + echo "orioledb.rewind_max_transactions = 100000" >> "/etc/postgresql/postgresql.conf" && \ + echo "orioledb.rewind_buffers = 1280" >> "/etc/postgresql/postgresql.conf" # # Include schema migrations diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index c930f71c2..862fe9260 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -51,6 +51,38 @@ line: "default_table_access_method = 'orioledb'" path: '/etc/postgresql/postgresql.conf' state: 'present' + + - name: Enable OrioleDB rewind feature + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "orioledb.enable_rewind = true" + state: present + when: is_psql_oriole and stage2_nix + become: yes + + - name: Set OrioleDB rewind max time (20 minutes) + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "orioledb.rewind_max_time = 1200" + state: present + when: is_psql_oriole and stage2_nix + become: yes + + - name: Set OrioleDB rewind max transactions + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "orioledb.rewind_max_transactions = 100000" + state: present + when: is_psql_oriole and stage2_nix + become: yes + + - name: Set OrioleDB rewind buffers (1280 buffers = 10MB) + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "orioledb.rewind_buffers = 1280" + state: present + when: is_psql_oriole and stage2_nix + become: yes - name: Add ORIOLEDB_ENABLED environment variable ansible.builtin.lineinfile: diff --git a/ansible/vars.yml b/ansible/vars.yml index 697b84cb2..201a6020a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.073-orioledb-plv8-1" - postgres17: "17.6.1.052-plv8-1" - postgres15: "15.14.1.052-plv8-1" + postgresorioledb-17: "17.6.0.010-orioledb" + postgres17: "17.6.1.053" + postgres15: "15.14.1.053" # Non Postgres Extensions pgbouncer_release: 1.19.0 diff --git a/migrations/schema-orioledb-17.sql b/migrations/schema-orioledb-17.sql index 5cfda417c..7abb63123 100644 --- a/migrations/schema-orioledb-17.sql +++ b/migrations/schema-orioledb-17.sql @@ -2,8 +2,10 @@ -- PostgreSQL database dump -- --- Dumped from database version 17.5 --- Dumped by pg_dump version 17.5 +\restrict SupabaseTestDumpKey123 + +-- Dumped from database version 17.6 +-- Dumped by pg_dump version 17.6 SET statement_timeout = 0; SET lock_timeout = 0; @@ -1012,3 +1014,5 @@ CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop -- PostgreSQL database dump complete -- +\unrestrict SupabaseTestDumpKey123 + diff --git a/nix/config.nix b/nix/config.nix index 4076f0cea..ca8a841f5 100644 --- a/nix/config.nix +++ b/nix/config.nix @@ -20,6 +20,10 @@ let options = { version = lib.mkOption { type = lib.types.str; }; hash = lib.mkOption { type = lib.types.str; }; + revision = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; }; }; supabaseSubmodule = lib.types.submodule { @@ -52,8 +56,8 @@ in }; orioledb = { "17" = { - version = "17_11"; - hash = "sha256-RZYU955PmGZExfX2JKw1dIQMMuuswtAXpXjZ9CLbOsw="; + version = "17_15"; + hash = "sha256-1v3FGIN0UW+E4ilLwbsV3nXvef3n9O8bVmgyjRFEJsU="; }; }; }; diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 549aa2ab1..a9170a9d0 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -15,10 +15,10 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "beta12"; - sha256 = "sha256-5dyVdKD1PzW+F5OPW3TR3OKBaJxxR3OhLbzK+o9Wf+Q="; + rev = "beta13"; + sha256 = "sha256-80RQHOgkEC7Hq3+N1VhsuKEUL+SNT/WfDN5vmXpaQG4="; }; - version = "beta12"; + version = "beta13"; buildInputs = [ curl libkrb5 @@ -26,7 +26,9 @@ stdenv.mkDerivation rec { python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=11"; + buildPhase = '' + make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=15 + ''; installPhase = '' runHook preInstall diff --git a/nix/ext/tests/orioledb.nix b/nix/ext/tests/orioledb.nix new file mode 100644 index 000000000..a6f5484c2 --- /dev/null +++ b/nix/ext/tests/orioledb.nix @@ -0,0 +1,82 @@ +{ self, pkgs }: +let + pname = "orioledb"; + inherit (pkgs) lib; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + self.packages.${pkgs.system}."psql_orioledb-17/exts/orioledb" + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; + psql_orioledb = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_orioledb-17; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = psql_orioledb; + settings = { + shared_preload_libraries = "orioledb"; + default_table_access_method = "orioledb"; + }; + initdbArgs = [ + "--allow-group-access" + "--locale-provider=icu" + "--encoding=UTF-8" + "--icu-locale=en_US.UTF-8" + ]; + initialScript = pkgs.writeText "init-postgres-with-orioledb" '' + CREATE EXTENSION orioledb CASCADE; + ''; + }; + }; + testScript = + { ... }: + '' + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + ''; +} diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index fdef115d3..0d65a1ad7 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -20,7 +20,7 @@ let pkgs.lib.nameValuePair "${namePrefix}${versionSuffix}" ( pkgs.callPackage ./generic.nix { inherit isOrioleDB; - inherit (config) version hash; + inherit (config) version hash revision; jitSupport = jitSupport; self = pkgs; } diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 61920c869..76904ced7 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -46,6 +46,7 @@ let # source specification version, hash, + revision ? null, muslPatches ? { }, # for tests @@ -88,11 +89,17 @@ let pname = pname + lib.optionalString jitSupport "-jit"; src = - if (builtins.match "[0-9][0-9]_.*" version != null) then - fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; - inherit hash; - } + if isOrioleDB then + if revision != null then + fetchurl { + url = "https://github.com/orioledb/postgres/archive/${revision}.tar.gz"; + inherit hash; + } + else + fetchurl { + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; + inherit hash; + } else fetchurl { url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; @@ -124,18 +131,15 @@ let ++ lib.optionals gssSupport [ libkrb5 ] ++ lib.optionals stdenv'.isLinux [ linux-pam ] ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ] - ++ - lib.optionals - ((builtins.match "[0-9][0-9]_.*" version != null) || (lib.versionAtLeast version "17")) - [ - perl - bison - flex - docbook_xsl - docbook_xml_dtd_45 - docbook_xsl_ns - libxslt - ]; + ++ lib.optionals (isOrioleDB || (lib.versionAtLeast version "17")) [ + perl + bison + flex + docbook_xsl + docbook_xml_dtd_45 + docbook_xsl_ns + libxslt + ]; nativeBuildInputs = [ @@ -285,6 +289,7 @@ let in { psqlSchema = lib.versions.major version; + inherit revision; withJIT = if jitSupport then this else jitToggle; withoutJIT = if jitSupport then jitToggle else this; @@ -292,6 +297,12 @@ let dlSuffix = if olderThan "16" then ".so" else stdenv.hostPlatform.extensions.sharedLibrary; inherit isOrioleDB; + patchset = + if isOrioleDB then + if revision != null then revision else builtins.elemAt (builtins.split "_" version) 2 + else + null; + pkgs = let scope = { @@ -404,7 +415,13 @@ let ''; passthru = { - inherit (postgresql) version psqlSchema isOrioleDB; + inherit (postgresql) + version + revision + patchset + psqlSchema + isOrioleDB + ; }; }; in diff --git a/nix/tools/dbmate-tool.sh.in b/nix/tools/dbmate-tool.sh.in index 934724c03..ec38ecf1a 100644 --- a/nix/tools/dbmate-tool.sh.in +++ b/nix/tools/dbmate-tool.sh.in @@ -182,7 +182,7 @@ perform_dump() { # Only use --restrict-key for standard PostgreSQL 15 and 17 versions # OrioleDB doesn't support this flag yet - if [ "$PSQL_VERSION" = "15" ] || [ "$PSQL_VERSION" = "17" ]; then + if [ "$PSQL_VERSION" = "15" ] || [ "$PSQL_VERSION" = "17" ] || [ "$PSQL_VERSION" = "orioledb-17" ]; then # Use a fixed restrict key for reproducible test dumps # This is safe in testing contexts but should not be used in production dump_cmd="$dump_cmd --restrict-key=SupabaseTestDumpKey123" diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 182cbe554..2ca05080e 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -248,6 +248,11 @@ orioledb_config_items() { sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ pgjwt,//g;' "$DATDIR/supautils.conf" sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "$DATDIR/postgresql.conf" echo "default_table_access_method = 'orioledb'" >> "$DATDIR/postgresql.conf" + # OrioleDB rewind configuration (20 minute window, 10MB buffer) + echo "orioledb.enable_rewind = true" >> "$DATDIR/postgresql.conf" + echo "orioledb.rewind_max_time = 1200" >> "$DATDIR/postgresql.conf" + echo "orioledb.rewind_max_transactions = 100000" >> "$DATDIR/postgresql.conf" + echo "orioledb.rewind_buffers = 1280" >> "$DATDIR/postgresql.conf" elif [[ "$1" = "orioledb-17" && "$CURRENT_SYSTEM" = "aarch64-darwin" ]]; then # macOS specific configuration echo "macOS detected, applying macOS specific configuration" @@ -263,6 +268,11 @@ orioledb_config_items() { perl -pi -e 's/(shared_preload_libraries\s*=\s*'\''.*?)'\''/\1, orioledb'\''/' "$DATDIR/postgresql.conf" echo "default_table_access_method = 'orioledb'" >> "$DATDIR/postgresql.conf" + # OrioleDB rewind configuration (20 minute window, 10MB buffer) + echo "orioledb.enable_rewind = true" >> "$DATDIR/postgresql.conf" + echo "orioledb.rewind_max_time = 1200" >> "$DATDIR/postgresql.conf" + echo "orioledb.rewind_max_transactions = 100000" >> "$DATDIR/postgresql.conf" + echo "orioledb.rewind_buffers = 1280" >> "$DATDIR/postgresql.conf" elif [[ "$VERSION" == "17" && "$CURRENT_SYSTEM" != "aarch64-darwin" ]]; then echo "non-macos pg 17 conf" sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf"