From 613fb250b3cd43865384e8f4e1f7cd3791ba9333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Wed, 18 Jun 2025 18:44:57 +0200 Subject: [PATCH 1/6] feat: multiple versions for the pgjwt extension Build multiple versions of the pgjwt extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17. --- nix/ext/pgjwt.nix | 97 +++++++++++++++++++------- nix/ext/tests/pgjwt.nix | 147 ++++++++++++++++++++++++++++++++++++++++ nix/ext/versions.json | 10 +++ 3 files changed, 230 insertions(+), 24 deletions(-) create mode 100644 nix/ext/tests/pgjwt.nix diff --git a/nix/ext/pgjwt.nix b/nix/ext/pgjwt.nix index 1a2f895b7..72f1614d2 100644 --- a/nix/ext/pgjwt.nix +++ b/nix/ext/pgjwt.nix @@ -1,36 +1,85 @@ { + buildEnv, lib, stdenv, fetchFromGitHub, postgresql, unstableGitUpdater, }: - -stdenv.mkDerivation rec { +let pname = "pgjwt"; - version = "9742dab1b2f297ad3811120db7b21451bca2d3c9"; + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.trace "Versions: ${toString (builtins.length versions)}" ( + builtins.length versions + ); + build = + version: hash: revision: + stdenv.mkDerivation { + inherit pname version; - src = fetchFromGitHub { - owner = "michelp"; - repo = "pgjwt"; - rev = "${version}"; - hash = "sha256-Hw3R9bMGDmh+dMzjmqZSy/rT4mX8cPU969OJiARFg10="; - }; + src = fetchFromGitHub { + owner = "michelp"; + repo = "pgjwt"; + rev = revision; + inherit hash; + }; + + dontBuild = true; + installPhase = '' + mkdir -p $out/share/postgresql/extension + create_sql_files() { + echo "Creating SQL files for previous versions..." + if [[ "${version}" == "${latestVersion}" ]]; then + cp *.sql $out/share/postgresql/extension + fi + } + + create_control_files() { + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \ + ${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control + + if [[ "${version}" == "${latestVersion}" ]]; then + { + echo "default_version = '${latestVersion}'" + cat $out/share/postgresql/extension/${pname}--${latestVersion}.control + } > $out/share/postgresql/extension/${pname}.control + fi + } + + create_sql_files + create_control_files + ''; + + passthru.updateScript = unstableGitUpdater { }; + + meta = with lib; { + description = "PostgreSQL implementation of JSON Web Tokens"; + longDescription = '' + sign() and verify() functions to create and verify JSON Web Tokens. + ''; + license = licenses.mit; + platforms = postgresql.meta.platforms; + }; + }; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash value.revision) supportedVersions + ); +in +buildEnv { + name = pname; + paths = packages; + pathsToLink = [ "/share/postgresql/extension" ]; - dontBuild = true; - installPhase = '' - mkdir -p $out/share/postgresql/extension - cp pg*sql *.control $out/share/postgresql/extension - ''; - - passthru.updateScript = unstableGitUpdater { }; - - meta = with lib; { - description = "PostgreSQL implementation of JSON Web Tokens"; - longDescription = '' - sign() and verify() functions to create and verify JSON Web Tokens. - ''; - license = licenses.mit; - platforms = postgresql.meta.platforms; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/tests/pgjwt.nix b/nix/ext/tests/pgjwt.nix new file mode 100644 index 000000000..39901a2f0 --- /dev/null +++ b/nix/ext/tests/pgjwt.nix @@ -0,0 +1,147 @@ +{ self, pkgs }: +let + pname = "pgjwt"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + self.packages.${pkgs.system}."psql_${majorVersion}/exts/hypopg" # dependency + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + }; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(pg_version): + with subtest("Check ${pname} upgrade path"): + firstVersion = versions[pg_version][0] + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") + run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" + for version in versions[pg_version][1:]: + run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path("15") + + with subtest("Check ${pname} latest extension version"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") + server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["15"][-1] + assert f"${pname},{latestVersion}" in installed_extensions + + with subtest("switch to postgresql 17"): + server.succeed( + "${pg17-configuration}/bin/switch-to-configuration test >&2" + ) + + check_upgrade_path("17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index 7d8093d7a..5a33a4fc3 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -709,6 +709,16 @@ "hash": "sha256-YNeIfmccT/DtOrwDmpYFCuV2/P6k3Zj23VWBDkOh6sw=" } }, + "pgjwt": { + "0.2.0": { + "revision": "f3d82fd30151e754e19ce5d6a06c71c20689ce3d", + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-nDZEDf5+sFc1HDcG2eBNQj+kGcdAYRXJseKi9oww+JU=" + } + }, "pg_repack": { "1.4.8": { "postgresql": [ From ba54206d903061c285d4a1d37262ba82117e50df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 12 Aug 2025 14:24:29 +0200 Subject: [PATCH 2/6] chore: add release suffix for testing --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 1a442d3e7..924093c81 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.070-orioledb" - postgres17: "17.6.1.049" - postgres15: "15.14.1.049" + postgresorioledb-17: "17.5.1.065-orioledb-pg-jwt-1" + postgres17: "17.6.1.044-pg-jwt-1" + postgres15: "15.14.1.044-pg-jwt-1" # Non Postgres Extensions pgbouncer_release: 1.19.0 From 25ad2216743bcd89500e698312e6e63919efe10e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 12 Aug 2025 14:24:29 +0200 Subject: [PATCH 3/6] feat: run pg_regress tests for pgjwt extension Reuse the existing test library for postgresql extensions to run pg_regress tests --- nix/ext/tests/pgjwt.nix | 89 ++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/nix/ext/tests/pgjwt.nix b/nix/ext/tests/pgjwt.nix index 39901a2f0..ac240a73d 100644 --- a/nix/ext/tests/pgjwt.nix +++ b/nix/ext/tests/pgjwt.nix @@ -15,7 +15,6 @@ let postgresql postgresql.lib (installedExtension majorVersion) - self.packages.${pkgs.system}."psql_${majorVersion}/exts/hypopg" # dependency ]; passthru = { inherit (postgresql) version psqlSchema; @@ -36,6 +35,8 @@ let }; in pkg; + psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; in self.inputs.nixpkgs.lib.nixos.runTest { name = pname; @@ -55,12 +56,17 @@ self.inputs.nixpkgs.lib.nixos.runTest { services.postgresql = { enable = true; - package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + package = psql_15; + # pg_regress test for pgjwt rely on the "extensions" schema to be present + initialScript = pkgs.writeText "init-postgres-with-schema" '' + CREATE SCHEMA IF NOT EXISTS extensions; + CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + ''; }; specialisation.postgresql17.configuration = { services.postgresql = { - package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + package = lib.mkForce psql_17; }; systemd.services.postgresql-migrate = { @@ -74,8 +80,8 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; script = let - oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldPostgresql = psql_15; + newPostgresql = psql_17; oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; in @@ -103,45 +109,72 @@ self.inputs.nixpkgs.lib.nixos.runTest { pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; in '' + from pathlib import Path versions = { "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } + extension_name = "${pname}" + support_upgrade = False + pg17_configuration = "${pg17-configuration}" + ext_has_background_worker = ${ + if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" + } + sql_test_directory = Path("${../../tests}") + pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}" - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + ${builtins.readFile ./lib.py} start_all() server.wait_for_unit("multi-user.target") server.wait_for_unit("postgresql.service") - check_upgrade_path("15") + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade) + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") + + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + # We need to uninstall the extension before running pg_regress + test.drop_extension() + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["15"][-1] - assert f"${pname},{latestVersion}" in installed_extensions + if ext_has_background_worker: + with subtest("Test switch_${pname}_version"): + test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15") + + with subtest("Check pg_regress with postgresql 15 after installing the last version"): + # We need to uninstall the extension before running pg_regress + test.drop_extension() + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) with subtest("switch to postgresql 17"): server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" + f"{pg17_configuration}/bin/switch-to-configuration test >&2" ) - check_upgrade_path("17") + with subtest("Check last version of the extension after postgresql upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + # We need to uninstall the extension before running pg_regress + test.drop_extension() + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Check the install of the last version of the extension"): + test.check_install_last_version("17") + + with subtest("Check pg_regress with postgresql 17 after installing the last version"): + # We need to uninstall the extension before running pg_regress + test.drop_extension() + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } From 7f725586525f0c71f57c97ffc7e459073dbb751f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 14 Nov 2025 12:27:16 -0500 Subject: [PATCH 4/6] tests: bypass on this package if only one version --- nix/ext/tests/pgjwt.nix | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/nix/ext/tests/pgjwt.nix b/nix/ext/tests/pgjwt.nix index ac240a73d..785187a22 100644 --- a/nix/ext/tests/pgjwt.nix +++ b/nix/ext/tests/pgjwt.nix @@ -132,13 +132,14 @@ self.inputs.nixpkgs.lib.nixos.runTest { test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade) - with subtest("Check upgrade path with postgresql 15"): - test.check_upgrade_path("15") + if support_upgrade: + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") - with subtest("Check pg_regress with postgresql 15 after extension upgrade"): - # We need to uninstall the extension before running pg_regress - test.drop_extension() - test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + # We need to uninstall the extension before running pg_regress + test.drop_extension() + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) last_version = None with subtest("Check the install of the last version of the extension"): @@ -161,13 +162,14 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check last version of the extension after postgresql upgrade"): test.assert_version_matches(last_version) - with subtest("Check upgrade path with postgresql 17"): - test.check_upgrade_path("17") + if support_upgrade: + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") - with subtest("Check pg_regress with postgresql 17 after extension upgrade"): - # We need to uninstall the extension before running pg_regress - test.drop_extension() - test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + # We need to uninstall the extension before running pg_regress + test.drop_extension() + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) with subtest("Check the install of the last version of the extension"): test.check_install_last_version("17") From 3d9fc81eb8a705b19c97a92783786565c6605909 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 14 Nov 2025 12:40:39 -0500 Subject: [PATCH 5/6] fix: conn rules --- nix/ext/tests/pgjwt.nix | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nix/ext/tests/pgjwt.nix b/nix/ext/tests/pgjwt.nix index 785187a22..98884f4c3 100644 --- a/nix/ext/tests/pgjwt.nix +++ b/nix/ext/tests/pgjwt.nix @@ -57,16 +57,35 @@ self.inputs.nixpkgs.lib.nixos.runTest { services.postgresql = { enable = true; package = psql_15; + enableTCPIP = true; + authentication = '' + local all postgres peer map=postgres + local all all peer map=root + ''; + identMap = '' + root root supabase_admin + postgres postgres postgres + ''; + ensureUsers = [ + { + name = "supabase_admin"; + ensureClauses.superuser = true; + } + ]; # pg_regress test for pgjwt rely on the "extensions" schema to be present initialScript = pkgs.writeText "init-postgres-with-schema" '' CREATE SCHEMA IF NOT EXISTS extensions; CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; ''; + settings = (installedExtension "15").defaultSettings or { }; }; + networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ]; + specialisation.postgresql17.configuration = { services.postgresql = { package = lib.mkForce psql_17; + settings = (installedExtension "17").defaultSettings or { }; }; systemd.services.postgresql-migrate = { From bf2643cbde60f46685a31c7013437f255a0033ed Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 14 Nov 2025 16:09:38 -0500 Subject: [PATCH 6/6] chore: bump to release --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 924093c81..ef1362bf7 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.065-orioledb-pg-jwt-1" - postgres17: "17.6.1.044-pg-jwt-1" - postgres15: "15.14.1.044-pg-jwt-1" + postgresorioledb-17: "17.5.1.072-orioledb" + postgres17: "17.6.1.051" + postgres15: "15.14.1.051" # Non Postgres Extensions pgbouncer_release: 1.19.0