diff --git a/ansible/vars.yml b/ansible/vars.yml index 18cb0a206..b05651fd3 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.035-orioledb" - postgres17: "17.6.1.014" - postgres15: "15.14.1.014" + postgresorioledb-17: 17.5.1.036-orioledb + postgres17: 17.6.1.015 + postgres15: 15.14.1.015 # Non Postgres Extensions pgbouncer_release: 1.19.0 diff --git a/nix/ext/pgrouting.nix b/nix/ext/pgrouting.nix index e51b8bada..b20b89c43 100644 --- a/nix/ext/pgrouting.nix +++ b/nix/ext/pgrouting.nix @@ -6,69 +6,139 @@ perl, cmake, boost, + buildEnv, }: - -stdenv.mkDerivation rec { +let pname = "pgrouting"; - version = "3.4.1"; - nativeBuildInputs = [ - cmake - perl - ]; - buildInputs = [ - postgresql - boost - ]; + # Load version configuration from external file + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; - src = fetchFromGitHub { - owner = "pgRouting"; - repo = pname; - rev = "v${version}"; - hash = "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k="; - }; + # Filter versions compatible with current PostgreSQL version + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; - #disable compile time warnings for incompatible pointer types only on macos and pg16 - NIX_CFLAGS_COMPILE = lib.optionalString ( - stdenv.isDarwin && lib.versionAtLeast postgresql.version "16" - ) "-Wno-error=int-conversion -Wno-error=incompatible-pointer-types"; - - cmakeFlags = - [ "-DPOSTGRESQL_VERSION=${postgresql.version}" ] - ++ lib.optionals (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") [ - "-DCMAKE_MACOSX_RPATH=ON" - "-DCMAKE_SHARED_MODULE_SUFFIX=.dylib" - "-DCMAKE_SHARED_LIBRARY_SUFFIX=.dylib" - ]; - - preConfigure = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' - export DLSUFFIX=.dylib - export CMAKE_SHARED_LIBRARY_SUFFIX=.dylib - export CMAKE_SHARED_MODULE_SUFFIX=.dylib - export MACOSX_RPATH=ON - ''; + # Derived version information + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); - postBuild = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' - shopt -s nullglob - for file in lib/libpgrouting-*.so; do - if [ -f "$file" ]; then - mv "$file" "''${file%.so}.dylib" - fi - done - shopt -u nullglob - ''; + # Build function for individual versions + build = + version: hash: + stdenv.mkDerivation rec { + inherit pname version; + + nativeBuildInputs = [ + cmake + perl + ]; + buildInputs = [ + postgresql + boost + ]; + + src = fetchFromGitHub { + owner = "pgRouting"; + repo = pname; + rev = "v${version}"; + inherit hash; + }; + + #disable compile time warnings for incompatible pointer types only on macos and pg16 + NIX_CFLAGS_COMPILE = lib.optionalString ( + stdenv.isDarwin && lib.versionAtLeast postgresql.version "16" + ) "-Wno-error=int-conversion -Wno-error=incompatible-pointer-types"; + + cmakeFlags = + [ "-DPOSTGRESQL_VERSION=${postgresql.version}" ] + ++ lib.optionals (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") [ + "-DCMAKE_MACOSX_RPATH=ON" + "-DCMAKE_SHARED_MODULE_SUFFIX=.dylib" + "-DCMAKE_SHARED_LIBRARY_SUFFIX=.dylib" + ]; + + preConfigure = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + export DLSUFFIX=.dylib + export CMAKE_SHARED_LIBRARY_SUFFIX=.dylib + export CMAKE_SHARED_MODULE_SUFFIX=.dylib + export MACOSX_RPATH=ON + ''; + + postBuild = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + shopt -s nullglob + for file in lib/libpgrouting-*.so; do + if [ -f "$file" ]; then + mv "$file" "''${file%.so}.dylib" + fi + done + shopt -u nullglob + ''; + + installPhase = '' + MAJ_MIN_VERSION=${lib.concatStringsSep "." (lib.take 2 (builtins.splitVersion version))} + + mkdir -p $out/{lib,share/postgresql/extension} + + # Install shared library with version suffix + install -D lib/libpgrouting-$MAJ_MIN_VERSION${postgresql.dlSuffix} -t $out/lib + + # Create version-specific control file + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/lib${pname}-$MAJ_MIN_VERSION'|" \ + sql/common/${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control + + # Copy SQL upgrade scripts + cp sql/${pname}--*.sql $out/share/postgresql/extension + + if [[ "${version}" == "${latestVersion}" ]]; then + { + echo "default_version = '${version}'" + cat $out/share/postgresql/extension/${pname}--${version}.control + } > $out/share/postgresql/extension/${pname}.control + ln -sfn $out/lib/lib${pname}-$MAJ_MIN_VERSION${postgresql.dlSuffix} $out/lib/lib${pname}${postgresql.dlSuffix} + fi + ''; + + meta = with lib; { + description = "A PostgreSQL/PostGIS extension that provides geospatial routing functionality"; + homepage = "https://pgrouting.org/"; + changelog = "https://github.com/pgRouting/pgrouting/releases/tag/v${version}"; + license = licenses.gpl2Plus; + inherit (postgresql.meta) platforms; + }; + }; +in +buildEnv { + name = pname; + paths = packages; + + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; + + postBuild = '' + #Verify all expected library files are present + expectedFiles=${toString (numberOfVersions + 1)} + actualFiles=$(ls -l $out/lib/lib${pname}*${postgresql.dlSuffix} | wc -l) - installPhase = '' - install -D lib/*${postgresql.dlSuffix} -t $out/lib - install -D sql/pgrouting--*.sql -t $out/share/postgresql/extension - install -D sql/common/pgrouting.control -t $out/share/postgresql/extension + if [[ "$actualFiles" != "$expectedFiles" ]]; then + echo "Error: Expected $expectedFiles library files, found $actualFiles" + echo "Files found:" + ls -la $out/lib/*${postgresql.dlSuffix} || true + exit 1 + fi ''; - meta = with lib; { - description = "A PostgreSQL/PostGIS extension that provides geospatial routing functionality"; - homepage = "https://pgrouting.org/"; - changelog = "https://github.com/pgRouting/pgrouting/releases/tag/v${version}"; - platforms = postgresql.meta.platforms; - license = licenses.gpl2Plus; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/tests/pgrouting.nix b/nix/ext/tests/pgrouting.nix new file mode 100644 index 000000000..f8775e4aa --- /dev/null +++ b/nix/ext/tests/pgrouting.nix @@ -0,0 +1,228 @@ +{ self, pkgs }: +let + pname = "pgrouting"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = + [ + postgresql + postgresql.lib + (installedExtension majorVersion) + self.packages.${pkgs.system}."psql_${majorVersion}/exts/postgis-all" + ] + ++ lib.optional (postgresql.isOrioleDB + ) self.packages.${pkgs.system}."psql_orioledb-17/exts/orioledb"; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; + pg_regress = pkgs.callPackage ../pg_regress.nix { + postgresql = self.packages.${pkgs.system}.postgresql_15; + }; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIo+ulCUfJjnCVgfM4946Ih5Nm8DeZZiayYeABHGPEl7 jfroche" + ]; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + }; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + + specialisation.orioledb17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_orioledb-17); + settings = { + shared_preload_libraries = "orioledb"; + default_table_access_method = "orioledb"; + }; + initdbArgs = [ + "--allow-group-access" + "--locale-provider=icu" + "--encoding=UTF-8" + "--icu-locale=en_US.UTF-8" + ]; + initialScript = pkgs.writeText "init-postgres-with-orioledb" '' + CREATE EXTENSION orioledb CASCADE; + ''; + }; + + systemd.services.postgresql-migrate = { + # we don't support migrating from postgresql 17 to orioledb-17 so we just reinit the datadir + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_orioledb-17; + in + '' + set -x + systemctl cat postgresql.service + rm -rf ${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema} + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + orioledb17-configuration = "${nodes.server.system.build.toplevel}/specialisation/orioledb17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + "orioledb-17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "orioledb-17"))}], + } + + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def run_pg_regress(sql_file, pg_version): + try: + server.succeed(f"""sudo -u postgres ${pg_regress}/bin/pg_regress --inputdir=${../../tests} --use-existing --dbname=postgres --outputdir=/tmp/regression_output_{pg_version} "{sql_file}" """) + except: + server.copy_from_vm(f"/tmp/regression_output_{pg_version}", "") + raise + + def check_upgrade_path(pg_version): + with subtest("Check ${pname} upgrade path"): + firstVersion = versions[pg_version][0] + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") + run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" + for version in versions[pg_version][1:]: + run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + run_pg_regress("${pname}", pg_version) + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path("15") + + with subtest("Check ${pname} latest extension version"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") + server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["15"][-1] + assert f"${pname},{latestVersion}" in installed_extensions + + with subtest("switch to postgresql 17"): + server.succeed( + "${pg17-configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check ${pname} latest extension version after upgrade"): + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["17"][-1] + assert f"${pname},{latestVersion}" in installed_extensions + + check_upgrade_path("17") + + with subtest("switch to orioledb 17"): + server.succeed( + "${orioledb17-configuration}/bin/switch-to-configuration test >&2" + ) + installed_extensions=run_sql(r"""SELECT extname FROM pg_extension WHERE extname = 'orioledb';""") + assert "orioledb" in installed_extensions + + check_upgrade_path("orioledb-17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index 4bcb4fd28..3068b857b 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -278,6 +278,15 @@ "hash": "sha256-Cpi2iASi1QJoED0Qs1dANqg/BNZTsz5S+pw8iYyW03Y=" } }, + "pgrouting": { + "3.4.1": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k=" + } + }, "pgsodium": { "3.0.4": { "postgresql": [ diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 272dc4f35..fdef115d3 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -10,6 +10,7 @@ let namePrefix, jitSupport, supportedVersions, + isOrioleDB, }: pkgs.lib.mapAttrs' ( version: config: @@ -18,6 +19,7 @@ let in pkgs.lib.nameValuePair "${namePrefix}${versionSuffix}" ( pkgs.callPackage ./generic.nix { + inherit isOrioleDB; inherit (config) version hash; jitSupport = jitSupport; self = pkgs; @@ -30,10 +32,12 @@ let { namePrefix = "postgresql_"; versions = supportedPostgresVersions.postgres; + isOrioleDB = false; } { namePrefix = "postgresql_orioledb-"; versions = supportedPostgresVersions.orioledb; + isOrioleDB = true; } ]; @@ -44,7 +48,7 @@ let acc: flavor: acc // (mkPostgresqlPackages { - inherit (flavor) namePrefix; + inherit (flavor) namePrefix isOrioleDB; inherit jitSupport; supportedVersions = flavor.versions; }) diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index d922895fa..61920c869 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -64,6 +64,8 @@ let # detection of crypt fails when using llvm stdenv, so we add it manually # for <13 (where it got removed: https://github.com/postgres/postgres/commit/c45643d618e35ec2fe91438df15abd4f3c0d85ca) libxcrypt, + + isOrioleDB ? false, }@args: let atLeast = lib.versionAtLeast version; @@ -288,6 +290,7 @@ let withoutJIT = if jitSupport then jitToggle else this; dlSuffix = if olderThan "16" then ".so" else stdenv.hostPlatform.extensions.sharedLibrary; + inherit isOrioleDB; pkgs = let @@ -347,6 +350,7 @@ let "libpq" ]; platforms = platforms.unix; + inherit isOrioleDB; # JIT support doesn't work with cross-compilation. It is attempted to build LLVM-bytecode # (`%.bc` is the corresponding `make(1)`-rule) for each sub-directory in `backend/` for @@ -399,8 +403,9 @@ let wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib ''; - passthru.version = postgresql.version; - passthru.psqlSchema = postgresql.psqlSchema; + passthru = { + inherit (postgresql) version psqlSchema isOrioleDB; + }; }; in generic