diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index d3209fc04..74da50a57 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -217,19 +217,8 @@ recurse: yes when: stage2_nix -- name: Check psql_version and run postgis linking if not oriole-xx - block: - - name: Check if psql_version is psql_orioledb-17 - set_fact: - is_psql_oriole: "{{ psql_version == 'psql_orioledb-17' }}" - - - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir - shell: > - sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ - sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ - && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" - become: yes - when: stage2_nix and not is_psql_oriole +# PostGIS contrib linking removed - PostGIS doesn't install to contrib directory +# It installs extensions to /share/postgresql/extension/ which is already linked above - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets shell: >- diff --git a/ansible/vars.yml b/ansible/vars.yml index 6e3170cf3..2fc0ff36f 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: 17.5.1.033-orioledb - postgres17: 17.6.1.012 - postgres15: 15.14.1.012 + postgresorioledb-17: 17.5.1.034-orioledb + postgres17: 17.6.1.013 + postgres15: 15.14.1.013 # Non Postgres Extensions pgbouncer_release: 1.19.0 diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index 27a449210..6a152ef43 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -15,88 +15,198 @@ pcre2, nixosTests, callPackage, + buildEnv, }: let sfcgal = callPackage ./sfcgal/sfcgal.nix { }; gdal = callPackage ./gdal.nix { inherit postgresql; }; -in -stdenv.mkDerivation rec { pname = "postgis"; - version = "3.3.7"; - outputs = [ - "out" - "doc" - ]; + # Load version configuration from external file + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; - src = fetchurl { - url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; - sha256 = "sha256-UHJKDd5JrcJT5Z4CTYsY/va+ToU0GUPG1eHhuXTkP84="; - }; + # Filter versions compatible with current PostgreSQL version + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; + + # Derived version information + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); - buildInputs = [ - libxml2 - postgresql - geos - proj - gdal - json_c - protobufc - pcre2.dev - sfcgal - ] ++ lib.optional stdenv.isDarwin libiconv; - nativeBuildInputs = [ - perl - pkg-config + # List of C extensions to be included in the build + cExtensions = [ + "address_standardizer" + "postgis" + "postgis_raster" + "postgis_sfcgal" + "postgis_topology" ]; - dontDisableStatic = true; - env.NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; + sqlExtensions = [ + "address_standardizer_data_us" + "postgis_tiger_geocoder" + ]; - preConfigure = '' - sed -i 's@/usr/bin/file@${file}/bin/file@' configure - configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install --with-sfcgal" + # Build function for individual versions + build = + version: hash: + stdenv.mkDerivation rec { + inherit pname version; - makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" - ''; + outputs = [ + "out" + "doc" + ]; - postConfigure = '' - sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; - s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g - " \ - "raster/loader/Makefile"; - sed -i "s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g - " \ - "raster/scripts/python/Makefile"; - mkdir -p $out/bin - ln -s ${postgresql}/bin/postgres $out/bin/postgres - ''; + src = fetchurl { + url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; + inherit hash; + }; - postInstall = '' - rm $out/bin/postgres - for prog in $out/bin/*; do # */ - ln -s $prog $prog-${version} - done - # Add function definition and usage to tiger geocoder files - for file in $out/share/postgresql/extension/postgis_tiger_geocoder*--${version}.sql; do - sed -i "/SELECT postgis_extension_AddToSearchPath('tiger');/a SELECT postgis_extension_AddToSearchPath('extensions');" "$file" - done - # Original topology patching - for file in $out/share/postgresql/extension/postgis_topology*--${version}.sql; do - sed -i "/SELECT topology.AddToSearchPath('topology');/i SELECT topology.AddToSearchPath('extensions');" "$file" - done - mkdir -p $doc/share/doc/postgis - mv doc/* $doc/share/doc/postgis/ - ''; + buildInputs = [ + libxml2 + postgresql + geos + proj + gdal + json_c + protobufc + pcre2.dev + sfcgal + ] ++ lib.optional stdenv.isDarwin libiconv; + nativeBuildInputs = [ + perl + pkg-config + ]; + dontDisableStatic = true; + + env.NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; + + preConfigure = '' + sed -i 's@/usr/bin/file@${file}/bin/file@' configure + configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --with-sfcgal" + + makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" + ''; + + postConfigure = '' + sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; + s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g + " \ + "raster/loader/Makefile"; + sed -i "s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g + " \ + "raster/scripts/python/Makefile"; + mkdir -p $out/bin + ln -s ${postgresql}/bin/postgres $out/bin/postgres + ''; + + postInstall = '' + MIN_MAJ_VERSION=${lib.concatStringsSep "." (lib.take 2 (builtins.splitVersion version))} + rm $out/bin/postgres + + # Rename C extension libraries with full version suffix + for ext in ${lib.concatStringsSep " " cExtensions}; do + if [ -f "$out/lib/$ext-3${postgresql.dlSuffix}" ]; then + mv $out/lib/$ext-3${postgresql.dlSuffix} $out/lib/$ext-${version}${postgresql.dlSuffix} + fi + done - passthru.tests.postgis = nixosTests.postgis; + # Create version-specific control files (without default_version, pointing to unversioned library) + for ext in ${lib.concatStringsSep " " (cExtensions ++ sqlExtensions)}; do + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/$ext-3'|" \ + $out/share/postgresql/extension/$ext.control > $out/share/postgresql/extension/$ext--${version}.control + rm $out/share/postgresql/extension/$ext.control + done + + # Add function definition and usage to tiger geocoder files + for file in $out/share/postgresql/extension/postgis_tiger_geocoder*--${version}.sql; do + sed -i "/SELECT postgis_extension_AddToSearchPath('tiger');/a SELECT postgis_extension_AddToSearchPath('extensions');" "$file" + done + # Original topology patching + for file in $out/share/postgresql/extension/postgis_topology*--${version}.sql; do + sed -i "/SELECT topology.AddToSearchPath('topology');/i SELECT topology.AddToSearchPath('extensions');" "$file" + done + + # For the latest version, create default control file and library symlinks + if [[ "${version}" == "${latestVersion}" ]]; then + # Copy all SQL upgrade scripts only for latest version + cp $out/share/postgresql/extension/*.sql $out/share/postgresql/extension/ 2>/dev/null || true + + for ext in ${lib.concatStringsSep " " (cExtensions ++ sqlExtensions)}; do + { + echo "default_version = '${version}'" + cat $out/share/postgresql/extension/$ext--${version}.control + } > $out/share/postgresql/extension/$ext.control + done + + # Create symlinks for C extension libraries (latest version becomes the default) + for ext in ${lib.concatStringsSep " " cExtensions}; do + ln -sfn $ext-${version}${postgresql.dlSuffix} $out/lib/$ext-3${postgresql.dlSuffix} + done + + for prog in $out/bin/*; do # */ + ln -s $prog $prog-${version} + done + else + # remove migration scripts for non-latest version + find $out/share/postgresql/extension -regex '.*--.*--.*\.sql' -delete + + for prog in $out/bin/*; do # */ + mv $prog $prog-${version} + done + fi + + mkdir -p $doc/share/doc/postgis + mv doc/* $doc/share/doc/postgis/ + ''; + + passthru.tests.postgis = nixosTests.postgis; + + meta = with lib; { + description = "Geographic Objects for PostgreSQL"; + homepage = "https://postgis.net/"; + changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; + license = licenses.gpl2; + inherit (postgresql.meta) platforms; + }; + }; +in +buildEnv { + name = pname; + paths = packages; + + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; + postBuild = '' + # Verify all expected library files are present + # We expect: (numberOfVersions * cExtensions) versioned libraries + cExtensions symlinks + expectedFiles=${ + toString ((numberOfVersions * builtins.length cExtensions) + builtins.length cExtensions) + } + actualFiles=$(ls -A $out/lib/*${postgresql.dlSuffix} | wc -l) + + if [[ "$actualFiles" != "$expectedFiles" ]]; then + echo "Error: Expected $expectedFiles library files, found $actualFiles" + echo "Files found:" + ls -la $out/lib/*${postgresql.dlSuffix} || true + exit 1 + fi + ''; - meta = with lib; { - description = "Geographic Objects for PostgreSQL"; - homepage = "https://postgis.net/"; - changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; - license = licenses.gpl2; - inherit (postgresql.meta) platforms; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/tests/postgis.nix b/nix/ext/tests/postgis.nix new file mode 100644 index 000000000..ab6a4b3f8 --- /dev/null +++ b/nix/ext/tests/postgis.nix @@ -0,0 +1,156 @@ +{ self, pkgs }: +let + pname = "postgis"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + }; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(pg_version): + with subtest("Check ${pname} upgrade path"): + firstVersion = versions[pg_version][0] + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") + run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" + for version in versions[pg_version][1:]: + run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path("15") + + with subtest("Check ${pname} latest extension version"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") + server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension where extname = '${pname}';""") + latestVersion = versions["15"][-1] + majMinVersion = ".".join(latestVersion.split('.')[:1]) + assert f"${pname},{majMinVersion}" in installed_extensions, f"Expected ${pname} version {latestVersion}, but found {installed_extensions}" + + with subtest("switch to postgresql 17"): + server.succeed( + "${pg17-configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check ${pname} latest extension version after upgrade"): + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["17"][-1] + majMinVersion = ".".join(latestVersion.split('.')[:1]) + assert f"${pname},{majMinVersion}" in installed_extensions + + check_upgrade_path("17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index f786fc3f5..fe34cb814 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -183,6 +183,20 @@ "hash": "sha256-j5F1PPdwfQRbV8XJ8Mloi8FvZF0MTl4eyIJcBYQy1E4=" } }, + "postgis": { + "3.3.2": { + "postgresql": [ + "15" + ], + "hash": "sha256-miohnaAFoXMKOdGVmhx87GGbHvsAm2W+gP/CW60pkGg=" + }, + "3.3.7": { + "postgresql": [ + "17" + ], + "hash": "sha256-UHJKDd5JrcJT5Z4CTYsY/va+ToU0GUPG1eHhuXTkP84=" + } + }, "rum": { "1.3": { "postgresql": [