diff --git a/Dockerfile-15 b/Dockerfile-15 index 3827deffb..a06292674 100644 --- a/Dockerfile-15 +++ b/Dockerfile-15 @@ -122,6 +122,20 @@ RUN nix profile install .#wal-g-3 && \ RUN nix store gc +WORKDIR / +#################### +# setup-groonga +#################### +FROM base as groonga + +WORKDIR /nixpg + +RUN nix profile install .#supabase-groonga && \ + mkdir -p /tmp/groonga-plugins && \ + cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ + +RUN nix store gc + WORKDIR / # #################### # # Download gosu for easy step-down from root @@ -153,6 +167,7 @@ FROM gosu as production RUN id postgres || (echo "postgres user does not exist" && exit 1) # # Setup extensions COPY --from=walg /tmp/wal-g /usr/local/bin/ +COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins # # Initialise configs COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf @@ -216,4 +231,7 @@ ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive RUN mkdir -p /usr/share/postgresql/extension/ && \ ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh + +ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins + CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/Dockerfile-17 b/Dockerfile-17 index 23989c7b9..0bd89256c 100644 --- a/Dockerfile-17 +++ b/Dockerfile-17 @@ -126,6 +126,20 @@ RUN nix profile install .#wal-g-3 && \ RUN nix store gc +WORKDIR / +#################### +# setup-groonga +#################### +FROM base as groonga + +WORKDIR /nixpg + +RUN nix profile install .#supabase-groonga && \ + mkdir -p /tmp/groonga-plugins && \ + cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ + +RUN nix store gc + WORKDIR / # #################### # # Download gosu for easy step-down from root @@ -157,6 +171,7 @@ FROM gosu as production RUN id postgres || (echo "postgres user does not exist" && exit 1) # # Setup extensions COPY --from=walg /tmp/wal-g /usr/local/bin/ +COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins # # Initialise configs COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf @@ -229,4 +244,7 @@ ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive RUN mkdir -p /usr/share/postgresql/extension/ && \ ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh + +ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins + CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index 1eb9400df..0b4f49d7e 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -126,6 +126,20 @@ RUN nix profile install .#wal-g-3 && \ RUN nix store gc +WORKDIR / +#################### +# setup-groonga +#################### +FROM base as groonga + +WORKDIR /nixpg + +RUN nix profile install .#supabase-groonga && \ + mkdir -p /tmp/groonga-plugins && \ + cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ + +RUN nix store gc + WORKDIR / # #################### # # Download gosu for easy step-down from root @@ -157,6 +171,7 @@ FROM gosu as production RUN id postgres || (echo "postgres user does not exist" && exit 1) # # Setup extensions COPY --from=walg /tmp/wal-g /usr/local/bin/ +COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins # # Initialise configs COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf @@ -235,4 +250,6 @@ RUN mkdir -p /usr/share/postgresql/extension/ && \ ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh +ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins + CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/ansible/vars.yml b/ansible/vars.yml index 7da38b910..33ea789d2 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.049-orioledb" - postgres17: "17.6.1.028" - postgres15: "15.14.1.028" + postgresorioledb-17: "17.5.1.050-orioledb" + postgres17: "17.6.1.029" + postgres15: "15.14.1.029" # Non Postgres Extensions pgbouncer_release: 1.19.0 diff --git a/flake.nix b/flake.nix index db14dac9a..de9048fe5 100644 --- a/flake.nix +++ b/flake.nix @@ -30,7 +30,6 @@ nix/checks.nix nix/config.nix nix/devShells.nix - nix/ext nix/fmt.nix nix/hooks.nix nix/nixpkgs.nix diff --git a/nix/checks.nix b/nix/checks.nix index 74b50a84f..aaf7fffce 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -13,7 +13,6 @@ psql_15 = self'.packages."psql_15/bin"; psql_17 = self'.packages."psql_17/bin"; psql_orioledb-17 = self'.packages."psql_orioledb-17/bin"; - pgroonga = self'.packages."psql_15/exts/pgroonga"; inherit (self.supabase) defaults; }; in @@ -68,9 +67,36 @@ ''; }; + # Get the major version for filtering + majorVersion = + let + version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version; + isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null; + isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null; + result = + if isOrioledbMatch then + "orioledb-17" + else if isSeventeenMatch then + "17" + else + "15"; + in + builtins.trace "Major version result: ${result}" result; + + # Select the appropriate pgroonga package for this PostgreSQL version + pgroonga = self'.packages."psql_${majorVersion}/exts/pgroonga-all"; + + pgPort = + if (majorVersion == "17") then + "5535" + else if (majorVersion == "15") then + "5536" + else + "5537"; + # Use the shared setup but with a test-specific name start-postgres-server-bin = pkgs-lib.makePostgresDevSetup { - inherit pkgs; + inherit pkgs pgroonga; name = "start-postgres-server-test"; extraSubstitutions = { PGSODIUM_GETKEY = "${getkey-script}/bin/pgsodium-getkey"; @@ -116,33 +142,9 @@ in pkgs.lib.filterAttrs (name: _: isValidFile name) files; - # Get the major version for filtering - majorVersion = - let - version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version; - isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null; - isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null; - result = - if isOrioledbMatch then - "orioledb-17" - else if isSeventeenMatch then - "17" - else - "15"; - in - builtins.trace "Major version result: ${result}" result; # Trace the result # For "15.8" - # Filter SQL test files filteredSqlTests = filterTestFiles majorVersion ./tests/sql; - pgPort = - if (majorVersion == "17") then - "5535" - else if (majorVersion == "15") then - "5536" - else - "5537"; - # Convert filtered tests to a sorted list of basenames (without extension) testList = pkgs.lib.mapAttrsToList ( name: _: builtins.substring 0 (pkgs.lib.stringLength name - 4) name diff --git a/nix/ext/default.nix b/nix/ext/default.nix deleted file mode 100644 index 9cd3de8bc..000000000 --- a/nix/ext/default.nix +++ /dev/null @@ -1,11 +0,0 @@ -{ ... }: -{ - perSystem = - { pkgs, ... }: - { - packages = { - sfcgal = pkgs.callPackage ./sfcgal/sfcgal.nix { }; - mecab_naist_jdic = pkgs.callPackage ./mecab-naist-jdic/default.nix { }; - }; - }; -} diff --git a/nix/ext/pgroonga.nix b/nix/ext/pgroonga.nix deleted file mode 100644 index d57efc8fb..000000000 --- a/nix/ext/pgroonga.nix +++ /dev/null @@ -1,95 +0,0 @@ -{ - lib, - stdenv, - fetchurl, - pkg-config, - postgresql, - msgpack-c, - mecab, - makeWrapper, - xxHash, - supabase-groonga, -}: -stdenv.mkDerivation rec { - pname = "pgroonga"; - version = "3.2.5"; - src = fetchurl { - url = "https://packages.groonga.org/source/${pname}/${pname}-${version}.tar.gz"; - sha256 = "sha256-GM9EOQty72hdE4Ecq8jpDudhZLiH3pP9ODLxs8DXcSY="; - }; - - nativeBuildInputs = [ - pkg-config - makeWrapper - ]; - - buildInputs = [ - postgresql - msgpack-c - supabase-groonga - mecab - ] ++ lib.optionals stdenv.isDarwin [ xxHash ]; - - propagatedBuildInputs = [ supabase-groonga ]; - configureFlags = [ - "--with-mecab=${mecab}" - "--enable-mecab" - "--with-groonga=${supabase-groonga}" - "--with-groonga-plugin-dir=${supabase-groonga}/lib/groonga/plugins" - ]; - - makeFlags = [ - "HAVE_MSGPACK=1" - "MSGPACK_PACKAGE_NAME=msgpack-c" - "HAVE_MECAB=1" - ]; - - NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin ( - builtins.concatStringsSep " " [ - "-Wno-error=incompatible-function-pointer-types" - "-Wno-error=format" - "-Wno-format" - "-I${supabase-groonga}/include/groonga" - "-I${xxHash}/include" - "-DPGRN_VERSION=\"${version}\"" - ] - ); - - preConfigure = '' - export GROONGA_LIBS="-L${supabase-groonga}/lib -lgroonga" - export GROONGA_CFLAGS="-I${supabase-groonga}/include" - export MECAB_CONFIG="${mecab}/bin/mecab-config" - ${lib.optionalString stdenv.isDarwin '' - export CPPFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" - export CFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" - export PG_CPPFLAGS="-Wno-error=incompatible-function-pointer-types -Wno-error=format" - ''} - ''; - - installPhase = '' - mkdir -p $out/lib $out/share/postgresql/extension $out/bin - install -D pgroonga${postgresql.dlSuffix} -t $out/lib/ - install -D pgroonga.control -t $out/share/postgresql/extension - install -D data/pgroonga-*.sql -t $out/share/postgresql/extension - install -D pgroonga_database${postgresql.dlSuffix} -t $out/lib/ - install -D pgroonga_database.control -t $out/share/postgresql/extension - install -D data/pgroonga_database-*.sql -t $out/share/postgresql/extension - - echo "Debug: Groonga plugins directory contents:" - ls -l ${supabase-groonga}/lib/groonga/plugins/tokenizers/ - ''; - - meta = with lib; { - description = "A PostgreSQL extension to use Groonga as the index"; - longDescription = '' - PGroonga is a PostgreSQL extension to use Groonga as the index. - PostgreSQL supports full text search against languages that use only alphabet and digit. - It means that PostgreSQL doesn't support full text search against Japanese, Chinese and so on. - You can use super fast full text search feature against all languages by installing PGroonga into your PostgreSQL. - ''; - homepage = "https://pgroonga.github.io/"; - changelog = "https://github.com/pgroonga/pgroonga/releases/tag/${version}"; - license = licenses.postgresql; - platforms = postgresql.meta.platforms; - }; -} diff --git a/nix/ext/pgroonga/default.nix b/nix/ext/pgroonga/default.nix new file mode 100644 index 000000000..55c0abedd --- /dev/null +++ b/nix/ext/pgroonga/default.nix @@ -0,0 +1,178 @@ +{ + lib, + stdenv, + fetchurl, + pkg-config, + postgresql, + msgpack-c, + mecab, + makeWrapper, + xxHash, + buildEnv, + supabase-groonga, + mecab-naist-jdic, +}: +let + pname = "pgroonga"; + + # Load version configuration from external file + allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).${pname}; + + # Filter versions compatible with current PostgreSQL version + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; + + # Derived version information + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); + + # List of C extensions to be included in the build + cExtensions = [ + "pgroonga" + "pgroonga_database" + ]; + + # Build function for individual versions + build = + version: hash: + stdenv.mkDerivation rec { + inherit pname version; + + src = fetchurl { + url = "https://packages.groonga.org/source/${pname}/${pname}-${version}.tar.gz"; + inherit hash; + }; + nativeBuildInputs = [ + pkg-config + makeWrapper + ]; + + buildInputs = [ + postgresql + msgpack-c + supabase-groonga + mecab + ] ++ lib.optionals stdenv.isDarwin [ xxHash ]; + + propagatedBuildInputs = [ + supabase-groonga + mecab-naist-jdic + ]; + configureFlags = [ + "--with-mecab=${mecab}" + "--enable-mecab" + "--with-groonga=${supabase-groonga}" + "--with-groonga-plugin-dir=${supabase-groonga}/lib/groonga/plugins" + ]; + + makeFlags = [ + "HAVE_MSGPACK=1" + "MSGPACK_PACKAGE_NAME=msgpack-c" + "HAVE_MECAB=1" + ]; + + NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin ( + builtins.concatStringsSep " " [ + "-Wno-error=incompatible-function-pointer-types" + "-Wno-error=format" + "-Wno-format" + "-I${supabase-groonga}/include/groonga" + "-I${xxHash}/include" + "-DPGRN_VERSION=\"${version}\"" + ] + ); + + preConfigure = '' + export GROONGA_LIBS="-L${supabase-groonga}/lib -lgroonga" + export GROONGA_CFLAGS="-I${supabase-groonga}/include" + export MECAB_CONFIG="${mecab}/bin/mecab-config" + export MECAB_DICDIR="${mecab-naist-jdic}/lib/mecab/dic/naist-jdic" + ${lib.optionalString stdenv.isDarwin '' + export CPPFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" + export CFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" + export PG_CPPFLAGS="-Wno-error=incompatible-function-pointer-types -Wno-error=format" + ''} + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/{lib,share/postgresql/extension} + + for ext in ${lib.concatStringsSep " " cExtensions}; do + # Install shared library with version suffix + mv $ext${postgresql.dlSuffix} $out/lib/$ext-${version}${postgresql.dlSuffix} + + # Create version-specific control file + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/$ext'|" \ + $ext.control > $out/share/postgresql/extension/$ext--${version}.control + + # Copy SQL file to install the specific version + cp data/$ext--${version}.sql $out/share/postgresql/extension + + # Create versioned control file with modified module path + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/$ext'|" \ + $ext.control > $out/share/postgresql/extension/$ext--${version}.control + + # For the latest version, create default control file and symlink and copy SQL upgrade scripts + if [[ "${version}" == "${latestVersion}" ]]; then + { + echo "default_version = '${version}'" + cat $out/share/postgresql/extension/$ext--${version}.control + } > $out/share/postgresql/extension/$ext.control + ln -sfn $ext-${version}${postgresql.dlSuffix} $out/lib/$ext${postgresql.dlSuffix} + cp data/$ext--*--*.sql $out/share/postgresql/extension + fi + done + ''; + + meta = with lib; { + description = "A PostgreSQL extension to use Groonga as the index"; + longDescription = '' + PGroonga is a PostgreSQL extension to use Groonga as the index. + PostgreSQL supports full text search against languages that use only alphabet and digit. + It means that PostgreSQL doesn't support full text search against Japanese, Chinese and so on. + You can use super fast full text search feature against all languages by installing PGroonga into your PostgreSQL. + ''; + homepage = "https://pgroonga.github.io/"; + changelog = "https://github.com/pgroonga/pgroonga/releases/tag/${version}"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; +in +buildEnv { + name = pname; + paths = packages; + + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; + postBuild = '' + # Verify all expected library files are present + expectedFiles=${toString ((numberOfVersions + 1) * (builtins.length cExtensions))} + actualFiles=$(ls -l $out/lib/${pname}*${postgresql.dlSuffix} | wc -l) + + if [[ "$actualFiles" != "$expectedFiles" ]]; then + echo "Error: Expected $expectedFiles library files, found $actualFiles" + echo "Files found:" + ls -la $out/lib/*${postgresql.dlSuffix} || true + exit 1 + fi + ''; + + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); + }; +} diff --git a/nix/ext/pgroonga/do-not-use-vendored-libraries.patch b/nix/ext/pgroonga/do-not-use-vendored-libraries.patch new file mode 100644 index 000000000..6a005349d --- /dev/null +++ b/nix/ext/pgroonga/do-not-use-vendored-libraries.patch @@ -0,0 +1,15 @@ +Do not use vendored libraries + +--- a/vendor/CMakeLists.txt ++++ b/vendor/CMakeLists.txt +@@ -14,10 +14,7 @@ + # License along with this library; if not, write to the Free Software + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + add_subdirectory(onigmo) +-add_subdirectory(mruby) +-add_subdirectory(mecab) +-add_subdirectory(message_pack) + if(GRN_WITH_MRUBY) + add_subdirectory(groonga-log) + endif() \ No newline at end of file diff --git a/nix/ext/pgroonga/fix-cmake-install-path.patch b/nix/ext/pgroonga/fix-cmake-install-path.patch new file mode 100644 index 000000000..1fe317b6c --- /dev/null +++ b/nix/ext/pgroonga/fix-cmake-install-path.patch @@ -0,0 +1,21 @@ +Fix CMake install path + +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1141,11 +1141,11 @@ + + set(prefix "${CMAKE_INSTALL_PREFIX}") + set(exec_prefix "\${prefix}") +-set(bindir "\${exec_prefix}/${CMAKE_INSTALL_BINDIR}") +-set(sbindir "\${exec_prefix}/${CMAKE_INSTALL_SBINDIR}") +-set(libdir "\${prefix}/${CMAKE_INSTALL_LIBDIR}") +-set(includedir "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") +-set(datarootdir "\${prefix}/${CMAKE_INSTALL_DATAROOTDIR}") ++set(bindir "${CMAKE_INSTALL_FULL_BINDIR}") ++set(sbindir "${CMAKE_INSTALL_FULL_SBINDIR}") ++set(libdir "${CMAKE_INSTALL_FULL_LIBDIR}") ++set(includedir "${CMAKE_INSTALL_FULL_INCLUDEDIR}") ++set(datarootdir "${CMAKE_INSTALL_FULL_DATAROOTDIR}") + set(datadir "\${datarootdir}") + set(expanded_pluginsdir "${GRN_PLUGINS_DIR}") + set(GRN_EXPANDED_DEFAULT_DOCUMENT_ROOT "${GRN_DEFAULT_DOCUMENT_ROOT}") \ No newline at end of file diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index 6a152ef43..ed1b738e4 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -19,7 +19,7 @@ }: let - sfcgal = callPackage ./sfcgal/sfcgal.nix { }; + sfcgal = callPackage ../packages/sfcgal.nix { }; gdal = callPackage ./gdal.nix { inherit postgresql; }; pname = "postgis"; diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index 9a379d587..51c22ff0e 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -131,7 +131,7 @@ let "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" - support_upgrade = True + support_upgrade = False pg17_configuration = "${pg17-configuration}" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" diff --git a/nix/ext/tests/pgroonga.nix b/nix/ext/tests/pgroonga.nix new file mode 100644 index 000000000..dd15b918d --- /dev/null +++ b/nix/ext/tests/pgroonga.nix @@ -0,0 +1,177 @@ +{ self, pkgs }: +let + pname = "pgroonga"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; + psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = psql_15; + }; + systemd.services.postgresql.environment.MECAB_DICDIR = "${ + self.packages.${pkgs.system}.mecab-naist-jdic + }/lib/mecab/dic/naist-jdic"; + systemd.services.postgresql.environment.MECAB_CONFIG = "${pkgs.mecab}/bin/mecab-config"; + systemd.services.postgresql.environment.GRN_PLUGINS_DIR = "${ + self.packages.${pkgs.system}.supabase-groonga + }/lib/groonga/plugins"; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce psql_17; + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = psql_15; + newPostgresql = psql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + from pathlib import Path + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + extension_name = "${pname}" + support_upgrade = False + pg17_configuration = "${pg17-configuration}" + ext_has_background_worker = ${ + if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" + } + sql_test_directory = Path("${../../tests}") + pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}" + + ${builtins.readFile ./lib.py} + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade) + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") + + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + if ext_has_background_worker: + with subtest("Test switch_${pname}_version"): + test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15") + + with subtest("Check pg_regress with postgresql 15 after installing the last version"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + with subtest("switch to postgresql 17"): + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check last version of the extension after postgresql upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Check the install of the last version of the extension"): + test.check_install_last_version("17") + + with subtest("Check pg_regress with postgresql 17 after installing the last version"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + ''; +} diff --git a/nix/ext/use-system-groonga.patch b/nix/ext/use-system-groonga.patch deleted file mode 100644 index 6d3042bc8..000000000 --- a/nix/ext/use-system-groonga.patch +++ /dev/null @@ -1,21 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 33b34477..f4ffefe5 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -12,7 +12,6 @@ if(MSVC_VERSION LESS 1800) - message(FATAL_ERROR "PGroonga supports only MSVC 2013 or later") - endif() - --add_subdirectory(vendor/groonga) - - set(PGRN_POSTGRESQL_DIR "${CMAKE_INSTALL_PREFIX}" - CACHE PATH "PostgreSQL binary directory") -@@ -52,8 +51,6 @@ string(REGEX REPLACE "([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\3" - string(REGEX REPLACE ".*comment = '([^']+)'.*" "\\1" - PGRN_DESCRIPTION "${PGRN_CONTROL}") - --file(READ "${CMAKE_CURRENT_SOURCE_DIR}/vendor/groonga/bundled_message_pack_version" -- PGRN_BUNDLED_MESSAGE_PACK_VERSION) - string(STRIP - "${PGRN_BUNDLED_MESSAGE_PACK_VERSION}" - PGRN_BUNDLED_MESSAGE_PACK_VERSION) \ No newline at end of file diff --git a/nix/ext/versions.json b/nix/ext/versions.json index aa8c2c9ca..be81e806d 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -336,6 +336,21 @@ "hash": "sha256-IU+i6ONPwtgsFKdzya6E+222ualR66gkbb0lDr+7Rb8=" } }, + "pgroonga": { + "3.0.7": { + "postgresql": [ + "15" + ], + "hash": "sha256-iF/zh4zDDpAw5fxW1WG8i2bfPt4VYsnYArwOoE/lwgM=" + }, + "3.2.5": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-GM9EOQty72hdE4Ecq8jpDudhZLiH3pP9ODLxs8DXcSY=" + } + }, "pgrouting": { "3.4.1": { "postgresql": [ @@ -345,6 +360,7 @@ "hash": "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k=" } }, + "pgsodium": { "3.0.4": { "postgresql": [ diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix index b508af8e3..26beece08 100644 --- a/nix/overlays/default.nix +++ b/nix/overlays/default.nix @@ -12,6 +12,7 @@ postgresql_orioledb-17 supabase-groonga switch-ext-version + mecab-naist-jdic ; xmrig = throw "The xmrig package has been explicitly disabled in this flake."; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index f297c8359..5aed24af9 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -23,7 +23,6 @@ psql_15 = self'.packages."psql_15/bin"; psql_17 = self'.packages."psql_17/bin"; psql_orioledb-17 = self'.packages."psql_orioledb-17/bin"; - pgroonga = self'.packages."psql_15/exts/pgroonga"; inherit (self.supabase) defaults; }; in @@ -37,6 +36,7 @@ supabase-groonga = pkgs.callPackage ./groonga { }; http-mock-server = pkgs.callPackage ./http-mock-server.nix { }; local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { }; + mecab-naist-jdic = pkgs.callPackage ./mecab-naist-jdic.nix { }; migrate-tool = pkgs.callPackage ./migrate-tool.nix { psql_15 = self'.packages."psql_15/bin"; }; overlayfs-on-package = pkgs.callPackage ./overlayfs-on-package.nix { }; packer = pkgs.callPackage ./packer.nix { inherit inputs; }; @@ -44,6 +44,7 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; pg_regress = makePgRegress activeVersion; run-testinfra = pkgs.callPackage ./run-testinfra.nix { }; + sfcgal = pkgs.callPackage ./sfcgal.nix { }; show-commands = pkgs.callPackage ./show-commands.nix { }; start-client = pkgs.callPackage ./start-client.nix { psql_15 = self'.packages."psql_15/bin"; @@ -58,6 +59,7 @@ start-server = pkgs-lib.makePostgresDevSetup { inherit pkgs; name = "start-postgres-server"; + pgroonga = self'.packages."psql_${activeVersion}/exts/pgroonga-all"; }; switch-ext-version = pkgs.callPackage ./switch-ext-version.nix { inherit (self'.packages) overlayfs-on-package; diff --git a/nix/packages/groonga/default.nix b/nix/packages/groonga/default.nix index 3fcd4b691..bef491109 100644 --- a/nix/packages/groonga/default.nix +++ b/nix/packages/groonga/default.nix @@ -18,11 +18,8 @@ lz4, zlibSupport ? true, zlib, - callPackage, + mecab-naist-jdic, }: -let - mecab-naist-jdic = callPackage ../../ext/mecab-naist-jdic { }; -in stdenv.mkDerivation (finalAttrs: { pname = "supabase-groonga"; version = "14.0.5"; diff --git a/nix/packages/lib.nix b/nix/packages/lib.nix index 971909162..0bd3d071e 100644 --- a/nix/packages/lib.nix +++ b/nix/packages/lib.nix @@ -5,13 +5,13 @@ defaults, supabase-groonga, system, - pgroonga, }: { makePostgresDevSetup = { pkgs, name, + pgroonga, extraSubstitutions ? { }, }: let diff --git a/nix/ext/mecab-naist-jdic/default.nix b/nix/packages/mecab-naist-jdic.nix similarity index 100% rename from nix/ext/mecab-naist-jdic/default.nix rename to nix/packages/mecab-naist-jdic.nix diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 576e82730..170180188 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -15,7 +15,7 @@ ourExtensions = [ ../ext/rum.nix ../ext/timescaledb.nix - ../ext/pgroonga.nix + ../ext/pgroonga ../ext/index_advisor.nix ../ext/wal2json.nix ../ext/pgmq diff --git a/nix/ext/sfcgal/sfcgal.nix b/nix/packages/sfcgal.nix similarity index 100% rename from nix/ext/sfcgal/sfcgal.nix rename to nix/packages/sfcgal.nix diff --git a/nix/tests/expected/pgroonga.out b/nix/tests/expected/pgroonga.out new file mode 100644 index 000000000..5ceeed254 --- /dev/null +++ b/nix/tests/expected/pgroonga.out @@ -0,0 +1,76 @@ +create schema v; +create table v.roon( + id serial primary key, + content text +); +with tokenizers as ( + select + x + from + jsonb_array_elements( + (select pgroonga_command('tokenizer_list'))::jsonb + ) x(val) + limit + 1 + offset + 1 -- first record is unrelated and not stable +) +select + t.x::jsonb ->> 'name' +from + jsonb_array_elements((select * from tokenizers)) t(x) +order by + t.x::jsonb ->> 'name'; + ?column? +--------------------------------------------- + TokenBigram + TokenBigramIgnoreBlank + TokenBigramIgnoreBlankSplitSymbol + TokenBigramIgnoreBlankSplitSymbolAlpha + TokenBigramIgnoreBlankSplitSymbolAlphaDigit + TokenBigramSplitSymbol + TokenBigramSplitSymbolAlpha + TokenBigramSplitSymbolAlphaDigit + TokenDelimit + TokenDelimitNull + TokenDocumentVectorBM25 + TokenDocumentVectorTFIDF + TokenMecab + TokenNgram + TokenPattern + TokenRegexp + TokenTable + TokenTrigram + TokenUnigram +(19 rows) + +insert into v.roon (content) +values + ('Hello World'), + ('PostgreSQL with PGroonga is a thing'), + ('This is a full-text search test'), + ('PGroonga supports various languages'); +-- Create default index +create index pgroonga_index on v.roon using pgroonga (content); +-- Create mecab tokenizer index since we had a bug with this one once +create index pgroonga_index_mecab on v.roon using pgroonga (content) with (tokenizer='TokenMecab'); +-- Run some queries to test the index +select * from v.roon where content &@~ 'Hello'; + id | content +----+------------- + 1 | Hello World +(1 row) + +select * from v.roon where content &@~ 'powerful'; + id | content +----+--------- +(0 rows) + +select * from v.roon where content &@~ 'supports'; + id | content +----+------------------------------------- + 4 | PGroonga supports various languages +(1 row) + +drop schema v cascade; +NOTICE: drop cascades to table v.roon diff --git a/nix/tests/sql/z_15_pgroonga.sql b/nix/tests/sql/pgroonga.sql similarity index 100% rename from nix/tests/sql/z_15_pgroonga.sql rename to nix/tests/sql/pgroonga.sql