Introduce FirstPassGroupingCollectorManager #15574

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

javanna merged 18 commits into apache:main from gaobinlong:firstPassManager

Jun 2, 2026

lucene/CHANGES.txt

-Original file line number
+Diff line change
@@ Expand Up / @@ -362,7 +362,7 @@ Improvements @@
     * GITHUB#15976: Store DocValuesSkipper data in their own file. (Alan Woodward)
-    * GITHUB#15574, GITHUB#15995: Introduce TopGroupsCollectorManager to parallelize search when using TopGroupsCollector. (Binlong Gao)
+    * GITHUB#15603, GITHUB#15995: Introduce TopGroupsCollectorManager to parallelize search when using TopGroupsCollector. (Binlong Gao)
     * GITHUB#15565: Introduce AllGroupHeadsCollectorManager to parallelize search when using AllGroupHeadsCollector. (Binlong Gao)
@@ Expand All / @@ -371,6 +371,8 @@ Improvements @@
     * GITHUB#16000: Clarify that Accountable#ramBytesUsed() reports JVM heap memory only. (Luca Cavanna)
+    * GITHUB#15574: Introduce FirstPassGroupingCollectorManager to parallelize search when using FirstPassGroupingCollector. (Binlong Gao)
     * GITHUB#15660: Introduce LargeNumHitsTopDocsCollectorManager to parallelize search when using LargeNumHitsTopDocsCollector. (Binlong Gao)
     * GITHUB#16088: SearchGroup#merge now returns an empty collection instead of null when topGroups is empty. (Luca Cavanna)
@@ Expand Down @@

...rouping/src/java/org/apache/lucene/search/grouping/FirstPassGroupingCollectorManager.java

-Original file line number
+Diff line change
@@ -0,0 +1,127 @@
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    package org.apache.lucene.search.grouping;
+    import java.io.IOException;
+    import java.util.ArrayList;
+    import java.util.Collection;
+    import java.util.List;
+    import java.util.function.Supplier;
+    import org.apache.lucene.search.CollectorManager;
+    import org.apache.lucene.search.Sort;
+    /**
+     * A CollectorManager implementation for {@link FirstPassGroupingCollector} that supports parallel
+     * collection and merges results across segments.
+     *
+     * <p>Example usage:
+     *
+     * <pre class="prettyprint">
+     * IndexSearcher searcher = new IndexSearcher(reader);
+     * Sort groupSort = Sort.RELEVANCE;
+     * int topNGroups = 10;
+     *
+     * FirstPassGroupingCollectorManager&lt;BytesRef&gt; manager =
+     *     new FirstPassGroupingCollectorManager&lt;&gt;(
+     *         () -&gt; new TermGroupSelector("category"),
+     *         groupSort,
+     *         0,
+     *         topNGroups);
+     *
+     * Collection&lt;SearchGroup&lt;BytesRef&gt;&gt; searchGroups = searcher.search(query, manager);
+     *
+     * // searchGroups can then be passed to a second pass collector manager like TopGroupsCollectorManager for full group results
+     * </pre>
+     *
+     * @lucene.experimental
+     */
+    public class FirstPassGroupingCollectorManager<T>
+        implements CollectorManager<FirstPassGroupingCollector<T>, Collection<SearchGroup<T>>> {
+      private final Supplier<GroupSelector<T>> groupSelectorFactory;
+      private final Sort groupSort;
+      private final int groupOffset;
+      private final int topNGroups;
+      private final boolean ignoreDocsWithoutGroupField;
+      /**
+       * Creates a new FirstPassGroupingCollectorManager.
+       *
+       * @param groupSelectorFactory factory to create group selectors for each collector
+       * @param groupSort the sort to use for groups
+       * @param groupOffset The offset in the collected groups
+       * @param topNGroups the number of top groups to collect
+       */
+      public FirstPassGroupingCollectorManager(
+          Supplier<GroupSelector<T>> groupSelectorFactory,
+          Sort groupSort,
+          int groupOffset,
+          int topNGroups) {
+        this(groupSelectorFactory, groupSort, groupOffset, topNGroups, false);
+      }
+      /**
+       * Creates a new FirstPassGroupingCollectorManager.
+       *
+       * @param groupSelectorFactory factory to create group selectors for each collector
+       * @param groupSort the sort to use for groups
+       * @param groupOffset The offset in the collected groups
+       * @param topNGroups the number of top groups to collect
+       * @param ignoreDocsWithoutGroupField whether to ignore documents without a group field
+       */
+      public FirstPassGroupingCollectorManager(
+          Supplier<GroupSelector<T>> groupSelectorFactory,
+          Sort groupSort,
+          int groupOffset,
+          int topNGroups,
+          boolean ignoreDocsWithoutGroupField) {
+        if (groupOffset < 0) {
+          throw new IllegalArgumentException("groupOffset must be >= 0 (got " + groupOffset + ")");
+        }
+        if (topNGroups < 1) {
+          throw new IllegalArgumentException("topNGroups must be >= 1 (got " + topNGroups + ")");
+        }
+        this.groupSelectorFactory = groupSelectorFactory;
+        this.groupSort = groupSort;
+        this.groupOffset = groupOffset;
+        this.topNGroups = topNGroups;
+        this.ignoreDocsWithoutGroupField = ignoreDocsWithoutGroupField;
+      }
+      @Override
+      public FirstPassGroupingCollector<T> newCollector() throws IOException {
+        return new FirstPassGroupingCollector<>(
+            groupSelectorFactory.get(),
+            groupSort,
+            groupOffset + topNGroups,
+            ignoreDocsWithoutGroupField);
+      }
+      @Override
+      public Collection<SearchGroup<T>> reduce(Collection<FirstPassGroupingCollector<T>> collectors)
+          throws IOException {
+        final List<Collection<SearchGroup<T>>> allGroups = new ArrayList<>();
+        for (FirstPassGroupingCollector<T> collector : collectors) {
+          Collection<SearchGroup<T>> groups = collector.getTopGroups(0);
+          if (groups != null) {
+            allGroups.add(groups);
+          }
+        }
+        return SearchGroup.merge(allGroups, groupOffset, topNGroups, groupSort);
+      }
+    }

lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -309,17 +309,17 @@ public void testShardedGrouping() throws IOException {
  
        // A grouped query run in two phases against the control should give us the same

        // result as the query run against shards and merged back together after each phase.

        FirstPassGroupingCollector<T> singletonFirstPass =

            new FirstPassGroupingCollector<>(getGroupSelector(), sort, 5);

        control.getIndexSearcher().search(topLevel, singletonFirstPass);

        Collection<SearchGroup<T>> singletonGroups = singletonFirstPass.getTopGroups(0);

        FirstPassGroupingCollectorManager<T> firstPassGroupingCollectorManager =

            new FirstPassGroupingCollectorManager<>(this::getGroupSelector, sort, 0, 5);

        Collection<SearchGroup<T>> singletonGroups =

            control.getIndexSearcher().search(topLevel, firstPassGroupingCollectorManager);

        List<Collection<SearchGroup<T>>> shardGroups = new ArrayList<>();

        for (Shard shard : shards) {

          FirstPassGroupingCollector<T> fc =

              new FirstPassGroupingCollector<>(getGroupSelector(), sort, 5);

          shard.getIndexSearcher().search(topLevel, fc);

          shardGroups.add(fc.getTopGroups(0));

          FirstPassGroupingCollectorManager<T> fcm =

              new FirstPassGroupingCollectorManager<>(this::getGroupSelector, sort, 0, 5);

          Collection<SearchGroup<T>> topGroups = shard.getIndexSearcher().search(topLevel, fcm);

          shardGroups.add(topGroups);

        }

        Collection<SearchGroup<T>> mergedGroups = SearchGroup.merge(shardGroups, 0, 5, sort);

        assertEquals(singletonGroups, mergedGroups);

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Introduce FirstPassGroupingCollectorManager #15574

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!