From 45e1b1a5bb60817d8d282082a9b0f8cab5cac82d Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 28 Jul 2026 11:14:33 +0200 Subject: [PATCH] GlobalOrdinalsWithScoreCollector to accumulate scores in double precision Accumulate join Total/Avg scores in double precision in GlobalOrdinalsWithScoreCollector, fixing an intermittent failure caused by non-associative float addition that #16378 missed. Closes #16408 --- lucene/CHANGES.txt | 4 ++ .../GlobalOrdinalsWithScoreCollector.java | 70 ++++++++++--------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 5916cb97d087..cc1166b5b3f3 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -99,6 +99,10 @@ Bug Fixes including the default codec. Since 10.4.0 the merger tested the per-field wrapper instead of the unwrapped reader, so every merge rebuilt the graph from scratch. (Jeho Jeong) +* GITHUB#16408: Accumulate join Total/Avg scores in double precision in + GlobalOrdinalsWithScoreCollector, fixing an intermittent failure caused by + non-associative float addition that #16378 missed. (Luca Cavanna) + Other --------------------- * GITHUB#16266: Remove deprecated search(Query, Collector) calls in QueryUtils by replacing diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java index 27becea124e2..d91a2f944b86 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java @@ -77,12 +77,12 @@ public boolean match(int globalOrd) { } public float score(int globalOrdinal) { - return scores.getScore(globalOrdinal); + return (float) scores.getScore(globalOrdinal); } - protected abstract void doScore(int globalOrd, float existingScore, float newScore); + protected abstract void doScore(int globalOrd, double existingScore, double newScore); - protected abstract float unset(); + protected abstract double unset(); @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { @@ -116,8 +116,8 @@ public void collect(int doc) throws IOException { if (docTermOrds.advanceExact(doc)) { final int globalOrd = (int) segmentOrdToGlobalOrdLookup.get(docTermOrds.ordValue()); collectedOrds.set(globalOrd); - float existingScore = scores.getScore(globalOrd); - float newScore = scorer.score(); + double existingScore = scores.getScore(globalOrd); + double newScore = scorer.score(); doScore(globalOrd, existingScore, newScore); if (occurrences != null) { occurrences.increment(globalOrd); @@ -145,8 +145,8 @@ public void collect(int doc) throws IOException { if (docTermOrds.advanceExact(doc)) { int segmentOrd = docTermOrds.ordValue(); collectedOrds.set(segmentOrd); - float existingScore = scores.getScore(segmentOrd); - float newScore = scorer.score(); + double existingScore = scores.getScore(segmentOrd); + double newScore = scorer.score(); doScore(segmentOrd, existingScore, newScore); if (occurrences != null) { occurrences.increment(segmentOrd); @@ -167,13 +167,13 @@ public Min(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma } @Override - protected void doScore(int globalOrd, float existingScore, float newScore) { + protected void doScore(int globalOrd, double existingScore, double newScore) { scores.setScore(globalOrd, Math.min(existingScore, newScore)); } @Override - protected float unset() { - return Float.POSITIVE_INFINITY; + protected double unset() { + return Double.POSITIVE_INFINITY; } } @@ -184,13 +184,13 @@ public Max(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma } @Override - protected void doScore(int globalOrd, float existingScore, float newScore) { + protected void doScore(int globalOrd, double existingScore, double newScore) { scores.setScore(globalOrd, Math.max(existingScore, newScore)); } @Override - protected float unset() { - return Float.NEGATIVE_INFINITY; + protected double unset() { + return Double.NEGATIVE_INFINITY; } } @@ -201,13 +201,13 @@ public Sum(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma } @Override - protected void doScore(int globalOrd, float existingScore, float newScore) { + protected void doScore(int globalOrd, double existingScore, double newScore) { scores.setScore(globalOrd, existingScore + newScore); } @Override - protected float unset() { - return 0f; + protected double unset() { + return 0d; } } @@ -218,18 +218,18 @@ public Avg(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma } @Override - protected void doScore(int globalOrd, float existingScore, float newScore) { + protected void doScore(int globalOrd, double existingScore, double newScore) { scores.setScore(globalOrd, existingScore + newScore); } @Override public float score(int globalOrdinal) { - return scores.getScore(globalOrdinal) / occurrences.getOccurrence(globalOrdinal); + return (float) (scores.getScore(globalOrdinal) / occurrences.getOccurrence(globalOrdinal)); } @Override - protected float unset() { - return 0f; + protected double unset() { + return 0d; } } @@ -276,7 +276,7 @@ public void collect(int doc) throws IOException { } @Override - protected void doScore(int globalOrd, float existingScore, float newScore) {} + protected void doScore(int globalOrd, double existingScore, double newScore) {} @Override public float score(int globalOrdinal) { @@ -284,8 +284,8 @@ public float score(int globalOrdinal) { } @Override - protected float unset() { - return 0f; + protected double unset() { + return 0d; } @Override @@ -309,33 +309,35 @@ public org.apache.lucene.search.ScoreMode scoreMode() { static final class Scores { - final float[][] blocks; - final float unset; + // Accumulated in double precision because float addition isn't associative, so summing the + // same per-document scores in a different order may round to a different float. + final double[][] blocks; + final double unset; - private Scores(long valueCount, float unset) { + private Scores(long valueCount, double unset) { long blockSize = valueCount + arraySize - 1; - blocks = new float[(int) ((blockSize) / arraySize)][]; + blocks = new double[(int) ((blockSize) / arraySize)][]; this.unset = unset; } - public void setScore(int globalOrdinal, float score) { + public void setScore(int globalOrdinal, double score) { int block = globalOrdinal / arraySize; int offset = globalOrdinal % arraySize; - float[] scores = blocks[block]; + double[] scores = blocks[block]; if (scores == null) { - blocks[block] = scores = new float[arraySize]; - if (unset != 0f) { + blocks[block] = scores = new double[arraySize]; + if (unset != 0d) { Arrays.fill(scores, unset); } } scores[offset] = score; } - public float getScore(int globalOrdinal) { + public double getScore(int globalOrdinal) { int block = globalOrdinal / arraySize; int offset = globalOrdinal % arraySize; - float[] scores = blocks[block]; - float score; + double[] scores = blocks[block]; + double score; if (scores != null) { score = scores[offset]; } else {