Add a de-duplicating flat vector format

lucene/CHANGES.txt

@@ -330,6 +330,10 @@ New Features
   and document length instead of corpus statistics such as document frequency. It also supports k3
   query-term frequency saturation. (Tianxiao Wei)
 
+* GITHUB#15979: Add a sandboxed de-duplicating HNSW vector format (DedupHnswVectorsFormat) that stores
+  each distinct full-precision vector once, shared across all documents and fields that reference it.
+  (Kaival Parikh)
+
 Improvements
 ---------------------
 

lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java

@@ -236,10 +236,10 @@ public void finish() throws IOException {
 
   @Override
   public long ramBytesUsed() {
-    long total = SHALLOW_RAM_BYTES_USED;
+    long total = SHALLOW_RAM_BYTES_USED + flatVectorWriter.ramBytesUsed();
     for (FieldWriter<?> field : fields) {
       // the field tracks the delegate field usage
-      total += field.ramBytesUsed();
+      total += field.ownRamBytesUsed();
     }
     return total;
   }
@@ -813,13 +813,17 @@ OnHeapHnswGraph getGraph() throws IOException {
       }
     }
 
-    @Override
-    public long ramBytesUsed() {
-      long total = SHALLOW_SIZE + flatFieldVectorsWriter.ramBytesUsed();
+    private long ownRamBytesUsed() {
+      long total = SHALLOW_SIZE;
       if (hnswGraphBuilder != null) {
         total += hnswGraphBuilder.getGraph().ramBytesUsed();
       }
       return total;
     }
+
+    @Override
+    public long ramBytesUsed() {
+      return ownRamBytesUsed() + flatFieldVectorsWriter.ramBytesUsed();
+    }
   }
 }

lucene/sandbox/src/java/module-info.java

@@ -22,6 +22,7 @@
   requires org.apache.lucene.facet;
 
   exports org.apache.lucene.payloads;
+  exports org.apache.lucene.sandbox.codecs.dedup;
   exports org.apache.lucene.sandbox.codecs.faiss;
   exports org.apache.lucene.sandbox.codecs.idversion;
   exports org.apache.lucene.sandbox.codecs.quantization;
@@ -41,5 +42,6 @@
   provides org.apache.lucene.codecs.PostingsFormat with
       org.apache.lucene.sandbox.codecs.idversion.IDVersionPostingsFormat;
   provides org.apache.lucene.codecs.KnnVectorsFormat with
-      org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormat;
+      org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormat,
+      org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat;
 }

lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatFieldVectorsWriter.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;
+import org.apache.lucene.index.DocsWithFieldSet;
+import org.apache.lucene.internal.hppc.IntArrayList;
+import org.apache.lucene.internal.hppc.ObjectCursor;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/**
+ * Buffers one field's vectors during flush, de-duplicating them through a shared {@link
+ * DedupGroup}.
+ *
+ * @lucene.experimental
+ */
+final class DedupFlatFieldVectorsWriter<T> extends FlatFieldVectorsWriter<T> {
+  private static final long SHALLOW_SIZE =
+      RamUsageEstimator.shallowSizeOfInstance(DedupFlatFieldVectorsWriter.class);
+
+  private final DedupGroup<T> group;
+  private final DocsWithFieldSet docsWithFieldSet;
+  private final List<T> vectors;
+  private final IntArrayList fieldOrdToGroupOrd;
+  private int lastDocID;
+  private boolean finished;
+
+  DedupFlatFieldVectorsWriter(DedupGroup<T> group) {
+    this.group = group;
+    this.docsWithFieldSet = new DocsWithFieldSet();
+    this.vectors = new ArrayList<>();
+    this.fieldOrdToGroupOrd = new IntArrayList();
+    this.lastDocID = -1;
+    this.finished = false;
+  }
+
+  @Override
+  public List<T> getVectors() {
+    return vectors;
+  }
+
+  @Override
+  public DocsWithFieldSet getDocsWithFieldSet() {
+    return docsWithFieldSet;
+  }
+
+  IntArrayList getFieldOrdToGroupOrd() {
+    return fieldOrdToGroupOrd;
+  }
+
+  @Override
+  public void finish() {
+    if (finished) {
+      throw new IllegalStateException("already finished");
+    }
+    finished = true;
+  }
+
+  @Override
+  public boolean isFinished() {
+    return finished;
+  }
+
+  @Override
+  public T copyValue(T vectorValue) {
+    throw new UnsupportedOperationException(); // handled inside group
+  }
+
+  @Override
+  public void addValue(int docID, T vectorValue) throws IOException {
+    if (finished) {
+      throw new IllegalStateException("already finished");
+    } else if (docID <= lastDocID) {
+      throw new IllegalArgumentException(
+          "docID=" + docID + " not going forwards, indexed lastDocID=" + lastDocID);
+    }
+
+    lastDocID = docID;
+    docsWithFieldSet.add(docID);
+
+    ObjectCursor<T> cursor = group.addUnique(vectorValue);
+    vectors.add(cursor.value); // owned vector value
+    fieldOrdToGroupOrd.add(cursor.index); // index in group
+  }
+
+  @Override
+  public long ramBytesUsed() {
+    return SHALLOW_SIZE
+        + docsWithFieldSet.ramBytesUsed()
+        + (long) vectors.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF
+        + fieldOrdToGroupOrd.ramBytesUsed();
+  }
+}

lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java

@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
+import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+/**
+ * Flat vector format that stores each distinct vector once.
+ *
+ * <p>Vectors that share the same dimension and encoding form a <i>group</i>. Within a group, an
+ * identical vector is stored a single time regardless of how many documents (across all fields that
+ * map to that group) reference it; each field then keeps an {@code fieldOrdToGroupOrd} map from its
+ * document ordinal to the group ordinal of the shared vector. This is well suited to indexes with
+ * repeated vectors, e.g. several fields derived from the same embedding, or heavily duplicated
+ * content.
+ *
+ * <h2>.vdd (vector de-dup data) file</h2>
+ *
+ * <ul>
+ *   <li>For each group, its distinct vectors, aligned to 4 bytes (BYTE) or 64 bytes (FLOAT32 and
+ *       FLOAT16). This a best-effort alignment, because Arm Neoverse machines incur a performance
+ *       penalty in reading data not aligned to 64 bytes. This penalty may be incurred for float
+ *       vectors that do not have a dimension of a multiple of 16 (equivalent to 64 bytes), because
+ *       the alignment will not hold for all vectors in the file.
+ *   <li>For each field:
+ *       <ul>
+ *         <li>The sparse-encoding data (only when some documents lack the field): DocIds encoded by
+ *             {@link
+ *             org.apache.lucene.codecs.lucene90.IndexedDISI#writeBitSet(org.apache.lucene.search.DocIdSetIterator,
+ *             org.apache.lucene.store.IndexOutput, byte)}, followed by the ordinal-to-doc mapping
+ *             encoded by {@link org.apache.lucene.util.packed.DirectMonotonicWriter}.
+ *         <li>The {@code fieldOrdToGroupOrd} map (aligned to 4 bytes): one entry per document
+ *             ordinal giving the group ordinal of the shared vector, packed by {@link
+ *             org.apache.lucene.util.packed.DirectWriter}.
+ *       </ul>
+ * </ul>
+ *
+ * <h2>.vdm (vector de-dup metadata) file</h2>
+ *
+ * <p>A list of groups, each:
+ *
+ * <ul>
+ *   <li><b>[int32]</b> group ordinal
+ *   <li><b>[int32]</b> vector dimension
+ *   <li><b>[int32]</b> vector encoding ordinal
+ *   <li><b>[int32]</b> group size (number of distinct vectors)
+ *   <li><b>[int64]</b> offset to this group's vectors in the .vdd file
+ *   <li><b>[int64]</b> length of this group's vectors, in bytes
+ * </ul>
+ *
+ * <p>terminated by <b>[int32]</b> {@code -1}, then a list of fields, each:
+ *
+ * <ul>
+ *   <li><b>[int32]</b> field number
+ *   <li><b>[int32]</b> vector similarity function ordinal
+ *   <li><b>[int32]</b> vector dimension
+ *   <li><b>[int32]</b> vector encoding ordinal
+ *   <li><b>[int32]</b> ordinal of the group holding this field's vectors
+ *   <li><b>[int32]</b> the number of documents having values for this field
+ *   <li>the sparse-encoding metadata (docs-with-field offset/length and ordToDoc configuration), as
+ *       written by {@link
+ *       org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration#writeStoredMeta}
+ *   <li><b>[int64]</b> offset to this field's {@code fieldOrdToGroupOrd} map in the .vdd file
+ *   <li><b>[int64]</b> length of this field's {@code fieldOrdToGroupOrd} map, in bytes
+ * </ul>
+ *
+ * <p>also terminated by <b>[int32]</b> {@code -1}.
+ *
+ * <h2>Complexity</h2>
+ *
+ * <p>Let {@code N} be the number of indexed vectors (one per document per field), {@code U} the
+ * number of distinct vectors, and {@code d} the dimension. De-duplication interns each vector via a
+ * hash lookup with linear probing, resolving hash collisions with a full equality check.
+ *
+ * <ul>
+ *   <li><b>Indexing (flush):</b> expected {@code O(N * d)} time (a hash plus occasional equality
+ *       check per vector). Heap is {@code O(U * d)} for the distinct vectors held in the group,
+ *       plus {@code O(N)} for the per-document references and {@code fieldOrdToGroupOrd} entries.
+ *   <li><b>Merge:</b> expected {@code O(N * d)} time; distinct vectors are written to disk as soon
+ *       as they are first seen rather than buffered, so heap stays {@code O(N)} (the per-field
+ *       {@code fieldOrdToGroupOrd} maps and light per-vector handles) with no {@code O(U * d)}
+ *       term. When a source segment is itself in this format, equality is decided by comparing
+ *       group ordinals in {@code O(1)} without reading the vectors back.
+ *   <li><b>Reading:</b> both the vectors and the {@code fieldOrdToGroupOrd} map stay off-heap; a
+ *       read resolves a document ordinal to its vector via one extra {@code fieldOrdToGroupOrd}
+ *       lookup.
+ * </ul>
+ *
+ * @lucene.experimental
+ */
+final class DedupFlatVectorsFormat extends FlatVectorsFormat {
+  static final String NAME = "DedupFlatVectorsFormat";
+
+  static final String META_CODEC_NAME = "DedupFlatVectorsFormatMeta";
+  static final String META_EXTENSION = "vdm";
+
+  static final String VECTOR_DATA_CODEC_NAME = "DedupFlatVectorsFormatVectorData";
+  static final String VECTOR_DATA_EXTENSION = "vdd";
+
+  static final int VERSION_START = 0;
+  static final int VERSION_CURRENT = VERSION_START;
+
+  private static final DedupFlatVectorsScorer FLAT_VECTORS_SCORER = new DedupFlatVectorsScorer();
+
+  DedupFlatVectorsFormat() {
+    super(NAME);
+  }
+
+  @Override
+  public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+    return new DedupFlatVectorsWriter(state, FLAT_VECTORS_SCORER);
+  }
+
+  @Override
+  public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+    return new DedupFlatVectorsReader(state, FLAT_VECTORS_SCORER);
+  }
+}