[MOD-9685] Introduce SVS Basic Benchmarks (#829)

* initial imp of training bm

introduce BM_VecSimSVSTrain class with 2 methods: Train and TrainAsync

add GoogleTest to benchmarks so we can use ASSERT_* API

tieredIndexMock: possible to initialize with a specific thread count

add train bemchmark to CI benchmark dispatcher

* make bm_files general for other algs

rename svs_training_fp32 ->svs_indices_training_fp32

add to bm_files.sh

* replace std::formtat only supported from gcc13 with ostringstream

* format

* revrt assert

* intialize quantBits
move svs params init to CreateTieredSVSIndex

only 5 iterations

* move iterawtion logic to runTrainBMIteration

add compressed index bm

* assert depdnding on HAVE_SVS_LVQ

* sepearate non compression and compression bm

* TO REVERT !!! test abort

* fix if else

* revrt timeoutgurard vhanges

* dont pause after training to see how it affrects performance

remove some prints

* fix #ifdef HAVE_SVS_LVQ to #if HAVE_SVS_LVQ

* use pause timers its faster

* do 3 iter instread of 5 and test if results are stable

* use 5 again

* fix download all all script

* fp16 bm

remove 100K

* remove 100K from fp32

* increase timeout

* try bigger machine

* try a bigger machine

* try 2 iter

move UNIT_AND_ITERATIONS and QUANT_BITS_ARGS to bm_vecsim_basics_Svs

* unify bm_training_initialize_fp32.h and bm_training_initialize_fp16.h to bm_training_initialize.h

define DATA_TYPE_INDEX_T in bm_svs_training_fp*.cpp

remove th 10k and 50k for arm

* reevet timeout to 10

fix include header for fp16

* move CreateTieredSVSParams and verifyNumThreads to svs params

* revert increease machine size

* change assert to log

* fix

* fix2

* format

* introduce bm_svs

* add tiered
add NewIndex from existing svs to tiered factory
imp AddLabel
add AddLabelBatches(not implmneted)

take bm_utils from meiravg_svs_training_bm

* introduce setUpdateTriggerThreshold in BUILD_TESTS

move initialize index to a function

introfuce bm function:
addlabel: insert one by one
AddLabelBatches: add in batches with one thread
AddLabelAsync: add in batches with multiple threads

* fix comment
add to yml

* remove lock

* format

* fix num threads in addlabelinplace
fix assertupdateTriggerThreshold in AddLabelAsync

* use train svs instead

* format

* small fixes

* rename BM_VecSimSVSTrain->BM_VecSimSVS

bm_vecsim_svs_train.h->bm_vecsim_svs

* remove unrelated

* align with new name

* revert unnecessary changes in bm_vecsim_index

add LVQ BM if HAVE_SVS_LVQ

* fix include

* fix quantbits

* extract general

* fix missing main on LVQ cpp

* replace vectors file

* run only BENCHMARK_MAIN

* try dummy for mac

* fix DATA_TYPE_INDEX_T definition LVQ

* quantBits is now static and needs to be intizlied by the CPP file
we hard code the name of the data file based on quantBits

* TO REVERT:
benchmark deletsion according to a when we exceed 0.5 index size
this benchmark takes 20K ms (20s) for 500 vectors!!!
that's a lot
after revert - benchmark only gc to detrmine

* REVERT svs.h change consolidation_threshold

runGC instead of delete label to not be depnd on consolidation_threshold that can't be controloed and runs for vrey very long!

* revert unrelated changes

* fix LVQ8 cpp for non LVQ

* foirmat

* cleanups

fix mac

* remove new line in cmake

* Update tests/benchmark/bm_vecsim_svs.h

Co-authored-by: BenGoldberger <[email protected]>

---------

Co-authored-by: BenGoldberger <[email protected]>

Author: meiravgri

.github/workflows/benchmark.yml

@@ -43,6 +43,7 @@ on:
           - bm-updated-fp32-single
           - bm-svs-train-fp32
           - bm-svs-train-fp16
+          - bm-basics-svs-fp32-single
           - bm-spaces
         description: 'Benchmarks set to run'
         default: benchmarks-all

src/VecSim/index_factories/tiered_factory.h

@@ -13,6 +13,7 @@
 #include "VecSim/memory/vecsim_malloc.h"
 #include "VecSim/vec_sim_index.h"
 #include "VecSim/algorithms/hnsw/hnsw_tiered.h"
+#include "VecSim/algorithms/svs/svs_tiered.h"
 #include "VecSim/algorithms/brute_force/brute_force.h"
 #include "VecSim/index_factories/factory_utils.h"
 
@@ -58,8 +59,35 @@ VecSimIndex *NewIndex(const TieredIndexParams *params, HNSWIndex<DataType, DistT
 // verification of the backend index algorithm. To be removed once a proper verification is
 // introduced.
 namespace TieredSVSFactory {
-BFParams NewBFParams(const TieredIndexParams *params);
+
+#if HAVE_SVS
+template <typename DataType>
+inline VecSimIndex *NewIndex(const TieredIndexParams *params,
+                             VecSimIndexAbstract<DataType, float> *svs_index) {
+    // Initialize brute force index.
+    BFParams bf_params = {.type = svs_index->getType(),
+                          .dim = svs_index->getDim(),
+                          .metric = svs_index->getMetric(),
+                          .multi = svs_index->isMultiValue(),
+                          .blockSize = svs_index->getBlockSize()};
+
+    AbstractIndexInitParams abstractInitParams =
+        VecSimFactory::NewAbstractInitParams(&bf_params, params->primaryIndexParams->logCtx, false);
+    assert(svs_index->getInputBlobSize() == abstractInitParams.storedDataSize);
+    assert(svs_index->getStoredDataSize() == abstractInitParams.storedDataSize);
+    auto frontendIndex = static_cast<BruteForceIndex<DataType, float> *>(
+        BruteForceFactory::NewIndex(&bf_params, abstractInitParams, false));
+
+    // Create new tiered svs index
+    std::shared_ptr<VecSimAllocator> management_layer_allocator =
+        VecSimAllocator::newVecsimAllocator();
+
+    return new (management_layer_allocator)
+        TieredSVSIndex<DataType>(svs_index, frontendIndex, *params, management_layer_allocator);
 }
 #endif
+BFParams NewBFParams(const TieredIndexParams *params);
+} // namespace TieredSVSFactory
+#endif
 
 }; // namespace TieredFactory

tests/benchmark/benchmarks.sh

@@ -11,6 +11,8 @@ if [ -z "$BM_TYPE"  ] || [ "$BM_TYPE" = "benchmarks-all" ]; then
     echo updated_index_single_fp32
     echo svs_training_fp32
     echo svs_training_fp16
+    echo basics_svs_single_fp32
+    echo basics_svs_single_fp32_LVQ8
     echo spaces_fp32
     echo spaces_fp64
     echo spaces_bf16
@@ -87,10 +89,14 @@ elif [ "$BM_TYPE" = "bm-batch-iter-uint8-multi" ] ; then
 elif [ "$BM_TYPE" = "bm-updated-fp32-single" ] ; then
     echo updated_index_single_fp32
 
+# SVS benchmarks
 elif [ "$BM_TYPE" = "bm-svs-train-fp32" ] ; then
     echo svs_training_fp32
 elif [ "$BM_TYPE" = "bm-svs-train-fp16" ] ; then
     echo svs_training_fp16
+elif [ "$BM_TYPE" = "bm-basics-svs-fp32-single" ] ; then
+    echo basics_svs_single_fp32
+    echo basics_svs_single_fp32_LVQ8
 
 # Spaces benchmarks
 elif [ "$BM_TYPE" = "bm-spaces" ] ; then

tests/benchmark/bm_files.sh

@@ -49,6 +49,9 @@ elif [ "$BM_TYPE" = "bm-svs-train-fp32" ] \
 then
     file_name="training"
     alg="svs"
+elif [ "$BM_TYPE" = "bm-basics-svs-fp32-single" ]; then
+    file_name="basic_fp32"
+    alg="svs"
 else
     echo "No files to download for BM_TYPE=$BM_TYPE"
     exit 0

tests/benchmark/bm_initialization/bm_basics_svs_initialize_fp32.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2006-Present, Redis Ltd.
+ * All rights reserved.
+ *
+ * Licensed under your choice of the Redis Source Available License 2.0
+ * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
+ * GNU Affero General Public License v3 (AGPLv3).
+ */
+
+#pragma once
+/**************************************
+  Define and register tests
+  NOTE: benchmarks' tests order can affect their results. Please add new benchmarks at the end of
+the file.
+***************************************/
+// deleteLabel one by one
+BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimSVS, BM_FUNC_NAME(BM_RunGC), DATA_TYPE_INDEX_T)
+(benchmark::State &st) { RunGC(st); }
+BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_RunGC))
+    ->Unit(benchmark::kMillisecond)
+    ->Iterations(1)
+    ->Arg(50)
+    ->Arg(100)
+    ->Arg(250)
+    ->Arg(500)
+    ->ArgName("num_deletions");
+
+// AddLabel one by one
+BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimSVS, BM_FUNC_NAME(BM_AddLabelOneByOne), DATA_TYPE_INDEX_T)
+(benchmark::State &st) { AddLabel(st); }
+BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_AddLabelOneByOne))
+    ->Unit(benchmark::kMillisecond)
+    ->Iterations(BM_VecSimGeneral::block_size);
+
+// Add vectors in batches via tiered index
+BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimSVS, BM_FUNC_NAME(BM_TriggerUpdateTiered), DATA_TYPE_INDEX_T)
+(benchmark::State &st) { TriggerUpdateTiered(st); }
+BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_TriggerUpdateTiered))
+    ->Unit(benchmark::kMillisecond)
+    ->Iterations(1)
+    ->ArgsProduct({{static_cast<long int>(BM_VecSimGeneral::block_size), 5000,
+                    static_cast<long int>(10 * BM_VecSimGeneral::block_size)},
+                   {2, 4, 8}})
+    ->ArgNames({"update_threshold", "thread_count"})
+    ->MeasureProcessCPUTime();

tests/benchmark/bm_utils.h

@@ -29,12 +29,12 @@ CreateTieredSVSParams(VecSimParams &svs_params, tieredIndexMock &mock_thread_poo
 
 template <typename data_t>
 static void verifyNumThreads(TieredSVSIndex<data_t> *tiered_index, size_t expected_num_threads,
-                             size_t expected_capcity) {
+                             size_t expected_capcity, std::string msg = "") {
     ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity)
-        << "thread pool capacity mismatch";
+        << msg << ": thread pool capacity mismatch";
     size_t num_reserved_threads = tiered_index->GetSVSIndex()->getNumThreads();
     if (num_reserved_threads < expected_num_threads) {
-        std::cout << "WARNING: last reserved threads (" << num_reserved_threads
+        std::cout << msg << ": WARNING: last reserved threads (" << num_reserved_threads
                   << ") is less than expected (" << expected_num_threads << ")." << std::endl;
     }
 }