Merge branch 'branch-24.10' into enh-cagra-separable-compilation

ci/release/update-version.sh

@@ -45,6 +45,7 @@ DEPENDENCIES=(
   dask-cuda
   cuvs
   pylibraft
+  librmm
   rmm
   rapids-dask-dependency
 )

conda/environments/all_cuda-118_arch-aarch64.yaml

@@ -35,6 +35,7 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
+- librmm==24.10.*,>=0.0.0a0
 - make
 - nccl>=2.9.9
 - ninja
@@ -49,7 +50,6 @@ dependencies:
 - pytest==7.*
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.10.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.10.0
 - scikit-learn

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -35,6 +35,7 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
+- librmm==24.10.*,>=0.0.0a0
 - make
 - nccl>=2.9.9
 - ninja
@@ -49,7 +50,6 @@ dependencies:
 - pytest==7.*
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.10.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.10.0
 - scikit-learn

conda/environments/all_cuda-125_arch-aarch64.yaml

@@ -32,6 +32,7 @@ dependencies:
 - libcurand-dev
 - libcusolver-dev
 - libcusparse-dev
+- librmm==24.10.*,>=0.0.0a0
 - make
 - nccl>=2.9.9
 - ninja
@@ -45,7 +46,6 @@ dependencies:
 - pytest==7.*
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.10.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.10.0
 - scikit-learn

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -32,6 +32,7 @@ dependencies:
 - libcurand-dev
 - libcusolver-dev
 - libcusparse-dev
+- librmm==24.10.*,>=0.0.0a0
 - make
 - nccl>=2.9.9
 - ninja
@@ -45,7 +46,6 @@ dependencies:
 - pytest==7.*
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.10.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.10.0
 - scikit-learn

conda/environments/bench_ann_cuda-118_arch-aarch64.yaml

@@ -33,6 +33,7 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
+- librmm==24.10.*,>=0.0.0a0
 - matplotlib
 - nccl>=2.9.9
 - ninja
@@ -42,6 +43,5 @@ dependencies:
 - pandas
 - pylibraft==24.10.*,>=0.0.0a0
 - pyyaml
-- rmm==24.10.*,>=0.0.0a0
 - sysroot_linux-aarch64==2.17
 name: bench_ann_cuda-118_arch-aarch64

conda/environments/bench_ann_cuda-118_arch-x86_64.yaml

@@ -33,6 +33,7 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
+- librmm==24.10.*,>=0.0.0a0
 - matplotlib
 - nccl>=2.9.9
 - ninja
@@ -42,6 +43,5 @@ dependencies:
 - pandas
 - pylibraft==24.10.*,>=0.0.0a0
 - pyyaml
-- rmm==24.10.*,>=0.0.0a0
 - sysroot_linux-64==2.17
 name: bench_ann_cuda-118_arch-x86_64

conda/environments/bench_ann_cuda-125_arch-aarch64.yaml

@@ -30,6 +30,7 @@ dependencies:
 - libcurand-dev
 - libcusolver-dev
 - libcusparse-dev
+- librmm==24.10.*,>=0.0.0a0
 - matplotlib
 - nccl>=2.9.9
 - ninja
@@ -38,6 +39,5 @@ dependencies:
 - pandas
 - pylibraft==24.10.*,>=0.0.0a0
 - pyyaml
-- rmm==24.10.*,>=0.0.0a0
 - sysroot_linux-aarch64==2.17
 name: bench_ann_cuda-125_arch-aarch64

conda/environments/bench_ann_cuda-125_arch-x86_64.yaml

@@ -30,6 +30,7 @@ dependencies:
 - libcurand-dev
 - libcusolver-dev
 - libcusparse-dev
+- librmm==24.10.*,>=0.0.0a0
 - matplotlib
 - nccl>=2.9.9
 - ninja
@@ -38,6 +39,5 @@ dependencies:
 - pandas
 - pylibraft==24.10.*,>=0.0.0a0
 - pyyaml
-- rmm==24.10.*,>=0.0.0a0
 - sysroot_linux-64==2.17
 name: bench_ann_cuda-125_arch-x86_64

cpp/CMakeLists.txt

@@ -568,6 +568,7 @@ if(BUILD_C_LIBRARY)
     src/neighbors/ivf_flat_c.cpp
     src/neighbors/ivf_pq_c.cpp
     src/neighbors/cagra_c.cpp
+    src/neighbors/hnsw_c.cpp
     src/neighbors/refine/refine_c.cpp
     src/distance/pairwise_distance_c.cpp
   )

cpp/include/cuvs/neighbors/cagra.h

@@ -337,7 +337,10 @@ cuvsError_t cuvsCagraBuild(cuvsResources_t res,
  *        It is also important to note that the CAGRA Index must have been built
  *        with the same type of `queries`, such that `index.dtype.code ==
  * queries.dl_tensor.dtype.code` Types for input are:
- *        1. `queries`: `kDLDataType.code == kDLFloat` and `kDLDataType.bits = 32`
+ *        1. `queries`:
+ *`         a. kDLDataType.code == kDLFloat` and `kDLDataType.bits = 32`
+ *          b. `kDLDataType.code == kDLInt` and `kDLDataType.bits = 8`
+ *          c. `kDLDataType.code == kDLUInt` and `kDLDataType.bits = 8`
  *        2. `neighbors`: `kDLDataType.code == kDLUInt` and `kDLDataType.bits = 32`
  *        3. `distances`: `kDLDataType.code == kDLFloat` and `kDLDataType.bits = 32`
  *
@@ -394,7 +397,7 @@ cuvsError_t cuvsCagraSearch(cuvsResources_t res,
  *
  * Experimental, both the API and the serialization format are subject to change.
  *
- * @code{.cpp}
+ * @code{.c}
  * #include <cuvs/neighbors/cagra.h>
  *
  * // Create cuvsResources_t
@@ -416,6 +419,34 @@ cuvsError_t cuvsCagraSerialize(cuvsResources_t res,
                                cuvsCagraIndex_t index,
                                bool include_dataset);
 
+/**
+ * Save the CAGRA index to file in hnswlib format.
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
+ *
+ * Experimental, both the API and the serialization format are subject to change.
+ *
+ * @code{.c}
+ * #include <cuvs/core/c_api.h>
+ * #include <cuvs/neighbors/cagra.h>
+ *
+ * // Create cuvsResources_t
+ * cuvsResources_t res;
+ * cuvsError_t res_create_status = cuvsResourcesCreate(&res);
+ *
+ * // create an index with `cuvsCagraBuild`
+ * cuvsCagraSerializeHnswlib(res, "/path/to/index", index);
+ * @endcode
+ *
+ * @param[in] res cuvsResources_t opaque C handle
+ * @param[in] filename the file name for saving the index
+ * @param[in] index CAGRA index
+ *
+ */
+cuvsError_t cuvsCagraSerializeToHnswlib(cuvsResources_t res,
+                                        const char* filename,
+                                        cuvsCagraIndex_t index);
+
 /**
  * Load index from file.
  *

cpp/include/cuvs/neighbors/cagra.hpp

@@ -1345,6 +1345,8 @@ void deserialize(raft::resources const& handle,
 
 /**
  * Write the CAGRA built index as a base layer HNSW index to an output stream
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
  *
  * Experimental, both the API and the serialization format are subject to change.
  *
@@ -1371,6 +1373,8 @@ void serialize_to_hnswlib(raft::resources const& handle,
 
 /**
  * Save a CAGRA build index in hnswlib base-layer-only serialized format
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
  *
  * Experimental, both the API and the serialization format are subject to change.
  *
@@ -1398,6 +1402,8 @@ void serialize_to_hnswlib(raft::resources const& handle,
 
 /**
  * Write the CAGRA built index as a base layer HNSW index to an output stream
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
  *
  * Experimental, both the API and the serialization format are subject to change.
  *
@@ -1424,6 +1430,8 @@ void serialize_to_hnswlib(raft::resources const& handle,
 
 /**
  * Save a CAGRA build index in hnswlib base-layer-only serialized format
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
  *
  * Experimental, both the API and the serialization format are subject to change.
  *
@@ -1451,6 +1459,8 @@ void serialize_to_hnswlib(raft::resources const& handle,
 
 /**
  * Write the CAGRA built index as a base layer HNSW index to an output stream
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
  *
  * Experimental, both the API and the serialization format are subject to change.
  *
@@ -1477,6 +1487,8 @@ void serialize_to_hnswlib(raft::resources const& handle,
 
 /**
  * Save a CAGRA build index in hnswlib base-layer-only serialized format
+ * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS,
+ *       as the serialization format is not compatible with the original hnswlib.
  *
  * Experimental, both the API and the serialization format are subject to change.
  *