From 1da5d9dc1b6803b6cedce53a99852ae78b7bb892 Mon Sep 17 00:00:00 2001 From: achirkin Date: Mon, 16 Sep 2024 11:56:42 +0200 Subject: [PATCH] CAGRA - runtime dispatch of distance functions --- cpp/CMakeLists.txt | 132 ++--- .../detail/cagra/compute_distance.hpp | 167 +++++- .../detail/cagra/compute_distance_vpq.cuh | 6 +- cpp/src/neighbors/detail/cagra/factory.cuh | 12 +- .../cagra/q_search_multi_cta_00_generate.py | 26 +- ...float_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ...float_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ...a_float_uint32_dim128_t8_8pq_2subd_half.cu | 36 -- ...a_float_uint32_dim128_t8_8pq_4subd_half.cu | 36 -- ..._float_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ..._float_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ..._float_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ..._float_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- ...float_uint64_dim1024_t32_8pq_2subd_half.cu | 36 -- ...float_uint64_dim1024_t32_8pq_4subd_half.cu | 36 -- ...a_float_uint64_dim128_t8_8pq_2subd_half.cu | 36 -- ...a_float_uint64_dim128_t8_8pq_4subd_half.cu | 36 -- ..._float_uint64_dim256_t16_8pq_2subd_half.cu | 36 -- ..._float_uint64_dim256_t16_8pq_4subd_half.cu | 36 -- ..._float_uint64_dim512_t32_8pq_2subd_half.cu | 36 -- ..._float_uint64_dim512_t32_8pq_4subd_half.cu | 36 -- ..._half_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ..._half_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ...ta_half_uint32_dim128_t8_8pq_2subd_half.cu | 36 -- ...ta_half_uint32_dim128_t8_8pq_4subd_half.cu | 36 -- ...a_half_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ...a_half_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ...a_half_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ...a_half_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- ..._half_uint64_dim1024_t32_8pq_2subd_half.cu | 36 -- ..._half_uint64_dim1024_t32_8pq_4subd_half.cu | 36 -- ...ta_half_uint64_dim128_t8_8pq_2subd_half.cu | 36 -- ...ta_half_uint64_dim128_t8_8pq_4subd_half.cu | 36 -- ...a_half_uint64_dim256_t16_8pq_2subd_half.cu | 36 -- ...a_half_uint64_dim256_t16_8pq_4subd_half.cu | 36 -- ...a_half_uint64_dim512_t32_8pq_2subd_half.cu | 36 -- ...a_half_uint64_dim512_t32_8pq_4subd_half.cu | 36 -- ..._int8_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ..._int8_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ...ta_int8_uint32_dim128_t8_8pq_2subd_half.cu | 36 -- ...ta_int8_uint32_dim128_t8_8pq_4subd_half.cu | 36 -- ...a_int8_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ...a_int8_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ...a_int8_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ...a_int8_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- ...uint8_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ...uint8_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ...a_uint8_uint32_dim128_t8_8pq_2subd_half.cu | 36 -- ...a_uint8_uint32_dim128_t8_8pq_4subd_half.cu | 36 -- ..._uint8_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ..._uint8_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ..._uint8_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ..._uint8_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- .../cagra/q_search_single_cta_00_generate.py | 26 +- ...single_cta_float_uint32_8pq_2subd_half.cu} | 4 +- ...single_cta_float_uint32_8pq_4subd_half.cu} | 4 +- ...float_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ...float_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ..._float_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ..._float_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ..._float_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ..._float_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- ...single_cta_float_uint64_8pq_2subd_half.cu} | 4 +- ...single_cta_float_uint64_8pq_4subd_half.cu} | 4 +- ...float_uint64_dim1024_t32_8pq_2subd_half.cu | 36 -- ...float_uint64_dim1024_t32_8pq_4subd_half.cu | 36 -- ..._float_uint64_dim256_t16_8pq_2subd_half.cu | 36 -- ..._float_uint64_dim256_t16_8pq_4subd_half.cu | 36 -- ..._float_uint64_dim512_t32_8pq_2subd_half.cu | 36 -- ..._float_uint64_dim512_t32_8pq_4subd_half.cu | 36 -- ..._single_cta_half_uint32_8pq_2subd_half.cu} | 4 +- ..._single_cta_half_uint32_8pq_4subd_half.cu} | 4 +- ..._half_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ..._half_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ...a_half_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ...a_half_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ...a_half_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ...a_half_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- ..._single_cta_half_uint64_8pq_2subd_half.cu} | 4 +- ..._single_cta_half_uint64_8pq_4subd_half.cu} | 4 +- ..._half_uint64_dim1024_t32_8pq_2subd_half.cu | 36 -- ..._half_uint64_dim1024_t32_8pq_4subd_half.cu | 36 -- ...a_half_uint64_dim256_t16_8pq_2subd_half.cu | 36 -- ...a_half_uint64_dim256_t16_8pq_4subd_half.cu | 36 -- ...a_half_uint64_dim512_t32_8pq_2subd_half.cu | 36 -- ...a_half_uint64_dim512_t32_8pq_4subd_half.cu | 36 -- ..._single_cta_int8_uint32_8pq_2subd_half.cu} | 4 +- ..._single_cta_int8_uint32_8pq_4subd_half.cu} | 4 +- ..._int8_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ..._int8_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ...a_int8_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ...a_int8_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ...a_int8_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ...a_int8_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- ...single_cta_uint8_uint32_8pq_2subd_half.cu} | 4 +- ...single_cta_uint8_uint32_8pq_4subd_half.cu} | 4 +- ...uint8_uint32_dim1024_t32_8pq_2subd_half.cu | 36 -- ...uint8_uint32_dim1024_t32_8pq_4subd_half.cu | 36 -- ..._uint8_uint32_dim256_t16_8pq_2subd_half.cu | 36 -- ..._uint8_uint32_dim256_t16_8pq_4subd_half.cu | 36 -- ..._uint8_uint32_dim512_t32_8pq_2subd_half.cu | 36 -- ..._uint8_uint32_dim512_t32_8pq_4subd_half.cu | 36 -- .../detail/cagra/search_multi_cta.cuh | 49 +- .../cagra/search_multi_cta_00_generate.py | 20 +- ...t8.cu => search_multi_cta_float_uint32.cu} | 2 - ...arch_multi_cta_float_uint32_dim1024_t32.cu | 37 -- ...earch_multi_cta_float_uint32_dim256_t16.cu | 37 -- ...earch_multi_cta_float_uint32_dim512_t32.cu | 37 -- ...t8.cu => search_multi_cta_float_uint64.cu} | 2 - ...arch_multi_cta_float_uint64_dim1024_t32.cu | 37 -- ...earch_multi_cta_float_uint64_dim256_t16.cu | 37 -- ...earch_multi_cta_float_uint64_dim512_t32.cu | 37 -- ..._t8.cu => search_multi_cta_half_uint32.cu} | 2 - ...earch_multi_cta_half_uint32_dim1024_t32.cu | 37 -- ...search_multi_cta_half_uint32_dim256_t16.cu | 37 -- ...search_multi_cta_half_uint32_dim512_t32.cu | 37 -- ..._t8.cu => search_multi_cta_half_uint64.cu} | 2 - ...earch_multi_cta_half_uint64_dim1024_t32.cu | 37 -- ...search_multi_cta_half_uint64_dim256_t16.cu | 37 -- ...search_multi_cta_half_uint64_dim512_t32.cu | 37 -- .../detail/cagra/search_multi_cta_inst.cuh | 47 +- ..._t8.cu => search_multi_cta_int8_uint32.cu} | 2 - ...earch_multi_cta_int8_uint32_dim1024_t32.cu | 37 -- ...search_multi_cta_int8_uint32_dim256_t16.cu | 37 -- ...search_multi_cta_int8_uint32_dim512_t32.cu | 37 -- .../cagra/search_multi_cta_kernel-ext.cuh | 317 ++--------- .../cagra/search_multi_cta_kernel-inl.cuh | 110 ++-- ...t8.cu => search_multi_cta_uint8_uint32.cu} | 2 - ...arch_multi_cta_uint8_uint32_dim1024_t32.cu | 37 -- ...earch_multi_cta_uint8_uint32_dim256_t16.cu | 37 -- ...earch_multi_cta_uint8_uint32_dim512_t32.cu | 37 -- .../detail/cagra/search_single_cta.cuh | 51 +- .../cagra/search_single_cta_00_generate.py | 19 +- ...8.cu => search_single_cta_float_uint32.cu} | 2 - ...rch_single_cta_float_uint32_dim1024_t32.cu | 37 -- ...arch_single_cta_float_uint32_dim256_t16.cu | 37 -- ...arch_single_cta_float_uint32_dim512_t32.cu | 37 -- ...8.cu => search_single_cta_float_uint64.cu} | 2 - ...rch_single_cta_float_uint64_dim1024_t32.cu | 37 -- ...arch_single_cta_float_uint64_dim256_t16.cu | 37 -- ...arch_single_cta_float_uint64_dim512_t32.cu | 37 -- ...t8.cu => search_single_cta_half_uint32.cu} | 2 - ...arch_single_cta_half_uint32_dim1024_t32.cu | 37 -- ...earch_single_cta_half_uint32_dim256_t16.cu | 37 -- ...earch_single_cta_half_uint32_dim512_t32.cu | 37 -- ...t8.cu => search_single_cta_half_uint64.cu} | 2 - ...arch_single_cta_half_uint64_dim1024_t32.cu | 37 -- ...earch_single_cta_half_uint64_dim256_t16.cu | 37 -- ...earch_single_cta_half_uint64_dim512_t32.cu | 37 -- .../detail/cagra/search_single_cta_inst.cuh | 49 +- ...t8.cu => search_single_cta_int8_uint32.cu} | 2 - ...arch_single_cta_int8_uint32_dim1024_t32.cu | 37 -- ...earch_single_cta_int8_uint32_dim256_t16.cu | 37 -- ...earch_single_cta_int8_uint32_dim512_t32.cu | 37 -- .../cagra/search_single_cta_kernel-ext.cuh | 499 ++---------------- .../cagra/search_single_cta_kernel-inl.cuh | 123 ++--- ...8.cu => search_single_cta_uint8_uint32.cu} | 2 - ...rch_single_cta_uint8_uint32_dim1024_t32.cu | 37 -- ...arch_single_cta_uint8_uint32_dim256_t16.cu | 37 -- ...arch_single_cta_uint8_uint32_dim512_t32.cu | 37 -- cpp/test/neighbors/ann_cagra.cuh | 2 +- 161 files changed, 531 insertions(+), 5552 deletions(-) delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu => q_search_single_cta_float_uint32_8pq_2subd_half.cu} (90%) rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu => q_search_single_cta_float_uint32_8pq_4subd_half.cu} (90%) delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu => q_search_single_cta_float_uint64_8pq_2subd_half.cu} (90%) rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu => q_search_single_cta_float_uint64_8pq_4subd_half.cu} (90%) delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu => q_search_single_cta_half_uint32_8pq_2subd_half.cu} (90%) rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu => q_search_single_cta_half_uint32_8pq_4subd_half.cu} (90%) delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu => q_search_single_cta_half_uint64_8pq_2subd_half.cu} (90%) rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu => q_search_single_cta_half_uint64_8pq_4subd_half.cu} (90%) delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu => q_search_single_cta_int8_uint32_8pq_2subd_half.cu} (90%) rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu => q_search_single_cta_int8_uint32_8pq_4subd_half.cu} (90%) delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu => q_search_single_cta_uint8_uint32_8pq_2subd_half.cu} (90%) rename cpp/src/neighbors/detail/cagra/{q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu => q_search_single_cta_uint8_uint32_8pq_4subd_half.cu} (90%) delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu rename cpp/src/neighbors/detail/cagra/{search_multi_cta_float_uint32_dim128_t8.cu => search_multi_cta_float_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_multi_cta_float_uint64_dim128_t8.cu => search_multi_cta_float_uint64.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_multi_cta_half_uint32_dim128_t8.cu => search_multi_cta_half_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_multi_cta_half_uint64_dim128_t8.cu => search_multi_cta_half_uint64.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_multi_cta_int8_uint32_dim128_t8.cu => search_multi_cta_int8_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_multi_cta_uint8_uint32_dim128_t8.cu => search_multi_cta_uint8_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_single_cta_float_uint32_dim128_t8.cu => search_single_cta_float_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_single_cta_float_uint64_dim128_t8.cu => search_single_cta_float_uint64.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_single_cta_half_uint32_dim128_t8.cu => search_single_cta_half_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_single_cta_half_uint64_dim128_t8.cu => search_single_cta_half_uint64.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_single_cta_int8_uint32_dim128_t8.cu => search_single_cta_int8_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu rename cpp/src/neighbors/detail/cagra/{search_single_cta_uint8_uint32_dim128_t8.cu => search_single_cta_uint8_uint32.cu} (98%) delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fec1248bb..079fadaa9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -273,102 +273,42 @@ add_library( src/neighbors/cagra_serialize_int8.cu src/neighbors/cagra_serialize_uint8.cu src/neighbors/detail/cagra/cagra_build.cpp - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_4subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_2subd_half.cu + src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_4subd_half.cu + src/neighbors/detail/cagra/search_multi_cta_float_uint32.cu + src/neighbors/detail/cagra/search_multi_cta_half_uint32.cu + src/neighbors/detail/cagra/search_multi_cta_int8_uint32.cu + src/neighbors/detail/cagra/search_multi_cta_uint8_uint32.cu + src/neighbors/detail/cagra/search_multi_cta_float_uint64.cu + src/neighbors/detail/cagra/search_multi_cta_half_uint64.cu + src/neighbors/detail/cagra/search_single_cta_float_uint32.cu + src/neighbors/detail/cagra/search_single_cta_half_uint32.cu + src/neighbors/detail/cagra/search_single_cta_int8_uint32.cu + src/neighbors/detail/cagra/search_single_cta_uint8_uint32.cu + src/neighbors/detail/cagra/search_single_cta_float_uint64.cu + src/neighbors/detail/cagra/search_single_cta_half_uint64.cu $<$:src/neighbors/hnsw.cpp> src/neighbors/ivf_flat_index.cpp src/neighbors/ivf_flat/ivf_flat_build_extend_float_int64_t.cu diff --git a/cpp/src/neighbors/detail/cagra/compute_distance.hpp b/cpp/src/neighbors/detail/cagra/compute_distance.hpp index 2b0c750ff..04671379a 100644 --- a/cpp/src/neighbors/detail/cagra/compute_distance.hpp +++ b/cpp/src/neighbors/detail/cagra/compute_distance.hpp @@ -32,6 +32,15 @@ namespace cuvs::neighbors::cagra::detail { namespace device { +constexpr inline auto get_DBD(uint32_t team_size) +{ + switch (team_size) { + case 32: return 512; + case 16: return 256; + default: return 128; + } +} + // using LOAD_256BIT_T = ulonglong4; using LOAD_128BIT_T = uint4; using LOAD_64BIT_T = uint64_t; @@ -122,6 +131,85 @@ _RAFT_DEVICE void compute_distance_to_random_nodes( } } +template +_RAFT_DEVICE void compute_distance_to_random_nodes( + unsigned team_size, + INDEX_T* const result_indices_ptr, // [num_pickup] + DISTANCE_T* const result_distances_ptr, // [num_pickup] + const typename DATASET_DESCRIPTOR_T::QUERY_T* const query_buffer, + const DATASET_DESCRIPTOR_T& dataset_desc, + const std::size_t num_pickup, + const unsigned num_distilation, + const uint64_t rand_xor_mask, + const INDEX_T* const seed_ptr, // [num_seeds] + const uint32_t num_seeds, + INDEX_T* const visited_hash_ptr, + const uint32_t hash_bitlen, + const cuvs::distance::DistanceType metric, + const uint32_t block_id = 0, + const uint32_t num_blocks = 1) +{ + switch (team_size) { + case 32: + return compute_distance_to_random_nodes<32, + get_DBD(32), + DATASET_DESCRIPTOR_T, + DISTANCE_T, + INDEX_T>(result_indices_ptr, + result_distances_ptr, + query_buffer, + dataset_desc, + num_pickup, + num_distilation, + rand_xor_mask, + seed_ptr, + num_seeds, + visited_hash_ptr, + hash_bitlen, + metric, + block_id, + num_blocks); + case 16: + return compute_distance_to_random_nodes<16, + get_DBD(16), + DATASET_DESCRIPTOR_T, + DISTANCE_T, + INDEX_T>(result_indices_ptr, + result_distances_ptr, + query_buffer, + dataset_desc, + num_pickup, + num_distilation, + rand_xor_mask, + seed_ptr, + num_seeds, + visited_hash_ptr, + hash_bitlen, + metric, + block_id, + num_blocks); + default: + return compute_distance_to_random_nodes<8, + get_DBD(8), + DATASET_DESCRIPTOR_T, + DISTANCE_T, + INDEX_T>(result_indices_ptr, + result_distances_ptr, + query_buffer, + dataset_desc, + num_pickup, + num_distilation, + rand_xor_mask, + seed_ptr, + num_seeds, + visited_hash_ptr, + hash_bitlen, + metric, + block_id, + num_blocks); + } +} + template +_RAFT_DEVICE void compute_distance_to_child_nodes( + unsigned team_size, + INDEX_T* const result_child_indices_ptr, + DISTANCE_T* const result_child_distances_ptr, + // query + const typename DATASET_DESCRIPTOR_T::QUERY_T* const query_buffer, + // [dataset_dim, dataset_size] + const DATASET_DESCRIPTOR_T& dataset_desc, + // [knn_k, dataset_size] + const INDEX_T* const knn_graph, + const std::uint32_t knn_k, + // hashmap + INDEX_T* const visited_hashmap_ptr, + const std::uint32_t hash_bitlen, + const INDEX_T* const parent_indices, + const INDEX_T* const internal_topk_list, + const std::uint32_t search_width, + const cuvs::distance::DistanceType metric) +{ + switch (team_size) { + case 32: + return compute_distance_to_child_nodes<32, + get_DBD(32), + MAX_N_FRAGS, + DATASET_DESCRIPTOR_T, + DISTANCE_T, + INDEX_T>(result_child_indices_ptr, + result_child_distances_ptr, + query_buffer, + dataset_desc, + knn_graph, + knn_k, + visited_hashmap_ptr, + hash_bitlen, + parent_indices, + internal_topk_list, + search_width, + metric); + case 16: + return compute_distance_to_child_nodes<16, + get_DBD(16), + MAX_N_FRAGS, + DATASET_DESCRIPTOR_T, + DISTANCE_T, + INDEX_T>(result_child_indices_ptr, + result_child_distances_ptr, + query_buffer, + dataset_desc, + knn_graph, + knn_k, + visited_hashmap_ptr, + hash_bitlen, + parent_indices, + internal_topk_list, + search_width, + metric); + default: + return compute_distance_to_child_nodes<8, + get_DBD(8), + MAX_N_FRAGS, + DATASET_DESCRIPTOR_T, + DISTANCE_T, + INDEX_T>(result_child_indices_ptr, + result_child_distances_ptr, + query_buffer, + dataset_desc, + knn_graph, + knn_k, + visited_hashmap_ptr, + hash_bitlen, + parent_indices, + internal_topk_list, + search_width, + metric); + } +} + } // namespace device template @@ -243,7 +409,6 @@ struct standard_dataset_descriptor_t static const std::uint32_t smem_buffer_size_in_byte = 0; __device__ void set_smem_ptr(void* const){}; - template __device__ void copy_query(const DATA_T* const dmem_query_ptr, QUERY_T* const smem_query_ptr, const std::uint32_t query_smem_buffer_length) diff --git a/cpp/src/neighbors/detail/cagra/compute_distance_vpq.cuh b/cpp/src/neighbors/detail/cagra/compute_distance_vpq.cuh index 68973662f..d5df451d4 100644 --- a/cpp/src/neighbors/detail/cagra/compute_distance_vpq.cuh +++ b/cpp/src/neighbors/detail/cagra/compute_distance_vpq.cuh @@ -92,7 +92,6 @@ struct cagra_q_dataset_descriptor_t : public dataset_descriptor_base_t __device__ void copy_query(const DATA_T* const dmem_query_ptr, QUERY_T* const smem_query_ptr, const std::uint32_t query_smem_buffer_length) @@ -105,8 +104,7 @@ struct cagra_q_dataset_descriptor_t : public dataset_descriptor_base_t(i / 2)] = - buf2; + ((half2*)smem_query_ptr)[device::swizzling(i / 2)] = buf2; } else { (reinterpret_cast(smem_query_ptr + i))[0] = buf2; } @@ -228,4 +226,4 @@ struct cagra_q_dataset_descriptor_t : public dataset_descriptor_base_t>( - new single_cta_search:: - search( - res, plan, plan.dim, plan.graph_degree, plan.topk, plan.metric)); + new single_cta_search::search( + res, plan, plan.dim, plan.graph_degree, plan.topk, plan.metric)); } else if (plan.algo == search_algo::MULTI_CTA) { return std::unique_ptr>( - new multi_cta_search:: - search( - res, plan, plan.dim, plan.graph_degree, plan.topk, plan.metric)); + new multi_cta_search::search( + res, plan, plan.dim, plan.graph_degree, plan.topk, plan.metric)); } else { return std::unique_ptr>( new multi_kernel_search:: diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py index 63171373f..3ac06b7f7 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py +++ b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py @@ -47,7 +47,6 @@ } // namespace cuvs::neighbors::cagra::detail::multi_cta_search """ -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] pq_bits = [8] subspace_dims = [2, 4] # block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] @@ -68,16 +67,15 @@ ) # knn for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - for code_book_t in code_book_types: - for subspace_dim in subspace_dims: - for pq_bit in pq_bits: - path = f"q_search_multi_cta_{type_path}_dim{mxdim}_t{team}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" - ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") + for code_book_t in code_book_types: + for subspace_dim in subspace_dims: + for pq_bit in pq_bits: + path = f"q_search_multi_cta_{type_path}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" + with open(path, "w") as f: + f.write(header) + f.write( + f"instantiate_kernel_selection(\n cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" + ) + f.write(trailer) + # For pasting into CMakeLists.txt + print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 9ec7ce3dd..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 292a1429a..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 1a5ad50e3..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 0ab23d7eb..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 5d94a501a..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 56534dc05..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 7ff962058..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 3387a32a3..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 2d3f2cb1d..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 73dd8cd4b..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index b5e33602d..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 32fe0d628..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index e2726ea26..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index b4ebd49c4..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 72f198c92..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index dfb667a7f..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index c583569f6..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index fedfb5146..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 1116eaaa4..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 7e3ec363d..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 2b6e8e3da..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 4a97fb752..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 675cd3c93..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index b42b3289c..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 0db4296f1..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 4a2610dc7..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index af60c776a..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 5dd79a79b..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index b1c15662e..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 201f68fb5..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 26744ed76..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 1bce71bef..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 694304f3c..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index e6a563731..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 5c554af3f..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 965b43c07..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 97a4f8092..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index bdd1719b3..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index e39bc1e2d..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 599cf327a..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 621c5a249..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index cbed3ef8a..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 7428bfd9e..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 70efefdb0..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 4039b8582..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 022eb0e05..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index e48b2ed71..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 64f08530f..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_multi_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py index bc5f506ac..4b809ba33 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py @@ -47,7 +47,6 @@ } // namespace cuvs::neighbors::cagra::detail::single_cta_search """ -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] # block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] # itopk_candidates = [64, 128, 256] # itopk_size = [64, 128, 256, 512] @@ -72,17 +71,16 @@ # knn for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - for code_book_t in code_book_types: - for subspace_dim in subspace_dims: - for pq_bit in pq_bits: - path = f"q_search_single_cta_{type_path}_dim{mxdim}_t{team}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" - ) + for code_book_t in code_book_types: + for subspace_dim in subspace_dims: + for pq_bit in pq_bits: + path = f"q_search_single_cta_{type_path}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" + with open(path, "w") as f: + f.write(header) + f.write( + f"instantiate_kernel_selection(\n cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" + ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") + f.write(trailer) + # For pasting into CMakeLists.txt + print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_2subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_2subd_half.cu index ef483437a..fb3bfe074 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_2subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_4subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_4subd_half.cu index d9ebb1b85..046cd4104 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_8pq_4subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index b40322741..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 36273d0d4..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index e86524ee0..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 9f2b7fbc7..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 1ce4f5520..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 2d6f93ef0..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_2subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_2subd_half.cu index ea8faee1c..528cffdd1 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_2subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_4subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_4subd_half.cu index 061b1a04e..259ab3d7a 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_8pq_4subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 5f3267410..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 631ac7938..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 15610d853..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index f984b46f0..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 45299f272..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index fcb91be8c..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_2subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_2subd_half.cu index d80fef52c..c0223cf44 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_2subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_4subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_4subd_half.cu index e2c3ef4f7..9855c44a4 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_8pq_4subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index b594fedab..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index a82be6b55..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 98889811d..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index f5e9d12c9..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 4f14910b4..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 67d52f8d5..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_2subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_2subd_half.cu index 7a98b59a9..df9ecbf8f 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_2subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_4subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_4subd_half.cu index 7e07033c7..4361bc3d8 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_8pq_4subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 1420918a1..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index eb0a72da3..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 857f32712..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 3c00c5223..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index e5c4c7b69..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 22359d71b..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_2subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_2subd_half.cu index 773f567c4..a80cda68e 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_2subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_4subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_4subd_half.cu index dfc176abd..c22542b30 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_8pq_4subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 37c783f19..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 0a4049d79..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 680c32655..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index e57881e82..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 525004f2e..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 7af2ef124..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_2subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_2subd_half.cu index aa58ac2b7..599e8a1e7 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_2subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_4subd_half.cu similarity index 90% rename from cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu rename to cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_4subd_half.cu index 189c3ed9c..bd5d6e370 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_8pq_4subd_half.cu @@ -27,9 +27,7 @@ #include "search_single_cta_inst.cuh" namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< +instantiate_kernel_selection(cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 0fd36c31b..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index d4cc5f449..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 9dc9aaae3..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 100110313..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 8d4e0aeee..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 4c7318735..000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "compute_distance_vpq.cuh" -#include "search_single_cta_inst.cuh" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - cuvs::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh b/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh index efbf9b56d..c1972f4c2 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh @@ -51,10 +51,7 @@ namespace cuvs::neighbors::cagra::detail { namespace multi_cta_search { -template +template struct search : public search_plan_impl { using DATA_T = typename DATASET_DESCRIPTOR_T::DATA_T; @@ -130,7 +127,7 @@ struct search : public search_plan_impl { RAFT_EXPECTS(result_buffer_size_32 <= 256, "Result buffer size cannot exceed 256"); const auto query_smem_buffer_length = - raft::ceildiv(dim, DATASET_BLOCK_DIM) * DATASET_BLOCK_DIM; + raft::round_up_safe(dim, device::get_DBD(team_size)); smem_size = sizeof(float) * query_smem_buffer_length + (sizeof(INDEX_T) + sizeof(DISTANCE_T)) * result_buffer_size_32 + @@ -221,27 +218,27 @@ struct search : public search_plan_impl { { cudaStream_t stream = raft::resource::get_cuda_stream(res); - select_and_run( - dataset_desc, - graph, - intermediate_indices.data(), - intermediate_distances.data(), - queries_ptr, - num_queries, - dev_seed_ptr, - num_executed_iterations, - *this, - topk, - thread_block_size, - result_buffer_size, - smem_size, - hash_bitlen, - hashmap.data(), - num_cta_per_query, - num_seeds, - sample_filter, - this->metric, - stream); + select_and_run(dataset_desc, + graph, + intermediate_indices.data(), + intermediate_distances.data(), + queries_ptr, + num_queries, + dev_seed_ptr, + num_executed_iterations, + *this, + topk, + thread_block_size, + result_buffer_size, + smem_size, + hash_bitlen, + hashmap.data(), + num_cta_per_query, + num_seeds, + sample_filter, + this->metric, + stream, + team_size); RAFT_CUDA_TRY(cudaPeekAtLastError()); // Select the top-k results from the intermediate results diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py index cb63c0e03..6d8ad63a6 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py @@ -48,7 +48,6 @@ } // namespace cuvs::neighbors::cagra::detail::multi_cta_search """ -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] # block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] # mxelem = [64, 128, 256] load_types = ["uint4"] @@ -66,13 +65,12 @@ ) # knn for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - path = f"search_multi_cta_{type_path}_dim{mxdim}_t{team}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" - ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") + path = f"search_multi_cta_{type_path}.cu" + with open(path, "w") as f: + f.write(header) + f.write( + f"instantiate_kernel_selection(\n cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" + ) + f.write(trailer) + # For pasting into CMakeLists.txt + print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32.cu index ea38b60c0..d85efae44 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::multi_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu deleted file mode 100644 index 0bf4a192f..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu deleted file mode 100644 index a77859b7d..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu deleted file mode 100644 index ab49fa9f2..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64.cu index 3094ddaeb..6661d781b 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::multi_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu deleted file mode 100644 index c38eeb009..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu deleted file mode 100644 index 91725d185..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu deleted file mode 100644 index 0f452a6fa..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32.cu index 157942dc5..b6498f7dc 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::multi_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu deleted file mode 100644 index cfe7a7aef..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu deleted file mode 100644 index ee2400037..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu deleted file mode 100644 index 292859382..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64.cu index 2a14699f4..09d2e3817 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::multi_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu deleted file mode 100644 index 13044f12d..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu deleted file mode 100644 index 2ce6f292d..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu deleted file mode 100644 index 2d607eb8d..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cuh b/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cuh index b1cfaf870..531401606 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cuh @@ -21,29 +21,30 @@ namespace cuvs::neighbors::cagra::detail::multi_cta_search { -#define instantiate_kernel_selection(TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \ - template void select_and_run( \ - DATASET_DESC_T dataset_desc, \ - raft::device_matrix_view \ - graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - const search_params& ps, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_seeds, \ - SAMPLE_FILTER_T sample_filter, \ - cuvs::distance::DistanceType metric, \ - cudaStream_t stream); +#define instantiate_kernel_selection(DATASET_DESC_T, SAMPLE_FILTER_T) \ + template void select_and_run( \ + DATASET_DESC_T dataset_desc, \ + raft::device_matrix_view \ + graph, \ + typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + const uint32_t num_queries, \ + const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ + uint32_t* const num_executed_iterations, \ + const search_params& ps, \ + uint32_t topk, \ + uint32_t block_size, \ + uint32_t result_buffer_size, \ + uint32_t smem_size, \ + int64_t hash_bitlen, \ + typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ + uint32_t num_cta_per_query, \ + uint32_t num_seeds, \ + SAMPLE_FILTER_T sample_filter, \ + cuvs::distance::DistanceType metric, \ + cudaStream_t stream, \ + uint32_t team_size); #define COMMA , diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32.cu index af5f13397..99120ee76 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::multi_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu deleted file mode 100644 index c28adbf80..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu deleted file mode 100644 index bcc7b9b8c..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu deleted file mode 100644 index 916196c35..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh b/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh index e907568f5..246f39c09 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh @@ -26,10 +26,7 @@ namespace multi_cta_search { #ifdef _CUVS_EXPLICIT_INSTANTIATE_ONLY -template +template void select_and_run( DATASET_DESCRIPTOR_T dataset_desc, raft::device_matrix_view @@ -52,14 +49,12 @@ void select_and_run( uint32_t num_seeds, SAMPLE_FILTER_T sample_filter, cuvs::distance::DistanceType metric, - cudaStream_t stream) RAFT_EXPLICIT; + cudaStream_t stream, + uint32_t team_size) RAFT_EXPLICIT; #endif // CUVS_EXPLICIT_INSTANTIATE_ONLY -#define instantiate_kernel_selection( \ - TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ +#define instantiate_kernel_selection(DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ extern template void select_and_run< \ - TEAM_SIZE, \ - MAX_DATASET_DIM, \ cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, \ SAMPLE_FILTER_T>( \ cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t \ @@ -82,56 +77,32 @@ void select_and_run( uint32_t num_seeds, \ SAMPLE_FILTER_T sample_filter, \ cuvs::distance::DistanceType metric, \ - cudaStream_t stream); + cudaStream_t stream, \ + uint32_t team_size); -instantiate_kernel_selection( - 32, 1024, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); +instantiate_kernel_selection(float, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); +instantiate_kernel_selection(half, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); +instantiate_kernel_selection(int8_t, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); +instantiate_kernel_selection(uint8_t, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_kernel_selection -#define instantiate_q_kernel_selection(TEAM_SIZE, \ - MAX_DATASET_DIM, \ - CODE_BOOK_T, \ - PQ_BITS, \ - PQ_CODE_BOOK_DIM, \ - DATA_T, \ - INDEX_T, \ - DISTANCE_T, \ - SAMPLE_FILTER_T) \ +#define instantiate_q_kernel_selection( \ + CODE_BOOK_T, PQ_BITS, PQ_CODE_BOOK_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ extern template void \ - select_and_run +template __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( typename DATASET_DESCRIPTOR_T::INDEX_T* const result_indices_ptr, // [num_queries, num_cta_per_query, itopk_size] @@ -157,7 +153,8 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( const uint32_t max_iteration, uint32_t* const num_executed_iterations, /* stats */ SAMPLE_FILTER_T sample_filter, - const cuvs::distance::DistanceType metric) + const cuvs::distance::DistanceType metric, + uint32_t team_size) { using DATA_T = typename DATASET_DESCRIPTOR_T::DATA_T; using INDEX_T = typename DATASET_DESCRIPTOR_T::INDEX_T; @@ -198,7 +195,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( assert(result_buffer_size_32 <= MAX_ELEMENTS); const auto query_smem_buffer_length = - raft::ceildiv(dataset_desc.dim, DATASET_BLOCK_DIM) * DATASET_BLOCK_DIM; + raft::round_up_safe(dataset_desc.dim, device::get_DBD(team_size)); auto query_buffer = reinterpret_cast(smem); auto result_indices_buffer = reinterpret_cast(query_buffer + query_smem_buffer_length); auto result_distances_buffer = @@ -221,8 +218,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( } #endif const DATA_T* const query_ptr = queries_ptr + (dataset_desc.dim * query_id); - dataset_desc.template copy_query( - query_ptr, query_buffer, query_smem_buffer_length); + dataset_desc.copy_query(query_ptr, query_buffer, query_smem_buffer_length); if (threadIdx.x == 0) { terminate_flag[0] = 0; } INDEX_T* const local_visited_hashmap_ptr = @@ -236,20 +232,21 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( uint32_t block_id = cta_id + (num_cta_per_query * query_id); uint32_t num_blocks = num_cta_per_query * num_queries; - device::compute_distance_to_random_nodes(result_indices_buffer, - result_distances_buffer, - query_buffer, - dataset_desc, - result_buffer_size, - num_distilation, - rand_xor_mask, - local_seed_ptr, - num_seeds, - local_visited_hashmap_ptr, - hash_bitlen, - metric, - block_id, - num_blocks); + device::compute_distance_to_random_nodes(team_size, + result_indices_buffer, + result_distances_buffer, + query_buffer, + dataset_desc, + result_buffer_size, + num_distilation, + rand_xor_mask, + local_seed_ptr, + num_seeds, + local_visited_hashmap_ptr, + hash_bitlen, + metric, + block_id, + num_blocks); __syncthreads(); _CLK_REC(clk_compute_1st_distance); @@ -281,19 +278,19 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( _CLK_START(); // constexpr unsigned max_n_frags = 16; constexpr unsigned max_n_frags = 0; - device::compute_distance_to_child_nodes( - result_indices_buffer + itopk_size, - result_distances_buffer + itopk_size, - query_buffer, - dataset_desc, - knn_graph, - graph_degree, - local_visited_hashmap_ptr, - hash_bitlen, - parent_indices_buffer, - result_indices_buffer, - search_width, - metric); + device::compute_distance_to_child_nodes(team_size, + result_indices_buffer + itopk_size, + result_distances_buffer + itopk_size, + query_buffer, + dataset_desc, + knn_graph, + graph_degree, + local_visited_hashmap_ptr, + hash_bitlen, + parent_indices_buffer, + result_indices_buffer, + search_width, + metric); _CLK_REC(clk_compute_distance); __syncthreads(); @@ -409,45 +406,27 @@ void set_value_batch(T* const dev_ptr, <<>>(dev_ptr, ld, val, count, batch_size); } -template +template struct search_kernel_config { // Search kernel function type. Note that the actual values for the template value // parameters do not matter, because they are not part of the function signature. The // second to fourth value parameters will be selected by the choose_* functions below. - using kernel_t = decltype(&search_kernel); + using kernel_t = decltype(&search_kernel<128, DATASET_DESCRIPTOR_T, SAMPLE_FILTER_T>); static auto choose_buffer_size(unsigned result_buffer_size, unsigned block_size) -> kernel_t { if (result_buffer_size <= 64) { - return search_kernel; + return search_kernel<64, DATASET_DESCRIPTOR_T, SAMPLE_FILTER_T>; } else if (result_buffer_size <= 128) { - return search_kernel; + return search_kernel<128, DATASET_DESCRIPTOR_T, SAMPLE_FILTER_T>; } else if (result_buffer_size <= 256) { - return search_kernel; + return search_kernel<256, DATASET_DESCRIPTOR_T, SAMPLE_FILTER_T>; } THROW("Result buffer size %u larger than max buffer size %u", result_buffer_size, 256); } }; -template +template void select_and_run( DATASET_DESCRIPTOR_T dataset_desc, raft::device_matrix_view @@ -470,11 +449,11 @@ void select_and_run( uint32_t num_seeds, SAMPLE_FILTER_T sample_filter, cuvs::distance::DistanceType metric, - cudaStream_t stream) + cudaStream_t stream, + uint32_t team_size) { - auto kernel = - search_kernel_config:: - choose_buffer_size(result_buffer_size, block_size); + auto kernel = search_kernel_config::choose_buffer_size( + result_buffer_size, block_size); RAFT_CUDA_TRY(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, @@ -514,7 +493,8 @@ void select_and_run( ps.max_iterations, num_executed_iterations, sample_filter, - metric); + metric, + team_size); } } // namespace multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32.cu index e2f25a1c2..f1ba37bb9 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::multi_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu deleted file mode 100644 index 3fa12d933..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu deleted file mode 100644 index 4cd206d8c..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu deleted file mode 100644 index 56989a1d5..000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta.cuh b/cpp/src/neighbors/detail/cagra/search_single_cta.cuh index 0a101cbfe..a864b8a99 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta.cuh +++ b/cpp/src/neighbors/detail/cagra/search_single_cta.cuh @@ -49,10 +49,7 @@ namespace cuvs::neighbors::cagra::detail { namespace single_cta_search { -template +template struct search : search_plan_impl { using DATA_T = typename DATASET_DESCRIPTOR_T::DATA_T; using INDEX_T = typename DATASET_DESCRIPTOR_T::INDEX_T; @@ -129,7 +126,7 @@ struct search : search_plan_impl { // const std::uint32_t topk_ws_size = 3; const auto query_smem_buffer_length = - raft::ceildiv(dim, DATASET_BLOCK_DIM) * DATASET_BLOCK_DIM; + raft::round_up_safe(dim, device::get_DBD(team_size)); const std::uint32_t base_smem_size = sizeof(float) * query_smem_buffer_length + (sizeof(INDEX_T) + sizeof(DISTANCE_T)) * result_buffer_size_32 + @@ -224,28 +221,28 @@ struct search : search_plan_impl { SAMPLE_FILTER_T sample_filter) { cudaStream_t stream = raft::resource::get_cuda_stream(res); - select_and_run( - dataset_desc, - graph, - result_indices_ptr, - result_distances_ptr, - queries_ptr, - num_queries, - dev_seed_ptr, - num_executed_iterations, - *this, - topk, - num_itopk_candidates, - static_cast(thread_block_size), - smem_size, - hash_bitlen, - hashmap.data(), - small_hash_bitlen, - small_hash_reset_interval, - num_seeds, - sample_filter, - this->metric, - stream); + select_and_run(dataset_desc, + graph, + result_indices_ptr, + result_distances_ptr, + queries_ptr, + num_queries, + dev_seed_ptr, + num_executed_iterations, + *this, + topk, + num_itopk_candidates, + static_cast(thread_block_size), + smem_size, + hash_bitlen, + hashmap.data(), + small_hash_bitlen, + small_hash_reset_interval, + num_seeds, + sample_filter, + this->metric, + stream, + team_size); } }; diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py index a361269a6..cd6be786c 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py @@ -69,14 +69,13 @@ # knn for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - path = f"search_single_cta_{type_path}_dim{mxdim}_t{team}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" - ) + path = f"search_single_cta_{type_path}.cu" + with open(path, "w") as f: + f.write(header) + f.write( + f"instantiate_kernel_selection(\n cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, cuvs::neighbors::filtering::none_cagra_sample_filter);\n" + ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") + f.write(trailer) + # For pasting into CMakeLists.txt + print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32.cu index 7d3e86f38..5ad16ed0f 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::single_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu deleted file mode 100644 index 4cf4a26f7..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu deleted file mode 100644 index 692710476..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu deleted file mode 100644 index ed3a900ff..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64.cu index 8b26a595f..6646127e2 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::single_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu deleted file mode 100644 index 2c4da00db..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu deleted file mode 100644 index a93f893d4..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu deleted file mode 100644 index 4a7502e3e..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32.cu index 1e2b83492..4c6ba93dc 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::single_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu deleted file mode 100644 index 6c13df91a..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu deleted file mode 100644 index cfae9e367..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu deleted file mode 100644 index 12aa72a24..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64.cu index c2cfb13c4..da9eb6d7c 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::single_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu deleted file mode 100644 index 84a173d6d..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu deleted file mode 100644 index d9c5198eb..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu deleted file mode 100644 index 3ba8f4e4d..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cuh b/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cuh index a4581d15e..492fa7a79 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cuh +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cuh @@ -21,30 +21,31 @@ namespace cuvs::neighbors::cagra::detail::single_cta_search { -#define instantiate_kernel_selection(TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \ - template void select_and_run( \ - DATASET_DESC_T dataset_desc, \ - raft::device_matrix_view \ - graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - const search_params& ps, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_seeds, \ - SAMPLE_FILTER_T sample_filter, \ - cuvs::distance::DistanceType metric, \ - cudaStream_t stream); +#define instantiate_kernel_selection(DATASET_DESC_T, SAMPLE_FILTER_T) \ + template void select_and_run( \ + DATASET_DESC_T dataset_desc, \ + raft::device_matrix_view \ + graph, \ + typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + const uint32_t num_queries, \ + const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ + uint32_t* const num_executed_iterations, \ + const search_params& ps, \ + uint32_t topk, \ + uint32_t num_itopk_candidates, \ + uint32_t block_size, \ + uint32_t smem_size, \ + int64_t hash_bitlen, \ + typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ + size_t small_hash_bitlen, \ + size_t small_hash_reset_interval, \ + uint32_t num_seeds, \ + SAMPLE_FILTER_T sample_filter, \ + cuvs::distance::DistanceType metric, \ + cudaStream_t stream, \ + uint32_t team_size); #define COMMA , diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32.cu index 6130a84bc..16ae10d79 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::single_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu deleted file mode 100644 index ad2ca16fc..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu deleted file mode 100644 index 1e7bee57c..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu deleted file mode 100644 index 7f789e3d0..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh b/cpp/src/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh index 10dda0389..b9f17302c 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh @@ -25,10 +25,7 @@ namespace single_cta_search { #ifdef _CUVS_EXPLICIT_INSTANTIATE_ONLY -template +template void select_and_run( DATASET_DESCRIPTOR_T dataset_desc, raft::device_matrix_view @@ -51,15 +48,13 @@ void select_and_run( uint32_t num_seeds, SAMPLE_FILTER_T sample_filter, cuvs::distance::DistanceType metric, - cudaStream_t stream) RAFT_EXPLICIT; + cudaStream_t stream, + uint32_t team_size) RAFT_EXPLICIT; #endif // CUVS_EXPLICIT_INSTANTIATE_ONLY -#define instantiate_single_cta_select_and_run( \ - TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ +#define instantiate_single_cta_select_and_run(DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ extern template void select_and_run< \ - TEAM_SIZE, \ - MAX_DATASET_DIM, \ cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, \ SAMPLE_FILTER_T>( \ cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t \ @@ -83,56 +78,32 @@ void select_and_run( uint32_t num_seeds, \ SAMPLE_FILTER_T sample_filter, \ cuvs::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_single_cta_select_and_run( - 32, 1024, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, half, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); + cudaStream_t stream, \ + uint32_t team_size); + +instantiate_single_cta_select_and_run(float, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run(half, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run(int8_t, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run(uint8_t, + uint32_t, + float, + cuvs::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_select_and_run -#define instantiate_q_single_cta_select_and_run(TEAM_SIZE, \ - MAX_DATASET_DIM, \ - CODE_BOOK_T, \ - PQ_BITS, \ - PQ_CODE_BOOK_DIM, \ - DATA_T, \ - INDEX_T, \ - DISTANCE_T, \ - SAMPLE_FILTER_T) \ +#define instantiate_q_single_cta_select_and_run( \ + CODE_BOOK_T, PQ_BITS, PQ_CODE_BOOK_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ extern template void \ - select_and_run(dataset_desc.dim, DATASET_BLOCK_DIM) * DATASET_BLOCK_DIM; + raft::round_up_safe(dataset_desc.dim, device::get_DBD(team_size)); auto query_buffer = reinterpret_cast(smem); auto result_indices_buffer = reinterpret_cast(query_buffer + query_smem_buffer_length); auto result_distances_buffer = @@ -548,8 +547,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( auto filter_flag = terminate_flag; const DATA_T* const query_ptr = queries_ptr + query_id * dataset_desc.dim; - dataset_desc.template copy_query( - query_ptr, query_buffer, query_smem_buffer_length); + dataset_desc.copy_query(query_ptr, query_buffer, query_smem_buffer_length); if (threadIdx.x == 0) { terminate_flag[0] = 0; @@ -570,18 +568,19 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( // compute distance to randomly selecting nodes _CLK_START(); const INDEX_T* const local_seed_ptr = seed_ptr ? seed_ptr + (num_seeds * query_id) : nullptr; - device::compute_distance_to_random_nodes(result_indices_buffer, - result_distances_buffer, - query_buffer, - dataset_desc, - result_buffer_size, - num_distilation, - rand_xor_mask, - local_seed_ptr, - num_seeds, - local_visited_hashmap_ptr, - hash_bitlen, - metric); + device::compute_distance_to_random_nodes(team_size, + result_indices_buffer, + result_distances_buffer, + query_buffer, + dataset_desc, + result_buffer_size, + num_distilation, + rand_xor_mask, + local_seed_ptr, + num_seeds, + local_visited_hashmap_ptr, + hash_bitlen, + metric); __syncthreads(); _CLK_REC(clk_compute_1st_distance); @@ -707,19 +706,19 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( // compute the norms between child nodes and query node _CLK_START(); constexpr unsigned max_n_frags = 8; - device::compute_distance_to_child_nodes( - result_indices_buffer + internal_topk, - result_distances_buffer + internal_topk, - query_buffer, - dataset_desc, - knn_graph, - graph_degree, - local_visited_hashmap_ptr, - hash_bitlen, - parent_list_buffer, - result_indices_buffer, - search_width, - metric); + device::compute_distance_to_child_nodes(team_size, + result_indices_buffer + internal_topk, + result_distances_buffer + internal_topk, + query_buffer, + dataset_desc, + knn_graph, + graph_degree, + local_visited_hashmap_ptr, + hash_bitlen, + parent_list_buffer, + result_indices_buffer, + search_width, + metric); __syncthreads(); _CLK_REC(clk_compute_distance); @@ -815,50 +814,33 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( #endif } -template +template struct search_kernel_config { - using kernel_t = decltype(&search_kernel); + using kernel_t = decltype(&search_kernel<64, 64, 0, DATASET_DESCRIPTOR_T, SAMPLE_FILTER_T>); template static auto choose_search_kernel(unsigned itopk_size) -> kernel_t { if (itopk_size <= 64) { - return search_kernel; } else if (itopk_size <= 128) { - return search_kernel; } else if (itopk_size <= 256) { - return search_kernel; } else if (itopk_size <= 512) { - return search_kernel; + return search_kernel<256, max_candidates, 0, DATASET_DESCRIPTOR_T, SAMPLE_FILTER_T>; } else if (itopk_size <= 512) { - return search_kernel; + return search_kernel<512, max_candidates, 0, DATASET_DESCRIPTOR_T, SAMPLE_FILTER_T>; } } THROW("No kernel for parametels itopk_size %u, num_itopk_candidates %u", @@ -905,10 +875,7 @@ struct search_kernel_config { } }; -template +template void select_and_run( DATASET_DESCRIPTOR_T dataset_desc, raft::device_matrix_view @@ -931,11 +898,12 @@ void select_and_run( uint32_t num_seeds, SAMPLE_FILTER_T sample_filter, cuvs::distance::DistanceType metric, - cudaStream_t stream) + cudaStream_t stream, + uint32_t team_size) { auto kernel = - search_kernel_config:: - choose_itopk_and_mx_candidates(ps.itopk_size, num_itopk_candidates, block_size); + search_kernel_config::choose_itopk_and_mx_candidates( + ps.itopk_size, num_itopk_candidates, block_size); RAFT_CUDA_TRY(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size + DATASET_DESCRIPTOR_T::smem_buffer_size_in_byte)); @@ -964,7 +932,8 @@ void select_and_run( small_hash_bitlen, small_hash_reset_interval, sample_filter, - metric); + metric, + team_size); RAFT_CUDA_TRY(cudaPeekAtLastError()); } } // namespace single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32.cu similarity index 98% rename from cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu rename to cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32.cu index 614e6ca01..34295d82e 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32.cu @@ -29,8 +29,6 @@ namespace cuvs::neighbors::cagra::detail::single_cta_search { instantiate_kernel_selection( - 8, - 128, cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, cuvs::neighbors::filtering::none_cagra_sample_filter); diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu deleted file mode 100644 index 35e04ea6a..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu deleted file mode 100644 index 005afb566..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu deleted file mode 100644 index af30b2e24..000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta_inst.cuh" - -#include "compute_distance.hpp" - -namespace cuvs::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - cuvs::neighbors::cagra::detail::standard_dataset_descriptor_t, - cuvs::neighbors::filtering::none_cagra_sample_filter); - -} // namespace cuvs::neighbors::cagra::detail::single_cta_search diff --git a/cpp/test/neighbors/ann_cagra.cuh b/cpp/test/neighbors/ann_cagra.cuh index 9d2f9c175..4ce0849fd 100644 --- a/cpp/test/neighbors/ann_cagra.cuh +++ b/cpp/test/neighbors/ann_cagra.cuh @@ -706,7 +706,7 @@ inline std::vector generate_inputs() {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, {search_algo::AUTO}, {10}, - {0, 4, 8, 16, 32}, // team_size + {0, 8, 16, 32}, // team_size {64}, {1}, {cuvs::distance::DistanceType::L2Expanded},