From e66616de2e37d7588178148b6c05f7f6ef1f1b13 Mon Sep 17 00:00:00 2001 From: Clemens Winter Date: Sun, 11 Aug 2024 10:15:44 -0700 Subject: [PATCH] Intern strings when serializing meta db --- Cargo.lock | 2 +- Cargo.toml | 2 +- locustdb-serialization/Cargo.lock | 2 +- locustdb-serialization/Cargo.toml | 2 +- locustdb-serialization/schemas/dbmeta.capnp | 4 +- locustdb-serialization/src/dbmeta_capnp.rs | 147 ++++++++++++++++---- src/disk_store/meta_store.rs | 39 ++++-- 7 files changed, 153 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1474054b..98592a34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1893,7 +1893,7 @@ dependencies = [ [[package]] name = "locustdb-serialization" -version = "0.2.1" +version = "0.2.2" dependencies = [ "capnp", "serde", diff --git a/Cargo.toml b/Cargo.toml index 8f26076e..ba1b8eca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ itertools = "0.5" lazy_static = "1.4.0" locustdb-compression-utils = {path = "./locustdb-compression-utils", version = "0.2.0"} locustdb-derive = {path = "./locustdb-derive", version = "0.2.1"} -locustdb-serialization = {path = "./locustdb-serialization", version = "0.2.1"} +locustdb-serialization = {path = "./locustdb-serialization", version = "0.2.2"} log = {features = ["max_level_trace", "release_max_level_debug"], version = "0.4"} lru = "0.12" lz4_flex = { version = "0.11" } diff --git a/locustdb-serialization/Cargo.lock b/locustdb-serialization/Cargo.lock index 19efd986..345bf15d 100644 --- a/locustdb-serialization/Cargo.lock +++ b/locustdb-serialization/Cargo.lock @@ -25,7 +25,7 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "locustdb-serialization" -version = "0.2.1" +version = "0.2.2" dependencies = [ "capnp", "pretty_assertions", diff --git a/locustdb-serialization/Cargo.toml b/locustdb-serialization/Cargo.toml index 24addcd4..28a9cf2c 100644 --- a/locustdb-serialization/Cargo.toml +++ b/locustdb-serialization/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "locustdb-serialization" -version = "0.2.1" +version = "0.2.2" edition = "2021" description = "Serialization formats used by LocustDB for peristent storage and client/server communication." license-file = "../LICENSE" diff --git a/locustdb-serialization/schemas/dbmeta.capnp b/locustdb-serialization/schemas/dbmeta.capnp index 7b65c7f2..d44d4de7 100644 --- a/locustdb-serialization/schemas/dbmeta.capnp +++ b/locustdb-serialization/schemas/dbmeta.capnp @@ -3,6 +3,7 @@ struct DBMeta { nextWalId @0 :UInt64; partitions @1 :List(PartitionMetadata); + strings @2 :List(Text); # unused in legacy format } struct PartitionMetadata { @@ -16,5 +17,6 @@ struct PartitionMetadata { struct SubpartitionMetadata { sizeBytes @0 :UInt64; subpartitionKey @1 :Text; - columns @2 :List(Text); + columns @2 :List(Text); # deprecated in favor of internedColumns + internedColumns @3 :List(UInt64); } \ No newline at end of file diff --git a/locustdb-serialization/src/dbmeta_capnp.rs b/locustdb-serialization/src/dbmeta_capnp.rs index 7a430e21..f6781166 100644 --- a/locustdb-serialization/src/dbmeta_capnp.rs +++ b/locustdb-serialization/src/dbmeta_capnp.rs @@ -76,11 +76,19 @@ pub mod d_b_meta { pub fn has_partitions(&self) -> bool { !self.reader.get_pointer_field(0).is_null() } + #[inline] + pub fn get_strings(self) -> ::capnp::Result<::capnp::text_list::Reader<'a>> { + ::capnp::traits::FromPointerReader::get_from_pointer(&self.reader.get_pointer_field(1), ::core::option::Option::None) + } + #[inline] + pub fn has_strings(&self) -> bool { + !self.reader.get_pointer_field(1).is_null() + } } pub struct Builder<'a> { builder: ::capnp::private::layout::StructBuilder<'a> } impl <'a,> ::capnp::traits::HasStructSize for Builder<'a,> { - const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 1 }; + const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 2 }; } impl <'a,> ::capnp::traits::HasTypeId for Builder<'a,> { const TYPE_ID: u64 = _private::TYPE_ID; @@ -154,6 +162,22 @@ pub mod d_b_meta { pub fn has_partitions(&self) -> bool { !self.builder.is_pointer_field_null(0) } + #[inline] + pub fn get_strings(self) -> ::capnp::Result<::capnp::text_list::Builder<'a>> { + ::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(1), ::core::option::Option::None) + } + #[inline] + pub fn set_strings(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::text_list::Owned>) -> ::capnp::Result<()> { + ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(1), value, false) + } + #[inline] + pub fn init_strings(self, size: u32) -> ::capnp::text_list::Builder<'a> { + ::capnp::traits::FromPointerBuilder::init_pointer(self.builder.get_pointer_field(1), size) + } + #[inline] + pub fn has_strings(&self) -> bool { + !self.builder.is_pointer_field_null(1) + } } pub struct Pipeline { _typeless: ::capnp::any_pointer::Pipeline } @@ -165,38 +189,45 @@ pub mod d_b_meta { impl Pipeline { } mod _private { - pub static ENCODED_NODE: [::capnp::Word; 53] = [ + pub static ENCODED_NODE: [::capnp::Word; 72] = [ ::capnp::word(0, 0, 0, 0, 5, 0, 6, 0), ::capnp::word(134, 233, 91, 102, 237, 22, 116, 146), ::capnp::word(13, 0, 0, 0, 1, 0, 1, 0), ::capnp::word(245, 46, 142, 94, 29, 184, 169, 175), - ::capnp::word(1, 0, 7, 0, 0, 0, 0, 0), + ::capnp::word(2, 0, 7, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(21, 0, 0, 0, 162, 0, 0, 0), ::capnp::word(29, 0, 0, 0, 7, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(25, 0, 0, 0, 119, 0, 0, 0), + ::capnp::word(25, 0, 0, 0, 175, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(100, 98, 109, 101, 116, 97, 46, 99), ::capnp::word(97, 112, 110, 112, 58, 68, 66, 77), ::capnp::word(101, 116, 97, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 1, 0, 1, 0), - ::capnp::word(8, 0, 0, 0, 3, 0, 4, 0), + ::capnp::word(12, 0, 0, 0, 3, 0, 4, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 1, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(41, 0, 0, 0, 82, 0, 0, 0), + ::capnp::word(69, 0, 0, 0, 82, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(40, 0, 0, 0, 3, 0, 1, 0), - ::capnp::word(52, 0, 0, 0, 2, 0, 1, 0), + ::capnp::word(68, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(80, 0, 0, 0, 2, 0, 1, 0), ::capnp::word(1, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 1, 0, 1, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(49, 0, 0, 0, 90, 0, 0, 0), + ::capnp::word(77, 0, 0, 0, 90, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(76, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(104, 0, 0, 0, 2, 0, 1, 0), + ::capnp::word(2, 0, 0, 0, 1, 0, 0, 0), + ::capnp::word(0, 0, 1, 0, 2, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(48, 0, 0, 0, 3, 0, 1, 0), - ::capnp::word(76, 0, 0, 0, 2, 0, 1, 0), + ::capnp::word(101, 0, 0, 0, 66, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(96, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(124, 0, 0, 0, 2, 0, 1, 0), ::capnp::word(110, 101, 120, 116, 87, 97, 108, 73), ::capnp::word(100, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(9, 0, 0, 0, 0, 0, 0, 0), @@ -219,11 +250,24 @@ pub mod d_b_meta { ::capnp::word(14, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(115, 116, 114, 105, 110, 103, 115, 0), + ::capnp::word(14, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(12, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(14, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ]; pub fn get_field_types(index: u16) -> ::capnp::introspect::Type { match index { 0 => ::introspect(), 1 => <::capnp::struct_list::Owned as ::capnp::introspect::Introspect>::introspect(), + 2 => <::capnp::text_list::Owned as ::capnp::introspect::Introspect>::introspect(), _ => panic!("invalid field index {}", index), } } @@ -236,9 +280,9 @@ pub mod d_b_meta { members_by_discriminant: MEMBERS_BY_DISCRIMINANT, members_by_name: MEMBERS_BY_NAME, }; - pub static NONUNION_MEMBERS : &[u16] = &[0,1]; + pub static NONUNION_MEMBERS : &[u16] = &[0,1,2]; pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[]; - pub static MEMBERS_BY_NAME : &[u16] = &[0,1]; + pub static MEMBERS_BY_NAME : &[u16] = &[0,1,2]; pub const TYPE_ID: u64 = 0x9274_16ed_665b_e986; } } @@ -661,11 +705,19 @@ pub mod subpartition_metadata { pub fn has_columns(&self) -> bool { !self.reader.get_pointer_field(1).is_null() } + #[inline] + pub fn get_interned_columns(self) -> ::capnp::Result<::capnp::primitive_list::Reader<'a,u64>> { + ::capnp::traits::FromPointerReader::get_from_pointer(&self.reader.get_pointer_field(2), ::core::option::Option::None) + } + #[inline] + pub fn has_interned_columns(&self) -> bool { + !self.reader.get_pointer_field(2).is_null() + } } pub struct Builder<'a> { builder: ::capnp::private::layout::StructBuilder<'a> } impl <'a,> ::capnp::traits::HasStructSize for Builder<'a,> { - const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 2 }; + const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 3 }; } impl <'a,> ::capnp::traits::HasTypeId for Builder<'a,> { const TYPE_ID: u64 = _private::TYPE_ID; @@ -755,6 +807,22 @@ pub mod subpartition_metadata { pub fn has_columns(&self) -> bool { !self.builder.is_pointer_field_null(1) } + #[inline] + pub fn get_interned_columns(self) -> ::capnp::Result<::capnp::primitive_list::Builder<'a,u64>> { + ::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(2), ::core::option::Option::None) + } + #[inline] + pub fn set_interned_columns(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::primitive_list::Owned>) -> ::capnp::Result<()> { + ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(2), value, false) + } + #[inline] + pub fn init_interned_columns(self, size: u32) -> ::capnp::primitive_list::Builder<'a,u64> { + ::capnp::traits::FromPointerBuilder::init_pointer(self.builder.get_pointer_field(2), size) + } + #[inline] + pub fn has_interned_columns(&self) -> bool { + !self.builder.is_pointer_field_null(2) + } } pub struct Pipeline { _typeless: ::capnp::any_pointer::Pipeline } @@ -766,17 +834,17 @@ pub mod subpartition_metadata { impl Pipeline { } mod _private { - pub static ENCODED_NODE: [::capnp::Word; 70] = [ + pub static ENCODED_NODE: [::capnp::Word; 90] = [ ::capnp::word(0, 0, 0, 0, 5, 0, 6, 0), ::capnp::word(48, 51, 123, 36, 100, 192, 248, 192), ::capnp::word(13, 0, 0, 0, 1, 0, 1, 0), ::capnp::word(245, 46, 142, 94, 29, 184, 169, 175), - ::capnp::word(2, 0, 7, 0, 0, 0, 0, 0), + ::capnp::word(3, 0, 7, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(21, 0, 0, 0, 18, 1, 0, 0), ::capnp::word(37, 0, 0, 0, 7, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(33, 0, 0, 0, 175, 0, 0, 0), + ::capnp::word(33, 0, 0, 0, 231, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(100, 98, 109, 101, 116, 97, 46, 99), @@ -785,28 +853,35 @@ pub mod subpartition_metadata { ::capnp::word(110, 77, 101, 116, 97, 100, 97, 116), ::capnp::word(97, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 1, 0, 1, 0), - ::capnp::word(12, 0, 0, 0, 3, 0, 4, 0), + ::capnp::word(16, 0, 0, 0, 3, 0, 4, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 1, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(69, 0, 0, 0, 82, 0, 0, 0), + ::capnp::word(97, 0, 0, 0, 82, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(68, 0, 0, 0, 3, 0, 1, 0), - ::capnp::word(80, 0, 0, 0, 2, 0, 1, 0), + ::capnp::word(96, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(108, 0, 0, 0, 2, 0, 1, 0), ::capnp::word(1, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 1, 0, 1, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(77, 0, 0, 0, 130, 0, 0, 0), + ::capnp::word(105, 0, 0, 0, 130, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(76, 0, 0, 0, 3, 0, 1, 0), - ::capnp::word(88, 0, 0, 0, 2, 0, 1, 0), + ::capnp::word(104, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(116, 0, 0, 0, 2, 0, 1, 0), ::capnp::word(2, 0, 0, 0, 1, 0, 0, 0), ::capnp::word(0, 0, 1, 0, 2, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(85, 0, 0, 0, 66, 0, 0, 0), + ::capnp::word(113, 0, 0, 0, 66, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), - ::capnp::word(80, 0, 0, 0, 3, 0, 1, 0), - ::capnp::word(108, 0, 0, 0, 2, 0, 1, 0), + ::capnp::word(108, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(136, 0, 0, 0, 2, 0, 1, 0), + ::capnp::word(3, 0, 0, 0, 2, 0, 0, 0), + ::capnp::word(0, 0, 1, 0, 3, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(133, 0, 0, 0, 130, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(132, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(160, 0, 0, 0, 2, 0, 1, 0), ::capnp::word(115, 105, 122, 101, 66, 121, 116, 101), ::capnp::word(115, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(9, 0, 0, 0, 0, 0, 0, 0), @@ -837,12 +912,26 @@ pub mod subpartition_metadata { ::capnp::word(14, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(105, 110, 116, 101, 114, 110, 101, 100), + ::capnp::word(67, 111, 108, 117, 109, 110, 115, 0), + ::capnp::word(14, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 3, 0, 1, 0), + ::capnp::word(9, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(14, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), + ::capnp::word(0, 0, 0, 0, 0, 0, 0, 0), ]; pub fn get_field_types(index: u16) -> ::capnp::introspect::Type { match index { 0 => ::introspect(), 1 => <::capnp::text::Owned as ::capnp::introspect::Introspect>::introspect(), 2 => <::capnp::text_list::Owned as ::capnp::introspect::Introspect>::introspect(), + 3 => <::capnp::primitive_list::Owned as ::capnp::introspect::Introspect>::introspect(), _ => panic!("invalid field index {}", index), } } @@ -855,9 +944,9 @@ pub mod subpartition_metadata { members_by_discriminant: MEMBERS_BY_DISCRIMINANT, members_by_name: MEMBERS_BY_NAME, }; - pub static NONUNION_MEMBERS : &[u16] = &[0,1,2]; + pub static NONUNION_MEMBERS : &[u16] = &[0,1,2,3]; pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[]; - pub static MEMBERS_BY_NAME : &[u16] = &[2,0,1]; + pub static MEMBERS_BY_NAME : &[u16] = &[2,3,0,1]; pub const TYPE_ID: u64 = 0xc0f8_c064_247b_3330; } } diff --git a/src/disk_store/meta_store.rs b/src/disk_store/meta_store.rs index 0424540c..8149df93 100644 --- a/src/disk_store/meta_store.rs +++ b/src/disk_store/meta_store.rs @@ -42,18 +42,24 @@ impl MetaStore { let mut dbmeta = builder.init_root::(); dbmeta.set_next_wal_id(self.next_wal_id); + let mut column_name_to_id: HashMap<&str, u64> = HashMap::new(); + let total_partitions = self.partitions.values().map(|x| x.len()).sum::(); assert!(total_partitions < std::u32::MAX as usize); - let mut partitions_builder = dbmeta.init_partitions(total_partitions as u32); + let mut partitions_builder = dbmeta.reborrow().init_partitions(total_partitions as u32); let mut i = 0; for table in self.partitions.values() { for partition in table.values() { - let mut subpartition_index_to_column_names = vec![Vec::new(); partition.subpartitions.len()]; + let mut subpartition_index_to_column_names = + vec![Vec::new(); partition.subpartitions.len()]; for (column_name, subpartition_index) in &partition.column_name_to_subpartition_index { - subpartition_index_to_column_names[*subpartition_index] - .push(column_name.clone()); + let next_column_id = column_name_to_id.len() as u64; + let column_id = *column_name_to_id + .entry(column_name.as_str()) + .or_insert_with(|| next_column_id); + subpartition_index_to_column_names[*subpartition_index].push(column_id); } let mut partition_builder = partitions_builder.reborrow().get(i); partition_builder.set_id(partition.id); @@ -67,19 +73,22 @@ impl MetaStore { let mut subpartition_builder = subpartitions_builder.reborrow().get(i as u32); subpartition_builder.set_size_bytes(subpartition.size_bytes); subpartition_builder.set_subpartition_key(&subpartition.subpartition_key); - let mut columns_builder = subpartition_builder - .init_columns(subpartition_index_to_column_names[i].len() as u32); - for (i, column_name) in - std::mem::take(&mut subpartition_index_to_column_names[i]) - .into_iter() - .enumerate() + let mut interned_columns_builder = subpartition_builder + .init_interned_columns(subpartition_index_to_column_names[i].len() as u32); + for (i, column_id) in std::mem::take(&mut subpartition_index_to_column_names[i]) + .into_iter() + .enumerate() { - columns_builder.set(i as u32, column_name); + interned_columns_builder.set(i as u32, column_id); } } i += 1; } } + let mut strings_builder = dbmeta.init_strings(column_name_to_id.len() as u32); + for (column_name, column_id) in column_name_to_id { + strings_builder.set(column_id as u32, column_name); + } let mut buf = Vec::new(); serialize_packed::write_message(&mut buf, &builder).unwrap(); @@ -91,6 +100,10 @@ impl MetaStore { serialize_packed::read_message(&mut &data[..], default_reader_options())?; let dbmeta = message_reader.get_root::()?; let next_wal_id = dbmeta.get_next_wal_id(); + let mut strings = Vec::new(); + for string in dbmeta.get_strings()? { + strings.push(string?.to_string().unwrap()); + } let mut partitions = HashMap::>::new(); for partition in dbmeta.get_partitions()? { let id = partition.get_id(); @@ -110,6 +123,10 @@ impl MetaStore { let column = column?.to_string().unwrap(); column_name_to_subpartition_index.insert(column, i); } + for column_string_id in subpartition.get_interned_columns()? { + let column = strings[column_string_id as usize].clone(); + column_name_to_subpartition_index.insert(column, i); + } } let partition = PartitionMetadata { id,