Skip to content

Commit

Permalink
Intern strings when serializing meta db
Browse files Browse the repository at this point in the history
  • Loading branch information
@cswinter
cswinter committed Aug 11, 2024
1 parent 6adab54 commit e66616d
Show file tree
Hide file tree
Showing 7 changed files with 153 additions and 45 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ itertools = "0.5"
lazy_static = "1.4.0"
locustdb-compression-utils = {path = "./locustdb-compression-utils", version = "0.2.0"}
locustdb-derive = {path = "./locustdb-derive", version = "0.2.1"}
locustdb-serialization = {path = "./locustdb-serialization", version = "0.2.1"}
locustdb-serialization = {path = "./locustdb-serialization", version = "0.2.2"}
log = {features = ["max_level_trace", "release_max_level_debug"], version = "0.4"}
lru = "0.12"
lz4_flex = { version = "0.11" }
Expand Down
2 changes: 1 addition & 1 deletion locustdb-serialization/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion locustdb-serialization/Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "locustdb-serialization"
version = "0.2.1"
version = "0.2.2"
edition = "2021"
description = "Serialization formats used by LocustDB for peristent storage and client/server communication."
license-file = "../LICENSE"
Expand Down
4 changes: 3 additions & 1 deletion locustdb-serialization/schemas/dbmeta.capnp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
struct DBMeta {
nextWalId @0 :UInt64;
partitions @1 :List(PartitionMetadata);
strings @2 :List(Text); # unused in legacy format
}

struct PartitionMetadata {
Expand All @@ -16,5 +17,6 @@ struct PartitionMetadata {
struct SubpartitionMetadata {
sizeBytes @0 :UInt64;
subpartitionKey @1 :Text;
columns @2 :List(Text);
columns @2 :List(Text); # deprecated in favor of internedColumns
internedColumns @3 :List(UInt64);
}
147 changes: 118 additions & 29 deletions locustdb-serialization/src/dbmeta_capnp.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,19 @@ pub mod d_b_meta {
pub fn has_partitions(&self) -> bool {
!self.reader.get_pointer_field(0).is_null()
}
#[inline]
pub fn get_strings(self) -> ::capnp::Result<::capnp::text_list::Reader<'a>> {
::capnp::traits::FromPointerReader::get_from_pointer(&self.reader.get_pointer_field(1), ::core::option::Option::None)
}
#[inline]
pub fn has_strings(&self) -> bool {
!self.reader.get_pointer_field(1).is_null()
}
}

pub struct Builder<'a> { builder: ::capnp::private::layout::StructBuilder<'a> }
impl <'a,> ::capnp::traits::HasStructSize for Builder<'a,> {
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 1 };
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 2 };
}
impl <'a,> ::capnp::traits::HasTypeId for Builder<'a,> {
const TYPE_ID: u64 = _private::TYPE_ID;
Expand Down Expand Up @@ -154,6 +162,22 @@ pub mod d_b_meta {
pub fn has_partitions(&self) -> bool {
!self.builder.is_pointer_field_null(0)
}
#[inline]
pub fn get_strings(self) -> ::capnp::Result<::capnp::text_list::Builder<'a>> {
::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(1), ::core::option::Option::None)
}
#[inline]
pub fn set_strings(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::text_list::Owned>) -> ::capnp::Result<()> {
::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(1), value, false)
}
#[inline]
pub fn init_strings(self, size: u32) -> ::capnp::text_list::Builder<'a> {
::capnp::traits::FromPointerBuilder::init_pointer(self.builder.get_pointer_field(1), size)
}
#[inline]
pub fn has_strings(&self) -> bool {
!self.builder.is_pointer_field_null(1)
}
}

pub struct Pipeline { _typeless: ::capnp::any_pointer::Pipeline }
Expand All @@ -165,38 +189,45 @@ pub mod d_b_meta {
impl Pipeline {
}
mod _private {
pub static ENCODED_NODE: [::capnp::Word; 53] = [
pub static ENCODED_NODE: [::capnp::Word; 72] = [
::capnp::word(0, 0, 0, 0, 5, 0, 6, 0),
::capnp::word(134, 233, 91, 102, 237, 22, 116, 146),
::capnp::word(13, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(245, 46, 142, 94, 29, 184, 169, 175),
::capnp::word(1, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(2, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(21, 0, 0, 0, 162, 0, 0, 0),
::capnp::word(29, 0, 0, 0, 7, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(25, 0, 0, 0, 119, 0, 0, 0),
::capnp::word(25, 0, 0, 0, 175, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(100, 98, 109, 101, 116, 97, 46, 99),
::capnp::word(97, 112, 110, 112, 58, 68, 66, 77),
::capnp::word(101, 116, 97, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(8, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(12, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(41, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(69, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(40, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(52, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(68, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(80, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(1, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(49, 0, 0, 0, 90, 0, 0, 0),
::capnp::word(77, 0, 0, 0, 90, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(76, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(104, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(2, 0, 0, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 2, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(48, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(76, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(101, 0, 0, 0, 66, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(96, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(124, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(110, 101, 120, 116, 87, 97, 108, 73),
::capnp::word(100, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(9, 0, 0, 0, 0, 0, 0, 0),
Expand All @@ -219,11 +250,24 @@ pub mod d_b_meta {
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(115, 116, 114, 105, 110, 103, 115, 0),
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(12, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
];
pub fn get_field_types(index: u16) -> ::capnp::introspect::Type {
match index {
0 => <u64 as ::capnp::introspect::Introspect>::introspect(),
1 => <::capnp::struct_list::Owned<crate::dbmeta_capnp::partition_metadata::Owned> as ::capnp::introspect::Introspect>::introspect(),
2 => <::capnp::text_list::Owned as ::capnp::introspect::Introspect>::introspect(),
_ => panic!("invalid field index {}", index),
}
}
Expand All @@ -236,9 +280,9 @@ pub mod d_b_meta {
members_by_discriminant: MEMBERS_BY_DISCRIMINANT,
members_by_name: MEMBERS_BY_NAME,
};
pub static NONUNION_MEMBERS : &[u16] = &[0,1];
pub static NONUNION_MEMBERS : &[u16] = &[0,1,2];
pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[];
pub static MEMBERS_BY_NAME : &[u16] = &[0,1];
pub static MEMBERS_BY_NAME : &[u16] = &[0,1,2];
pub const TYPE_ID: u64 = 0x9274_16ed_665b_e986;
}
}
Expand Down Expand Up @@ -661,11 +705,19 @@ pub mod subpartition_metadata {
pub fn has_columns(&self) -> bool {
!self.reader.get_pointer_field(1).is_null()
}
#[inline]
pub fn get_interned_columns(self) -> ::capnp::Result<::capnp::primitive_list::Reader<'a,u64>> {
::capnp::traits::FromPointerReader::get_from_pointer(&self.reader.get_pointer_field(2), ::core::option::Option::None)
}
#[inline]
pub fn has_interned_columns(&self) -> bool {
!self.reader.get_pointer_field(2).is_null()
}
}

pub struct Builder<'a> { builder: ::capnp::private::layout::StructBuilder<'a> }
impl <'a,> ::capnp::traits::HasStructSize for Builder<'a,> {
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 2 };
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 3 };
}
impl <'a,> ::capnp::traits::HasTypeId for Builder<'a,> {
const TYPE_ID: u64 = _private::TYPE_ID;
Expand Down Expand Up @@ -755,6 +807,22 @@ pub mod subpartition_metadata {
pub fn has_columns(&self) -> bool {
!self.builder.is_pointer_field_null(1)
}
#[inline]
pub fn get_interned_columns(self) -> ::capnp::Result<::capnp::primitive_list::Builder<'a,u64>> {
::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(2), ::core::option::Option::None)
}
#[inline]
pub fn set_interned_columns(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::primitive_list::Owned<u64>>) -> ::capnp::Result<()> {
::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(2), value, false)
}
#[inline]
pub fn init_interned_columns(self, size: u32) -> ::capnp::primitive_list::Builder<'a,u64> {
::capnp::traits::FromPointerBuilder::init_pointer(self.builder.get_pointer_field(2), size)
}
#[inline]
pub fn has_interned_columns(&self) -> bool {
!self.builder.is_pointer_field_null(2)
}
}

pub struct Pipeline { _typeless: ::capnp::any_pointer::Pipeline }
Expand All @@ -766,17 +834,17 @@ pub mod subpartition_metadata {
impl Pipeline {
}
mod _private {
pub static ENCODED_NODE: [::capnp::Word; 70] = [
pub static ENCODED_NODE: [::capnp::Word; 90] = [
::capnp::word(0, 0, 0, 0, 5, 0, 6, 0),
::capnp::word(48, 51, 123, 36, 100, 192, 248, 192),
::capnp::word(13, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(245, 46, 142, 94, 29, 184, 169, 175),
::capnp::word(2, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(3, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(21, 0, 0, 0, 18, 1, 0, 0),
::capnp::word(37, 0, 0, 0, 7, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(33, 0, 0, 0, 175, 0, 0, 0),
::capnp::word(33, 0, 0, 0, 231, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(100, 98, 109, 101, 116, 97, 46, 99),
Expand All @@ -785,28 +853,35 @@ pub mod subpartition_metadata {
::capnp::word(110, 77, 101, 116, 97, 100, 97, 116),
::capnp::word(97, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(12, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(16, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(69, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(97, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(68, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(80, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(96, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(108, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(1, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(77, 0, 0, 0, 130, 0, 0, 0),
::capnp::word(105, 0, 0, 0, 130, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(76, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(88, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(104, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(116, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(2, 0, 0, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 2, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(85, 0, 0, 0, 66, 0, 0, 0),
::capnp::word(113, 0, 0, 0, 66, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(80, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(108, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(108, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(136, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(3, 0, 0, 0, 2, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 3, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(133, 0, 0, 0, 130, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(132, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(160, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(115, 105, 122, 101, 66, 121, 116, 101),
::capnp::word(115, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(9, 0, 0, 0, 0, 0, 0, 0),
Expand Down Expand Up @@ -837,12 +912,26 @@ pub mod subpartition_metadata {
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(105, 110, 116, 101, 114, 110, 101, 100),
::capnp::word(67, 111, 108, 117, 109, 110, 115, 0),
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(9, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
];
pub fn get_field_types(index: u16) -> ::capnp::introspect::Type {
match index {
0 => <u64 as ::capnp::introspect::Introspect>::introspect(),
1 => <::capnp::text::Owned as ::capnp::introspect::Introspect>::introspect(),
2 => <::capnp::text_list::Owned as ::capnp::introspect::Introspect>::introspect(),
3 => <::capnp::primitive_list::Owned<u64> as ::capnp::introspect::Introspect>::introspect(),
_ => panic!("invalid field index {}", index),
}
}
Expand All @@ -855,9 +944,9 @@ pub mod subpartition_metadata {
members_by_discriminant: MEMBERS_BY_DISCRIMINANT,
members_by_name: MEMBERS_BY_NAME,
};
pub static NONUNION_MEMBERS : &[u16] = &[0,1,2];
pub static NONUNION_MEMBERS : &[u16] = &[0,1,2,3];
pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[];
pub static MEMBERS_BY_NAME : &[u16] = &[2,0,1];
pub static MEMBERS_BY_NAME : &[u16] = &[2,3,0,1];
pub const TYPE_ID: u64 = 0xc0f8_c064_247b_3330;
}
}
Loading

0 comments on commit e66616d

Please sign in to comment.