Skip to content

Commit

Permalink
Meta store compression
Browse files Browse the repository at this point in the history
  • Loading branch information
@cswinter
cswinter committed Aug 18, 2024
1 parent 11be3b6 commit 0d4d741
Show file tree
Hide file tree
Showing 3 changed files with 233 additions and 54 deletions.
9 changes: 6 additions & 3 deletions locustdb-serialization/schemas/dbmeta.capnp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
struct DBMeta {
nextWalId @0 :UInt64;
partitions @1 :List(PartitionMetadata);
strings @2 :List(Text); # unused in legacy format
strings @2 :List(Text); # [v1] unused in legacy format and deprecated in new format
compressedStrings @3 :Data; # [v2..]
lengthsCompressedStrings@4 :List(UInt16); # [v2..]
}

struct PartitionMetadata {
Expand All @@ -17,6 +19,7 @@ struct PartitionMetadata {
struct SubpartitionMetadata {
sizeBytes @0 :UInt64;
subpartitionKey @1 :Text;
columns @2 :List(Text); # deprecated in favor of internedColumns
internedColumns @3 :List(UInt64);
columns @2 :List(Text); # [v0] deprecated in favor of internedColumns
internedColumns @3 :List(UInt64); # [v1] unused in legacy format and deprecated in new format
compressedInternedColumns @4 :Data; # [v2..]
}
202 changes: 167 additions & 35 deletions locustdb-serialization/src/dbmeta_capnp.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,27 @@ pub mod d_b_meta {
pub fn has_strings(&self) -> bool {
!self.reader.get_pointer_field(1).is_null()
}
#[inline]
pub fn get_compressed_strings(self) -> ::capnp::Result<::capnp::data::Reader<'a>> {
::capnp::traits::FromPointerReader::get_from_pointer(&self.reader.get_pointer_field(2), ::core::option::Option::None)
}
#[inline]
pub fn has_compressed_strings(&self) -> bool {
!self.reader.get_pointer_field(2).is_null()
}
#[inline]
pub fn get_lengths_compressed_strings(self) -> ::capnp::Result<::capnp::primitive_list::Reader<'a,u16>> {
::capnp::traits::FromPointerReader::get_from_pointer(&self.reader.get_pointer_field(3), ::core::option::Option::None)
}
#[inline]
pub fn has_lengths_compressed_strings(&self) -> bool {
!self.reader.get_pointer_field(3).is_null()
}
}

pub struct Builder<'a> { builder: ::capnp::private::layout::StructBuilder<'a> }
impl <'a,> ::capnp::traits::HasStructSize for Builder<'a,> {
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 2 };
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 4 };
}
impl <'a,> ::capnp::traits::HasTypeId for Builder<'a,> {
const TYPE_ID: u64 = _private::TYPE_ID;
Expand Down Expand Up @@ -178,6 +194,38 @@ pub mod d_b_meta {
pub fn has_strings(&self) -> bool {
!self.builder.is_pointer_field_null(1)
}
#[inline]
pub fn get_compressed_strings(self) -> ::capnp::Result<::capnp::data::Builder<'a>> {
::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(2), ::core::option::Option::None)
}
#[inline]
pub fn set_compressed_strings(&mut self, value: ::capnp::data::Reader<'_>) {
self.builder.reborrow().get_pointer_field(2).set_data(value);
}
#[inline]
pub fn init_compressed_strings(self, size: u32) -> ::capnp::data::Builder<'a> {
self.builder.get_pointer_field(2).init_data(size)
}
#[inline]
pub fn has_compressed_strings(&self) -> bool {
!self.builder.is_pointer_field_null(2)
}
#[inline]
pub fn get_lengths_compressed_strings(self) -> ::capnp::Result<::capnp::primitive_list::Builder<'a,u16>> {
::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(3), ::core::option::Option::None)
}
#[inline]
pub fn set_lengths_compressed_strings(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::primitive_list::Owned<u16>>) -> ::capnp::Result<()> {
::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(3), value, false)
}
#[inline]
pub fn init_lengths_compressed_strings(self, size: u32) -> ::capnp::primitive_list::Builder<'a,u16> {
::capnp::traits::FromPointerBuilder::init_pointer(self.builder.get_pointer_field(3), size)
}
#[inline]
pub fn has_lengths_compressed_strings(&self) -> bool {
!self.builder.is_pointer_field_null(3)
}
}

pub struct Pipeline { _typeless: ::capnp::any_pointer::Pipeline }
Expand All @@ -189,45 +237,59 @@ pub mod d_b_meta {
impl Pipeline {
}
mod _private {
pub static ENCODED_NODE: [::capnp::Word; 72] = [
pub static ENCODED_NODE: [::capnp::Word; 111] = [
::capnp::word(0, 0, 0, 0, 5, 0, 6, 0),
::capnp::word(134, 233, 91, 102, 237, 22, 116, 146),
::capnp::word(13, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(245, 46, 142, 94, 29, 184, 169, 175),
::capnp::word(2, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(4, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(21, 0, 0, 0, 162, 0, 0, 0),
::capnp::word(29, 0, 0, 0, 7, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(25, 0, 0, 0, 175, 0, 0, 0),
::capnp::word(25, 0, 0, 0, 31, 1, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(100, 98, 109, 101, 116, 97, 46, 99),
::capnp::word(97, 112, 110, 112, 58, 68, 66, 77),
::capnp::word(101, 116, 97, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(12, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(20, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(69, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(125, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(68, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(80, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(124, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(136, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(1, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(77, 0, 0, 0, 90, 0, 0, 0),
::capnp::word(133, 0, 0, 0, 90, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(76, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(104, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(132, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(160, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(2, 0, 0, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 2, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(101, 0, 0, 0, 66, 0, 0, 0),
::capnp::word(157, 0, 0, 0, 66, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(152, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(180, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(3, 0, 0, 0, 2, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 3, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(177, 0, 0, 0, 146, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(180, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(192, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(4, 0, 0, 0, 3, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 4, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(189, 0, 0, 0, 202, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(96, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(124, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(196, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(224, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(110, 101, 120, 116, 87, 97, 108, 73),
::capnp::word(100, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(9, 0, 0, 0, 0, 0, 0, 0),
Expand Down Expand Up @@ -262,12 +324,39 @@ pub mod d_b_meta {
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(99, 111, 109, 112, 114, 101, 115, 115),
::capnp::word(101, 100, 83, 116, 114, 105, 110, 103),
::capnp::word(115, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(13, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(13, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(108, 101, 110, 103, 116, 104, 115, 67),
::capnp::word(111, 109, 112, 114, 101, 115, 115, 101),
::capnp::word(100, 83, 116, 114, 105, 110, 103, 115),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(7, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
];
pub fn get_field_types(index: u16) -> ::capnp::introspect::Type {
match index {
0 => <u64 as ::capnp::introspect::Introspect>::introspect(),
1 => <::capnp::struct_list::Owned<crate::dbmeta_capnp::partition_metadata::Owned> as ::capnp::introspect::Introspect>::introspect(),
2 => <::capnp::text_list::Owned as ::capnp::introspect::Introspect>::introspect(),
3 => <::capnp::data::Owned as ::capnp::introspect::Introspect>::introspect(),
4 => <::capnp::primitive_list::Owned<u16> as ::capnp::introspect::Introspect>::introspect(),
_ => panic!("invalid field index {}", index),
}
}
Expand All @@ -280,9 +369,9 @@ pub mod d_b_meta {
members_by_discriminant: MEMBERS_BY_DISCRIMINANT,
members_by_name: MEMBERS_BY_NAME,
};
pub static NONUNION_MEMBERS : &[u16] = &[0,1,2];
pub static NONUNION_MEMBERS : &[u16] = &[0,1,2,3,4];
pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[];
pub static MEMBERS_BY_NAME : &[u16] = &[0,1,2];
pub static MEMBERS_BY_NAME : &[u16] = &[3,4,0,1,2];
pub const TYPE_ID: u64 = 0x9274_16ed_665b_e986;
}
}
Expand Down Expand Up @@ -713,11 +802,19 @@ pub mod subpartition_metadata {
pub fn has_interned_columns(&self) -> bool {
!self.reader.get_pointer_field(2).is_null()
}
#[inline]
pub fn get_compressed_interned_columns(self) -> ::capnp::Result<::capnp::data::Reader<'a>> {
::capnp::traits::FromPointerReader::get_from_pointer(&self.reader.get_pointer_field(3), ::core::option::Option::None)
}
#[inline]
pub fn has_compressed_interned_columns(&self) -> bool {
!self.reader.get_pointer_field(3).is_null()
}
}

pub struct Builder<'a> { builder: ::capnp::private::layout::StructBuilder<'a> }
impl <'a,> ::capnp::traits::HasStructSize for Builder<'a,> {
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 3 };
const STRUCT_SIZE: ::capnp::private::layout::StructSize = ::capnp::private::layout::StructSize { data: 1, pointers: 4 };
}
impl <'a,> ::capnp::traits::HasTypeId for Builder<'a,> {
const TYPE_ID: u64 = _private::TYPE_ID;
Expand Down Expand Up @@ -823,6 +920,22 @@ pub mod subpartition_metadata {
pub fn has_interned_columns(&self) -> bool {
!self.builder.is_pointer_field_null(2)
}
#[inline]
pub fn get_compressed_interned_columns(self) -> ::capnp::Result<::capnp::data::Builder<'a>> {
::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(3), ::core::option::Option::None)
}
#[inline]
pub fn set_compressed_interned_columns(&mut self, value: ::capnp::data::Reader<'_>) {
self.builder.reborrow().get_pointer_field(3).set_data(value);
}
#[inline]
pub fn init_compressed_interned_columns(self, size: u32) -> ::capnp::data::Builder<'a> {
self.builder.get_pointer_field(3).init_data(size)
}
#[inline]
pub fn has_compressed_interned_columns(&self) -> bool {
!self.builder.is_pointer_field_null(3)
}
}

pub struct Pipeline { _typeless: ::capnp::any_pointer::Pipeline }
Expand All @@ -834,17 +947,17 @@ pub mod subpartition_metadata {
impl Pipeline {
}
mod _private {
pub static ENCODED_NODE: [::capnp::Word; 90] = [
pub static ENCODED_NODE: [::capnp::Word; 108] = [
::capnp::word(0, 0, 0, 0, 5, 0, 6, 0),
::capnp::word(48, 51, 123, 36, 100, 192, 248, 192),
::capnp::word(13, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(245, 46, 142, 94, 29, 184, 169, 175),
::capnp::word(3, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(4, 0, 7, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(21, 0, 0, 0, 18, 1, 0, 0),
::capnp::word(37, 0, 0, 0, 7, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(33, 0, 0, 0, 231, 0, 0, 0),
::capnp::word(33, 0, 0, 0, 31, 1, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(100, 98, 109, 101, 116, 97, 46, 99),
Expand All @@ -853,35 +966,42 @@ pub mod subpartition_metadata {
::capnp::word(110, 77, 101, 116, 97, 100, 97, 116),
::capnp::word(97, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 1, 0, 1, 0),
::capnp::word(16, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(20, 0, 0, 0, 3, 0, 4, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(97, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(125, 0, 0, 0, 82, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(96, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(108, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(124, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(136, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(1, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(105, 0, 0, 0, 130, 0, 0, 0),
::capnp::word(133, 0, 0, 0, 130, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(104, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(116, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(132, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(144, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(2, 0, 0, 0, 1, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 2, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(113, 0, 0, 0, 66, 0, 0, 0),
::capnp::word(141, 0, 0, 0, 66, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(108, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(136, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(136, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(164, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(3, 0, 0, 0, 2, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 3, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(133, 0, 0, 0, 130, 0, 0, 0),
::capnp::word(161, 0, 0, 0, 130, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(132, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(160, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(160, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(188, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(4, 0, 0, 0, 3, 0, 0, 0),
::capnp::word(0, 0, 1, 0, 4, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(185, 0, 0, 0, 210, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(192, 0, 0, 0, 3, 0, 1, 0),
::capnp::word(204, 0, 0, 0, 2, 0, 1, 0),
::capnp::word(115, 105, 122, 101, 66, 121, 116, 101),
::capnp::word(115, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(9, 0, 0, 0, 0, 0, 0, 0),
Expand Down Expand Up @@ -925,13 +1045,25 @@ pub mod subpartition_metadata {
::capnp::word(14, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(99, 111, 109, 112, 114, 101, 115, 115),
::capnp::word(101, 100, 73, 110, 116, 101, 114, 110),
::capnp::word(101, 100, 67, 111, 108, 117, 109, 110),
::capnp::word(115, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(13, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(13, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
::capnp::word(0, 0, 0, 0, 0, 0, 0, 0),
];
pub fn get_field_types(index: u16) -> ::capnp::introspect::Type {
match index {
0 => <u64 as ::capnp::introspect::Introspect>::introspect(),
1 => <::capnp::text::Owned as ::capnp::introspect::Introspect>::introspect(),
2 => <::capnp::text_list::Owned as ::capnp::introspect::Introspect>::introspect(),
3 => <::capnp::primitive_list::Owned<u64> as ::capnp::introspect::Introspect>::introspect(),
4 => <::capnp::data::Owned as ::capnp::introspect::Introspect>::introspect(),
_ => panic!("invalid field index {}", index),
}
}
Expand All @@ -944,9 +1076,9 @@ pub mod subpartition_metadata {
members_by_discriminant: MEMBERS_BY_DISCRIMINANT,
members_by_name: MEMBERS_BY_NAME,
};
pub static NONUNION_MEMBERS : &[u16] = &[0,1,2,3];
pub static NONUNION_MEMBERS : &[u16] = &[0,1,2,3,4];
pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[];
pub static MEMBERS_BY_NAME : &[u16] = &[2,3,0,1];
pub static MEMBERS_BY_NAME : &[u16] = &[2,4,3,0,1];
pub const TYPE_ID: u64 = 0xc0f8_c064_247b_3330;
}
}
Loading

0 comments on commit 0d4d741

Please sign in to comment.