From 5d96890fdc5cd70cd0de6d453cb2620ec6bed058 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Thu, 8 Aug 2024 15:45:54 +0200 Subject: [PATCH 1/6] feat: meta_struct support --- ddapm_test_agent/trace.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ddapm_test_agent/trace.py b/ddapm_test_agent/trace.py index e1dc6be..7d2d6a5 100644 --- a/ddapm_test_agent/trace.py +++ b/ddapm_test_agent/trace.py @@ -71,6 +71,7 @@ class Span(TypedDict): meta: NotRequired[Dict[str, str]] metrics: NotRequired[Dict[str, MetricType]] span_links: NotRequired[List[SpanLink]] + meta_struct: NotRequired[Dict[str, dict]] SpanAttr = Literal[ @@ -87,6 +88,7 @@ class Span(TypedDict): "meta", "metrics", "span_links", + "meta_struct", ] TopLevelSpanValue = Union[None, SpanId, TraceId, int, str, Dict[str, str], Dict[str, MetricType], List[SpanLink]] Trace = List[Span] @@ -124,6 +126,16 @@ def verify_span(d: Any) -> Span: for k, v in d["meta"].items(): assert isinstance(k, str), f"Expected key 'meta.{k}' to be of type: 'str', got: {type(k)}" assert isinstance(v, str), f"Expected value of key 'meta.{k}' to be of type: 'str', got: {type(v)}" + if "meta_struct" in d: + assert isinstance(d["meta_struct"], dict) + for k, v in d["meta_struct"].items(): + assert isinstance(k, str), f"Expected key 'meta_struct.{k}' to be of type: 'str', got: {type(k)}" + assert isinstance(v, bytes), f"Expected msgpack'd value of key 'meta_struct.{k}' to be of type: 'bytes', got: {type(v)}" + # Decode meta_struct msgpack values + decoded_meta_struct = { key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"] } + for k, val in decoded_meta_struct.items(): + assert isinstance(v, dict), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(v)}" + d["meta_struct"] = decoded_meta_struct if "metrics" in d: assert isinstance(d["metrics"], dict) for k, v in d["metrics"].items(): @@ -383,7 +395,6 @@ def json_decoder(maybe_span): parsed_data: Dict[str, Any] = json.loads(json_string, object_hook=json_decoder) return parsed_data - def decode_v04(content_type: str, data: bytes, suppress_errors: bool) -> v04TracePayload: if content_type == "application/msgpack": payload = msgpack.unpackb(data) @@ -391,6 +402,7 @@ def decode_v04(content_type: str, data: bytes, suppress_errors: bool) -> v04Trac payload = _trace_decoder_flexible(data) if suppress_errors else json.loads(data) else: raise TypeError("Content type %r not supported" % content_type) + return _verify_v04_payload(payload) From ba308e2a9ed1efe70ee3fc88faca9ace70921373 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana Date: Fri, 9 Aug 2024 14:46:54 +0200 Subject: [PATCH 2/6] deserialize meta struct values Signed-off-by: Eliott Bouhana --- ddapm_test_agent/trace.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ddapm_test_agent/trace.py b/ddapm_test_agent/trace.py index 7d2d6a5..1d0e5f1 100644 --- a/ddapm_test_agent/trace.py +++ b/ddapm_test_agent/trace.py @@ -71,7 +71,7 @@ class Span(TypedDict): meta: NotRequired[Dict[str, str]] metrics: NotRequired[Dict[str, MetricType]] span_links: NotRequired[List[SpanLink]] - meta_struct: NotRequired[Dict[str, dict]] + meta_struct: NotRequired[Dict[str, Dict[str, Any]]] SpanAttr = Literal[ @@ -132,9 +132,13 @@ def verify_span(d: Any) -> Span: assert isinstance(k, str), f"Expected key 'meta_struct.{k}' to be of type: 'str', got: {type(k)}" assert isinstance(v, bytes), f"Expected msgpack'd value of key 'meta_struct.{k}' to be of type: 'bytes', got: {type(v)}" # Decode meta_struct msgpack values - decoded_meta_struct = { key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"] } + decoded_meta_struct = { key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"].items() } for k, val in decoded_meta_struct.items(): - assert isinstance(v, dict), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(v)}" + assert isinstance(val, dict), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(v)}" + + for inner_k in val.keys(): + assert isinstance(inner_k, str), f"Expected key 'meta_struct.{k}.{inner_k}' to be of type: 'str', got: {type(inner_k)}" + d["meta_struct"] = decoded_meta_struct if "metrics" in d: assert isinstance(d["metrics"], dict) From 74aef8e0826015b00f959f351a9f8eac22fc44d2 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana Date: Tue, 13 Aug 2024 15:18:44 +0200 Subject: [PATCH 3/6] test the feature Signed-off-by: Eliott Bouhana --- ddapm_test_agent/trace.py | 5 ++-- tests/test_trace.py | 62 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/ddapm_test_agent/trace.py b/ddapm_test_agent/trace.py index 1d0e5f1..4fb4cdb 100644 --- a/ddapm_test_agent/trace.py +++ b/ddapm_test_agent/trace.py @@ -126,6 +126,7 @@ def verify_span(d: Any) -> Span: for k, v in d["meta"].items(): assert isinstance(k, str), f"Expected key 'meta.{k}' to be of type: 'str', got: {type(k)}" assert isinstance(v, str), f"Expected value of key 'meta.{k}' to be of type: 'str', got: {type(v)}" + if "meta_struct" in d: assert isinstance(d["meta_struct"], dict) for k, v in d["meta_struct"].items(): @@ -134,9 +135,9 @@ def verify_span(d: Any) -> Span: # Decode meta_struct msgpack values decoded_meta_struct = { key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"].items() } for k, val in decoded_meta_struct.items(): - assert isinstance(val, dict), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(v)}" + assert isinstance(val, dict), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(val)}" - for inner_k in val.keys(): + for inner_k in val: assert isinstance(inner_k, str), f"Expected key 'meta_struct.{k}.{inner_k}' to be of type: 'str', got: {type(inner_k)}" d["meta_struct"] = decoded_meta_struct diff --git a/tests/test_trace.py b/tests/test_trace.py index 44c26a4..0446e82 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -49,6 +49,21 @@ def test_trace_chunk(): ] ), ), + ( + "application/msgpack", + msgpack.packb( + [ + [ + { + "name": "span", + "span_id": 1234, + "trace_id": 321, + "meta_struct": {"key": msgpack.packb({"subkey": "value"})}, + } + ] + ] + ), + ), ], ) def test_decode_v04(content_type, payload): @@ -58,8 +73,53 @@ def test_decode_v04(content_type, payload): @pytest.mark.parametrize( "content_type, payload", [ - ("application/msgpack", msgpack.packb([{"name": "test"}])), ("application/json", json.dumps([{"name": "test"}])), + ("application/msgpack", msgpack.packb([{"name": "test"}])), + ( + "application/msgpack", + msgpack.packb( + [ + [ + { + "name": "span", + "span_id": 1234, + "trace_id": 321, + "meta_struct": "not a valid msgpack", + } + ] + ] + ), + ), + ( + "application/msgpack", + msgpack.packb( + [ + [ + { + "name": "span", + "span_id": 1234, + "trace_id": 321, + "meta_struct": ["this is not a dict"], + } + ] + ] + ), + ), + ( + "application/msgpack", + msgpack.packb( + [ + [ + { + "name": "span", + "span_id": 1234, + "trace_id": 321, + "meta_struct": {"key": msgpack.packb(["this is not a dict"])}, + } + ] + ] + ), + ), ], ) def test_decode_v04_bad(content_type, payload): From 382a42b8d5a9b803745fbee008f2a603121927a3 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana Date: Tue, 13 Aug 2024 15:26:06 +0200 Subject: [PATCH 4/6] remove empty lines Signed-off-by: Eliott Bouhana --- ddapm_test_agent/trace.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ddapm_test_agent/trace.py b/ddapm_test_agent/trace.py index 4fb4cdb..837bfcd 100644 --- a/ddapm_test_agent/trace.py +++ b/ddapm_test_agent/trace.py @@ -126,7 +126,6 @@ def verify_span(d: Any) -> Span: for k, v in d["meta"].items(): assert isinstance(k, str), f"Expected key 'meta.{k}' to be of type: 'str', got: {type(k)}" assert isinstance(v, str), f"Expected value of key 'meta.{k}' to be of type: 'str', got: {type(v)}" - if "meta_struct" in d: assert isinstance(d["meta_struct"], dict) for k, v in d["meta_struct"].items(): @@ -136,10 +135,8 @@ def verify_span(d: Any) -> Span: decoded_meta_struct = { key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"].items() } for k, val in decoded_meta_struct.items(): assert isinstance(val, dict), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(val)}" - for inner_k in val: assert isinstance(inner_k, str), f"Expected key 'meta_struct.{k}.{inner_k}' to be of type: 'str', got: {type(inner_k)}" - d["meta_struct"] = decoded_meta_struct if "metrics" in d: assert isinstance(d["metrics"], dict) @@ -400,6 +397,7 @@ def json_decoder(maybe_span): parsed_data: Dict[str, Any] = json.loads(json_string, object_hook=json_decoder) return parsed_data + def decode_v04(content_type: str, data: bytes, suppress_errors: bool) -> v04TracePayload: if content_type == "application/msgpack": payload = msgpack.unpackb(data) @@ -407,7 +405,6 @@ def decode_v04(content_type: str, data: bytes, suppress_errors: bool) -> v04Trac payload = _trace_decoder_flexible(data) if suppress_errors else json.loads(data) else: raise TypeError("Content type %r not supported" % content_type) - return _verify_v04_payload(payload) From 53ef9130ce679f1f809a6a3bcb6671d748af3837 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana Date: Tue, 13 Aug 2024 16:11:55 +0200 Subject: [PATCH 5/6] release notes + format Signed-off-by: Eliott Bouhana --- ddapm_test_agent/trace.py | 14 ++++-- .../notes/meta-struct-2cce08475cb05470.yaml | 6 +++ tests/test_trace.py | 44 +++++++++---------- 3 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 releasenotes/notes/meta-struct-2cce08475cb05470.yaml diff --git a/ddapm_test_agent/trace.py b/ddapm_test_agent/trace.py index 837bfcd..02a94aa 100644 --- a/ddapm_test_agent/trace.py +++ b/ddapm_test_agent/trace.py @@ -130,13 +130,19 @@ def verify_span(d: Any) -> Span: assert isinstance(d["meta_struct"], dict) for k, v in d["meta_struct"].items(): assert isinstance(k, str), f"Expected key 'meta_struct.{k}' to be of type: 'str', got: {type(k)}" - assert isinstance(v, bytes), f"Expected msgpack'd value of key 'meta_struct.{k}' to be of type: 'bytes', got: {type(v)}" + assert isinstance( + v, bytes + ), f"Expected msgpack'd value of key 'meta_struct.{k}' to be of type: 'bytes', got: {type(v)}" # Decode meta_struct msgpack values - decoded_meta_struct = { key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"].items() } + decoded_meta_struct = {key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"].items()} for k, val in decoded_meta_struct.items(): - assert isinstance(val, dict), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(val)}" + assert isinstance( + val, dict + ), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(val)}" for inner_k in val: - assert isinstance(inner_k, str), f"Expected key 'meta_struct.{k}.{inner_k}' to be of type: 'str', got: {type(inner_k)}" + assert isinstance( + inner_k, str + ), f"Expected key 'meta_struct.{k}.{inner_k}' to be of type: 'str', got: {type(inner_k)}" d["meta_struct"] = decoded_meta_struct if "metrics" in d: assert isinstance(d["metrics"], dict) diff --git a/releasenotes/notes/meta-struct-2cce08475cb05470.yaml b/releasenotes/notes/meta-struct-2cce08475cb05470.yaml new file mode 100644 index 0000000..fc26dd4 --- /dev/null +++ b/releasenotes/notes/meta-struct-2cce08475cb05470.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add support for parsing the `meta_struct` field in traces. This field just like the `meta` field map but the values + are arbitrary inner msgpack-encoded values. When the `meta_struct` field is present, its inner messages will be + parsed and added to the trace as a dictionary. \ No newline at end of file diff --git a/tests/test_trace.py b/tests/test_trace.py index 0446e82..74f6609 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -91,34 +91,34 @@ def test_decode_v04(content_type, payload): ), ), ( - "application/msgpack", - msgpack.packb( + "application/msgpack", + msgpack.packb( + [ [ - [ - { - "name": "span", - "span_id": 1234, - "trace_id": 321, - "meta_struct": ["this is not a dict"], - } - ] + { + "name": "span", + "span_id": 1234, + "trace_id": 321, + "meta_struct": ["this is not a dict"], + } ] - ), + ] + ), ), ( - "application/msgpack", - msgpack.packb( + "application/msgpack", + msgpack.packb( + [ [ - [ - { - "name": "span", - "span_id": 1234, - "trace_id": 321, - "meta_struct": {"key": msgpack.packb(["this is not a dict"])}, - } - ] + { + "name": "span", + "span_id": 1234, + "trace_id": 321, + "meta_struct": {"key": msgpack.packb(["this is not a dict"])}, + } ] - ), + ] + ), ), ], ) From 6ed45f8f1e08f5ebc351f80f11b26ef6f82f3719 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana Date: Wed, 14 Aug 2024 16:46:49 +0200 Subject: [PATCH 6/6] split parsing and verification Signed-off-by: Eliott Bouhana --- ddapm_test_agent/trace.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/ddapm_test_agent/trace.py b/ddapm_test_agent/trace.py index 02a94aa..0752af2 100644 --- a/ddapm_test_agent/trace.py +++ b/ddapm_test_agent/trace.py @@ -133,9 +133,7 @@ def verify_span(d: Any) -> Span: assert isinstance( v, bytes ), f"Expected msgpack'd value of key 'meta_struct.{k}' to be of type: 'bytes', got: {type(v)}" - # Decode meta_struct msgpack values - decoded_meta_struct = {key: msgpack.unpackb(val_bytes) for key, val_bytes in d["meta_struct"].items()} - for k, val in decoded_meta_struct.items(): + for k, val in d["meta_struct"].items(): assert isinstance( val, dict ), f"Expected msgpack decoded value of key 'meta_struct.{k}' to be of type: 'dict', got: {type(val)}" @@ -143,7 +141,6 @@ def verify_span(d: Any) -> Span: assert isinstance( inner_k, str ), f"Expected key 'meta_struct.{k}.{inner_k}' to be of type: 'str', got: {type(inner_k)}" - d["meta_struct"] = decoded_meta_struct if "metrics" in d: assert isinstance(d["metrics"], dict) for k, v in d["metrics"].items(): @@ -188,6 +185,26 @@ def verify_span(d: Any) -> Span: raise TypeError(*e.args) from e +def _parse_meta_struct(value: Any) -> Dict[str, Dict[str, Any]]: + if not isinstance(value, dict): + raise TypeError("Expected meta_struct to be of type: 'dict', got: %s" % type(value)) + + return {key: msgpack.unpackb(val_bytes) for key, val_bytes in value.items()} + + +def _flexible_decode_meta_struct(value: Any) -> None: + if not isinstance(value, list): + return + for maybe_trace in value: + if not isinstance(maybe_trace, list): + continue + for maybe_span in maybe_trace: + if not isinstance(maybe_span, dict): + continue + if "meta_struct" in maybe_span: + maybe_span["meta_struct"] = _parse_meta_struct(maybe_span["meta_struct"]) + + def v04_verify_trace(maybe_trace: Any) -> Trace: if not isinstance(maybe_trace, list): raise TypeError("Trace must be a list.") @@ -411,6 +428,7 @@ def decode_v04(content_type: str, data: bytes, suppress_errors: bool) -> v04Trac payload = _trace_decoder_flexible(data) if suppress_errors else json.loads(data) else: raise TypeError("Content type %r not supported" % content_type) + _flexible_decode_meta_struct(payload) return _verify_v04_payload(payload)