[Internal] OpenTelemetry : Adds Telemetry Distributed Tracing functionality

Directory.Build.props

@@ -3,7 +3,7 @@
 		<ClientOfficialVersion>3.34.0</ClientOfficialVersion>
 		<ClientPreviewVersion>3.34.0</ClientPreviewVersion>
 		<ClientPreviewSuffixVersion>preview</ClientPreviewSuffixVersion>
-		<DirectVersion>3.31.0</DirectVersion>
+		<DirectVersion>3.31.1</DirectVersion>
 		<EncryptionOfficialVersion>2.0.2</EncryptionOfficialVersion>
 		<EncryptionPreviewVersion>2.0.2</EncryptionPreviewVersion>
 		<EncryptionPreviewSuffixVersion>preview</EncryptionPreviewSuffixVersion>

Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs

@@ -497,6 +497,7 @@ private async Task<TResult> RunWithDiagnosticsHelperAsync<TResult>(
                                     databaseName: databaseName,
                                     operationType: operationType,
                                     requestOptions: requestOptions,
+                                    trace: trace,
                                     clientContext: this.isDisposed ? null : this))
             using (new ActivityScope(Guid.NewGuid()))
             {

Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryAttributeKeys.cs

@@ -10,6 +10,7 @@ internal sealed class OpenTelemetryAttributeKeys
         public const string DiagnosticNamespace = "Azure.Cosmos";
         public const string ResourceProviderNamespace = "Microsoft.DocumentDB";
         public const string OperationPrefix = "Operation";
+        public const string NetworkLevelPrefix = "Request";
 
         // Common database attributes
         public const string DbSystemName = "db.system";

Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryCoreRecorder.cs

@@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry
 {
     using System;
     using System.Collections.Generic;
+    using System.Diagnostics;
     using global::Azure.Core;
 
     /// <summary>
@@ -14,11 +15,12 @@ namespace Microsoft.Azure.Cosmos.Telemetry
     internal struct OpenTelemetryCoreRecorder : IDisposable
     {
         private const string CosmosDb = "cosmosdb";
-
-        private readonly DiagnosticScope scope;
-        private readonly DistributedTracingOptions config;
 
-        private readonly Documents.OperationType operationType;
+        private readonly DiagnosticScope scope = default;
+        private readonly DistributedTracingOptions config = null;
+        private readonly Activity activity = null;
+
+        private readonly Documents.OperationType operationType = Documents.OperationType.Invalid;
         private OpenTelemetryAttributes response = null;
 
         internal static IDictionary<Type, Action<Exception, DiagnosticScope>> OTelCompatibleExceptions = new Dictionary<Type, Action<Exception, DiagnosticScope>>()
@@ -30,7 +32,19 @@ internal struct OpenTelemetryCoreRecorder : IDisposable
             { typeof(ChangeFeedProcessorUserException), (exception, scope) => ChangeFeedProcessorUserException.RecordOtelAttributes((ChangeFeedProcessorUserException)exception, scope)}
         };
 
-        public OpenTelemetryCoreRecorder(
+        private OpenTelemetryCoreRecorder(DiagnosticScope scope)
+        {
+            this.scope = scope;
+            this.scope.Start();
+        }
+
+        private OpenTelemetryCoreRecorder(string operationName)
+        {
+            this.activity = new Activity(operationName);
+            this.activity.Start();
+        }
+
+        private OpenTelemetryCoreRecorder(
             DiagnosticScope scope,
             string operationName,
             string containerName,
@@ -54,6 +68,45 @@ public OpenTelemetryCoreRecorder(
             }
         }
 
+        /// <summary>
+        /// Used for creating parent activity in scenario where there are no listeners at operation level 
+        /// but they are present at network level
+        /// </summary>
+        public static OpenTelemetryCoreRecorder CreateNetworkLevelParentActivity(DiagnosticScope networkScope)
+        {
+            return new OpenTelemetryCoreRecorder(networkScope);
+        }
+
+        /// <summary>
+        /// Used for creating parent activity in scenario where there are no listeners at operation level and network level
+        /// </summary>
+        public static OpenTelemetryCoreRecorder CreateParentActivity(string operationName)
+        {
+            return new OpenTelemetryCoreRecorder(operationName);
+        }
+
+        /// <summary>
+        /// Used for creating parent activity in scenario where there are listeners at operation level 
+        /// </summary>
+        public static OpenTelemetryCoreRecorder CreateOperationLevelParentActivity(
+            DiagnosticScope operationScope,
+            string operationName,
+            string containerName,
+            string databaseName,
+            Documents.OperationType operationType,
+            CosmosClientContext clientContext,
+            DistributedTracingOptions config)
+        {
+            return new OpenTelemetryCoreRecorder(
+                        operationScope,
+                        operationName,
+                        containerName,
+                        databaseName,
+                        operationType,
+                        clientContext,
+                        config);
+        }
+
         public bool IsEnabled => this.scope.IsEnabled;
 
         public void Record(string key, string value)
@@ -179,6 +232,10 @@ Documents.OperationType operationType
 
                 this.scope.Dispose();
             }
+            else
+            {
+                this.activity?.Stop();
+            }
         }
     }
 }

Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryRecorderFactory.cs

@@ -5,7 +5,9 @@
 namespace Microsoft.Azure.Cosmos.Telemetry
 {
     using System;
+    using System.Diagnostics;
     using global::Azure.Core;
+    using Microsoft.Azure.Cosmos.Tracing;
 
     /// <summary>
     /// This class is used to generate Activities with Azure.Cosmos.Operation Source Name
@@ -15,43 +17,60 @@ internal static class OpenTelemetryRecorderFactory
         /// <summary>
         /// Singleton to make sure we only have one instance of the DiagnosticScopeFactory and pattern matching of listener happens only once
         /// </summary>
-        private static DiagnosticScopeFactory ScopeFactory { get; set; } 
-
+        private static readonly Lazy<DiagnosticScopeFactory> LazyOperationScopeFactory = new Lazy<DiagnosticScopeFactory>(
+            valueFactory: () => new DiagnosticScopeFactory(
+                           clientNamespace: $"{OpenTelemetryAttributeKeys.DiagnosticNamespace}.{OpenTelemetryAttributeKeys.OperationPrefix}",
+                           resourceProviderNamespace: OpenTelemetryAttributeKeys.ResourceProviderNamespace,
+                           isActivityEnabled: true,
+                           suppressNestedClientActivities: true),
+            isThreadSafe: true);
+
+        private static readonly Lazy<DiagnosticScopeFactory> LazyNetworkScopeFactory = new Lazy<DiagnosticScopeFactory>(
+            valueFactory: () => new DiagnosticScopeFactory(
+                           clientNamespace: $"{OpenTelemetryAttributeKeys.DiagnosticNamespace}.{OpenTelemetryAttributeKeys.NetworkLevelPrefix}",
+                           resourceProviderNamespace: OpenTelemetryAttributeKeys.ResourceProviderNamespace,
+                           isActivityEnabled: true,
+                           suppressNestedClientActivities: true),
+            isThreadSafe: true);
+
         public static OpenTelemetryCoreRecorder CreateRecorder(string operationName,
             string containerName,
             string databaseName,
             Documents.OperationType operationType,
             RequestOptions requestOptions, 
+            ITrace trace,
             CosmosClientContext clientContext)
         {
+            OpenTelemetryCoreRecorder openTelemetryRecorder = default;
             if (clientContext is { ClientOptions.IsDistributedTracingEnabled: true })
             {
-                OpenTelemetryRecorderFactory.ScopeFactory ??= new DiagnosticScopeFactory(clientNamespace: $"{OpenTelemetryAttributeKeys.DiagnosticNamespace}.{OpenTelemetryAttributeKeys.OperationPrefix}",
-                        resourceProviderNamespace: OpenTelemetryAttributeKeys.ResourceProviderNamespace,
-                        isActivityEnabled: true,
-                        suppressNestedClientActivities: true);
-
                 // If there is no source then it will return default otherwise a valid diagnostic scope
-                DiagnosticScope scope = OpenTelemetryRecorderFactory
-                    .ScopeFactory
-                    .CreateScope(name: operationName,
+                DiagnosticScope scope = LazyOperationScopeFactory.Value.CreateScope(name: operationName,
                                  kind: clientContext.ClientOptions.ConnectionMode == ConnectionMode.Gateway ? DiagnosticScope.ActivityKind.Internal : DiagnosticScope.ActivityKind.Client);
 
                 // Record values only when we have a valid Diagnostic Scope
                 if (scope.IsEnabled)
                 {
-                    return new OpenTelemetryCoreRecorder(
-                        scope: scope,
+                    openTelemetryRecorder = OpenTelemetryCoreRecorder.CreateOperationLevelParentActivity(
+                        operationScope: scope,
                         operationName: operationName,
                         containerName: containerName,
                         databaseName: databaseName,
                         operationType: operationType,
                         clientContext: clientContext,
                         config: requestOptions?.DistributedTracingOptions ?? clientContext.ClientOptions?.DistributedTracingOptions);
                 }
-            }
+#if !INTERNAL
+                else if (Activity.Current is null)
+                {
+                    DiagnosticScope requestScope = LazyNetworkScopeFactory.Value.CreateScope(name: operationName);
 
-            return default;
+                    openTelemetryRecorder = requestScope.IsEnabled ? OpenTelemetryCoreRecorder.CreateNetworkLevelParentActivity(networkScope: requestScope) : OpenTelemetryCoreRecorder.CreateParentActivity(operationName);
+                }
+#endif
+                trace.AddDatum("DistributedTraceId", Activity.Current?.TraceId);
+            }
+            return openTelemetryRecorder;
         }
     }
 }

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/EndToEndTraceWriterBaselineTests.BatchOperationsAsync.xml

@@ -38,6 +38,8 @@
     │   (
     │       [Client Configuration]
     │       Redacted To Not Change The Baselines From Run To Run
+    │       [DistributedTraceId]
+    │       Redacted To Not Change The Baselines From Run To Run
     │   )
     └── Execute Next Batch(00000000-0000-0000-0000-000000000000)  Batch-Component  00:00:00:000  0.00 milliseconds  
         ├── Create Batch Request(00000000-0000-0000-0000-000000000000)  Batch-Component  00:00:00:000  0.00 milliseconds  
@@ -64,7 +66,8 @@
   "start datetime": "0001-01-01T00:00:00Z",
   "duration in milliseconds": 0,
   "data": {
-    "Client Configuration": "Redacted To Not Change The Baselines From Run To Run"
+    "Client Configuration": "Redacted To Not Change The Baselines From Run To Run",
+    "DistributedTraceId": "Redacted To Not Change The Baselines From Run To Run"
   },
   "children": [
     {