Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@sourabh1007
sourabh1007 committed Mar 13, 2023
1 parent 9c4db46 commit d1a8c35
Show file tree
Hide file tree
Showing 6 changed files with 42 additions and 50 deletions.
4 changes: 2 additions & 2 deletions Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetry.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ ConcurrentDictionary<RequestInfo, LongConcurrentHistogram> requestInfoSnapshot
try
{
await this.processor
.ProcessAndSendAsync(
.ProcessAndSendAsync(
clientTelemetryInfo: this.clientTelemetryInfo,
operationInfoSnapshot: operationInfoSnapshot,
cacheRefreshInfoSnapshot: cacheRefreshInfoSnapshot,
Expand Down Expand Up @@ -352,7 +352,7 @@ internal void CollectOperationInfo(CosmosDiagnostics cosmosDiagnostics,
/// <param name="droppedRntbdRequestCount"></param>
private void RecordRntbdResponses(string containerId, string databaseId, List<StoreResponseStatistics> storeResponseStatistics, out int droppedRntbdRequestCount)
{
IClientTelemetrySampler<RequestInfo> networkRequestSampler
using IClientTelemetrySampler<RequestInfo> networkRequestSampler
= new NetworkRequestSampler(ClientTelemetryOptions.NetworkTelemetrySampleSize);

droppedRntbdRequestCount = 0;
Expand Down
21 changes: 10 additions & 11 deletions Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ internal static class ClientTelemetryOptions

internal const string DroppedRntbdRequestsName = "DroppedRntbdRequests";
internal const string DroppedRntbdRequestsUnit = "Count";

internal const double DefaultTimeStampInSeconds = 600;
internal const double Percentile50 = 50.0;
internal const double Percentile90 = 90.0;
Expand All @@ -89,20 +89,20 @@ internal static class ClientTelemetryOptions
internal const string EnvPropsClientTelemetryEndpoint = "COSMOS.CLIENT_TELEMETRY_ENDPOINT";
internal const string EnvPropsClientTelemetryEnvironmentName = "COSMOS.ENVIRONMENT_NAME";

internal static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings
{
internal static readonly ResourceType AllowedResourceTypes = ResourceType.Document;
// Why 5 sec? As of now, if any network request is taking more than 5 millisecond sec, we will consider it slow request this value can be revisited in future
internal static readonly TimeSpan NetworkLatencyThreshold = TimeSpan.FromMilliseconds(5);
internal static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr
};

internal static readonly ResourceType AllowedResourceTypes = ResourceType.Document;

// Why 5 millisec? As of now, if any network request is taking more than 5 millisecond sec, we will consider it slow request this value can be revisited in future
internal static readonly TimeSpan NetworkLatencyThreshold = TimeSpan.FromMilliseconds(5);
internal static readonly List<int> ExcludedStatusCodes = new List<int> { 404, 409 };

internal static readonly int NetworkTelemetrySampleSize = 200;
internal static readonly List<int> NetworkRequestExcludedStatusCodes = new List<int> { 404, 409 };
internal static int ClientTelemetryServicePayloadSizeThreshold = 1024 * 1024 * 2; // 2MB

internal static int PayloadSizeThreshold = 1024 * 1024 * 2; // 2MB

private static Uri clientTelemetryEndpoint;
private static string environmentName;
private static TimeSpan scheduledTimeSpan = TimeSpan.Zero;
Expand Down Expand Up @@ -184,6 +184,5 @@ internal static string GetEnvironmentName()
}
return environmentName;
}

}
}
11 changes: 6 additions & 5 deletions Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryPayloadWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public static async Task SerializedPayloadChunksAsync(
throw new ArgumentNullException(nameof(properties));
}

StringBuilder stringBuilder = new StringBuilder(ClientTelemetryOptions.ClientTelemetryServicePayloadSizeThreshold);
StringBuilder stringBuilder = new StringBuilder(ClientTelemetryOptions.PayloadSizeThreshold);

JsonWriter writer = ClientTelemetryPayloadWriter.GetWriterWithSectionStartTag(stringBuilder, properties, "operationInfo");

Expand Down Expand Up @@ -58,7 +58,8 @@ public static async Task SerializedPayloadChunksAsync(
string requestChargeMetrics = JsonConvert.SerializeObject(payloadForRequestCharge);
string droppedRntbdRequestsMetrics = JsonConvert.SerializeObject(payloadForDroppedRntbdRequests);

if (lengthNow + latencyMetrics.Length + requestChargeMetrics.Length > ClientTelemetryOptions.ClientTelemetryServicePayloadSizeThreshold)
int thisSectionLength = latencyMetrics.Length + requestChargeMetrics.Length + droppedRntbdRequestsMetrics.Length;
if (lengthNow + thisSectionLength > ClientTelemetryOptions.PayloadSizeThreshold)
{
writer.WriteEndArray();
writer.WriteEndObject();
Expand Down Expand Up @@ -91,7 +92,7 @@ public static async Task SerializedPayloadChunksAsync(

string latencyMetrics = JsonConvert.SerializeObject(payloadForLatency);

if (lengthNow + latencyMetrics.Length > ClientTelemetryOptions.ClientTelemetryServicePayloadSizeThreshold)
if (lengthNow + latencyMetrics.Length > ClientTelemetryOptions.PayloadSizeThreshold)
{
writer.WriteEndArray();
writer.WriteEndObject();
Expand All @@ -118,12 +119,12 @@ public static async Task SerializedPayloadChunksAsync(

MetricInfo metricInfo = new MetricInfo(ClientTelemetryOptions.RequestLatencyName, ClientTelemetryOptions.RequestLatencyUnit);
metricInfo.SetAggregators(entry.Value, ClientTelemetryOptions.TicksToMsFactor);

RequestInfo payloadForLatency = entry.Key;
payloadForLatency.Metrics.Add(metricInfo);
string latencyMetrics = JsonConvert.SerializeObject(payloadForLatency);

if (lengthNow + latencyMetrics.Length > ClientTelemetryOptions.ClientTelemetryServicePayloadSizeThreshold)
if (lengthNow + latencyMetrics.Length > ClientTelemetryOptions.PayloadSizeThreshold)
{
writer.WriteEndArray();
writer.WriteEndObject();
Expand Down
6 changes: 3 additions & 3 deletions Microsoft.Azure.Cosmos/src/Telemetry/Sampler/IClientTelemetrySampler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@

namespace Microsoft.Azure.Cosmos.Telemetry.Sampler
{
internal interface IClientTelemetrySampler<T>
using System;

internal interface IClientTelemetrySampler<T> : IDisposable
{
internal bool ShouldSample(T statisticsObj);

internal void Clear();
}
}
37 changes: 14 additions & 23 deletions Microsoft.Azure.Cosmos/src/Telemetry/Sampler/NetworkRequestSampler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ internal sealed class NetworkRequestSampler : IClientTelemetrySampler<RequestInf
{
private readonly int NetworkFailuresCount;
private readonly ISet<RequestInfo> TempStorage;

private readonly object lockObject = new object();

public NetworkRequestSampler(int maxNumberOfFailures)
{
Expand All @@ -28,29 +26,18 @@ public bool ShouldSample(RequestInfo requestInfo)
return false;
}

lock (this.lockObject)
if (this.TempStorage.Count < this.NetworkFailuresCount)
{
if (this.TempStorage.Count < this.NetworkFailuresCount)
{
return this.TempStorage.Add(requestInfo);
}
else
{
bool isAdded = this.TempStorage.Add(requestInfo);
if (isAdded)
{
this.TempStorage.Remove(requestInfo);
}
return !isAdded;
}
return this.TempStorage.Add(requestInfo);
}
}

public void Clear()
{
lock (this.lockObject)
else
{
this.TempStorage.Clear();
bool isAdded = this.TempStorage.Add(requestInfo);
if (isAdded)
{
this.TempStorage.Remove(requestInfo);
}
return !isAdded;
}
}

Expand All @@ -77,8 +64,12 @@ private static bool IsUserOrServerError(int statusCode)

private static bool IsStatusCodeNotExcluded(int statusCode, int subStatusCode)
{
return !(ClientTelemetryOptions.NetworkRequestExcludedStatusCodes.Contains(statusCode) && subStatusCode == 0);
return !(ClientTelemetryOptions.ExcludedStatusCodes.Contains(statusCode) && subStatusCode == 0);
}

public void Dispose()
{
this.TempStorage.Clear();
}
}
}
13 changes: 7 additions & 6 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Telemetry/ClientTelemetryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,14 +119,14 @@ public void CheckJsonSerializerContractWithPreferredRegions()
}

[TestMethod]
[DataRow(100, 50, 200)] // When operation, cacherefresh and request info is there in payload
[DataRow(150, 50, 200)] // When operation, cacherefresh and request info is there in payload
[DataRow(0, 50, 0)] // When only cacherefresh info is there in payload
[DataRow(100, 50, 0)] // When only operation and cacherefresh info is there in payload
[DataRow(100, 0, 0)] // When only operation info is there in payload
[DataRow(150, 50, 0)] // When only operation and cacherefresh info is there in payload
[DataRow(150, 0, 0)] // When only operation info is there in payload
public async Task CheckIfPayloadIsDividedCorrectlyAsync(int expectedOperationInfoSize, int expectedCacheRefreshInfoSize, int expectedRequestInfoSize)
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, "http://dummy.telemetry.endpoint/");
ClientTelemetryOptions.ClientTelemetryServicePayloadSizeThreshold = 1024 * 15; //15 Kb
ClientTelemetryOptions.PayloadSizeThreshold = 1024 * 15; //15 Kb

string data = File.ReadAllText("Telemetry/ClientTelemetryPayloadWithoutMetrics.json", Encoding.UTF8);
ClientTelemetryProperties clientTelemetryProperties = JsonConvert.DeserializeObject<ClientTelemetryProperties>(data);
Expand All @@ -145,7 +145,7 @@ public async Task CheckIfPayloadIsDividedCorrectlyAsync(int expectedOperationInf
(request, cancellationToken) =>
{
string payloadJson = request.Content.ReadAsStringAsync().Result;
Assert.IsTrue(payloadJson.Length <= ClientTelemetryOptions.ClientTelemetryServicePayloadSizeThreshold, "Payload Size is " + payloadJson.Length);
Assert.IsTrue(payloadJson.Length <= ClientTelemetryOptions.PayloadSizeThreshold, "Payload Size is " + payloadJson.Length);
ClientTelemetryProperties propertiesToSend = JsonConvert.DeserializeObject<ClientTelemetryProperties>(payloadJson);
Expand All @@ -164,7 +164,8 @@ public async Task CheckIfPayloadIsDividedCorrectlyAsync(int expectedOperationInf
ConcurrentDictionary<OperationInfo, (LongConcurrentHistogram latency, LongConcurrentHistogram requestcharge, int droppedrequestCount)> operationInfoSnapshot
= new ConcurrentDictionary<OperationInfo, (LongConcurrentHistogram latency, LongConcurrentHistogram requestcharge, int droppedrequestCount)> ();

for (int i = 0; i < (expectedOperationInfoSize/2); i++)
int numberOfMetricsInOperatiionSection = 3;
for (int i = 0; i < (expectedOperationInfoSize/ numberOfMetricsInOperatiionSection); i++)
{
OperationInfo opeInfo = new OperationInfo(Regions.WestUS,
0,
Expand Down

0 comments on commit d1a8c35

Please sign in to comment.