[IngestionClient] refactor: Injecting environment variables instead o…

…f using static env file (#2501)

* DI for env variables

* Removing static Env files

* updating the correct value for MaxPollingDelayInMinutes

samples/ingestion/ingestion-client/FetchTranscription/Config/AppConfig.cs

@@ -0,0 +1,109 @@
+// <copyright file="AppConfig.cs" company="Microsoft Corporation">
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+// </copyright>
+
+namespace FetchTranscription
+{
+    using System;
+    using Connector.Constants;
+    using Connector.Enums;
+    using Connector.Serializable.Language.Conversations;
+
+    public class AppConfig
+    {
+        private ConversationSummarizationOptions conversationSummarizationOptions;
+
+        private int initialPollingDelayInMinutes = Constants.DefaultInitialPollingDelayInMinutes;
+
+        private int retryLimit = Constants.DefaultRetryLimit;
+        private int maxPollingDelayInMinutes = Constants.DefaultMaxPollingDelayInMinutes;
+
+        public SentimentAnalysisSetting SentimentAnalysisSetting { get; set; } = SentimentAnalysisSetting.None;
+
+        public PiiRedactionSetting PiiRedactionSetting { get; set; } = PiiRedactionSetting.None;
+
+        public bool CreateHtmlResultFile { get; set; }
+
+        public ConversationPiiSetting ConversationPiiSetting { get; set; } = ConversationPiiSetting.None;
+
+        public string ConversationPiiCategories { get; set; }
+
+        public string ConversationPiiInferenceSource { get; set; }
+
+        public int ConversationPiiMaxChunkSize { get; set; } = Constants.DefaultConversationAnalysisMaxChunkSize;
+
+        public ConversationSummarizationOptions ConversationSummarizationOptions
+        {
+            get
+            {
+                if (this.conversationSummarizationOptions == null)
+                {
+                    var envVarValue = Environment.GetEnvironmentVariable(nameof(this.ConversationSummarizationOptions), EnvironmentVariableTarget.Process);
+                    this.conversationSummarizationOptions = string.IsNullOrEmpty(envVarValue)
+                        ? new ConversationSummarizationOptions()
+                        : System.Text.Json.JsonSerializer.Deserialize<ConversationSummarizationOptions>(envVarValue);
+                }
+
+                return this.conversationSummarizationOptions;
+            }
+
+            set
+            {
+                this.conversationSummarizationOptions = value;
+            }
+        }
+
+        public bool UseSqlDatabase { get; set; }
+
+        public int InitialPollingDelayInMinutes
+        {
+            get => this.initialPollingDelayInMinutes;
+            set => this.initialPollingDelayInMinutes = Math.Clamp(value, 2, Constants.MaxInitialPollingDelayInMinutes);
+        }
+
+        public int MaxPollingDelayInMinutes { get; set; } = Constants.DefaultMaxPollingDelayInMinutes;
+
+        public int RetryLimit
+        {
+            get => this.retryLimit;
+            set => this.retryLimit = Math.Clamp(value, 1, Constants.MaxRetryLimit);
+        }
+
+        public string AudioInputContainer { get; set; }
+
+        public string AzureSpeechServicesKey { get; set; }
+
+        public string AzureWebJobsStorage { get; set; }
+
+        public string DatabaseConnectionString { get; set; }
+
+        public string ErrorFilesOutputContainer { get; set; }
+
+        public string ErrorReportOutputContainer { get; set; }
+
+        public string FetchTranscriptionServiceBusConnectionString { get; set; }
+
+        public string CompletedServiceBusConnectionString { get; set; }
+
+        public string HtmlResultOutputContainer { get; set; }
+
+        public string JsonResultOutputContainer { get; set; }
+
+        public string StartTranscriptionServiceBusConnectionString { get; set; }
+
+        public string TextAnalyticsKey { get; set; }
+
+        public string TextAnalyticsEndpoint { get; set; }
+
+        public string PiiCategories { get; set; }
+
+        public bool CreateConsolidatedOutputFiles { get; set; }
+
+        public string ConsolidatedFilesOutputContainer { get; set; }
+
+        public bool CreateAudioProcessedContainer { get; set; }
+
+        public string AudioProcessedContainer { get; set; }
+    }
+}

samples/ingestion/ingestion-client/FetchTranscription/FetchTranscription.cs

@@ -16,6 +16,7 @@ namespace FetchTranscription
     using Microsoft.Extensions.Azure;
     using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.Logging;
+    using Microsoft.Extensions.Options;
 
     /// <summary>
     /// Fetch Transcription class.
@@ -26,6 +27,7 @@ public class FetchTranscription
         private readonly IStorageConnector storageConnector;
         private readonly IAzureClientFactory<ServiceBusClient> serviceBusClientFactory;
         private readonly ILogger<FetchTranscription> logger;
+        private readonly AppConfig appConfig;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="FetchTranscription"/> class.
@@ -34,16 +36,19 @@ public class FetchTranscription
         /// <param name="logger">The FetchTranscription logger.</param>
         /// <param name="storageConnector">Storage Connector dependency</param>
         /// <param name="serviceBusClientFactory">Azure client factory for service bus clients</param>
+        /// <param name="appConfig">Environment configuration</param>
         public FetchTranscription(
             IServiceProvider serviceProvider,
             ILogger<FetchTranscription> logger,
             IStorageConnector storageConnector,
-            IAzureClientFactory<ServiceBusClient> serviceBusClientFactory)
+            IAzureClientFactory<ServiceBusClient> serviceBusClientFactory,
+            IOptions<AppConfig> appConfig)
         {
             this.serviceProvider = serviceProvider;
             this.logger = logger;
             this.storageConnector = storageConnector;
             this.serviceBusClientFactory = serviceBusClientFactory;
+            this.appConfig = appConfig?.Value;
         }
 
         /// <summary>
@@ -65,9 +70,9 @@ public async Task Run([ServiceBusTrigger("fetch_transcription_queue", Connection
 
             var serviceBusMessage = TranscriptionStartedMessage.DeserializeMessage(message);
 
-            var databaseContext = FetchTranscriptionEnvironmentVariables.UseSqlDatabase ? this.serviceProvider.GetRequiredService<IngestionClientDbContext>() : null;
+            var databaseContext = this.appConfig.UseSqlDatabase ? this.serviceProvider.GetRequiredService<IngestionClientDbContext>() : null;
 
-            var transcriptionProcessor = new TranscriptionProcessor(this.storageConnector, this.serviceBusClientFactory, databaseContext);
+            var transcriptionProcessor = new TranscriptionProcessor(this.storageConnector, this.serviceBusClientFactory, databaseContext, Options.Create(this.appConfig));
 
             await transcriptionProcessor.ProcessTranscriptionJobAsync(serviceBusMessage, this.serviceProvider,  this.logger).ConfigureAwait(false);
         }

samples/ingestion/ingestion-client/FetchTranscription/Program.cs

@@ -5,7 +5,7 @@
 
 namespace FetchTranscription
 {
-    using System;
+    using System.IO;
 
     using Azure.Storage;
     using Azure.Storage.Blobs;
@@ -16,55 +16,68 @@ namespace FetchTranscription
 
     using Microsoft.EntityFrameworkCore;
     using Microsoft.Extensions.Azure;
+    using Microsoft.Extensions.Configuration;
     using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.Hosting;
+    using Microsoft.Extensions.Options;
 
     public static class Program
     {
         public static void Main(string[] args)
         {
-            var useSqlDatabase = FetchTranscriptionEnvironmentVariables.UseSqlDatabase;
-            var storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
-            var blobServiceClient = new BlobServiceClient(storageConnectionString);
-            var storageCredential = new StorageSharedKeyCredential(
-                AzureStorageConnectionExtensions.GetValueFromConnectionString("AccountName", storageConnectionString),
-                AzureStorageConnectionExtensions.GetValueFromConnectionString("AccountKey", storageConnectionString));
-
             var host = new HostBuilder()
                 .ConfigureFunctionsWorkerDefaults()
-                .ConfigureServices(s =>
+                .ConfigureAppConfiguration((context, config) =>
+                {
+                    config.SetBasePath(Directory.GetCurrentDirectory())
+                     .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
+                     .AddEnvironmentVariables();
+                })
+                .ConfigureServices((context, services) =>
                 {
+                    var configuration = context.Configuration;
+                    var config = new AppConfig();
+                    configuration.GetSection("Values").Bind(config);
+
+                    var blobServiceClient = new BlobServiceClient(config.AzureWebJobsStorage);
+                    var storageCredential = new StorageSharedKeyCredential(
+                    AzureStorageConnectionExtensions.GetValueFromConnectionString("AccountName", config.AzureWebJobsStorage),
+                    AzureStorageConnectionExtensions.GetValueFromConnectionString("AccountKey", config.AzureWebJobsStorage));
+
                     // This is a unified way to configure logging filter for all functions.
-                    s.ConfigureIngestionClientLogging();
+                    services.ConfigureIngestionClientLogging();
 
-                    if (useSqlDatabase)
+                    if (config.UseSqlDatabase)
                     {
-                        s.AddDbContext<IngestionClientDbContext>(
-                        options => SqlServerDbContextOptionsExtensions.UseSqlServer(options, FetchTranscriptionEnvironmentVariables.DatabaseConnectionString));
+                        services.AddDbContext<IngestionClientDbContext>(
+                        options => SqlServerDbContextOptionsExtensions.UseSqlServer(options, config.DatabaseConnectionString));
                     }
 
-                    s.AddSingleton(blobServiceClient);
-                    s.AddSingleton(storageCredential);
-                    s.AddTransient<IStorageConnector, StorageConnector>();
+                    services.AddSingleton(blobServiceClient);
+                    services.AddSingleton(storageCredential);
+                    services.AddTransient<IStorageConnector, StorageConnector>();
 
-                    s.AddAzureClients(clientBuilder =>
+                    services.AddAzureClients(clientBuilder =>
                     {
-                        clientBuilder.AddServiceBusClient(FetchTranscriptionEnvironmentVariables.StartTranscriptionServiceBusConnectionString)
+                        clientBuilder.AddServiceBusClient(config.StartTranscriptionServiceBusConnectionString)
                             .WithName(ServiceBusClientName.StartTranscriptionServiceBusClient.ToString());
-                        clientBuilder.AddServiceBusClient(FetchTranscriptionEnvironmentVariables.FetchTranscriptionServiceBusConnectionString)
+                        clientBuilder.AddServiceBusClient(config.FetchTranscriptionServiceBusConnectionString)
                             .WithName(ServiceBusClientName.FetchTranscriptionServiceBusClient.ToString());
 
-                        if (!string.IsNullOrWhiteSpace(FetchTranscriptionEnvironmentVariables.CompletedServiceBusConnectionString))
+                        if (!string.IsNullOrWhiteSpace(config.CompletedServiceBusConnectionString))
                         {
-                            clientBuilder.AddServiceBusClient(FetchTranscriptionEnvironmentVariables.CompletedServiceBusConnectionString)
+                            clientBuilder.AddServiceBusClient(config.CompletedServiceBusConnectionString)
                                 .WithName(ServiceBusClientName.CompletedTranscriptionServiceBusClient.ToString());
                         }
                     });
+                    services.Configure<AppConfig>(configuration.GetSection("Values"));
                 })
                 .Build();
 
+            var config = host.Services.GetService<IOptions<AppConfig>>().Value;
+
             // apply database migrations once during startup (not with every function execution):
-            if (useSqlDatabase)
+            if (config.UseSqlDatabase)
             {
                 using var scope = host.Services.CreateScope();
                 var ingestionClientDbContext = scope.ServiceProvider.GetRequiredService<IngestionClientDbContext>();