From fdcca0655cfe2adbcb180ad04c1c736586f7aaea Mon Sep 17 00:00:00 2001 From: "l.gabrysiak" Date: Mon, 24 Mar 2025 21:14:46 +0100 Subject: [PATCH] Add User Agent --- AzureFunction/GoogleStorageController.cs | 20 ++----- AzureFunction/Program.cs | 8 +++ .../Services/IGoogleBigQueryClientFactory.cs | 8 +++ .../GoogleBigQueryClientFactory.cs | 60 +++++++++++++++++++ Migration/GoogleStorageController.cs | 11 +--- Migration/Program.cs | 8 +++ Migration/Services/IDataverseProvider.cs | 10 ++-- .../Services/IGoogleBigQueryClientFactory.cs | 8 +++ .../Implementation/DataverseProvider.cs | 7 --- .../GoogleBigQueryClientFactory.cs | 60 +++++++++++++++++++ Migration/appsettings.json | 1 + 11 files changed, 165 insertions(+), 36 deletions(-) create mode 100644 AzureFunction/Services/IGoogleBigQueryClientFactory.cs create mode 100644 AzureFunction/Services/Implementation/GoogleBigQueryClientFactory.cs create mode 100644 Migration/Services/IGoogleBigQueryClientFactory.cs create mode 100644 Migration/Services/Implementation/GoogleBigQueryClientFactory.cs diff --git a/AzureFunction/GoogleStorageController.cs b/AzureFunction/GoogleStorageController.cs index be2ce5d..8e6b418 100644 --- a/AzureFunction/GoogleStorageController.cs +++ b/AzureFunction/GoogleStorageController.cs @@ -1,7 +1,6 @@ -using Microsoft.Extensions.Configuration; -using Google.Apis.Auth.OAuth2; +using Google.Apis.Auth.OAuth2; using Google.Cloud.BigQuery.V2; -using System.Linq.Expressions; +using Google.Apis.Http; namespace OSAFeedXML { @@ -38,22 +37,13 @@ namespace OSAFeedXML private readonly string DomainRedirect; private readonly string GoogleProjectId, GoogleDatabase; - public GoogleBigQuery() + public GoogleBigQuery(BigQueryClient bigQueryClient) { - string googleCredentialsJson = Environment.GetEnvironmentVariable("GOOGLE_CREDENTIALS_JSON") ?? throw new Exception("Brak danych uwierzytelniających Google."); - - if (string.IsNullOrEmpty(googleCredentialsJson)) - { - throw new InvalidOperationException("Brak danych uwierzytelniających Google."); - } - - GoogleCredential googleCredential = GoogleCredential.FromJson(googleCredentialsJson); - GoogleProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID") ?? throw new Exception("Brak identyfikatora projektu Google."); GoogleDatabase = Environment.GetEnvironmentVariable("GOOGLE_DATABASE") ?? throw new Exception("Brak nazwy bazy danych Google."); - - bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential); DomainRedirect = Environment.GetEnvironmentVariable("DOMAIN_REDIRECT") ?? throw new Exception("Brak domeny przekierowania."); + + this.bigqueryClient = bigQueryClient; } public dynamic GetDataBigQuery(List Destinations, bool WithoutBg = false) diff --git a/AzureFunction/Program.cs b/AzureFunction/Program.cs index c5fe62f..d7673f6 100644 --- a/AzureFunction/Program.cs +++ b/AzureFunction/Program.cs @@ -1,6 +1,7 @@ using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Migration.Services; using OSAFeedXML; var host = new HostBuilder() @@ -10,6 +11,13 @@ var host = new HostBuilder() services.AddApplicationInsightsTelemetryWorkerService(); services.ConfigureFunctionsApplicationInsights(); services.AddSingleton(); + + services.AddSingleton(); + services.AddSingleton(sp => + { + var factory = sp.GetRequiredService(); + return factory.CreateClient(); + }); }) .Build(); diff --git a/AzureFunction/Services/IGoogleBigQueryClientFactory.cs b/AzureFunction/Services/IGoogleBigQueryClientFactory.cs new file mode 100644 index 0000000..4284f3d --- /dev/null +++ b/AzureFunction/Services/IGoogleBigQueryClientFactory.cs @@ -0,0 +1,8 @@ +using Google.Cloud.BigQuery.V2; + +namespace Migration.Services; +public interface IGoogleBigQueryClientFactory +{ + BigQueryClient CreateClient(); + BigQueryTable GetTable(string datasetId, string tableId); +} \ No newline at end of file diff --git a/AzureFunction/Services/Implementation/GoogleBigQueryClientFactory.cs b/AzureFunction/Services/Implementation/GoogleBigQueryClientFactory.cs new file mode 100644 index 0000000..7c546b3 --- /dev/null +++ b/AzureFunction/Services/Implementation/GoogleBigQueryClientFactory.cs @@ -0,0 +1,60 @@ +using Google.Apis.Auth.OAuth2; +using Google.Apis.Http; +using Google.Cloud.BigQuery.V2; +using System.Net.Http.Headers; + +namespace Migration.Services; + +public class CustomHttpClientFactory : HttpClientFactory +{ + private readonly string userAgent; + + public CustomHttpClientFactory(string userAgent) + { + if (string.IsNullOrWhiteSpace(userAgent)) + { + throw new ArgumentException("User-Agent cannot be null or empty", nameof(userAgent)); + } + this.userAgent = userAgent; + } + + public new ConfigurableHttpClient CreateHttpClient(CreateHttpClientArgs args) + { + var httpClient = base.CreateHttpClient(args); + httpClient.DefaultRequestHeaders.UserAgent.Clear(); + httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(userAgent, "1.0")); + return httpClient; + } +} + +public class GoogleBigQueryClientFactory : IGoogleBigQueryClientFactory +{ + private readonly Lazy bigqueryClient; + + public GoogleBigQueryClientFactory() + { + bigqueryClient = new Lazy(CreateClient); + } + + public BigQueryClient CreateClient() + { + var projectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID") ?? throw new Exception("Brak identyfikatora projektu Google."); + var userAgent = Environment.GetEnvironmentVariable("USER_AGENT") ?? throw new Exception("Nie podano UserAgent."); + var credentialFile = Environment.GetEnvironmentVariable("GOOGLE_CREDENTIALS_JSON") ?? throw new Exception("Nie podano ścieżki do pliku GoogleCredentials."); + + var credential = GoogleCredential.FromJson(credentialFile); + var httpClientFactory = new CustomHttpClientFactory(userAgent); + + return new BigQueryClientBuilder + { + ProjectId = projectId, + Credential = credential, + HttpClientFactory = httpClientFactory + }.Build(); + } + + public BigQueryTable GetTable(string datasetId, string tableId) + { + return bigqueryClient.Value.GetTable(datasetId, tableId); + } +} \ No newline at end of file diff --git a/Migration/GoogleStorageController.cs b/Migration/GoogleStorageController.cs index d994bd8..84809b7 100644 --- a/Migration/GoogleStorageController.cs +++ b/Migration/GoogleStorageController.cs @@ -1,5 +1,4 @@ using System.Text.RegularExpressions; -using Google.Apis.Auth.OAuth2; using Google.Cloud.BigQuery.V2; using Microsoft.AspNetCore.Mvc; using Migration.Services; @@ -15,18 +14,14 @@ public interface IBigQuery public class GoogleBigQuery : IBigQuery { - private readonly GoogleCredential googleCredential; private readonly BigQueryClient bigqueryClient; - private readonly string GoogleProjectId, GoogleDatabase; + private readonly string GoogleDatabase; private readonly IHttpClientFactory httpClientFactory; - public GoogleBigQuery(IConfiguration configuration, IHttpClientFactory httpClientFactory) + public GoogleBigQuery(IConfiguration configuration, IHttpClientFactory httpClientFactory, BigQueryClient bigQueryClient) { - GoogleProjectId = configuration.GetValue("BigQuery:ProjectId") ?? throw new Exception("Nie podano parametru identyfikatora projektu Google."); + this.bigqueryClient = bigQueryClient; GoogleDatabase = configuration.GetValue("BigQuery:DbName") ?? throw new Exception("Nie podano parametru nazwy bazy danych BigQuery w konfiguracji"); - - googleCredential = GoogleCredential.FromFile(configuration.GetValue("GoogleCredentialFile")); - bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential); this.httpClientFactory = httpClientFactory; } diff --git a/Migration/Program.cs b/Migration/Program.cs index 56962e7..f4e3318 100644 --- a/Migration/Program.cs +++ b/Migration/Program.cs @@ -11,6 +11,14 @@ builder.Services.Configure(builder.Configuration.GetSection(MainSe builder.Services.AddSingleton(); builder.Services.AddHttpClient(DataverseProvider.HTTP_CLIENT, DataverseProvider.Setup); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => +{ + var factory = sp.GetRequiredService(); + var configuration = sp.GetRequiredService(); + return factory.CreateClient(configuration); +}); + var app = builder.Build(); if (app.Environment.IsDevelopment()) diff --git a/Migration/Services/IDataverseProvider.cs b/Migration/Services/IDataverseProvider.cs index a779f57..170ff8c 100644 --- a/Migration/Services/IDataverseProvider.cs +++ b/Migration/Services/IDataverseProvider.cs @@ -1,10 +1,8 @@ using Migration.Settings; -namespace Migration.Services +namespace Migration.Services; +public interface IDataverseProvider { - public interface IDataverseProvider - { - Uri Url { get; } - string Token { get; } - } + Uri Url { get; } + string Token { get; } } \ No newline at end of file diff --git a/Migration/Services/IGoogleBigQueryClientFactory.cs b/Migration/Services/IGoogleBigQueryClientFactory.cs new file mode 100644 index 0000000..e2e053a --- /dev/null +++ b/Migration/Services/IGoogleBigQueryClientFactory.cs @@ -0,0 +1,8 @@ +using Google.Cloud.BigQuery.V2; + +namespace Migration.Services; +public interface IGoogleBigQueryClientFactory +{ + BigQueryClient CreateClient(IConfiguration configuration); + BigQueryTable GetTable(string datasetId, string tableId); +} \ No newline at end of file diff --git a/Migration/Services/Implementation/DataverseProvider.cs b/Migration/Services/Implementation/DataverseProvider.cs index a994ced..9c9b0d3 100644 --- a/Migration/Services/Implementation/DataverseProvider.cs +++ b/Migration/Services/Implementation/DataverseProvider.cs @@ -2,27 +2,21 @@ using System.Net.Mime; using Migration.Settings; using Microsoft.Extensions.Options; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; namespace Migration.Services; - public class DataverseProvider : IDataverseProvider { public const string HTTP_CLIENT = "FeedDataverseBQ"; - private static readonly ProductInfoHeaderValue USER_AGENT = ProductInfoHeaderValue.Parse("FeedDataverseBQ"); private static readonly MediaTypeWithQualityHeaderValue JSON_MEDIA_TYPE = new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json); public Uri Url { get; init; } public string Token { get; init; } - public DataverseProvider(IOptions mainSettingsOptions, IHttpClientFactory httpClientFactory) { var DataverseSettings = mainSettingsOptions.Value; Url = new Uri(DataverseSettings.Url); Token = DataverseSettings.Token; } - public static void Setup(IServiceProvider serviceProvider, HttpClient httpClient) { var dataverseConnectionsProvider = serviceProvider.GetRequiredService(); @@ -39,7 +33,6 @@ public class DataverseDelegatingHandler : DelegatingHandler { return SendAsync(request, cancellationToken).GetAwaiter().GetResult(); } - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { return await base.SendAsync(request, cancellationToken); diff --git a/Migration/Services/Implementation/GoogleBigQueryClientFactory.cs b/Migration/Services/Implementation/GoogleBigQueryClientFactory.cs new file mode 100644 index 0000000..5747117 --- /dev/null +++ b/Migration/Services/Implementation/GoogleBigQueryClientFactory.cs @@ -0,0 +1,60 @@ +using Google.Apis.Auth.OAuth2; +using Google.Apis.Http; +using Google.Cloud.BigQuery.V2; +using System.Net.Http.Headers; + +namespace Migration.Services; + +public class CustomHttpClientFactory : HttpClientFactory +{ + private readonly string userAgent; + + public CustomHttpClientFactory(string userAgent) + { + if (string.IsNullOrWhiteSpace(userAgent)) + { + throw new ArgumentException("User-Agent cannot be null or empty", nameof(userAgent)); + } + this.userAgent = userAgent; + } + + public new ConfigurableHttpClient CreateHttpClient(CreateHttpClientArgs args) + { + var httpClient = base.CreateHttpClient(args); + httpClient.DefaultRequestHeaders.UserAgent.Clear(); + httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(userAgent, "1.0")); + return httpClient; + } +} + +public class GoogleBigQueryClientFactory : IGoogleBigQueryClientFactory +{ + private readonly Lazy bigqueryClient; + + public GoogleBigQueryClientFactory(IConfiguration configuration) + { + bigqueryClient = new Lazy(CreateClient(configuration)); + } + + public BigQueryClient CreateClient(IConfiguration configuration) + { + var projectId = configuration.GetValue("BigQuery:ProjectId") ?? throw new Exception("Nie podano identyfikatora projektu Google."); + var userAgent = configuration.GetValue("BigQuery:UserAgent") ?? throw new Exception("Nie podano UserAgent."); + var credentialFile = configuration.GetValue("GoogleCredentialFile") ?? throw new Exception("Nie podano ścieżki do pliku GoogleCredentials."); + + var credential = GoogleCredential.FromFile(credentialFile); + var httpClientFactory = new CustomHttpClientFactory(userAgent); + + return new BigQueryClientBuilder + { + ProjectId = projectId, + Credential = credential, + HttpClientFactory = httpClientFactory + }.Build(); + } + + public BigQueryTable GetTable(string datasetId, string tableId) + { + return bigqueryClient.Value.GetTable(datasetId, tableId); + } +} \ No newline at end of file diff --git a/Migration/appsettings.json b/Migration/appsettings.json index 9887d4e..61bafd2 100644 --- a/Migration/appsettings.json +++ b/Migration/appsettings.json @@ -13,6 +13,7 @@ }, "BigQuery": { "ProjectId": "", + "UserAgent": "", "DbName": "" } }