This commit is contained in:
l.gabrysiak 2025-03-24 14:45:46 +01:00
commit 2ab789ac19
26 changed files with 1735 additions and 0 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

25
.dockerignore Normal file
View File

@ -0,0 +1,25 @@
**/.classpath
**/.dockerignore
**/.env
**/.git
**/.gitignore
**/.project
**/.settings
**/.toolstarget
**/.vs
**/.vscode
**/*.*proj.user
**/*.dbmdl
**/*.jfm
**/bin
**/charts
**/docker-compose*
**/compose*
**/Dockerfile*
**/node_modules
**/npm-debug.log
**/obj
**/secrets.dev.yaml
**/values.dev.yaml
LICENSE
README.md

6
.vscode/extensions.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"recommendations": [
"ms-azuretools.vscode-azurefunctions",
"ms-dotnettools.csharp"
]
}

45
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,45 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Attach to .NET Functions",
"type": "coreclr",
"request": "attach",
"processId": "${command:azureFunctions.pickProcess}"
},
{
"name": ".NET Core Launch (web)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/Migration/bin/Debug/net8.0/bigqueryconnector.dll",
"args": [],
"cwd": "${workspaceFolder}/Migration",
"stopAtEntry": false,
"serverReadyAction": {
"action": "openExternally",
"pattern": "\\bNow listening on:\\s+(https?://\\S+)"
},
"env": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"sourceFileMap": {
"/Views": "${workspaceFolder}/Views"
}
},
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach"
},
{
"name": "Docker .NET Launch",
"type": "docker",
"request": "launch",
"preLaunchTask": "docker-run: debug",
"netCore": {
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
}
}
]
}

7
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,7 @@
{
"azureFunctions.deploySubpath": "AzureFunction/bin/Release/net8.0/publish",
"azureFunctions.projectLanguage": "C#",
"azureFunctions.projectRuntime": "~4",
"debug.internalConsoleOptions": "neverOpen",
"azureFunctions.preDeployTask": "publish (functions)"
}

177
.vscode/tasks.json vendored Normal file
View File

@ -0,0 +1,177 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "clean (functions)",
"command": "dotnet",
"args": [
"clean",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/AzureFunction"
}
},
{
"label": "build (functions)",
"command": "dotnet",
"args": [
"build",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"dependsOn": "clean (functions)",
"group": {
"kind": "build",
"isDefault": true
},
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/AzureFunction"
}
},
{
"label": "clean release (functions)",
"command": "dotnet",
"args": [
"clean",
"--configuration",
"Release",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/AzureFunction"
}
},
{
"label": "publish (functions)",
"command": "dotnet",
"args": [
"publish",
"--configuration",
"Release",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"dependsOn": "clean release (functions)",
"problemMatcher": "$msCompile",
"options": {
"cwd": "${workspaceFolder}/AzureFunction"
}
},
{
"type": "func",
"dependsOn": "build (functions)",
"options": {
"cwd": "${workspaceFolder}/AzureFunction/bin/Debug/net8.0"
},
"command": "host start",
"isBackground": true,
"problemMatcher": "$func-dotnet-watch"
},
{
"label": "build",
"command": "dotnet",
"type": "process",
"args": [
"build",
"${workspaceFolder}/FeedDataverseBQ.sln",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary;ForceNoAlign"
],
"problemMatcher": "$msCompile"
},
{
"label": "publish",
"command": "dotnet",
"type": "process",
"args": [
"publish",
"${workspaceFolder}/FeedDataverseBQ.sln",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary;ForceNoAlign"
],
"problemMatcher": "$msCompile"
},
{
"label": "watch",
"command": "dotnet",
"type": "process",
"args": [
"watch",
"run",
"--project",
"${workspaceFolder}/FeedDataverseBQ.sln"
],
"problemMatcher": "$msCompile"
},
{
"type": "docker-build",
"label": "docker-build: debug",
"dependsOn": [
"build"
],
"dockerBuild": {
"tag": "feeddataversebq:dev",
"target": "base",
"dockerfile": "${workspaceFolder}/Migration/Dockerfile",
"context": "${workspaceFolder}",
"pull": true
},
"netCore": {
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
}
},
{
"type": "docker-build",
"label": "docker-build: release",
"dependsOn": [
"build"
],
"dockerBuild": {
"tag": "feeddataversebq:latest",
"dockerfile": "${workspaceFolder}/Migration/Dockerfile",
"context": "${workspaceFolder}",
"platform": {
"os": "linux",
"architecture": "amd64"
},
"pull": true
},
"netCore": {
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
}
},
{
"type": "docker-run",
"label": "docker-run: debug",
"dependsOn": [
"docker-build: debug"
],
"dockerRun": {},
"netCore": {
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj",
"enableDebugging": true
}
},
{
"type": "docker-run",
"label": "docker-run: release",
"dependsOn": [
"docker-build: release"
],
"dockerRun": {},
"netCore": {
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
}
}
]
}

268
AzureFunction/.gitignore vendored Normal file
View File

@ -0,0 +1,268 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# Azure Functions localsettings file
local.settings.json
osadkowski-hd-2f3bd9b0ab5d.json
appsettings.json
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
.DS_Store
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
project.fragment.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
#*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush
.cr/
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc

118
AzureFunction/Function1.cs Normal file
View File

@ -0,0 +1,118 @@
using System.Globalization;
using System.Net;
using System.Xml.Linq;
using Google.Cloud.BigQuery.V2;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json.Linq;
namespace OSAFeedXML
{
public class Function1
{
private readonly ILogger _logger;
private readonly IBigQuery _bq;
private readonly string DomainRedirect, ShortDescription, Title;
public Function1(ILoggerFactory loggerFactory, IBigQuery bq)
{
_logger = loggerFactory.CreateLogger<Function1>();
DomainRedirect = Environment.GetEnvironmentVariable("DOMAIN_REDIRECT") ?? throw new Exception("Brak domeny przekierowania.");
ShortDescription = Environment.GetEnvironmentVariable("SHORT_DESCRIPTION") ?? throw new Exception("Brak krótkiego opisu.");
Title = Environment.GetEnvironmentVariable("TITLE") ?? throw new Exception("Brak tytułu.");
_bq = bq;
}
[Function("getfeeds")]
public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get")] HttpRequestData req)
{
try
{
var queryParams = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
var sourceParam = queryParams["source"];
var destinationList = new List<Destinations>();
Boolean.TryParse(queryParams["withoutbg"], out var withoutbg);
if (sourceParam != null)
{
var sources = sourceParam.Split(',');
destinationList = sources
.Select(src => Enum.TryParse(typeof(Destinations), src, true, out var result) ? result : null)
.Where(v => v != null)
.Cast<Destinations>()
.ToList();
}
else
{
destinationList = new List<Destinations>() { Destinations.google, Destinations.facebook };
}
var result = _bq.GetDataBigQuery(destinationList, withoutbg);
var jsonArray = new JArray(
((BigQueryResults)result).Select(row => new JObject(
row.Schema.Fields.Select(field => new JProperty(field.Name, JToken.FromObject(row[field.Name] ?? string.Empty)))
))
);
JArray products = JArray.Parse(jsonArray.ToString());
XNamespace g = "http://base.google.com/ns/1.0";
XDocument xmlDoc = new XDocument(
new XElement("rss", new XAttribute("version", "2.0"),
new XAttribute(XNamespace.Xmlns + "g", g),
new XElement("channel",
new XElement("title", Title),
new XElement("link", DomainRedirect),
new XElement("description", ShortDescription),
from product in products
select new XElement("item",
new XElement(g + "id", product["id"]?.ToString()),
new XElement(g + "title", product["title"]?.ToString()),
new XElement(g + "description", product["description"]?.ToString()),
new XElement(g + "link", product["link"]?.ToString()),
new XElement(g + "image_link", product["image_link"]?.ToString()),
new XElement(g + "availability", product["availability"]?.ToString()),
new XElement(g + "price", FormatPrice(product["price"] ?? 0)),
new XElement(g + "brand", product["brand"]?.ToString()),
new XElement(g + "gtin", product["gtin"]?.ToString()),
new XElement(g + "mpn", product["mpn"]?.ToString()),
new XElement(g + "product_type", product["product_type"]?.ToString()),
new XElement(g + "google_product_category", product["google_product_category"]?.ToString()),
new XElement(g + "unit_pricing_measure", product["unit_pricing_measure"]?.ToString()),
new XElement(g + "condition", product["condition"]?.ToString()),
product["gtin"]?.ToString() == "" ? new XElement(g + "identifier_exists", "no") : new XElement(g + "identifier_exists", "yes")
)
)
)
);
var response = req.CreateResponse(HttpStatusCode.OK);
response.Headers.Add("Content-Type", "text/xml; charset=utf-8");
response.WriteString(xmlDoc.ToString());
return response;
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
return req.CreateResponse(HttpStatusCode.InternalServerError);
}
}
static string FormatPrice(JToken priceToken)
{
if (priceToken == null || string.IsNullOrWhiteSpace(priceToken.ToString()))
{
return "0.00 PLN";
}
decimal price = priceToken.Value<decimal>();
return price.ToString("0.00", CultureInfo.InvariantCulture) + " PLN";
}
}
}

View File

@ -0,0 +1,99 @@
using Microsoft.Extensions.Configuration;
using Google.Apis.Auth.OAuth2;
using Google.Cloud.BigQuery.V2;
using System.Linq.Expressions;
namespace OSAFeedXML
{
public interface IBigQuery
{
dynamic GetDataBigQuery(List<Destinations> destinations, bool withoutbg = false);
}
public enum Destinations
{
google = 0,
facebook = 1
}
internal enum PublicationStatus
{
HIDDEN = 1,
OFFLINE_OFFER = 2,
UNAVAILABLE = 3,
TEMPORARILY_UNAVAILABLE = 4,
SALE_IN_EC = 5,
PRESALE_IN_EC = 6
}
internal enum PriceConversion
{
NOT_CONVERTION = 0,
CONVERTION = 1
}
internal class GoogleBigQuery : IBigQuery
{
private readonly BigQueryClient bigqueryClient;
private readonly string DomainRedirect;
private readonly string GoogleProjectId, GoogleDatabase;
public GoogleBigQuery()
{
string googleCredentialsJson = Environment.GetEnvironmentVariable("GOOGLE_CREDENTIALS_JSON") ?? throw new Exception("Brak danych uwierzytelniających Google.");
if (string.IsNullOrEmpty(googleCredentialsJson))
{
throw new InvalidOperationException("Brak danych uwierzytelniających Google.");
}
GoogleCredential googleCredential = GoogleCredential.FromJson(googleCredentialsJson);
GoogleProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID") ?? throw new Exception("Brak identyfikatora projektu Google.");
GoogleDatabase = Environment.GetEnvironmentVariable("GOOGLE_DATABASE") ?? throw new Exception("Brak nazwy bazy danych Google.");
bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential);
DomainRedirect = Environment.GetEnvironmentVariable("DOMAIN_REDIRECT") ?? throw new Exception("Brak domeny przekierowania.");
}
public dynamic GetDataBigQuery(List<Destinations> Destinations, bool WithoutBg = false)
{
var status = new List<PublicationStatus>() { PublicationStatus.SALE_IN_EC, PublicationStatus.PRESALE_IN_EC };
string sql = @$"
SELECT
bon_gid AS id,
bon_ec_name AS title,
bon_seo_description AS description,
CONCAT('{DomainRedirect}/produkt/', bon_slug_url, '--s-', bon_gid) AS link,
COALESCE(ic.bon_url, bn.bon_url) AS image_link,
CASE
WHEN bon_warehouse_state_central IS NULL OR bon_warehouse_state_central = 0 THEN 'out_of_stock'
WHEN NOT EXISTS (
SELECT 1
FROM UNNEST(bon_marketplace) AS marketplace_value
WHERE marketplace_value IN UNNEST([{string.Join(", ", Destinations.Select(dest => (int)dest))}])
)
OR bon_ecommerce_status NOT IN ({string.Join(", ", status.Select(stat => (int)stat))}) THEN 'out_of_stock'
ELSE 'in_stock'
END AS availability,
ROUND(
bon_cash_price *
(CASE WHEN bon_price_for = {(int)PriceConversion.CONVERTION} THEN bon_converter ELSE 1 END) *
((100 + COALESCE(bon_vat_rate, 0)) / 100), 2
) AS price,
bon_category_path AS product_type,
bon_name AS brand,
bon_ean AS gtin,
bon_index AS mpn,
osa_gmc_id AS google_product_category
FROM `{GoogleProjectId}.{GoogleDatabase}.bon_main_product` bn
LEFT JOIN `{GoogleProjectId}.{GoogleDatabase}.images_clearbg` ic
ON bn.bon_commodity_indexid = ic.bon_commodity_indexid
";
BigQueryResults results = bigqueryClient.ExecuteQuery(sql, null);
return results;
}
}
}

View File

@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
<OutputType>Exe</OutputType>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Google.Apis.Auth" Version="1.69.0" />
<PackageReference Include="Google.Cloud.BigQuery.V2" Version="3.11.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.3.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.1" />
<PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.23.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.0.0" />
</ItemGroup>
<ItemGroup>
<None Update="host.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<Using Include="System.Threading.ExecutionContext" Alias="ExecutionContext" />
</ItemGroup>
</Project>

16
AzureFunction/Program.cs Normal file
View File

@ -0,0 +1,16 @@
using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using OSAFeedXML;
var host = new HostBuilder()
.ConfigureFunctionsWorkerDefaults()
.ConfigureServices(services =>
{
services.AddApplicationInsightsTelemetryWorkerService();
services.ConfigureFunctionsApplicationInsights();
services.AddSingleton<IBigQuery, GoogleBigQuery>();
})
.Build();
host.Run();

View File

@ -0,0 +1,9 @@
{
"profiles": {
"OSAFeedXML": {
"commandName": "Project",
"commandLineArgs": "--port 7168",
"launchBrowser": false
}
}
}

View File

@ -0,0 +1,11 @@
{
"dependencies": {
"appInsights1": {
"type": "appInsights"
},
"storage1": {
"type": "storage",
"connectionId": "AzureWebJobsStorage"
}
}
}

View File

@ -0,0 +1,11 @@
{
"dependencies": {
"appInsights1": {
"type": "appInsights.sdk"
},
"storage1": {
"type": "storage.emulator",
"connectionId": "AzureWebJobsStorage"
}
}
}

12
AzureFunction/host.json Normal file
View File

@ -0,0 +1,12 @@
{
"version": "2.0",
"logging": {
"applicationInsights": {
"samplingSettings": {
"isEnabled": true,
"excludedTypes": "Request"
},
"enableLiveMetricsFilters": true
}
}
}

30
FeedDataverseBQ.sln Normal file
View File

@ -0,0 +1,30 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.5.2.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "bigqueryconnector", "Migration\bigqueryconnector.csproj", "{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSAFeedXML", "AzureFunction\OSAFeedXML.csproj", "{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Release|Any CPU.Build.0 = Release|Any CPU
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {726D97D5-9D11-447F-BEB7-33A706B3BACF}
EndGlobalSection
EndGlobal

268
Migration/.gitignore vendored Normal file
View File

@ -0,0 +1,268 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# Azure Functions localsettings file
local.settings.json
osadkowski-hd-2f3bd9b0ab5d.json
appsettings.Development.json
appsettings.Production.json
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
.DS_Store
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
project.fragment.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
#*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush
.cr/
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc

24
Migration/Dockerfile Normal file
View File

@ -0,0 +1,24 @@
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
WORKDIR /app
EXPOSE 5020
ENV ASPNETCORE_URLS=http://+:5020
USER app
FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build
ARG configuration=Release
WORKDIR /src
COPY ["Migration/bigqueryconnector.csproj", "Migration/"]
RUN dotnet restore "Migration/bigqueryconnector.csproj"
COPY . .
WORKDIR "/src/Migration"
RUN dotnet build "bigqueryconnector.csproj" -c $configuration -o /app/build
FROM build AS publish
ARG configuration=Release
RUN dotnet publish "bigqueryconnector.csproj" -c $configuration -o /app/publish /p:UseAppHost=false
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
ENTRYPOINT ["dotnet", "bigqueryconnector.dll"]

View File

@ -0,0 +1,412 @@
using System.Text.RegularExpressions;
using Google.Apis.Auth.OAuth2;
using Google.Cloud.BigQuery.V2;
using Microsoft.AspNetCore.Mvc;
using Migration.Services;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json;
using System.Text;
public interface IBigQuery
{
Task<IActionResult> GetProducts();
}
public class GoogleBigQuery : IBigQuery
{
private readonly GoogleCredential googleCredential;
private readonly BigQueryClient bigqueryClient;
private readonly string GoogleProjectId, GoogleDatabase;
private readonly IHttpClientFactory httpClientFactory;
public GoogleBigQuery(IConfiguration configuration, IHttpClientFactory httpClientFactory)
{
GoogleProjectId = configuration.GetValue<string>("BigQuery:ProjectId") ?? throw new Exception("Nie podano parametru identyfikatora projektu Google.");
GoogleDatabase = configuration.GetValue<string>("BigQuery:DbName") ?? throw new Exception("Nie podano parametru nazwy bazy danych BigQuery w konfiguracji");
googleCredential = GoogleCredential.FromFile(configuration.GetValue<string>("GoogleCredentialFile"));
bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential);
this.httpClientFactory = httpClientFactory;
}
public async Task<IActionResult> GetProducts()
{
try
{
int from = 1, limit = 100;
string me = string.Empty;
var httpClient = httpClientFactory.CreateClient(DataverseProvider.HTTP_CLIENT);
do
{
var jsonBody = new JObject
{
{"entity_name", "bon_commodity_index"},
{"attributes", new JArray {
"bon_main_product_id",
"bon_index",
"bon_gid",
"bon_ean",
"bon_product_name",
"bon_slug_url",
"bon_warehouse_state_central",
"bon_vat_rate",
"bon_converter",
"bon_calculaltion_unit",
"bon_marketplace",
"bon_price_for"
}},
{"makeFilter", true},
{"conditions", new JArray {
new JObject {
{"attribute_name", "bon_slug_url"},
{"condition_operator", "NotNull"}
},
new JObject {
{"attribute_name", "bon_converter"},
{"condition_operator", "NotNull"}
}
}},
{"links", new JArray {
new JObject {
{"primary_entity_name", "bon_commodity_index"},
{"related_entity_name", "bon_main_product"},
{"primary_key", "bon_main_product_id"},
{"foreign_key", "bon_main_productid"},
{"join_operator", "Inner"},
{"columns", new JArray {
"bon_ec_name",
"bon_seo_description"
}},
{"entity_alias", "bon_main_product"}
},
new JObject {
{"primary_entity_name", "bon_commodity_index"},
{"related_entity_name", "bon_purchase_index"},
{"primary_key", "bon_commodity_indexid"},
{"foreign_key", "bon_commodity_index_id"},
{"join_operator", "LeftOuter"},
{"columns", new JArray {
"bon_ecommerce_status"
}},
{"entity_alias", "bon_purchase_index"}
},
new JObject {
{"primary_entity_name", "bon_main_product"},
{"related_entity_name", "bon_producer_farmer"},
{"primary_key", "bon_producer_id"},
{"foreign_key", "bon_producer_farmerid"},
{"join_operator", "Inner"},
{"columns", new JArray {
"bon_name"
}},
{"entity_alias", "bon_producer_farmer"}
},
new JObject {
{"primary_entity_name", "bon_commodity_index"},
{"related_entity_name", "bon_public_document"},
{"primary_key", "bon_commodity_indexid"},
{"foreign_key", "bon_commodity_index_id"},
{"join_operator", "Inner"},
{"columns", new JArray {
"bon_url"
}},
{"entity_alias", "bon_public_document"},
{"conditions", new JArray {
new JObject {
{"attribute_name", "bon_order"},
{"attribute_value", "1"},
{"condition_operator", "Equal"}
},
new JObject {
{"attribute_name", "bon_file_type_id"},
{"attribute_value", "c62765b3-2d0c-ee11-8f6e-6045bd8c9e0f"},
{"condition_operator", "Equal"}
}
}}
},
new JObject {
{"primary_entity_name", "bon_commodity_index"},
{"related_entity_name", "bon_sales_price_list"},
{"primary_key", "bon_commodity_indexid"},
{"foreign_key", "bon_commodity_index_id"},
{"join_operator", "Inner"},
{"columns", new JArray {
"bon_cash_price"
}},
{"entity_alias", "bon_sales_price_list"},
{"conditions", new JArray {
new JObject {
{"attribute_name", "bon_publication_type"},
{"attribute_value", "2"},
{"condition_operator", "Equal"}
},
new JObject {
{"attribute_name", "statuscode"},
{"attribute_value", new JArray {"1", "180430002"}},
{"condition_operator", "In"}
},
new JObject {
{"attribute_name", "bon_pricelistid"},
{"attribute_value", "48e17eb1-9495-e811-a835-000d3ab48443"},
{"condition_operator", "Equal"}
}
}}
},
new JObject {
{"primary_entity_name", "bon_main_product"},
{"related_entity_name", "bon_categories"},
{"primary_key", "bon_category_id"},
{"foreign_key", "bon_categoriesid"},
{"join_operator", "LeftOuter"},
{"columns", new JArray {
"bon_categoriesid",
"osa_gmc_id"
}},
{"entity_alias", "bon_categories"}
},
new JObject{
{"primary_entity_name", "bon_main_product"},
{"related_entity_name", "bon_categories"},
{"primary_key", "bon_category_id"},
{"foreign_key", "bon_categoriesid"},
{"join_operator", "LeftOuter"},
{"columns", new JArray {
"bon_short_name"
}},
{"entity_alias", "bon_category_path"},
{"recurece", true}
}
}}
};
var response = await httpClient.PostAsync($"getdata?enviroment=prod&from={from}&limit={limit}", new StringContent(
JsonConvert.SerializeObject(jsonBody, Formatting.None),
Encoding.UTF8,
"application/json"
));
if ((int)response.StatusCode == 500 && JObject.Parse(await response.Content.ReadAsStringAsync())?["errorCode"]?.Value<int>() == 501)
break;
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync() ?? throw new Exception("Brak odpowiedzi z Dataverse");
var jObject = JsonConvert.DeserializeObject<JObject>(content) ?? throw new Exception("Nieprawidłowy format JSON");
var resultArray = jObject["result"] as JArray ?? throw new Exception("Brak pola 'result' w odpowiedzi");
var resultList = resultArray.ToObject<List<Dictionary<string, object>>>() ?? new List<Dictionary<string, object>>();
List<BigQueryInsertRow> insertRows = resultArray!.ToObject<List<JObject>>()!
.ConvertAll(row => ConvertToBigQueryInsertRow(row));
string upsertQuery = $@"
MERGE `{GoogleDatabase}.bon_main_product` AS T
USING (
SELECT DISTINCT * FROM UNNEST([
{{0}}
])
) AS S
ON T.bon_commodity_indexid = S.bon_commodity_indexid
WHEN MATCHED THEN
UPDATE SET
bon_cash_price = S.bon_cash_price,
bon_categoriesid = S.bon_categoriesid,
transactioncurrencyid = S.transactioncurrencyid,
bon_warehouse_state_central = S.bon_warehouse_state_central,
bon_ec_name = S.bon_ec_name,
bon_ean = S.bon_ean,
bon_gid = S.bon_gid,
bon_index = S.bon_index,
bon_name = S.bon_name,
bon_url = S.bon_url,
bon_category_path = S.bon_category_path,
bon_seo_description = S.bon_seo_description,
bon_main_product_id = S.bon_main_product_id,
bon_slug_url = S.bon_slug_url,
bon_converter = S.bon_converter,
bon_vat_rate = S.bon_vat_rate,
bon_product_name = S.bon_product_name,
osa_gmc_id = S.osa_gmc_id,
bon_calculaltion_unit = S.bon_calculaltion_unit,
bon_marketplace = S.bon_marketplace,
bon_price_for = S.bon_price_for,
bon_ecommerce_status = S.bon_ecommerce_status
WHEN NOT MATCHED THEN
INSERT (
bon_commodity_indexid,
bon_cash_price,
bon_categoriesid,
transactioncurrencyid,
bon_warehouse_state_central,
bon_ec_name,
bon_ean,
bon_gid,
bon_index,
bon_name,
bon_url,
bon_category_path,
bon_seo_description,
bon_main_product_id,
bon_slug_url,
bon_converter,
bon_vat_rate,
bon_product_name,
osa_gmc_id,
bon_calculaltion_unit,
bon_marketplace,
bon_price_for,
bon_ecommerce_status
)
VALUES (
S.bon_commodity_indexid,
S.bon_cash_price,
S.bon_categoriesid,
S.transactioncurrencyid,
S.bon_warehouse_state_central,
S.bon_ec_name,
S.bon_ean,
S.bon_gid,
S.bon_index,
S.bon_name,
S.bon_url,
S.bon_category_path,
S.bon_seo_description,
S.bon_main_product_id,
S.bon_slug_url,
S.bon_converter,
S.bon_vat_rate,
S.bon_product_name,
S.osa_gmc_id,
S.bon_calculaltion_unit,
S.bon_marketplace,
S.bon_price_for,
S.bon_ecommerce_status
)";
await PutDataBigQuery(upsertQuery, insertRows);
from++;
}
while (true);
return new OkObjectResult(new { message = me });
}
catch (Exception ex)
{
Console.WriteLine($"PutDataBigQuery_bon_commodity_index: {ex.Message}");
return new BadRequestObjectResult(new { message = ex.Message });
}
}
internal async Task PutDataBigQuery(string upsertQuery, List<BigQueryInsertRow> rows)
{
try
{
var table = bigqueryClient.GetTable(GoogleDatabase, "bon_main_product");
string CleanString(object value)
{
return value != null
? Regex.Replace(value.ToString() ?? "", @"\t|\n|\r", "")
: "";
}
var insertRows = rows.Select(row => new BigQueryInsertRow
{
{ "bon_commodity_indexid", row["bon_commodity_indexid"] },
{ "bon_cash_price", row["bon_cash_price"] },
{ "bon_categoriesid", row["bon_categoriesid"] },
{ "transactioncurrencyid", row["transactioncurrencyid"] },
{ "bon_warehouse_state_central", row["bon_warehouse_state_central"] },
{ "bon_ec_name", row["bon_ec_name"] },
{ "bon_ean", row["bon_ean"] },
{ "bon_gid", row["bon_gid"] },
{ "bon_index", row["bon_index"] },
{ "bon_name", row["bon_name"] },
{ "bon_url", row["bon_url"] },
{ "bon_category_path", row["bon_category_path"] },
{ "bon_seo_description", CleanString(row["bon_seo_description"]) },
{ "bon_main_product_id", row["bon_main_product_id"] },
{ "bon_slug_url", row["bon_slug_url"] },
{ "bon_converter", row["bon_converter"] },
{ "bon_vat_rate", row["bon_vat_rate"] },
{ "bon_product_name", row["bon_product_name"] },
{ "osa_gmc_id", row["osa_gmc_id"] },
{ "bon_calculaltion_unit", row["bon_calculaltion_unit"] },
{ "bon_marketplace", row["bon_marketplace"] },
{ "bon_price_for", row["bon_price_for"] },
{ "bon_ecommerce_status", row["bon_ecommerce_status"] }
}).ToList();
var mergeValues = insertRows.Select(row =>
$"STRUCT('{row["bon_commodity_indexid"]}' as bon_commodity_indexid, " +
$"cast({row["bon_cash_price"] ?? "NULL"} as numeric) as bon_cash_price, " +
$"'{row["bon_categoriesid"]}' as bon_categoriesid, " +
$"'{row["transactioncurrencyid"]}' as transactioncurrencyid, " +
$"cast({row["bon_warehouse_state_central"] ?? "NULL"} as numeric) as bon_warehouse_state_central, " +
$"'{row["bon_ec_name"]}' as bon_ec_name, " +
$"'{row["bon_ean"]}' as bon_ean, " +
$"{row["bon_gid"] ?? "NULL"} as bon_gid, " +
$"'{row["bon_index"]}' as bon_index, " +
$"'{row["bon_name"]}' as bon_name, " +
$"'{row["bon_url"]}' as bon_url, " +
$"'{row["bon_category_path"]}' as bon_category_path, " +
$"'{row["bon_seo_description"]}' as bon_seo_description, " +
$"'{row["bon_main_product_id"]}' as bon_main_product_id, " +
$"'{row["bon_slug_url"]}' as bon_slug_url, " +
$"cast({row["bon_converter"] ?? "NULL"} as numeric) as bon_converter, " +
$"{row["bon_vat_rate"] ?? "NULL"} as bon_vat_rate, " +
$"'{row["bon_product_name"]}' as bon_product_name, " +
$"{row["osa_gmc_id"] ?? "NULL"} as osa_gmc_id, " +
$"'{row["bon_calculaltion_unit"]}' as bon_calculaltion_unit, " +
$"ARRAY[{row["bon_marketplace"]}] as bon_marketplace, " +
$"{row["bon_price_for"] ?? "NULL"} as bon_price_for, " +
$"{row["bon_ecommerce_status"] ?? "NULL"} as bon_ecommerce_status)");
var queryOptions = new QueryOptions { UseQueryCache = false };
await bigqueryClient.ExecuteQueryAsync(string.Format(upsertQuery, string.Join(", ", mergeValues)), null, queryOptions);
}
catch (Exception ex)
{
Console.WriteLine($"PutDataBigQuery: {ex.Message}");
}
}
static BigQueryInsertRow ConvertToBigQueryInsertRow(JObject item)
{
BigQueryInsertRow insertRow = new BigQueryInsertRow();
insertRow.Add("bon_commodity_indexid", (string?)item["bon_commodity_indexid"]);
insertRow.Add("bon_cash_price", BigQueryNumeric.FromDecimal((decimal)item["bon_cash_price"]!, LossOfPrecisionHandling.Throw));
insertRow.Add("bon_categoriesid", (string?)item["bon_categoriesid"]);
insertRow.Add("transactioncurrencyid", (string?)item["transactioncurrencyid"]);
insertRow.Add("bon_warehouse_state_central", BigQueryNumeric.FromDecimal((decimal)item["bon_warehouse_state_central"]!, LossOfPrecisionHandling.Truncate));
insertRow.Add("bon_ec_name", (string?)item["bon_ec_name"]);
insertRow.Add("bon_ean", (string?)item["bon_ean"]);
insertRow.Add("bon_gid", (int?)item["bon_gid"]);
insertRow.Add("bon_index", (string?)item["bon_index"]);
insertRow.Add("bon_name", (string?)item["bon_name"]);
insertRow.Add("bon_url", (string?)item["bon_url"]);
insertRow.Add("bon_category_path", (string?)item["bon_category_path"]);
insertRow.Add("bon_seo_description", (string?)item["bon_seo_description"]);
insertRow.Add("bon_main_product_id", (string?)item["bon_main_product_id"]);
insertRow.Add("bon_slug_url", (string?)item["bon_slug_url"]);
insertRow.Add("bon_converter", BigQueryNumeric.FromDecimal((decimal)item["bon_converter"]!, LossOfPrecisionHandling.Truncate));
insertRow.Add("bon_vat_rate", (int?)item["bon_vat_rate"] ?? null);
insertRow.Add("bon_product_name", (string?)item["bon_product_name"]);
insertRow.Add("osa_gmc_id", (long?)item["osa_gmc_id"]);
insertRow.Add("bon_calculaltion_unit", (string?)item["bon_calculaltion_unit"]);
insertRow.Add("bon_marketplace",
item["bon_marketplace"] is JArray marketplaceArray
? string.Join(", ", marketplaceArray.Select(mp => mp["value"]?.ToString()))
: null);
insertRow.Add("bon_price_for",
item["bon_price_for"] is JObject priceForObject
? (int?)priceForObject["id"]
: null);
insertRow.Add("bon_ecommerce_status",
item["bon_ecommerce_status"] is JObject statusForObject
? (int?)statusForObject["id"]
: -1);
return insertRow;
}
}

30
Migration/Program.cs Normal file
View File

@ -0,0 +1,30 @@
using Migration.Services;
using Migration.Settings;
var builder = WebApplication.CreateBuilder(args);
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();
builder.Services.AddSingleton<IBigQuery, GoogleBigQuery>();
builder.Services.Configure<MainSettings>(builder.Configuration.GetSection(MainSettings.ConfigName));
builder.Services.AddSingleton<IDataverseProvider, DataverseProvider>();
builder.Services.AddHttpClient(DataverseProvider.HTTP_CLIENT, DataverseProvider.Setup);
var app = builder.Build();
if (app.Environment.IsDevelopment())
{
app.UseSwagger();
app.UseSwaggerUI();
}
app.UseHttpsRedirection();
app.MapPut("/GetProducts", async (IBigQuery _bigQuery) =>
{
await _bigQuery.GetProducts();
})
.WithName("GetProducts");
app.Run();

View File

@ -0,0 +1,41 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"iisSettings": {
"windowsAuthentication": false,
"anonymousAuthentication": true,
"iisExpress": {
"applicationUrl": "http://localhost:33574",
"sslPort": 44377
}
},
"profiles": {
"http": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "swagger",
"applicationUrl": "http://localhost:5020",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"https": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "swagger",
"applicationUrl": "https://localhost:7027;http://localhost:5020",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"IIS Express": {
"commandName": "IISExpress",
"launchBrowser": true,
"launchUrl": "swagger",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}

View File

@ -0,0 +1,10 @@
using Migration.Settings;
namespace Migration.Services
{
public interface IDataverseProvider
{
Uri Url { get; }
string Token { get; }
}
}

View File

@ -0,0 +1,47 @@
using System.Net.Http.Headers;
using System.Net.Mime;
using Migration.Settings;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
namespace Migration.Services;
public class DataverseProvider : IDataverseProvider
{
public const string HTTP_CLIENT = "FeedDataverseBQ";
private static readonly ProductInfoHeaderValue USER_AGENT = ProductInfoHeaderValue.Parse("FeedDataverseBQ");
private static readonly MediaTypeWithQualityHeaderValue JSON_MEDIA_TYPE = new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json);
public Uri Url { get; init; }
public string Token { get; init; }
public DataverseProvider(IOptions<MainSettings> mainSettingsOptions, IHttpClientFactory httpClientFactory)
{
var DataverseSettings = mainSettingsOptions.Value;
Url = new Uri(DataverseSettings.Url);
Token = DataverseSettings.Token;
}
public static void Setup(IServiceProvider serviceProvider, HttpClient httpClient)
{
var dataverseConnectionsProvider = serviceProvider.GetRequiredService<IDataverseProvider>();
httpClient.BaseAddress = dataverseConnectionsProvider.Url;
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", dataverseConnectionsProvider.Token);
httpClient.DefaultRequestHeaders.UserAgent.Add(USER_AGENT);
httpClient.DefaultRequestHeaders.Accept.Add(JSON_MEDIA_TYPE);
}
}
public class DataverseDelegatingHandler : DelegatingHandler
{
protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
{
return SendAsync(request, cancellationToken).GetAwaiter().GetResult();
}
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
return await base.SendAsync(request, cancellationToken);
}
}

View File

@ -0,0 +1,9 @@
namespace Migration.Settings
{
public class MainSettings
{
internal const string ConfigName = "Dataverse";
public string Url { get; set; } = null!;
public string Token { get; set; } = null!;
}
}

View File

@ -0,0 +1,18 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*",
"GoogleCredentialFile": "osadkowski-hd-2f3bd9b0ab5d.json",
"Dataverse": {
"Url": "",
"Token": ""
},
"BigQuery": {
"ProjectId": "",
"DbName": ""
}
}

View File

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.14" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="8.0.0" />
<PackageReference Include="Google.Cloud.BigQuery.V2" Version="3.11.0" />
</ItemGroup>
</Project>