commit 2ab789ac19b60a4301909e32666f5f4f488158fd Author: l.gabrysiak Date: Mon Mar 24 14:45:46 2025 +0100 init diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..f935f9d Binary files /dev/null and b/.DS_Store differ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3dbbcf3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..bb76300 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "ms-azuretools.vscode-azurefunctions", + "ms-dotnettools.csharp" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..f6ea872 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,45 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to .NET Functions", + "type": "coreclr", + "request": "attach", + "processId": "${command:azureFunctions.pickProcess}" + }, + { + "name": ".NET Core Launch (web)", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/Migration/bin/Debug/net8.0/bigqueryconnector.dll", + "args": [], + "cwd": "${workspaceFolder}/Migration", + "stopAtEntry": false, + "serverReadyAction": { + "action": "openExternally", + "pattern": "\\bNow listening on:\\s+(https?://\\S+)" + }, + "env": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "sourceFileMap": { + "/Views": "${workspaceFolder}/Views" + } + }, + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach" + }, + { + "name": "Docker .NET Launch", + "type": "docker", + "request": "launch", + "preLaunchTask": "docker-run: debug", + "netCore": { + "appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj" + } + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..d239fe3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "azureFunctions.deploySubpath": "AzureFunction/bin/Release/net8.0/publish", + "azureFunctions.projectLanguage": "C#", + "azureFunctions.projectRuntime": "~4", + "debug.internalConsoleOptions": "neverOpen", + "azureFunctions.preDeployTask": "publish (functions)" +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..db20153 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,177 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "clean (functions)", + "command": "dotnet", + "args": [ + "clean", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "type": "process", + "problemMatcher": "$msCompile", + "options": { + "cwd": "${workspaceFolder}/AzureFunction" + } + }, + { + "label": "build (functions)", + "command": "dotnet", + "args": [ + "build", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "type": "process", + "dependsOn": "clean (functions)", + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": "$msCompile", + "options": { + "cwd": "${workspaceFolder}/AzureFunction" + } + }, + { + "label": "clean release (functions)", + "command": "dotnet", + "args": [ + "clean", + "--configuration", + "Release", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "type": "process", + "problemMatcher": "$msCompile", + "options": { + "cwd": "${workspaceFolder}/AzureFunction" + } + }, + { + "label": "publish (functions)", + "command": "dotnet", + "args": [ + "publish", + "--configuration", + "Release", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "type": "process", + "dependsOn": "clean release (functions)", + "problemMatcher": "$msCompile", + "options": { + "cwd": "${workspaceFolder}/AzureFunction" + } + }, + { + "type": "func", + "dependsOn": "build (functions)", + "options": { + "cwd": "${workspaceFolder}/AzureFunction/bin/Debug/net8.0" + }, + "command": "host start", + "isBackground": true, + "problemMatcher": "$func-dotnet-watch" + }, + { + "label": "build", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/FeedDataverseBQ.sln", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary;ForceNoAlign" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "publish", + "command": "dotnet", + "type": "process", + "args": [ + "publish", + "${workspaceFolder}/FeedDataverseBQ.sln", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary;ForceNoAlign" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "watch", + "command": "dotnet", + "type": "process", + "args": [ + "watch", + "run", + "--project", + "${workspaceFolder}/FeedDataverseBQ.sln" + ], + "problemMatcher": "$msCompile" + }, + { + "type": "docker-build", + "label": "docker-build: debug", + "dependsOn": [ + "build" + ], + "dockerBuild": { + "tag": "feeddataversebq:dev", + "target": "base", + "dockerfile": "${workspaceFolder}/Migration/Dockerfile", + "context": "${workspaceFolder}", + "pull": true + }, + "netCore": { + "appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj" + } + }, + { + "type": "docker-build", + "label": "docker-build: release", + "dependsOn": [ + "build" + ], + "dockerBuild": { + "tag": "feeddataversebq:latest", + "dockerfile": "${workspaceFolder}/Migration/Dockerfile", + "context": "${workspaceFolder}", + "platform": { + "os": "linux", + "architecture": "amd64" + }, + "pull": true + }, + "netCore": { + "appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj" + } + }, + { + "type": "docker-run", + "label": "docker-run: debug", + "dependsOn": [ + "docker-build: debug" + ], + "dockerRun": {}, + "netCore": { + "appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj", + "enableDebugging": true + } + }, + { + "type": "docker-run", + "label": "docker-run: release", + "dependsOn": [ + "docker-build: release" + ], + "dockerRun": {}, + "netCore": { + "appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj" + } + } + ] +} \ No newline at end of file diff --git a/AzureFunction/.gitignore b/AzureFunction/.gitignore new file mode 100644 index 0000000..7d54999 --- /dev/null +++ b/AzureFunction/.gitignore @@ -0,0 +1,268 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json +osadkowski-hd-2f3bd9b0ab5d.json +appsettings.json + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +.DS_Store + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc \ No newline at end of file diff --git a/AzureFunction/Function1.cs b/AzureFunction/Function1.cs new file mode 100644 index 0000000..caf41d3 --- /dev/null +++ b/AzureFunction/Function1.cs @@ -0,0 +1,118 @@ +using System.Globalization; +using System.Net; +using System.Xml.Linq; +using Google.Cloud.BigQuery.V2; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; + +namespace OSAFeedXML +{ + public class Function1 + { + private readonly ILogger _logger; + private readonly IBigQuery _bq; + private readonly string DomainRedirect, ShortDescription, Title; + + public Function1(ILoggerFactory loggerFactory, IBigQuery bq) + { + _logger = loggerFactory.CreateLogger(); + DomainRedirect = Environment.GetEnvironmentVariable("DOMAIN_REDIRECT") ?? throw new Exception("Brak domeny przekierowania."); + ShortDescription = Environment.GetEnvironmentVariable("SHORT_DESCRIPTION") ?? throw new Exception("Brak krótkiego opisu."); + Title = Environment.GetEnvironmentVariable("TITLE") ?? throw new Exception("Brak tytułu."); + _bq = bq; + } + + [Function("getfeeds")] + public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get")] HttpRequestData req) + { + try + { + var queryParams = System.Web.HttpUtility.ParseQueryString(req.Url.Query); + var sourceParam = queryParams["source"]; + var destinationList = new List(); + Boolean.TryParse(queryParams["withoutbg"], out var withoutbg); + + if (sourceParam != null) + { + var sources = sourceParam.Split(','); + destinationList = sources + .Select(src => Enum.TryParse(typeof(Destinations), src, true, out var result) ? result : null) + .Where(v => v != null) + .Cast() + .ToList(); + } + else + { + destinationList = new List() { Destinations.google, Destinations.facebook }; + } + + var result = _bq.GetDataBigQuery(destinationList, withoutbg); + + var jsonArray = new JArray( + ((BigQueryResults)result).Select(row => new JObject( + row.Schema.Fields.Select(field => new JProperty(field.Name, JToken.FromObject(row[field.Name] ?? string.Empty))) + )) + ); + + JArray products = JArray.Parse(jsonArray.ToString()); + + XNamespace g = "http://base.google.com/ns/1.0"; + + XDocument xmlDoc = new XDocument( + new XElement("rss", new XAttribute("version", "2.0"), + new XAttribute(XNamespace.Xmlns + "g", g), + new XElement("channel", + new XElement("title", Title), + new XElement("link", DomainRedirect), + new XElement("description", ShortDescription), + from product in products + select new XElement("item", + new XElement(g + "id", product["id"]?.ToString()), + new XElement(g + "title", product["title"]?.ToString()), + new XElement(g + "description", product["description"]?.ToString()), + new XElement(g + "link", product["link"]?.ToString()), + new XElement(g + "image_link", product["image_link"]?.ToString()), + new XElement(g + "availability", product["availability"]?.ToString()), + new XElement(g + "price", FormatPrice(product["price"] ?? 0)), + new XElement(g + "brand", product["brand"]?.ToString()), + new XElement(g + "gtin", product["gtin"]?.ToString()), + new XElement(g + "mpn", product["mpn"]?.ToString()), + new XElement(g + "product_type", product["product_type"]?.ToString()), + new XElement(g + "google_product_category", product["google_product_category"]?.ToString()), + new XElement(g + "unit_pricing_measure", product["unit_pricing_measure"]?.ToString()), + new XElement(g + "condition", product["condition"]?.ToString()), + product["gtin"]?.ToString() == "" ? new XElement(g + "identifier_exists", "no") : new XElement(g + "identifier_exists", "yes") + ) + ) + ) + ); + + + var response = req.CreateResponse(HttpStatusCode.OK); + response.Headers.Add("Content-Type", "text/xml; charset=utf-8"); + + response.WriteString(xmlDoc.ToString()); + + return response; + } + catch (Exception ex) + { + _logger.LogError(ex, ex.Message); + return req.CreateResponse(HttpStatusCode.InternalServerError); + } + } + + static string FormatPrice(JToken priceToken) + { + if (priceToken == null || string.IsNullOrWhiteSpace(priceToken.ToString())) + { + return "0.00 PLN"; + } + + decimal price = priceToken.Value(); + return price.ToString("0.00", CultureInfo.InvariantCulture) + " PLN"; + } + } +} diff --git a/AzureFunction/GoogleStorageController.cs b/AzureFunction/GoogleStorageController.cs new file mode 100644 index 0000000..bda9f9f --- /dev/null +++ b/AzureFunction/GoogleStorageController.cs @@ -0,0 +1,99 @@ +using Microsoft.Extensions.Configuration; +using Google.Apis.Auth.OAuth2; +using Google.Cloud.BigQuery.V2; +using System.Linq.Expressions; + +namespace OSAFeedXML +{ + public interface IBigQuery + { + dynamic GetDataBigQuery(List destinations, bool withoutbg = false); + } + + public enum Destinations + { + google = 0, + facebook = 1 + } + + internal enum PublicationStatus + { + HIDDEN = 1, + OFFLINE_OFFER = 2, + UNAVAILABLE = 3, + TEMPORARILY_UNAVAILABLE = 4, + SALE_IN_EC = 5, + PRESALE_IN_EC = 6 + } + + internal enum PriceConversion + { + NOT_CONVERTION = 0, + CONVERTION = 1 + } + + internal class GoogleBigQuery : IBigQuery + { + private readonly BigQueryClient bigqueryClient; + private readonly string DomainRedirect; + private readonly string GoogleProjectId, GoogleDatabase; + + public GoogleBigQuery() + { + string googleCredentialsJson = Environment.GetEnvironmentVariable("GOOGLE_CREDENTIALS_JSON") ?? throw new Exception("Brak danych uwierzytelniających Google."); + + if (string.IsNullOrEmpty(googleCredentialsJson)) + { + throw new InvalidOperationException("Brak danych uwierzytelniających Google."); + } + + GoogleCredential googleCredential = GoogleCredential.FromJson(googleCredentialsJson); + + GoogleProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID") ?? throw new Exception("Brak identyfikatora projektu Google."); + GoogleDatabase = Environment.GetEnvironmentVariable("GOOGLE_DATABASE") ?? throw new Exception("Brak nazwy bazy danych Google."); + + bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential); + DomainRedirect = Environment.GetEnvironmentVariable("DOMAIN_REDIRECT") ?? throw new Exception("Brak domeny przekierowania."); + } + + public dynamic GetDataBigQuery(List Destinations, bool WithoutBg = false) + { + var status = new List() { PublicationStatus.SALE_IN_EC, PublicationStatus.PRESALE_IN_EC }; + string sql = @$" + SELECT + bon_gid AS id, + bon_ec_name AS title, + bon_seo_description AS description, + CONCAT('{DomainRedirect}/produkt/', bon_slug_url, '--s-', bon_gid) AS link, + COALESCE(ic.bon_url, bn.bon_url) AS image_link, + CASE + WHEN bon_warehouse_state_central IS NULL OR bon_warehouse_state_central = 0 THEN 'out_of_stock' + WHEN NOT EXISTS ( + SELECT 1 + FROM UNNEST(bon_marketplace) AS marketplace_value + WHERE marketplace_value IN UNNEST([{string.Join(", ", Destinations.Select(dest => (int)dest))}]) + ) + OR bon_ecommerce_status NOT IN ({string.Join(", ", status.Select(stat => (int)stat))}) THEN 'out_of_stock' + ELSE 'in_stock' + END AS availability, + ROUND( + bon_cash_price * + (CASE WHEN bon_price_for = {(int)PriceConversion.CONVERTION} THEN bon_converter ELSE 1 END) * + ((100 + COALESCE(bon_vat_rate, 0)) / 100), 2 + ) AS price, + bon_category_path AS product_type, + bon_name AS brand, + bon_ean AS gtin, + bon_index AS mpn, + osa_gmc_id AS google_product_category + FROM `{GoogleProjectId}.{GoogleDatabase}.bon_main_product` bn + LEFT JOIN `{GoogleProjectId}.{GoogleDatabase}.images_clearbg` ic + ON bn.bon_commodity_indexid = ic.bon_commodity_indexid + "; + + BigQueryResults results = bigqueryClient.ExecuteQuery(sql, null); + + return results; + } + } +} diff --git a/AzureFunction/OSAFeedXML.csproj b/AzureFunction/OSAFeedXML.csproj new file mode 100644 index 0000000..b9be35d --- /dev/null +++ b/AzureFunction/OSAFeedXML.csproj @@ -0,0 +1,26 @@ + + + net8.0 + v4 + Exe + enable + enable + + + + + + + + + + + + + PreserveNewest + + + + + + \ No newline at end of file diff --git a/AzureFunction/Program.cs b/AzureFunction/Program.cs new file mode 100644 index 0000000..c5fe62f --- /dev/null +++ b/AzureFunction/Program.cs @@ -0,0 +1,16 @@ +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using OSAFeedXML; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .ConfigureServices(services => + { + services.AddApplicationInsightsTelemetryWorkerService(); + services.ConfigureFunctionsApplicationInsights(); + services.AddSingleton(); + }) + .Build(); + +host.Run(); diff --git a/AzureFunction/Properties/launchSettings.json b/AzureFunction/Properties/launchSettings.json new file mode 100644 index 0000000..bbc8b4f --- /dev/null +++ b/AzureFunction/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "profiles": { + "OSAFeedXML": { + "commandName": "Project", + "commandLineArgs": "--port 7168", + "launchBrowser": false + } + } +} \ No newline at end of file diff --git a/AzureFunction/Properties/serviceDependencies.json b/AzureFunction/Properties/serviceDependencies.json new file mode 100644 index 0000000..df4dcc9 --- /dev/null +++ b/AzureFunction/Properties/serviceDependencies.json @@ -0,0 +1,11 @@ +{ + "dependencies": { + "appInsights1": { + "type": "appInsights" + }, + "storage1": { + "type": "storage", + "connectionId": "AzureWebJobsStorage" + } + } +} \ No newline at end of file diff --git a/AzureFunction/Properties/serviceDependencies.local.json b/AzureFunction/Properties/serviceDependencies.local.json new file mode 100644 index 0000000..b804a28 --- /dev/null +++ b/AzureFunction/Properties/serviceDependencies.local.json @@ -0,0 +1,11 @@ +{ + "dependencies": { + "appInsights1": { + "type": "appInsights.sdk" + }, + "storage1": { + "type": "storage.emulator", + "connectionId": "AzureWebJobsStorage" + } + } +} \ No newline at end of file diff --git a/AzureFunction/host.json b/AzureFunction/host.json new file mode 100644 index 0000000..ee5cf5f --- /dev/null +++ b/AzureFunction/host.json @@ -0,0 +1,12 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true + } + } +} \ No newline at end of file diff --git a/FeedDataverseBQ.sln b/FeedDataverseBQ.sln new file mode 100644 index 0000000..6a8144e --- /dev/null +++ b/FeedDataverseBQ.sln @@ -0,0 +1,30 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "bigqueryconnector", "Migration\bigqueryconnector.csproj", "{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSAFeedXML", "AzureFunction\OSAFeedXML.csproj", "{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Release|Any CPU.Build.0 = Release|Any CPU + {8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {726D97D5-9D11-447F-BEB7-33A706B3BACF} + EndGlobalSection +EndGlobal diff --git a/Migration/.gitignore b/Migration/.gitignore new file mode 100644 index 0000000..4c5a583 --- /dev/null +++ b/Migration/.gitignore @@ -0,0 +1,268 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json +osadkowski-hd-2f3bd9b0ab5d.json +appsettings.Development.json +appsettings.Production.json +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +.DS_Store + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc \ No newline at end of file diff --git a/Migration/Dockerfile b/Migration/Dockerfile new file mode 100644 index 0000000..f1acba3 --- /dev/null +++ b/Migration/Dockerfile @@ -0,0 +1,24 @@ +FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base +WORKDIR /app +EXPOSE 5020 + +ENV ASPNETCORE_URLS=http://+:5020 + +USER app +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build +ARG configuration=Release +WORKDIR /src +COPY ["Migration/bigqueryconnector.csproj", "Migration/"] +RUN dotnet restore "Migration/bigqueryconnector.csproj" +COPY . . +WORKDIR "/src/Migration" +RUN dotnet build "bigqueryconnector.csproj" -c $configuration -o /app/build + +FROM build AS publish +ARG configuration=Release +RUN dotnet publish "bigqueryconnector.csproj" -c $configuration -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "bigqueryconnector.dll"] diff --git a/Migration/GoogleStorageController.cs b/Migration/GoogleStorageController.cs new file mode 100644 index 0000000..d994bd8 --- /dev/null +++ b/Migration/GoogleStorageController.cs @@ -0,0 +1,412 @@ +using System.Text.RegularExpressions; +using Google.Apis.Auth.OAuth2; +using Google.Cloud.BigQuery.V2; +using Microsoft.AspNetCore.Mvc; +using Migration.Services; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json; +using System.Text; + + +public interface IBigQuery +{ + Task GetProducts(); +} + +public class GoogleBigQuery : IBigQuery +{ + private readonly GoogleCredential googleCredential; + private readonly BigQueryClient bigqueryClient; + private readonly string GoogleProjectId, GoogleDatabase; + private readonly IHttpClientFactory httpClientFactory; + + public GoogleBigQuery(IConfiguration configuration, IHttpClientFactory httpClientFactory) + { + GoogleProjectId = configuration.GetValue("BigQuery:ProjectId") ?? throw new Exception("Nie podano parametru identyfikatora projektu Google."); + GoogleDatabase = configuration.GetValue("BigQuery:DbName") ?? throw new Exception("Nie podano parametru nazwy bazy danych BigQuery w konfiguracji"); + + googleCredential = GoogleCredential.FromFile(configuration.GetValue("GoogleCredentialFile")); + bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential); + this.httpClientFactory = httpClientFactory; + } + + public async Task GetProducts() + { + try + { + int from = 1, limit = 100; + string me = string.Empty; + var httpClient = httpClientFactory.CreateClient(DataverseProvider.HTTP_CLIENT); + do + { + var jsonBody = new JObject + { + {"entity_name", "bon_commodity_index"}, + {"attributes", new JArray { + "bon_main_product_id", + "bon_index", + "bon_gid", + "bon_ean", + "bon_product_name", + "bon_slug_url", + "bon_warehouse_state_central", + "bon_vat_rate", + "bon_converter", + "bon_calculaltion_unit", + "bon_marketplace", + "bon_price_for" + }}, + {"makeFilter", true}, + {"conditions", new JArray { + new JObject { + {"attribute_name", "bon_slug_url"}, + {"condition_operator", "NotNull"} + }, + new JObject { + {"attribute_name", "bon_converter"}, + {"condition_operator", "NotNull"} + } + }}, + {"links", new JArray { + new JObject { + {"primary_entity_name", "bon_commodity_index"}, + {"related_entity_name", "bon_main_product"}, + {"primary_key", "bon_main_product_id"}, + {"foreign_key", "bon_main_productid"}, + {"join_operator", "Inner"}, + {"columns", new JArray { + "bon_ec_name", + "bon_seo_description" + }}, + {"entity_alias", "bon_main_product"} + }, + new JObject { + {"primary_entity_name", "bon_commodity_index"}, + {"related_entity_name", "bon_purchase_index"}, + {"primary_key", "bon_commodity_indexid"}, + {"foreign_key", "bon_commodity_index_id"}, + {"join_operator", "LeftOuter"}, + {"columns", new JArray { + "bon_ecommerce_status" + }}, + {"entity_alias", "bon_purchase_index"} + }, + new JObject { + {"primary_entity_name", "bon_main_product"}, + {"related_entity_name", "bon_producer_farmer"}, + {"primary_key", "bon_producer_id"}, + {"foreign_key", "bon_producer_farmerid"}, + {"join_operator", "Inner"}, + {"columns", new JArray { + "bon_name" + }}, + {"entity_alias", "bon_producer_farmer"} + }, + new JObject { + {"primary_entity_name", "bon_commodity_index"}, + {"related_entity_name", "bon_public_document"}, + {"primary_key", "bon_commodity_indexid"}, + {"foreign_key", "bon_commodity_index_id"}, + {"join_operator", "Inner"}, + {"columns", new JArray { + "bon_url" + }}, + {"entity_alias", "bon_public_document"}, + {"conditions", new JArray { + new JObject { + {"attribute_name", "bon_order"}, + {"attribute_value", "1"}, + {"condition_operator", "Equal"} + }, + new JObject { + {"attribute_name", "bon_file_type_id"}, + {"attribute_value", "c62765b3-2d0c-ee11-8f6e-6045bd8c9e0f"}, + {"condition_operator", "Equal"} + } + }} + }, + new JObject { + {"primary_entity_name", "bon_commodity_index"}, + {"related_entity_name", "bon_sales_price_list"}, + {"primary_key", "bon_commodity_indexid"}, + {"foreign_key", "bon_commodity_index_id"}, + {"join_operator", "Inner"}, + {"columns", new JArray { + "bon_cash_price" + }}, + {"entity_alias", "bon_sales_price_list"}, + {"conditions", new JArray { + new JObject { + {"attribute_name", "bon_publication_type"}, + {"attribute_value", "2"}, + {"condition_operator", "Equal"} + }, + new JObject { + {"attribute_name", "statuscode"}, + {"attribute_value", new JArray {"1", "180430002"}}, + {"condition_operator", "In"} + }, + new JObject { + {"attribute_name", "bon_pricelistid"}, + {"attribute_value", "48e17eb1-9495-e811-a835-000d3ab48443"}, + {"condition_operator", "Equal"} + } + }} + }, + new JObject { + {"primary_entity_name", "bon_main_product"}, + {"related_entity_name", "bon_categories"}, + {"primary_key", "bon_category_id"}, + {"foreign_key", "bon_categoriesid"}, + {"join_operator", "LeftOuter"}, + {"columns", new JArray { + "bon_categoriesid", + "osa_gmc_id" + }}, + {"entity_alias", "bon_categories"} + }, + new JObject{ + {"primary_entity_name", "bon_main_product"}, + {"related_entity_name", "bon_categories"}, + {"primary_key", "bon_category_id"}, + {"foreign_key", "bon_categoriesid"}, + {"join_operator", "LeftOuter"}, + {"columns", new JArray { + "bon_short_name" + }}, + {"entity_alias", "bon_category_path"}, + {"recurece", true} + } + }} + }; + + var response = await httpClient.PostAsync($"getdata?enviroment=prod&from={from}&limit={limit}", new StringContent( + JsonConvert.SerializeObject(jsonBody, Formatting.None), + Encoding.UTF8, + "application/json" + )); + + if ((int)response.StatusCode == 500 && JObject.Parse(await response.Content.ReadAsStringAsync())?["errorCode"]?.Value() == 501) + break; + + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync() ?? throw new Exception("Brak odpowiedzi z Dataverse"); + var jObject = JsonConvert.DeserializeObject(content) ?? throw new Exception("Nieprawidłowy format JSON"); + + var resultArray = jObject["result"] as JArray ?? throw new Exception("Brak pola 'result' w odpowiedzi"); + var resultList = resultArray.ToObject>>() ?? new List>(); + + List insertRows = resultArray!.ToObject>()! + .ConvertAll(row => ConvertToBigQueryInsertRow(row)); + + string upsertQuery = $@" + MERGE `{GoogleDatabase}.bon_main_product` AS T + USING ( + SELECT DISTINCT * FROM UNNEST([ + {{0}} + ]) + ) AS S + ON T.bon_commodity_indexid = S.bon_commodity_indexid + WHEN MATCHED THEN + UPDATE SET + bon_cash_price = S.bon_cash_price, + bon_categoriesid = S.bon_categoriesid, + transactioncurrencyid = S.transactioncurrencyid, + bon_warehouse_state_central = S.bon_warehouse_state_central, + bon_ec_name = S.bon_ec_name, + bon_ean = S.bon_ean, + bon_gid = S.bon_gid, + bon_index = S.bon_index, + bon_name = S.bon_name, + bon_url = S.bon_url, + bon_category_path = S.bon_category_path, + bon_seo_description = S.bon_seo_description, + bon_main_product_id = S.bon_main_product_id, + bon_slug_url = S.bon_slug_url, + bon_converter = S.bon_converter, + bon_vat_rate = S.bon_vat_rate, + bon_product_name = S.bon_product_name, + osa_gmc_id = S.osa_gmc_id, + bon_calculaltion_unit = S.bon_calculaltion_unit, + bon_marketplace = S.bon_marketplace, + bon_price_for = S.bon_price_for, + bon_ecommerce_status = S.bon_ecommerce_status + WHEN NOT MATCHED THEN + INSERT ( + bon_commodity_indexid, + bon_cash_price, + bon_categoriesid, + transactioncurrencyid, + bon_warehouse_state_central, + bon_ec_name, + bon_ean, + bon_gid, + bon_index, + bon_name, + bon_url, + bon_category_path, + bon_seo_description, + bon_main_product_id, + bon_slug_url, + bon_converter, + bon_vat_rate, + bon_product_name, + osa_gmc_id, + bon_calculaltion_unit, + bon_marketplace, + bon_price_for, + bon_ecommerce_status + ) + VALUES ( + S.bon_commodity_indexid, + S.bon_cash_price, + S.bon_categoriesid, + S.transactioncurrencyid, + S.bon_warehouse_state_central, + S.bon_ec_name, + S.bon_ean, + S.bon_gid, + S.bon_index, + S.bon_name, + S.bon_url, + S.bon_category_path, + S.bon_seo_description, + S.bon_main_product_id, + S.bon_slug_url, + S.bon_converter, + S.bon_vat_rate, + S.bon_product_name, + S.osa_gmc_id, + S.bon_calculaltion_unit, + S.bon_marketplace, + S.bon_price_for, + S.bon_ecommerce_status + )"; + + await PutDataBigQuery(upsertQuery, insertRows); + + from++; + } + while (true); + return new OkObjectResult(new { message = me }); + } + catch (Exception ex) + { + Console.WriteLine($"PutDataBigQuery_bon_commodity_index: {ex.Message}"); + return new BadRequestObjectResult(new { message = ex.Message }); + } + } + + internal async Task PutDataBigQuery(string upsertQuery, List rows) + { + try + { + var table = bigqueryClient.GetTable(GoogleDatabase, "bon_main_product"); + + string CleanString(object value) + { + return value != null + ? Regex.Replace(value.ToString() ?? "", @"\t|\n|\r", "") + : ""; + } + + var insertRows = rows.Select(row => new BigQueryInsertRow + { + { "bon_commodity_indexid", row["bon_commodity_indexid"] }, + { "bon_cash_price", row["bon_cash_price"] }, + { "bon_categoriesid", row["bon_categoriesid"] }, + { "transactioncurrencyid", row["transactioncurrencyid"] }, + { "bon_warehouse_state_central", row["bon_warehouse_state_central"] }, + { "bon_ec_name", row["bon_ec_name"] }, + { "bon_ean", row["bon_ean"] }, + { "bon_gid", row["bon_gid"] }, + { "bon_index", row["bon_index"] }, + { "bon_name", row["bon_name"] }, + { "bon_url", row["bon_url"] }, + { "bon_category_path", row["bon_category_path"] }, + { "bon_seo_description", CleanString(row["bon_seo_description"]) }, + { "bon_main_product_id", row["bon_main_product_id"] }, + { "bon_slug_url", row["bon_slug_url"] }, + { "bon_converter", row["bon_converter"] }, + { "bon_vat_rate", row["bon_vat_rate"] }, + { "bon_product_name", row["bon_product_name"] }, + { "osa_gmc_id", row["osa_gmc_id"] }, + { "bon_calculaltion_unit", row["bon_calculaltion_unit"] }, + { "bon_marketplace", row["bon_marketplace"] }, + { "bon_price_for", row["bon_price_for"] }, + { "bon_ecommerce_status", row["bon_ecommerce_status"] } + }).ToList(); + + var mergeValues = insertRows.Select(row => + $"STRUCT('{row["bon_commodity_indexid"]}' as bon_commodity_indexid, " + + $"cast({row["bon_cash_price"] ?? "NULL"} as numeric) as bon_cash_price, " + + $"'{row["bon_categoriesid"]}' as bon_categoriesid, " + + $"'{row["transactioncurrencyid"]}' as transactioncurrencyid, " + + $"cast({row["bon_warehouse_state_central"] ?? "NULL"} as numeric) as bon_warehouse_state_central, " + + $"'{row["bon_ec_name"]}' as bon_ec_name, " + + $"'{row["bon_ean"]}' as bon_ean, " + + $"{row["bon_gid"] ?? "NULL"} as bon_gid, " + + $"'{row["bon_index"]}' as bon_index, " + + $"'{row["bon_name"]}' as bon_name, " + + $"'{row["bon_url"]}' as bon_url, " + + $"'{row["bon_category_path"]}' as bon_category_path, " + + $"'{row["bon_seo_description"]}' as bon_seo_description, " + + $"'{row["bon_main_product_id"]}' as bon_main_product_id, " + + $"'{row["bon_slug_url"]}' as bon_slug_url, " + + $"cast({row["bon_converter"] ?? "NULL"} as numeric) as bon_converter, " + + $"{row["bon_vat_rate"] ?? "NULL"} as bon_vat_rate, " + + $"'{row["bon_product_name"]}' as bon_product_name, " + + $"{row["osa_gmc_id"] ?? "NULL"} as osa_gmc_id, " + + $"'{row["bon_calculaltion_unit"]}' as bon_calculaltion_unit, " + + $"ARRAY[{row["bon_marketplace"]}] as bon_marketplace, " + + $"{row["bon_price_for"] ?? "NULL"} as bon_price_for, " + + $"{row["bon_ecommerce_status"] ?? "NULL"} as bon_ecommerce_status)"); + + var queryOptions = new QueryOptions { UseQueryCache = false }; + await bigqueryClient.ExecuteQueryAsync(string.Format(upsertQuery, string.Join(", ", mergeValues)), null, queryOptions); + } + catch (Exception ex) + { + Console.WriteLine($"PutDataBigQuery: {ex.Message}"); + } + } + + static BigQueryInsertRow ConvertToBigQueryInsertRow(JObject item) + { + BigQueryInsertRow insertRow = new BigQueryInsertRow(); + insertRow.Add("bon_commodity_indexid", (string?)item["bon_commodity_indexid"]); + insertRow.Add("bon_cash_price", BigQueryNumeric.FromDecimal((decimal)item["bon_cash_price"]!, LossOfPrecisionHandling.Throw)); + insertRow.Add("bon_categoriesid", (string?)item["bon_categoriesid"]); + insertRow.Add("transactioncurrencyid", (string?)item["transactioncurrencyid"]); + insertRow.Add("bon_warehouse_state_central", BigQueryNumeric.FromDecimal((decimal)item["bon_warehouse_state_central"]!, LossOfPrecisionHandling.Truncate)); + insertRow.Add("bon_ec_name", (string?)item["bon_ec_name"]); + insertRow.Add("bon_ean", (string?)item["bon_ean"]); + insertRow.Add("bon_gid", (int?)item["bon_gid"]); + insertRow.Add("bon_index", (string?)item["bon_index"]); + insertRow.Add("bon_name", (string?)item["bon_name"]); + insertRow.Add("bon_url", (string?)item["bon_url"]); + insertRow.Add("bon_category_path", (string?)item["bon_category_path"]); + insertRow.Add("bon_seo_description", (string?)item["bon_seo_description"]); + insertRow.Add("bon_main_product_id", (string?)item["bon_main_product_id"]); + insertRow.Add("bon_slug_url", (string?)item["bon_slug_url"]); + insertRow.Add("bon_converter", BigQueryNumeric.FromDecimal((decimal)item["bon_converter"]!, LossOfPrecisionHandling.Truncate)); + insertRow.Add("bon_vat_rate", (int?)item["bon_vat_rate"] ?? null); + insertRow.Add("bon_product_name", (string?)item["bon_product_name"]); + insertRow.Add("osa_gmc_id", (long?)item["osa_gmc_id"]); + insertRow.Add("bon_calculaltion_unit", (string?)item["bon_calculaltion_unit"]); + insertRow.Add("bon_marketplace", + item["bon_marketplace"] is JArray marketplaceArray + ? string.Join(", ", marketplaceArray.Select(mp => mp["value"]?.ToString())) + : null); + insertRow.Add("bon_price_for", + item["bon_price_for"] is JObject priceForObject + ? (int?)priceForObject["id"] + : null); + insertRow.Add("bon_ecommerce_status", + item["bon_ecommerce_status"] is JObject statusForObject + ? (int?)statusForObject["id"] + : -1); + return insertRow; + } +} \ No newline at end of file diff --git a/Migration/Program.cs b/Migration/Program.cs new file mode 100644 index 0000000..56962e7 --- /dev/null +++ b/Migration/Program.cs @@ -0,0 +1,30 @@ +using Migration.Services; +using Migration.Settings; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(); +builder.Services.AddSingleton(); +builder.Services.Configure(builder.Configuration.GetSection(MainSettings.ConfigName)); + +builder.Services.AddSingleton(); +builder.Services.AddHttpClient(DataverseProvider.HTTP_CLIENT, DataverseProvider.Setup); + +var app = builder.Build(); + +if (app.Environment.IsDevelopment()) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} + +app.UseHttpsRedirection(); + +app.MapPut("/GetProducts", async (IBigQuery _bigQuery) => +{ + await _bigQuery.GetProducts(); +}) +.WithName("GetProducts"); + +app.Run(); \ No newline at end of file diff --git a/Migration/Properties/launchSettings.json b/Migration/Properties/launchSettings.json new file mode 100644 index 0000000..bcbfe29 --- /dev/null +++ b/Migration/Properties/launchSettings.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:33574", + "sslPort": 44377 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "http://localhost:5020", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7027;http://localhost:5020", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/Migration/Services/IDataverseProvider.cs b/Migration/Services/IDataverseProvider.cs new file mode 100644 index 0000000..a779f57 --- /dev/null +++ b/Migration/Services/IDataverseProvider.cs @@ -0,0 +1,10 @@ +using Migration.Settings; + +namespace Migration.Services +{ + public interface IDataverseProvider + { + Uri Url { get; } + string Token { get; } + } +} \ No newline at end of file diff --git a/Migration/Services/Implementation/DataverseProvider.cs b/Migration/Services/Implementation/DataverseProvider.cs new file mode 100644 index 0000000..a994ced --- /dev/null +++ b/Migration/Services/Implementation/DataverseProvider.cs @@ -0,0 +1,47 @@ +using System.Net.Http.Headers; +using System.Net.Mime; +using Migration.Settings; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +namespace Migration.Services; + +public class DataverseProvider : IDataverseProvider +{ + public const string HTTP_CLIENT = "FeedDataverseBQ"; + + private static readonly ProductInfoHeaderValue USER_AGENT = ProductInfoHeaderValue.Parse("FeedDataverseBQ"); + private static readonly MediaTypeWithQualityHeaderValue JSON_MEDIA_TYPE = new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json); + public Uri Url { get; init; } + public string Token { get; init; } + + public DataverseProvider(IOptions mainSettingsOptions, IHttpClientFactory httpClientFactory) + { + var DataverseSettings = mainSettingsOptions.Value; + Url = new Uri(DataverseSettings.Url); + Token = DataverseSettings.Token; + } + + public static void Setup(IServiceProvider serviceProvider, HttpClient httpClient) + { + var dataverseConnectionsProvider = serviceProvider.GetRequiredService(); + httpClient.BaseAddress = dataverseConnectionsProvider.Url; + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", dataverseConnectionsProvider.Token); + httpClient.DefaultRequestHeaders.UserAgent.Add(USER_AGENT); + httpClient.DefaultRequestHeaders.Accept.Add(JSON_MEDIA_TYPE); + } +} + +public class DataverseDelegatingHandler : DelegatingHandler +{ + protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken) + { + return SendAsync(request, cancellationToken).GetAwaiter().GetResult(); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await base.SendAsync(request, cancellationToken); + } +} \ No newline at end of file diff --git a/Migration/Settings/MainSettings.cs b/Migration/Settings/MainSettings.cs new file mode 100644 index 0000000..77e9899 --- /dev/null +++ b/Migration/Settings/MainSettings.cs @@ -0,0 +1,9 @@ +namespace Migration.Settings +{ + public class MainSettings + { + internal const string ConfigName = "Dataverse"; + public string Url { get; set; } = null!; + public string Token { get; set; } = null!; + } +} \ No newline at end of file diff --git a/Migration/appsettings.json b/Migration/appsettings.json new file mode 100644 index 0000000..9887d4e --- /dev/null +++ b/Migration/appsettings.json @@ -0,0 +1,18 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "GoogleCredentialFile": "osadkowski-hd-2f3bd9b0ab5d.json", + "Dataverse": { + "Url": "", + "Token": "" + }, + "BigQuery": { + "ProjectId": "", + "DbName": "" + } +} diff --git a/Migration/bigqueryconnector.csproj b/Migration/bigqueryconnector.csproj new file mode 100644 index 0000000..8bbeb89 --- /dev/null +++ b/Migration/bigqueryconnector.csproj @@ -0,0 +1,16 @@ + + + + net8.0 + enable + enable + true + + + + + + + + +