init
This commit is contained in:
l.gabrysiak
commit
2ab789ac19
|
|
@ -0,0 +1,25 @@
|
||||||
|
**/.classpath
|
||||||
|
**/.dockerignore
|
||||||
|
**/.env
|
||||||
|
**/.git
|
||||||
|
**/.gitignore
|
||||||
|
**/.project
|
||||||
|
**/.settings
|
||||||
|
**/.toolstarget
|
||||||
|
**/.vs
|
||||||
|
**/.vscode
|
||||||
|
**/*.*proj.user
|
||||||
|
**/*.dbmdl
|
||||||
|
**/*.jfm
|
||||||
|
**/bin
|
||||||
|
**/charts
|
||||||
|
**/docker-compose*
|
||||||
|
**/compose*
|
||||||
|
**/Dockerfile*
|
||||||
|
**/node_modules
|
||||||
|
**/npm-debug.log
|
||||||
|
**/obj
|
||||||
|
**/secrets.dev.yaml
|
||||||
|
**/values.dev.yaml
|
||||||
|
LICENSE
|
||||||
|
README.md
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
{
|
||||||
|
"recommendations": [
|
||||||
|
"ms-azuretools.vscode-azurefunctions",
|
||||||
|
"ms-dotnettools.csharp"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
{
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Attach to .NET Functions",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "attach",
|
||||||
|
"processId": "${command:azureFunctions.pickProcess}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": ".NET Core Launch (web)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"preLaunchTask": "build",
|
||||||
|
"program": "${workspaceFolder}/Migration/bin/Debug/net8.0/bigqueryconnector.dll",
|
||||||
|
"args": [],
|
||||||
|
"cwd": "${workspaceFolder}/Migration",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"serverReadyAction": {
|
||||||
|
"action": "openExternally",
|
||||||
|
"pattern": "\\bNow listening on:\\s+(https?://\\S+)"
|
||||||
|
},
|
||||||
|
"env": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||||
|
},
|
||||||
|
"sourceFileMap": {
|
||||||
|
"/Views": "${workspaceFolder}/Views"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": ".NET Core Attach",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "attach"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Docker .NET Launch",
|
||||||
|
"type": "docker",
|
||||||
|
"request": "launch",
|
||||||
|
"preLaunchTask": "docker-run: debug",
|
||||||
|
"netCore": {
|
||||||
|
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
{
|
||||||
|
"azureFunctions.deploySubpath": "AzureFunction/bin/Release/net8.0/publish",
|
||||||
|
"azureFunctions.projectLanguage": "C#",
|
||||||
|
"azureFunctions.projectRuntime": "~4",
|
||||||
|
"debug.internalConsoleOptions": "neverOpen",
|
||||||
|
"azureFunctions.preDeployTask": "publish (functions)"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,177 @@
|
||||||
|
{
|
||||||
|
"version": "2.0.0",
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"label": "clean (functions)",
|
||||||
|
"command": "dotnet",
|
||||||
|
"args": [
|
||||||
|
"clean",
|
||||||
|
"/property:GenerateFullPaths=true",
|
||||||
|
"/consoleloggerparameters:NoSummary"
|
||||||
|
],
|
||||||
|
"type": "process",
|
||||||
|
"problemMatcher": "$msCompile",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/AzureFunction"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "build (functions)",
|
||||||
|
"command": "dotnet",
|
||||||
|
"args": [
|
||||||
|
"build",
|
||||||
|
"/property:GenerateFullPaths=true",
|
||||||
|
"/consoleloggerparameters:NoSummary"
|
||||||
|
],
|
||||||
|
"type": "process",
|
||||||
|
"dependsOn": "clean (functions)",
|
||||||
|
"group": {
|
||||||
|
"kind": "build",
|
||||||
|
"isDefault": true
|
||||||
|
},
|
||||||
|
"problemMatcher": "$msCompile",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/AzureFunction"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "clean release (functions)",
|
||||||
|
"command": "dotnet",
|
||||||
|
"args": [
|
||||||
|
"clean",
|
||||||
|
"--configuration",
|
||||||
|
"Release",
|
||||||
|
"/property:GenerateFullPaths=true",
|
||||||
|
"/consoleloggerparameters:NoSummary"
|
||||||
|
],
|
||||||
|
"type": "process",
|
||||||
|
"problemMatcher": "$msCompile",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/AzureFunction"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "publish (functions)",
|
||||||
|
"command": "dotnet",
|
||||||
|
"args": [
|
||||||
|
"publish",
|
||||||
|
"--configuration",
|
||||||
|
"Release",
|
||||||
|
"/property:GenerateFullPaths=true",
|
||||||
|
"/consoleloggerparameters:NoSummary"
|
||||||
|
],
|
||||||
|
"type": "process",
|
||||||
|
"dependsOn": "clean release (functions)",
|
||||||
|
"problemMatcher": "$msCompile",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/AzureFunction"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "func",
|
||||||
|
"dependsOn": "build (functions)",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/AzureFunction/bin/Debug/net8.0"
|
||||||
|
},
|
||||||
|
"command": "host start",
|
||||||
|
"isBackground": true,
|
||||||
|
"problemMatcher": "$func-dotnet-watch"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "build",
|
||||||
|
"command": "dotnet",
|
||||||
|
"type": "process",
|
||||||
|
"args": [
|
||||||
|
"build",
|
||||||
|
"${workspaceFolder}/FeedDataverseBQ.sln",
|
||||||
|
"/property:GenerateFullPaths=true",
|
||||||
|
"/consoleloggerparameters:NoSummary;ForceNoAlign"
|
||||||
|
],
|
||||||
|
"problemMatcher": "$msCompile"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "publish",
|
||||||
|
"command": "dotnet",
|
||||||
|
"type": "process",
|
||||||
|
"args": [
|
||||||
|
"publish",
|
||||||
|
"${workspaceFolder}/FeedDataverseBQ.sln",
|
||||||
|
"/property:GenerateFullPaths=true",
|
||||||
|
"/consoleloggerparameters:NoSummary;ForceNoAlign"
|
||||||
|
],
|
||||||
|
"problemMatcher": "$msCompile"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "watch",
|
||||||
|
"command": "dotnet",
|
||||||
|
"type": "process",
|
||||||
|
"args": [
|
||||||
|
"watch",
|
||||||
|
"run",
|
||||||
|
"--project",
|
||||||
|
"${workspaceFolder}/FeedDataverseBQ.sln"
|
||||||
|
],
|
||||||
|
"problemMatcher": "$msCompile"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "docker-build",
|
||||||
|
"label": "docker-build: debug",
|
||||||
|
"dependsOn": [
|
||||||
|
"build"
|
||||||
|
],
|
||||||
|
"dockerBuild": {
|
||||||
|
"tag": "feeddataversebq:dev",
|
||||||
|
"target": "base",
|
||||||
|
"dockerfile": "${workspaceFolder}/Migration/Dockerfile",
|
||||||
|
"context": "${workspaceFolder}",
|
||||||
|
"pull": true
|
||||||
|
},
|
||||||
|
"netCore": {
|
||||||
|
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "docker-build",
|
||||||
|
"label": "docker-build: release",
|
||||||
|
"dependsOn": [
|
||||||
|
"build"
|
||||||
|
],
|
||||||
|
"dockerBuild": {
|
||||||
|
"tag": "feeddataversebq:latest",
|
||||||
|
"dockerfile": "${workspaceFolder}/Migration/Dockerfile",
|
||||||
|
"context": "${workspaceFolder}",
|
||||||
|
"platform": {
|
||||||
|
"os": "linux",
|
||||||
|
"architecture": "amd64"
|
||||||
|
},
|
||||||
|
"pull": true
|
||||||
|
},
|
||||||
|
"netCore": {
|
||||||
|
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "docker-run",
|
||||||
|
"label": "docker-run: debug",
|
||||||
|
"dependsOn": [
|
||||||
|
"docker-build: debug"
|
||||||
|
],
|
||||||
|
"dockerRun": {},
|
||||||
|
"netCore": {
|
||||||
|
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj",
|
||||||
|
"enableDebugging": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "docker-run",
|
||||||
|
"label": "docker-run: release",
|
||||||
|
"dependsOn": [
|
||||||
|
"docker-build: release"
|
||||||
|
],
|
||||||
|
"dockerRun": {},
|
||||||
|
"netCore": {
|
||||||
|
"appProject": "${workspaceFolder}/Migration/bigqueryconnector.csproj"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,268 @@
|
||||||
|
## Ignore Visual Studio temporary files, build results, and
|
||||||
|
## files generated by popular Visual Studio add-ons.
|
||||||
|
|
||||||
|
# Azure Functions localsettings file
|
||||||
|
local.settings.json
|
||||||
|
osadkowski-hd-2f3bd9b0ab5d.json
|
||||||
|
appsettings.json
|
||||||
|
|
||||||
|
# User-specific files
|
||||||
|
*.suo
|
||||||
|
*.user
|
||||||
|
*.userosscache
|
||||||
|
*.sln.docstates
|
||||||
|
|
||||||
|
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||||
|
*.userprefs
|
||||||
|
|
||||||
|
# Build results
|
||||||
|
[Dd]ebug/
|
||||||
|
[Dd]ebugPublic/
|
||||||
|
[Rr]elease/
|
||||||
|
[Rr]eleases/
|
||||||
|
x64/
|
||||||
|
x86/
|
||||||
|
bld/
|
||||||
|
[Bb]in/
|
||||||
|
[Oo]bj/
|
||||||
|
[Ll]og/
|
||||||
|
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# Visual Studio 2015 cache/options directory
|
||||||
|
.vs/
|
||||||
|
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||||
|
#wwwroot/
|
||||||
|
|
||||||
|
# MSTest test Results
|
||||||
|
[Tt]est[Rr]esult*/
|
||||||
|
[Bb]uild[Ll]og.*
|
||||||
|
|
||||||
|
# NUNIT
|
||||||
|
*.VisualState.xml
|
||||||
|
TestResult.xml
|
||||||
|
|
||||||
|
# Build Results of an ATL Project
|
||||||
|
[Dd]ebugPS/
|
||||||
|
[Rr]eleasePS/
|
||||||
|
dlldata.c
|
||||||
|
|
||||||
|
# DNX
|
||||||
|
project.lock.json
|
||||||
|
project.fragment.lock.json
|
||||||
|
artifacts/
|
||||||
|
|
||||||
|
*_i.c
|
||||||
|
*_p.c
|
||||||
|
*_i.h
|
||||||
|
*.ilk
|
||||||
|
*.meta
|
||||||
|
*.obj
|
||||||
|
*.pch
|
||||||
|
*.pdb
|
||||||
|
*.pgc
|
||||||
|
*.pgd
|
||||||
|
*.rsp
|
||||||
|
*.sbr
|
||||||
|
*.tlb
|
||||||
|
*.tli
|
||||||
|
*.tlh
|
||||||
|
*.tmp
|
||||||
|
*.tmp_proj
|
||||||
|
*.log
|
||||||
|
*.vspscc
|
||||||
|
*.vssscc
|
||||||
|
.builds
|
||||||
|
*.pidb
|
||||||
|
*.svclog
|
||||||
|
*.scc
|
||||||
|
|
||||||
|
# Chutzpah Test files
|
||||||
|
_Chutzpah*
|
||||||
|
|
||||||
|
# Visual C++ cache files
|
||||||
|
ipch/
|
||||||
|
*.aps
|
||||||
|
*.ncb
|
||||||
|
*.opendb
|
||||||
|
*.opensdf
|
||||||
|
*.sdf
|
||||||
|
*.cachefile
|
||||||
|
*.VC.db
|
||||||
|
*.VC.VC.opendb
|
||||||
|
|
||||||
|
# Visual Studio profiler
|
||||||
|
*.psess
|
||||||
|
*.vsp
|
||||||
|
*.vspx
|
||||||
|
*.sap
|
||||||
|
|
||||||
|
# TFS 2012 Local Workspace
|
||||||
|
$tf/
|
||||||
|
|
||||||
|
# Guidance Automation Toolkit
|
||||||
|
*.gpState
|
||||||
|
|
||||||
|
# ReSharper is a .NET coding add-in
|
||||||
|
_ReSharper*/
|
||||||
|
*.[Rr]e[Ss]harper
|
||||||
|
*.DotSettings.user
|
||||||
|
|
||||||
|
# JustCode is a .NET coding add-in
|
||||||
|
.JustCode
|
||||||
|
|
||||||
|
# TeamCity is a build add-in
|
||||||
|
_TeamCity*
|
||||||
|
|
||||||
|
# DotCover is a Code Coverage Tool
|
||||||
|
*.dotCover
|
||||||
|
|
||||||
|
# NCrunch
|
||||||
|
_NCrunch_*
|
||||||
|
.*crunch*.local.xml
|
||||||
|
nCrunchTemp_*
|
||||||
|
|
||||||
|
# MightyMoose
|
||||||
|
*.mm.*
|
||||||
|
AutoTest.Net/
|
||||||
|
|
||||||
|
# Web workbench (sass)
|
||||||
|
.sass-cache/
|
||||||
|
|
||||||
|
# Installshield output folder
|
||||||
|
[Ee]xpress/
|
||||||
|
|
||||||
|
# DocProject is a documentation generator add-in
|
||||||
|
DocProject/buildhelp/
|
||||||
|
DocProject/Help/*.HxT
|
||||||
|
DocProject/Help/*.HxC
|
||||||
|
DocProject/Help/*.hhc
|
||||||
|
DocProject/Help/*.hhk
|
||||||
|
DocProject/Help/*.hhp
|
||||||
|
DocProject/Help/Html2
|
||||||
|
DocProject/Help/html
|
||||||
|
|
||||||
|
# Click-Once directory
|
||||||
|
publish/
|
||||||
|
|
||||||
|
# Publish Web Output
|
||||||
|
*.[Pp]ublish.xml
|
||||||
|
*.azurePubxml
|
||||||
|
# TODO: Comment the next line if you want to checkin your web deploy settings
|
||||||
|
# but database connection strings (with potential passwords) will be unencrypted
|
||||||
|
#*.pubxml
|
||||||
|
*.publishproj
|
||||||
|
|
||||||
|
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||||
|
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||||
|
# in these scripts will be unencrypted
|
||||||
|
PublishScripts/
|
||||||
|
|
||||||
|
# NuGet Packages
|
||||||
|
*.nupkg
|
||||||
|
# The packages folder can be ignored because of Package Restore
|
||||||
|
**/packages/*
|
||||||
|
# except build/, which is used as an MSBuild target.
|
||||||
|
!**/packages/build/
|
||||||
|
# Uncomment if necessary however generally it will be regenerated when needed
|
||||||
|
#!**/packages/repositories.config
|
||||||
|
# NuGet v3's project.json files produces more ignoreable files
|
||||||
|
*.nuget.props
|
||||||
|
*.nuget.targets
|
||||||
|
|
||||||
|
# Microsoft Azure Build Output
|
||||||
|
csx/
|
||||||
|
*.build.csdef
|
||||||
|
|
||||||
|
# Microsoft Azure Emulator
|
||||||
|
ecf/
|
||||||
|
rcf/
|
||||||
|
|
||||||
|
# Windows Store app package directories and files
|
||||||
|
AppPackages/
|
||||||
|
BundleArtifacts/
|
||||||
|
Package.StoreAssociation.xml
|
||||||
|
_pkginfo.txt
|
||||||
|
|
||||||
|
# Visual Studio cache files
|
||||||
|
# files ending in .cache can be ignored
|
||||||
|
*.[Cc]ache
|
||||||
|
# but keep track of directories ending in .cache
|
||||||
|
!*.[Cc]ache/
|
||||||
|
|
||||||
|
# Others
|
||||||
|
ClientBin/
|
||||||
|
~$*
|
||||||
|
*~
|
||||||
|
*.dbmdl
|
||||||
|
*.dbproj.schemaview
|
||||||
|
*.jfm
|
||||||
|
*.pfx
|
||||||
|
*.publishsettings
|
||||||
|
node_modules/
|
||||||
|
orleans.codegen.cs
|
||||||
|
|
||||||
|
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||||
|
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||||
|
#bower_components/
|
||||||
|
|
||||||
|
# RIA/Silverlight projects
|
||||||
|
Generated_Code/
|
||||||
|
|
||||||
|
# Backup & report files from converting an old project file
|
||||||
|
# to a newer Visual Studio version. Backup files are not needed,
|
||||||
|
# because we have git ;-)
|
||||||
|
_UpgradeReport_Files/
|
||||||
|
Backup*/
|
||||||
|
UpgradeLog*.XML
|
||||||
|
UpgradeLog*.htm
|
||||||
|
|
||||||
|
# SQL Server files
|
||||||
|
*.mdf
|
||||||
|
*.ldf
|
||||||
|
|
||||||
|
# Business Intelligence projects
|
||||||
|
*.rdl.data
|
||||||
|
*.bim.layout
|
||||||
|
*.bim_*.settings
|
||||||
|
|
||||||
|
# Microsoft Fakes
|
||||||
|
FakesAssemblies/
|
||||||
|
|
||||||
|
# GhostDoc plugin setting file
|
||||||
|
*.GhostDoc.xml
|
||||||
|
|
||||||
|
# Node.js Tools for Visual Studio
|
||||||
|
.ntvs_analysis.dat
|
||||||
|
|
||||||
|
# Visual Studio 6 build log
|
||||||
|
*.plg
|
||||||
|
|
||||||
|
# Visual Studio 6 workspace options file
|
||||||
|
*.opt
|
||||||
|
|
||||||
|
# Visual Studio LightSwitch build output
|
||||||
|
**/*.HTMLClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/ModelManifest.xml
|
||||||
|
**/*.Server/GeneratedArtifacts
|
||||||
|
**/*.Server/ModelManifest.xml
|
||||||
|
_Pvt_Extensions
|
||||||
|
|
||||||
|
# Paket dependency manager
|
||||||
|
.paket/paket.exe
|
||||||
|
paket-files/
|
||||||
|
|
||||||
|
# FAKE - F# Make
|
||||||
|
.fake/
|
||||||
|
|
||||||
|
# JetBrains Rider
|
||||||
|
.idea/
|
||||||
|
*.sln.iml
|
||||||
|
|
||||||
|
# CodeRush
|
||||||
|
.cr/
|
||||||
|
|
||||||
|
# Python Tools for Visual Studio (PTVS)
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
@ -0,0 +1,118 @@
|
||||||
|
using System.Globalization;
|
||||||
|
using System.Net;
|
||||||
|
using System.Xml.Linq;
|
||||||
|
using Google.Cloud.BigQuery.V2;
|
||||||
|
using Microsoft.Azure.Functions.Worker;
|
||||||
|
using Microsoft.Azure.Functions.Worker.Http;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Newtonsoft.Json.Linq;
|
||||||
|
|
||||||
|
namespace OSAFeedXML
|
||||||
|
{
|
||||||
|
public class Function1
|
||||||
|
{
|
||||||
|
private readonly ILogger _logger;
|
||||||
|
private readonly IBigQuery _bq;
|
||||||
|
private readonly string DomainRedirect, ShortDescription, Title;
|
||||||
|
|
||||||
|
public Function1(ILoggerFactory loggerFactory, IBigQuery bq)
|
||||||
|
{
|
||||||
|
_logger = loggerFactory.CreateLogger<Function1>();
|
||||||
|
DomainRedirect = Environment.GetEnvironmentVariable("DOMAIN_REDIRECT") ?? throw new Exception("Brak domeny przekierowania.");
|
||||||
|
ShortDescription = Environment.GetEnvironmentVariable("SHORT_DESCRIPTION") ?? throw new Exception("Brak krótkiego opisu.");
|
||||||
|
Title = Environment.GetEnvironmentVariable("TITLE") ?? throw new Exception("Brak tytułu.");
|
||||||
|
_bq = bq;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Function("getfeeds")]
|
||||||
|
public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get")] HttpRequestData req)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var queryParams = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
|
||||||
|
var sourceParam = queryParams["source"];
|
||||||
|
var destinationList = new List<Destinations>();
|
||||||
|
Boolean.TryParse(queryParams["withoutbg"], out var withoutbg);
|
||||||
|
|
||||||
|
if (sourceParam != null)
|
||||||
|
{
|
||||||
|
var sources = sourceParam.Split(',');
|
||||||
|
destinationList = sources
|
||||||
|
.Select(src => Enum.TryParse(typeof(Destinations), src, true, out var result) ? result : null)
|
||||||
|
.Where(v => v != null)
|
||||||
|
.Cast<Destinations>()
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
destinationList = new List<Destinations>() { Destinations.google, Destinations.facebook };
|
||||||
|
}
|
||||||
|
|
||||||
|
var result = _bq.GetDataBigQuery(destinationList, withoutbg);
|
||||||
|
|
||||||
|
var jsonArray = new JArray(
|
||||||
|
((BigQueryResults)result).Select(row => new JObject(
|
||||||
|
row.Schema.Fields.Select(field => new JProperty(field.Name, JToken.FromObject(row[field.Name] ?? string.Empty)))
|
||||||
|
))
|
||||||
|
);
|
||||||
|
|
||||||
|
JArray products = JArray.Parse(jsonArray.ToString());
|
||||||
|
|
||||||
|
XNamespace g = "http://base.google.com/ns/1.0";
|
||||||
|
|
||||||
|
XDocument xmlDoc = new XDocument(
|
||||||
|
new XElement("rss", new XAttribute("version", "2.0"),
|
||||||
|
new XAttribute(XNamespace.Xmlns + "g", g),
|
||||||
|
new XElement("channel",
|
||||||
|
new XElement("title", Title),
|
||||||
|
new XElement("link", DomainRedirect),
|
||||||
|
new XElement("description", ShortDescription),
|
||||||
|
from product in products
|
||||||
|
select new XElement("item",
|
||||||
|
new XElement(g + "id", product["id"]?.ToString()),
|
||||||
|
new XElement(g + "title", product["title"]?.ToString()),
|
||||||
|
new XElement(g + "description", product["description"]?.ToString()),
|
||||||
|
new XElement(g + "link", product["link"]?.ToString()),
|
||||||
|
new XElement(g + "image_link", product["image_link"]?.ToString()),
|
||||||
|
new XElement(g + "availability", product["availability"]?.ToString()),
|
||||||
|
new XElement(g + "price", FormatPrice(product["price"] ?? 0)),
|
||||||
|
new XElement(g + "brand", product["brand"]?.ToString()),
|
||||||
|
new XElement(g + "gtin", product["gtin"]?.ToString()),
|
||||||
|
new XElement(g + "mpn", product["mpn"]?.ToString()),
|
||||||
|
new XElement(g + "product_type", product["product_type"]?.ToString()),
|
||||||
|
new XElement(g + "google_product_category", product["google_product_category"]?.ToString()),
|
||||||
|
new XElement(g + "unit_pricing_measure", product["unit_pricing_measure"]?.ToString()),
|
||||||
|
new XElement(g + "condition", product["condition"]?.ToString()),
|
||||||
|
product["gtin"]?.ToString() == "" ? new XElement(g + "identifier_exists", "no") : new XElement(g + "identifier_exists", "yes")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
|
||||||
|
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||||
|
response.Headers.Add("Content-Type", "text/xml; charset=utf-8");
|
||||||
|
|
||||||
|
response.WriteString(xmlDoc.ToString());
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, ex.Message);
|
||||||
|
return req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static string FormatPrice(JToken priceToken)
|
||||||
|
{
|
||||||
|
if (priceToken == null || string.IsNullOrWhiteSpace(priceToken.ToString()))
|
||||||
|
{
|
||||||
|
return "0.00 PLN";
|
||||||
|
}
|
||||||
|
|
||||||
|
decimal price = priceToken.Value<decimal>();
|
||||||
|
return price.ToString("0.00", CultureInfo.InvariantCulture) + " PLN";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,99 @@
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Google.Apis.Auth.OAuth2;
|
||||||
|
using Google.Cloud.BigQuery.V2;
|
||||||
|
using System.Linq.Expressions;
|
||||||
|
|
||||||
|
namespace OSAFeedXML
|
||||||
|
{
|
||||||
|
public interface IBigQuery
|
||||||
|
{
|
||||||
|
dynamic GetDataBigQuery(List<Destinations> destinations, bool withoutbg = false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum Destinations
|
||||||
|
{
|
||||||
|
google = 0,
|
||||||
|
facebook = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
internal enum PublicationStatus
|
||||||
|
{
|
||||||
|
HIDDEN = 1,
|
||||||
|
OFFLINE_OFFER = 2,
|
||||||
|
UNAVAILABLE = 3,
|
||||||
|
TEMPORARILY_UNAVAILABLE = 4,
|
||||||
|
SALE_IN_EC = 5,
|
||||||
|
PRESALE_IN_EC = 6
|
||||||
|
}
|
||||||
|
|
||||||
|
internal enum PriceConversion
|
||||||
|
{
|
||||||
|
NOT_CONVERTION = 0,
|
||||||
|
CONVERTION = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
internal class GoogleBigQuery : IBigQuery
|
||||||
|
{
|
||||||
|
private readonly BigQueryClient bigqueryClient;
|
||||||
|
private readonly string DomainRedirect;
|
||||||
|
private readonly string GoogleProjectId, GoogleDatabase;
|
||||||
|
|
||||||
|
public GoogleBigQuery()
|
||||||
|
{
|
||||||
|
string googleCredentialsJson = Environment.GetEnvironmentVariable("GOOGLE_CREDENTIALS_JSON") ?? throw new Exception("Brak danych uwierzytelniających Google.");
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(googleCredentialsJson))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Brak danych uwierzytelniających Google.");
|
||||||
|
}
|
||||||
|
|
||||||
|
GoogleCredential googleCredential = GoogleCredential.FromJson(googleCredentialsJson);
|
||||||
|
|
||||||
|
GoogleProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID") ?? throw new Exception("Brak identyfikatora projektu Google.");
|
||||||
|
GoogleDatabase = Environment.GetEnvironmentVariable("GOOGLE_DATABASE") ?? throw new Exception("Brak nazwy bazy danych Google.");
|
||||||
|
|
||||||
|
bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential);
|
||||||
|
DomainRedirect = Environment.GetEnvironmentVariable("DOMAIN_REDIRECT") ?? throw new Exception("Brak domeny przekierowania.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public dynamic GetDataBigQuery(List<Destinations> Destinations, bool WithoutBg = false)
|
||||||
|
{
|
||||||
|
var status = new List<PublicationStatus>() { PublicationStatus.SALE_IN_EC, PublicationStatus.PRESALE_IN_EC };
|
||||||
|
string sql = @$"
|
||||||
|
SELECT
|
||||||
|
bon_gid AS id,
|
||||||
|
bon_ec_name AS title,
|
||||||
|
bon_seo_description AS description,
|
||||||
|
CONCAT('{DomainRedirect}/produkt/', bon_slug_url, '--s-', bon_gid) AS link,
|
||||||
|
COALESCE(ic.bon_url, bn.bon_url) AS image_link,
|
||||||
|
CASE
|
||||||
|
WHEN bon_warehouse_state_central IS NULL OR bon_warehouse_state_central = 0 THEN 'out_of_stock'
|
||||||
|
WHEN NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM UNNEST(bon_marketplace) AS marketplace_value
|
||||||
|
WHERE marketplace_value IN UNNEST([{string.Join(", ", Destinations.Select(dest => (int)dest))}])
|
||||||
|
)
|
||||||
|
OR bon_ecommerce_status NOT IN ({string.Join(", ", status.Select(stat => (int)stat))}) THEN 'out_of_stock'
|
||||||
|
ELSE 'in_stock'
|
||||||
|
END AS availability,
|
||||||
|
ROUND(
|
||||||
|
bon_cash_price *
|
||||||
|
(CASE WHEN bon_price_for = {(int)PriceConversion.CONVERTION} THEN bon_converter ELSE 1 END) *
|
||||||
|
((100 + COALESCE(bon_vat_rate, 0)) / 100), 2
|
||||||
|
) AS price,
|
||||||
|
bon_category_path AS product_type,
|
||||||
|
bon_name AS brand,
|
||||||
|
bon_ean AS gtin,
|
||||||
|
bon_index AS mpn,
|
||||||
|
osa_gmc_id AS google_product_category
|
||||||
|
FROM `{GoogleProjectId}.{GoogleDatabase}.bon_main_product` bn
|
||||||
|
LEFT JOIN `{GoogleProjectId}.{GoogleDatabase}.images_clearbg` ic
|
||||||
|
ON bn.bon_commodity_indexid = ic.bon_commodity_indexid
|
||||||
|
";
|
||||||
|
|
||||||
|
BigQueryResults results = bigqueryClient.ExecuteQuery(sql, null);
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
|
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Google.Apis.Auth" Version="1.69.0" />
|
||||||
|
<PackageReference Include="Google.Cloud.BigQuery.V2" Version="3.11.0" />
|
||||||
|
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.3.0" />
|
||||||
|
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.1" />
|
||||||
|
<PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.23.0" />
|
||||||
|
<PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.0.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<None Update="host.json">
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</None>
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Using Include="System.Threading.ExecutionContext" Alias="ExecutionContext" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
using Microsoft.Azure.Functions.Worker;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using OSAFeedXML;
|
||||||
|
|
||||||
|
var host = new HostBuilder()
|
||||||
|
.ConfigureFunctionsWorkerDefaults()
|
||||||
|
.ConfigureServices(services =>
|
||||||
|
{
|
||||||
|
services.AddApplicationInsightsTelemetryWorkerService();
|
||||||
|
services.ConfigureFunctionsApplicationInsights();
|
||||||
|
services.AddSingleton<IBigQuery, GoogleBigQuery>();
|
||||||
|
})
|
||||||
|
.Build();
|
||||||
|
|
||||||
|
host.Run();
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"profiles": {
|
||||||
|
"OSAFeedXML": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"commandLineArgs": "--port 7168",
|
||||||
|
"launchBrowser": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
{
|
||||||
|
"dependencies": {
|
||||||
|
"appInsights1": {
|
||||||
|
"type": "appInsights"
|
||||||
|
},
|
||||||
|
"storage1": {
|
||||||
|
"type": "storage",
|
||||||
|
"connectionId": "AzureWebJobsStorage"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
{
|
||||||
|
"dependencies": {
|
||||||
|
"appInsights1": {
|
||||||
|
"type": "appInsights.sdk"
|
||||||
|
},
|
||||||
|
"storage1": {
|
||||||
|
"type": "storage.emulator",
|
||||||
|
"connectionId": "AzureWebJobsStorage"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"version": "2.0",
|
||||||
|
"logging": {
|
||||||
|
"applicationInsights": {
|
||||||
|
"samplingSettings": {
|
||||||
|
"isEnabled": true,
|
||||||
|
"excludedTypes": "Request"
|
||||||
|
},
|
||||||
|
"enableLiveMetricsFilters": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
|
# Visual Studio Version 17
|
||||||
|
VisualStudioVersion = 17.5.2.0
|
||||||
|
MinimumVisualStudioVersion = 10.0.40219.1
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "bigqueryconnector", "Migration\bigqueryconnector.csproj", "{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSAFeedXML", "AzureFunction\OSAFeedXML.csproj", "{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}"
|
||||||
|
EndProject
|
||||||
|
Global
|
||||||
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
Release|Any CPU = Release|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
|
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{BEEF5A15-8C35-8C55-4D84-2528FDF180FE}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{8A2CD958-D8F3-48A4-BD25-792C9EC1E48E}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
|
HideSolutionNode = FALSE
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||||
|
SolutionGuid = {726D97D5-9D11-447F-BEB7-33A706B3BACF}
|
||||||
|
EndGlobalSection
|
||||||
|
EndGlobal
|
||||||
|
|
@ -0,0 +1,268 @@
|
||||||
|
## Ignore Visual Studio temporary files, build results, and
|
||||||
|
## files generated by popular Visual Studio add-ons.
|
||||||
|
|
||||||
|
# Azure Functions localsettings file
|
||||||
|
local.settings.json
|
||||||
|
osadkowski-hd-2f3bd9b0ab5d.json
|
||||||
|
appsettings.Development.json
|
||||||
|
appsettings.Production.json
|
||||||
|
# User-specific files
|
||||||
|
*.suo
|
||||||
|
*.user
|
||||||
|
*.userosscache
|
||||||
|
*.sln.docstates
|
||||||
|
|
||||||
|
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||||
|
*.userprefs
|
||||||
|
|
||||||
|
# Build results
|
||||||
|
[Dd]ebug/
|
||||||
|
[Dd]ebugPublic/
|
||||||
|
[Rr]elease/
|
||||||
|
[Rr]eleases/
|
||||||
|
x64/
|
||||||
|
x86/
|
||||||
|
bld/
|
||||||
|
[Bb]in/
|
||||||
|
[Oo]bj/
|
||||||
|
[Ll]og/
|
||||||
|
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# Visual Studio 2015 cache/options directory
|
||||||
|
.vs/
|
||||||
|
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||||
|
#wwwroot/
|
||||||
|
|
||||||
|
# MSTest test Results
|
||||||
|
[Tt]est[Rr]esult*/
|
||||||
|
[Bb]uild[Ll]og.*
|
||||||
|
|
||||||
|
# NUNIT
|
||||||
|
*.VisualState.xml
|
||||||
|
TestResult.xml
|
||||||
|
|
||||||
|
# Build Results of an ATL Project
|
||||||
|
[Dd]ebugPS/
|
||||||
|
[Rr]eleasePS/
|
||||||
|
dlldata.c
|
||||||
|
|
||||||
|
# DNX
|
||||||
|
project.lock.json
|
||||||
|
project.fragment.lock.json
|
||||||
|
artifacts/
|
||||||
|
|
||||||
|
*_i.c
|
||||||
|
*_p.c
|
||||||
|
*_i.h
|
||||||
|
*.ilk
|
||||||
|
*.meta
|
||||||
|
*.obj
|
||||||
|
*.pch
|
||||||
|
*.pdb
|
||||||
|
*.pgc
|
||||||
|
*.pgd
|
||||||
|
*.rsp
|
||||||
|
*.sbr
|
||||||
|
*.tlb
|
||||||
|
*.tli
|
||||||
|
*.tlh
|
||||||
|
*.tmp
|
||||||
|
*.tmp_proj
|
||||||
|
*.log
|
||||||
|
*.vspscc
|
||||||
|
*.vssscc
|
||||||
|
.builds
|
||||||
|
*.pidb
|
||||||
|
*.svclog
|
||||||
|
*.scc
|
||||||
|
|
||||||
|
# Chutzpah Test files
|
||||||
|
_Chutzpah*
|
||||||
|
|
||||||
|
# Visual C++ cache files
|
||||||
|
ipch/
|
||||||
|
*.aps
|
||||||
|
*.ncb
|
||||||
|
*.opendb
|
||||||
|
*.opensdf
|
||||||
|
*.sdf
|
||||||
|
*.cachefile
|
||||||
|
*.VC.db
|
||||||
|
*.VC.VC.opendb
|
||||||
|
|
||||||
|
# Visual Studio profiler
|
||||||
|
*.psess
|
||||||
|
*.vsp
|
||||||
|
*.vspx
|
||||||
|
*.sap
|
||||||
|
|
||||||
|
# TFS 2012 Local Workspace
|
||||||
|
$tf/
|
||||||
|
|
||||||
|
# Guidance Automation Toolkit
|
||||||
|
*.gpState
|
||||||
|
|
||||||
|
# ReSharper is a .NET coding add-in
|
||||||
|
_ReSharper*/
|
||||||
|
*.[Rr]e[Ss]harper
|
||||||
|
*.DotSettings.user
|
||||||
|
|
||||||
|
# JustCode is a .NET coding add-in
|
||||||
|
.JustCode
|
||||||
|
|
||||||
|
# TeamCity is a build add-in
|
||||||
|
_TeamCity*
|
||||||
|
|
||||||
|
# DotCover is a Code Coverage Tool
|
||||||
|
*.dotCover
|
||||||
|
|
||||||
|
# NCrunch
|
||||||
|
_NCrunch_*
|
||||||
|
.*crunch*.local.xml
|
||||||
|
nCrunchTemp_*
|
||||||
|
|
||||||
|
# MightyMoose
|
||||||
|
*.mm.*
|
||||||
|
AutoTest.Net/
|
||||||
|
|
||||||
|
# Web workbench (sass)
|
||||||
|
.sass-cache/
|
||||||
|
|
||||||
|
# Installshield output folder
|
||||||
|
[Ee]xpress/
|
||||||
|
|
||||||
|
# DocProject is a documentation generator add-in
|
||||||
|
DocProject/buildhelp/
|
||||||
|
DocProject/Help/*.HxT
|
||||||
|
DocProject/Help/*.HxC
|
||||||
|
DocProject/Help/*.hhc
|
||||||
|
DocProject/Help/*.hhk
|
||||||
|
DocProject/Help/*.hhp
|
||||||
|
DocProject/Help/Html2
|
||||||
|
DocProject/Help/html
|
||||||
|
|
||||||
|
# Click-Once directory
|
||||||
|
publish/
|
||||||
|
|
||||||
|
# Publish Web Output
|
||||||
|
*.[Pp]ublish.xml
|
||||||
|
*.azurePubxml
|
||||||
|
# TODO: Comment the next line if you want to checkin your web deploy settings
|
||||||
|
# but database connection strings (with potential passwords) will be unencrypted
|
||||||
|
#*.pubxml
|
||||||
|
*.publishproj
|
||||||
|
|
||||||
|
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||||
|
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||||
|
# in these scripts will be unencrypted
|
||||||
|
PublishScripts/
|
||||||
|
|
||||||
|
# NuGet Packages
|
||||||
|
*.nupkg
|
||||||
|
# The packages folder can be ignored because of Package Restore
|
||||||
|
**/packages/*
|
||||||
|
# except build/, which is used as an MSBuild target.
|
||||||
|
!**/packages/build/
|
||||||
|
# Uncomment if necessary however generally it will be regenerated when needed
|
||||||
|
#!**/packages/repositories.config
|
||||||
|
# NuGet v3's project.json files produces more ignoreable files
|
||||||
|
*.nuget.props
|
||||||
|
*.nuget.targets
|
||||||
|
|
||||||
|
# Microsoft Azure Build Output
|
||||||
|
csx/
|
||||||
|
*.build.csdef
|
||||||
|
|
||||||
|
# Microsoft Azure Emulator
|
||||||
|
ecf/
|
||||||
|
rcf/
|
||||||
|
|
||||||
|
# Windows Store app package directories and files
|
||||||
|
AppPackages/
|
||||||
|
BundleArtifacts/
|
||||||
|
Package.StoreAssociation.xml
|
||||||
|
_pkginfo.txt
|
||||||
|
|
||||||
|
# Visual Studio cache files
|
||||||
|
# files ending in .cache can be ignored
|
||||||
|
*.[Cc]ache
|
||||||
|
# but keep track of directories ending in .cache
|
||||||
|
!*.[Cc]ache/
|
||||||
|
|
||||||
|
# Others
|
||||||
|
ClientBin/
|
||||||
|
~$*
|
||||||
|
*~
|
||||||
|
*.dbmdl
|
||||||
|
*.dbproj.schemaview
|
||||||
|
*.jfm
|
||||||
|
*.pfx
|
||||||
|
*.publishsettings
|
||||||
|
node_modules/
|
||||||
|
orleans.codegen.cs
|
||||||
|
|
||||||
|
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||||
|
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||||
|
#bower_components/
|
||||||
|
|
||||||
|
# RIA/Silverlight projects
|
||||||
|
Generated_Code/
|
||||||
|
|
||||||
|
# Backup & report files from converting an old project file
|
||||||
|
# to a newer Visual Studio version. Backup files are not needed,
|
||||||
|
# because we have git ;-)
|
||||||
|
_UpgradeReport_Files/
|
||||||
|
Backup*/
|
||||||
|
UpgradeLog*.XML
|
||||||
|
UpgradeLog*.htm
|
||||||
|
|
||||||
|
# SQL Server files
|
||||||
|
*.mdf
|
||||||
|
*.ldf
|
||||||
|
|
||||||
|
# Business Intelligence projects
|
||||||
|
*.rdl.data
|
||||||
|
*.bim.layout
|
||||||
|
*.bim_*.settings
|
||||||
|
|
||||||
|
# Microsoft Fakes
|
||||||
|
FakesAssemblies/
|
||||||
|
|
||||||
|
# GhostDoc plugin setting file
|
||||||
|
*.GhostDoc.xml
|
||||||
|
|
||||||
|
# Node.js Tools for Visual Studio
|
||||||
|
.ntvs_analysis.dat
|
||||||
|
|
||||||
|
# Visual Studio 6 build log
|
||||||
|
*.plg
|
||||||
|
|
||||||
|
# Visual Studio 6 workspace options file
|
||||||
|
*.opt
|
||||||
|
|
||||||
|
# Visual Studio LightSwitch build output
|
||||||
|
**/*.HTMLClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/ModelManifest.xml
|
||||||
|
**/*.Server/GeneratedArtifacts
|
||||||
|
**/*.Server/ModelManifest.xml
|
||||||
|
_Pvt_Extensions
|
||||||
|
|
||||||
|
# Paket dependency manager
|
||||||
|
.paket/paket.exe
|
||||||
|
paket-files/
|
||||||
|
|
||||||
|
# FAKE - F# Make
|
||||||
|
.fake/
|
||||||
|
|
||||||
|
# JetBrains Rider
|
||||||
|
.idea/
|
||||||
|
*.sln.iml
|
||||||
|
|
||||||
|
# CodeRush
|
||||||
|
.cr/
|
||||||
|
|
||||||
|
# Python Tools for Visual Studio (PTVS)
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
|
||||||
|
WORKDIR /app
|
||||||
|
EXPOSE 5020
|
||||||
|
|
||||||
|
ENV ASPNETCORE_URLS=http://+:5020
|
||||||
|
|
||||||
|
USER app
|
||||||
|
FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build
|
||||||
|
ARG configuration=Release
|
||||||
|
WORKDIR /src
|
||||||
|
COPY ["Migration/bigqueryconnector.csproj", "Migration/"]
|
||||||
|
RUN dotnet restore "Migration/bigqueryconnector.csproj"
|
||||||
|
COPY . .
|
||||||
|
WORKDIR "/src/Migration"
|
||||||
|
RUN dotnet build "bigqueryconnector.csproj" -c $configuration -o /app/build
|
||||||
|
|
||||||
|
FROM build AS publish
|
||||||
|
ARG configuration=Release
|
||||||
|
RUN dotnet publish "bigqueryconnector.csproj" -c $configuration -o /app/publish /p:UseAppHost=false
|
||||||
|
|
||||||
|
FROM base AS final
|
||||||
|
WORKDIR /app
|
||||||
|
COPY --from=publish /app/publish .
|
||||||
|
ENTRYPOINT ["dotnet", "bigqueryconnector.dll"]
|
||||||
|
|
@ -0,0 +1,412 @@
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using Google.Apis.Auth.OAuth2;
|
||||||
|
using Google.Cloud.BigQuery.V2;
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
using Migration.Services;
|
||||||
|
using Newtonsoft.Json.Linq;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
|
|
||||||
|
public interface IBigQuery
|
||||||
|
{
|
||||||
|
Task<IActionResult> GetProducts();
|
||||||
|
}
|
||||||
|
|
||||||
|
public class GoogleBigQuery : IBigQuery
|
||||||
|
{
|
||||||
|
private readonly GoogleCredential googleCredential;
|
||||||
|
private readonly BigQueryClient bigqueryClient;
|
||||||
|
private readonly string GoogleProjectId, GoogleDatabase;
|
||||||
|
private readonly IHttpClientFactory httpClientFactory;
|
||||||
|
|
||||||
|
public GoogleBigQuery(IConfiguration configuration, IHttpClientFactory httpClientFactory)
|
||||||
|
{
|
||||||
|
GoogleProjectId = configuration.GetValue<string>("BigQuery:ProjectId") ?? throw new Exception("Nie podano parametru identyfikatora projektu Google.");
|
||||||
|
GoogleDatabase = configuration.GetValue<string>("BigQuery:DbName") ?? throw new Exception("Nie podano parametru nazwy bazy danych BigQuery w konfiguracji");
|
||||||
|
|
||||||
|
googleCredential = GoogleCredential.FromFile(configuration.GetValue<string>("GoogleCredentialFile"));
|
||||||
|
bigqueryClient = BigQueryClient.Create(GoogleProjectId, googleCredential);
|
||||||
|
this.httpClientFactory = httpClientFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<IActionResult> GetProducts()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int from = 1, limit = 100;
|
||||||
|
string me = string.Empty;
|
||||||
|
var httpClient = httpClientFactory.CreateClient(DataverseProvider.HTTP_CLIENT);
|
||||||
|
do
|
||||||
|
{
|
||||||
|
var jsonBody = new JObject
|
||||||
|
{
|
||||||
|
{"entity_name", "bon_commodity_index"},
|
||||||
|
{"attributes", new JArray {
|
||||||
|
"bon_main_product_id",
|
||||||
|
"bon_index",
|
||||||
|
"bon_gid",
|
||||||
|
"bon_ean",
|
||||||
|
"bon_product_name",
|
||||||
|
"bon_slug_url",
|
||||||
|
"bon_warehouse_state_central",
|
||||||
|
"bon_vat_rate",
|
||||||
|
"bon_converter",
|
||||||
|
"bon_calculaltion_unit",
|
||||||
|
"bon_marketplace",
|
||||||
|
"bon_price_for"
|
||||||
|
}},
|
||||||
|
{"makeFilter", true},
|
||||||
|
{"conditions", new JArray {
|
||||||
|
new JObject {
|
||||||
|
{"attribute_name", "bon_slug_url"},
|
||||||
|
{"condition_operator", "NotNull"}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"attribute_name", "bon_converter"},
|
||||||
|
{"condition_operator", "NotNull"}
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{"links", new JArray {
|
||||||
|
new JObject {
|
||||||
|
{"primary_entity_name", "bon_commodity_index"},
|
||||||
|
{"related_entity_name", "bon_main_product"},
|
||||||
|
{"primary_key", "bon_main_product_id"},
|
||||||
|
{"foreign_key", "bon_main_productid"},
|
||||||
|
{"join_operator", "Inner"},
|
||||||
|
{"columns", new JArray {
|
||||||
|
"bon_ec_name",
|
||||||
|
"bon_seo_description"
|
||||||
|
}},
|
||||||
|
{"entity_alias", "bon_main_product"}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"primary_entity_name", "bon_commodity_index"},
|
||||||
|
{"related_entity_name", "bon_purchase_index"},
|
||||||
|
{"primary_key", "bon_commodity_indexid"},
|
||||||
|
{"foreign_key", "bon_commodity_index_id"},
|
||||||
|
{"join_operator", "LeftOuter"},
|
||||||
|
{"columns", new JArray {
|
||||||
|
"bon_ecommerce_status"
|
||||||
|
}},
|
||||||
|
{"entity_alias", "bon_purchase_index"}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"primary_entity_name", "bon_main_product"},
|
||||||
|
{"related_entity_name", "bon_producer_farmer"},
|
||||||
|
{"primary_key", "bon_producer_id"},
|
||||||
|
{"foreign_key", "bon_producer_farmerid"},
|
||||||
|
{"join_operator", "Inner"},
|
||||||
|
{"columns", new JArray {
|
||||||
|
"bon_name"
|
||||||
|
}},
|
||||||
|
{"entity_alias", "bon_producer_farmer"}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"primary_entity_name", "bon_commodity_index"},
|
||||||
|
{"related_entity_name", "bon_public_document"},
|
||||||
|
{"primary_key", "bon_commodity_indexid"},
|
||||||
|
{"foreign_key", "bon_commodity_index_id"},
|
||||||
|
{"join_operator", "Inner"},
|
||||||
|
{"columns", new JArray {
|
||||||
|
"bon_url"
|
||||||
|
}},
|
||||||
|
{"entity_alias", "bon_public_document"},
|
||||||
|
{"conditions", new JArray {
|
||||||
|
new JObject {
|
||||||
|
{"attribute_name", "bon_order"},
|
||||||
|
{"attribute_value", "1"},
|
||||||
|
{"condition_operator", "Equal"}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"attribute_name", "bon_file_type_id"},
|
||||||
|
{"attribute_value", "c62765b3-2d0c-ee11-8f6e-6045bd8c9e0f"},
|
||||||
|
{"condition_operator", "Equal"}
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"primary_entity_name", "bon_commodity_index"},
|
||||||
|
{"related_entity_name", "bon_sales_price_list"},
|
||||||
|
{"primary_key", "bon_commodity_indexid"},
|
||||||
|
{"foreign_key", "bon_commodity_index_id"},
|
||||||
|
{"join_operator", "Inner"},
|
||||||
|
{"columns", new JArray {
|
||||||
|
"bon_cash_price"
|
||||||
|
}},
|
||||||
|
{"entity_alias", "bon_sales_price_list"},
|
||||||
|
{"conditions", new JArray {
|
||||||
|
new JObject {
|
||||||
|
{"attribute_name", "bon_publication_type"},
|
||||||
|
{"attribute_value", "2"},
|
||||||
|
{"condition_operator", "Equal"}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"attribute_name", "statuscode"},
|
||||||
|
{"attribute_value", new JArray {"1", "180430002"}},
|
||||||
|
{"condition_operator", "In"}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"attribute_name", "bon_pricelistid"},
|
||||||
|
{"attribute_value", "48e17eb1-9495-e811-a835-000d3ab48443"},
|
||||||
|
{"condition_operator", "Equal"}
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
new JObject {
|
||||||
|
{"primary_entity_name", "bon_main_product"},
|
||||||
|
{"related_entity_name", "bon_categories"},
|
||||||
|
{"primary_key", "bon_category_id"},
|
||||||
|
{"foreign_key", "bon_categoriesid"},
|
||||||
|
{"join_operator", "LeftOuter"},
|
||||||
|
{"columns", new JArray {
|
||||||
|
"bon_categoriesid",
|
||||||
|
"osa_gmc_id"
|
||||||
|
}},
|
||||||
|
{"entity_alias", "bon_categories"}
|
||||||
|
},
|
||||||
|
new JObject{
|
||||||
|
{"primary_entity_name", "bon_main_product"},
|
||||||
|
{"related_entity_name", "bon_categories"},
|
||||||
|
{"primary_key", "bon_category_id"},
|
||||||
|
{"foreign_key", "bon_categoriesid"},
|
||||||
|
{"join_operator", "LeftOuter"},
|
||||||
|
{"columns", new JArray {
|
||||||
|
"bon_short_name"
|
||||||
|
}},
|
||||||
|
{"entity_alias", "bon_category_path"},
|
||||||
|
{"recurece", true}
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
};
|
||||||
|
|
||||||
|
var response = await httpClient.PostAsync($"getdata?enviroment=prod&from={from}&limit={limit}", new StringContent(
|
||||||
|
JsonConvert.SerializeObject(jsonBody, Formatting.None),
|
||||||
|
Encoding.UTF8,
|
||||||
|
"application/json"
|
||||||
|
));
|
||||||
|
|
||||||
|
if ((int)response.StatusCode == 500 && JObject.Parse(await response.Content.ReadAsStringAsync())?["errorCode"]?.Value<int>() == 501)
|
||||||
|
break;
|
||||||
|
|
||||||
|
response.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
|
var content = await response.Content.ReadAsStringAsync() ?? throw new Exception("Brak odpowiedzi z Dataverse");
|
||||||
|
var jObject = JsonConvert.DeserializeObject<JObject>(content) ?? throw new Exception("Nieprawidłowy format JSON");
|
||||||
|
|
||||||
|
var resultArray = jObject["result"] as JArray ?? throw new Exception("Brak pola 'result' w odpowiedzi");
|
||||||
|
var resultList = resultArray.ToObject<List<Dictionary<string, object>>>() ?? new List<Dictionary<string, object>>();
|
||||||
|
|
||||||
|
List<BigQueryInsertRow> insertRows = resultArray!.ToObject<List<JObject>>()!
|
||||||
|
.ConvertAll(row => ConvertToBigQueryInsertRow(row));
|
||||||
|
|
||||||
|
string upsertQuery = $@"
|
||||||
|
MERGE `{GoogleDatabase}.bon_main_product` AS T
|
||||||
|
USING (
|
||||||
|
SELECT DISTINCT * FROM UNNEST([
|
||||||
|
{{0}}
|
||||||
|
])
|
||||||
|
) AS S
|
||||||
|
ON T.bon_commodity_indexid = S.bon_commodity_indexid
|
||||||
|
WHEN MATCHED THEN
|
||||||
|
UPDATE SET
|
||||||
|
bon_cash_price = S.bon_cash_price,
|
||||||
|
bon_categoriesid = S.bon_categoriesid,
|
||||||
|
transactioncurrencyid = S.transactioncurrencyid,
|
||||||
|
bon_warehouse_state_central = S.bon_warehouse_state_central,
|
||||||
|
bon_ec_name = S.bon_ec_name,
|
||||||
|
bon_ean = S.bon_ean,
|
||||||
|
bon_gid = S.bon_gid,
|
||||||
|
bon_index = S.bon_index,
|
||||||
|
bon_name = S.bon_name,
|
||||||
|
bon_url = S.bon_url,
|
||||||
|
bon_category_path = S.bon_category_path,
|
||||||
|
bon_seo_description = S.bon_seo_description,
|
||||||
|
bon_main_product_id = S.bon_main_product_id,
|
||||||
|
bon_slug_url = S.bon_slug_url,
|
||||||
|
bon_converter = S.bon_converter,
|
||||||
|
bon_vat_rate = S.bon_vat_rate,
|
||||||
|
bon_product_name = S.bon_product_name,
|
||||||
|
osa_gmc_id = S.osa_gmc_id,
|
||||||
|
bon_calculaltion_unit = S.bon_calculaltion_unit,
|
||||||
|
bon_marketplace = S.bon_marketplace,
|
||||||
|
bon_price_for = S.bon_price_for,
|
||||||
|
bon_ecommerce_status = S.bon_ecommerce_status
|
||||||
|
WHEN NOT MATCHED THEN
|
||||||
|
INSERT (
|
||||||
|
bon_commodity_indexid,
|
||||||
|
bon_cash_price,
|
||||||
|
bon_categoriesid,
|
||||||
|
transactioncurrencyid,
|
||||||
|
bon_warehouse_state_central,
|
||||||
|
bon_ec_name,
|
||||||
|
bon_ean,
|
||||||
|
bon_gid,
|
||||||
|
bon_index,
|
||||||
|
bon_name,
|
||||||
|
bon_url,
|
||||||
|
bon_category_path,
|
||||||
|
bon_seo_description,
|
||||||
|
bon_main_product_id,
|
||||||
|
bon_slug_url,
|
||||||
|
bon_converter,
|
||||||
|
bon_vat_rate,
|
||||||
|
bon_product_name,
|
||||||
|
osa_gmc_id,
|
||||||
|
bon_calculaltion_unit,
|
||||||
|
bon_marketplace,
|
||||||
|
bon_price_for,
|
||||||
|
bon_ecommerce_status
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
S.bon_commodity_indexid,
|
||||||
|
S.bon_cash_price,
|
||||||
|
S.bon_categoriesid,
|
||||||
|
S.transactioncurrencyid,
|
||||||
|
S.bon_warehouse_state_central,
|
||||||
|
S.bon_ec_name,
|
||||||
|
S.bon_ean,
|
||||||
|
S.bon_gid,
|
||||||
|
S.bon_index,
|
||||||
|
S.bon_name,
|
||||||
|
S.bon_url,
|
||||||
|
S.bon_category_path,
|
||||||
|
S.bon_seo_description,
|
||||||
|
S.bon_main_product_id,
|
||||||
|
S.bon_slug_url,
|
||||||
|
S.bon_converter,
|
||||||
|
S.bon_vat_rate,
|
||||||
|
S.bon_product_name,
|
||||||
|
S.osa_gmc_id,
|
||||||
|
S.bon_calculaltion_unit,
|
||||||
|
S.bon_marketplace,
|
||||||
|
S.bon_price_for,
|
||||||
|
S.bon_ecommerce_status
|
||||||
|
)";
|
||||||
|
|
||||||
|
await PutDataBigQuery(upsertQuery, insertRows);
|
||||||
|
|
||||||
|
from++;
|
||||||
|
}
|
||||||
|
while (true);
|
||||||
|
return new OkObjectResult(new { message = me });
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.WriteLine($"PutDataBigQuery_bon_commodity_index: {ex.Message}");
|
||||||
|
return new BadRequestObjectResult(new { message = ex.Message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal async Task PutDataBigQuery(string upsertQuery, List<BigQueryInsertRow> rows)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var table = bigqueryClient.GetTable(GoogleDatabase, "bon_main_product");
|
||||||
|
|
||||||
|
string CleanString(object value)
|
||||||
|
{
|
||||||
|
return value != null
|
||||||
|
? Regex.Replace(value.ToString() ?? "", @"\t|\n|\r", "")
|
||||||
|
: "";
|
||||||
|
}
|
||||||
|
|
||||||
|
var insertRows = rows.Select(row => new BigQueryInsertRow
|
||||||
|
{
|
||||||
|
{ "bon_commodity_indexid", row["bon_commodity_indexid"] },
|
||||||
|
{ "bon_cash_price", row["bon_cash_price"] },
|
||||||
|
{ "bon_categoriesid", row["bon_categoriesid"] },
|
||||||
|
{ "transactioncurrencyid", row["transactioncurrencyid"] },
|
||||||
|
{ "bon_warehouse_state_central", row["bon_warehouse_state_central"] },
|
||||||
|
{ "bon_ec_name", row["bon_ec_name"] },
|
||||||
|
{ "bon_ean", row["bon_ean"] },
|
||||||
|
{ "bon_gid", row["bon_gid"] },
|
||||||
|
{ "bon_index", row["bon_index"] },
|
||||||
|
{ "bon_name", row["bon_name"] },
|
||||||
|
{ "bon_url", row["bon_url"] },
|
||||||
|
{ "bon_category_path", row["bon_category_path"] },
|
||||||
|
{ "bon_seo_description", CleanString(row["bon_seo_description"]) },
|
||||||
|
{ "bon_main_product_id", row["bon_main_product_id"] },
|
||||||
|
{ "bon_slug_url", row["bon_slug_url"] },
|
||||||
|
{ "bon_converter", row["bon_converter"] },
|
||||||
|
{ "bon_vat_rate", row["bon_vat_rate"] },
|
||||||
|
{ "bon_product_name", row["bon_product_name"] },
|
||||||
|
{ "osa_gmc_id", row["osa_gmc_id"] },
|
||||||
|
{ "bon_calculaltion_unit", row["bon_calculaltion_unit"] },
|
||||||
|
{ "bon_marketplace", row["bon_marketplace"] },
|
||||||
|
{ "bon_price_for", row["bon_price_for"] },
|
||||||
|
{ "bon_ecommerce_status", row["bon_ecommerce_status"] }
|
||||||
|
}).ToList();
|
||||||
|
|
||||||
|
var mergeValues = insertRows.Select(row =>
|
||||||
|
$"STRUCT('{row["bon_commodity_indexid"]}' as bon_commodity_indexid, " +
|
||||||
|
$"cast({row["bon_cash_price"] ?? "NULL"} as numeric) as bon_cash_price, " +
|
||||||
|
$"'{row["bon_categoriesid"]}' as bon_categoriesid, " +
|
||||||
|
$"'{row["transactioncurrencyid"]}' as transactioncurrencyid, " +
|
||||||
|
$"cast({row["bon_warehouse_state_central"] ?? "NULL"} as numeric) as bon_warehouse_state_central, " +
|
||||||
|
$"'{row["bon_ec_name"]}' as bon_ec_name, " +
|
||||||
|
$"'{row["bon_ean"]}' as bon_ean, " +
|
||||||
|
$"{row["bon_gid"] ?? "NULL"} as bon_gid, " +
|
||||||
|
$"'{row["bon_index"]}' as bon_index, " +
|
||||||
|
$"'{row["bon_name"]}' as bon_name, " +
|
||||||
|
$"'{row["bon_url"]}' as bon_url, " +
|
||||||
|
$"'{row["bon_category_path"]}' as bon_category_path, " +
|
||||||
|
$"'{row["bon_seo_description"]}' as bon_seo_description, " +
|
||||||
|
$"'{row["bon_main_product_id"]}' as bon_main_product_id, " +
|
||||||
|
$"'{row["bon_slug_url"]}' as bon_slug_url, " +
|
||||||
|
$"cast({row["bon_converter"] ?? "NULL"} as numeric) as bon_converter, " +
|
||||||
|
$"{row["bon_vat_rate"] ?? "NULL"} as bon_vat_rate, " +
|
||||||
|
$"'{row["bon_product_name"]}' as bon_product_name, " +
|
||||||
|
$"{row["osa_gmc_id"] ?? "NULL"} as osa_gmc_id, " +
|
||||||
|
$"'{row["bon_calculaltion_unit"]}' as bon_calculaltion_unit, " +
|
||||||
|
$"ARRAY[{row["bon_marketplace"]}] as bon_marketplace, " +
|
||||||
|
$"{row["bon_price_for"] ?? "NULL"} as bon_price_for, " +
|
||||||
|
$"{row["bon_ecommerce_status"] ?? "NULL"} as bon_ecommerce_status)");
|
||||||
|
|
||||||
|
var queryOptions = new QueryOptions { UseQueryCache = false };
|
||||||
|
await bigqueryClient.ExecuteQueryAsync(string.Format(upsertQuery, string.Join(", ", mergeValues)), null, queryOptions);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.WriteLine($"PutDataBigQuery: {ex.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static BigQueryInsertRow ConvertToBigQueryInsertRow(JObject item)
|
||||||
|
{
|
||||||
|
BigQueryInsertRow insertRow = new BigQueryInsertRow();
|
||||||
|
insertRow.Add("bon_commodity_indexid", (string?)item["bon_commodity_indexid"]);
|
||||||
|
insertRow.Add("bon_cash_price", BigQueryNumeric.FromDecimal((decimal)item["bon_cash_price"]!, LossOfPrecisionHandling.Throw));
|
||||||
|
insertRow.Add("bon_categoriesid", (string?)item["bon_categoriesid"]);
|
||||||
|
insertRow.Add("transactioncurrencyid", (string?)item["transactioncurrencyid"]);
|
||||||
|
insertRow.Add("bon_warehouse_state_central", BigQueryNumeric.FromDecimal((decimal)item["bon_warehouse_state_central"]!, LossOfPrecisionHandling.Truncate));
|
||||||
|
insertRow.Add("bon_ec_name", (string?)item["bon_ec_name"]);
|
||||||
|
insertRow.Add("bon_ean", (string?)item["bon_ean"]);
|
||||||
|
insertRow.Add("bon_gid", (int?)item["bon_gid"]);
|
||||||
|
insertRow.Add("bon_index", (string?)item["bon_index"]);
|
||||||
|
insertRow.Add("bon_name", (string?)item["bon_name"]);
|
||||||
|
insertRow.Add("bon_url", (string?)item["bon_url"]);
|
||||||
|
insertRow.Add("bon_category_path", (string?)item["bon_category_path"]);
|
||||||
|
insertRow.Add("bon_seo_description", (string?)item["bon_seo_description"]);
|
||||||
|
insertRow.Add("bon_main_product_id", (string?)item["bon_main_product_id"]);
|
||||||
|
insertRow.Add("bon_slug_url", (string?)item["bon_slug_url"]);
|
||||||
|
insertRow.Add("bon_converter", BigQueryNumeric.FromDecimal((decimal)item["bon_converter"]!, LossOfPrecisionHandling.Truncate));
|
||||||
|
insertRow.Add("bon_vat_rate", (int?)item["bon_vat_rate"] ?? null);
|
||||||
|
insertRow.Add("bon_product_name", (string?)item["bon_product_name"]);
|
||||||
|
insertRow.Add("osa_gmc_id", (long?)item["osa_gmc_id"]);
|
||||||
|
insertRow.Add("bon_calculaltion_unit", (string?)item["bon_calculaltion_unit"]);
|
||||||
|
insertRow.Add("bon_marketplace",
|
||||||
|
item["bon_marketplace"] is JArray marketplaceArray
|
||||||
|
? string.Join(", ", marketplaceArray.Select(mp => mp["value"]?.ToString()))
|
||||||
|
: null);
|
||||||
|
insertRow.Add("bon_price_for",
|
||||||
|
item["bon_price_for"] is JObject priceForObject
|
||||||
|
? (int?)priceForObject["id"]
|
||||||
|
: null);
|
||||||
|
insertRow.Add("bon_ecommerce_status",
|
||||||
|
item["bon_ecommerce_status"] is JObject statusForObject
|
||||||
|
? (int?)statusForObject["id"]
|
||||||
|
: -1);
|
||||||
|
return insertRow;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
using Migration.Services;
|
||||||
|
using Migration.Settings;
|
||||||
|
|
||||||
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
|
builder.Services.AddEndpointsApiExplorer();
|
||||||
|
builder.Services.AddSwaggerGen();
|
||||||
|
builder.Services.AddSingleton<IBigQuery, GoogleBigQuery>();
|
||||||
|
builder.Services.Configure<MainSettings>(builder.Configuration.GetSection(MainSettings.ConfigName));
|
||||||
|
|
||||||
|
builder.Services.AddSingleton<IDataverseProvider, DataverseProvider>();
|
||||||
|
builder.Services.AddHttpClient(DataverseProvider.HTTP_CLIENT, DataverseProvider.Setup);
|
||||||
|
|
||||||
|
var app = builder.Build();
|
||||||
|
|
||||||
|
if (app.Environment.IsDevelopment())
|
||||||
|
{
|
||||||
|
app.UseSwagger();
|
||||||
|
app.UseSwaggerUI();
|
||||||
|
}
|
||||||
|
|
||||||
|
app.UseHttpsRedirection();
|
||||||
|
|
||||||
|
app.MapPut("/GetProducts", async (IBigQuery _bigQuery) =>
|
||||||
|
{
|
||||||
|
await _bigQuery.GetProducts();
|
||||||
|
})
|
||||||
|
.WithName("GetProducts");
|
||||||
|
|
||||||
|
app.Run();
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json.schemastore.org/launchsettings.json",
|
||||||
|
"iisSettings": {
|
||||||
|
"windowsAuthentication": false,
|
||||||
|
"anonymousAuthentication": true,
|
||||||
|
"iisExpress": {
|
||||||
|
"applicationUrl": "http://localhost:33574",
|
||||||
|
"sslPort": 44377
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"profiles": {
|
||||||
|
"http": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"dotnetRunMessages": true,
|
||||||
|
"launchBrowser": true,
|
||||||
|
"launchUrl": "swagger",
|
||||||
|
"applicationUrl": "http://localhost:5020",
|
||||||
|
"environmentVariables": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"https": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"dotnetRunMessages": true,
|
||||||
|
"launchBrowser": true,
|
||||||
|
"launchUrl": "swagger",
|
||||||
|
"applicationUrl": "https://localhost:7027;http://localhost:5020",
|
||||||
|
"environmentVariables": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"IIS Express": {
|
||||||
|
"commandName": "IISExpress",
|
||||||
|
"launchBrowser": true,
|
||||||
|
"launchUrl": "swagger",
|
||||||
|
"environmentVariables": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
using Migration.Settings;
|
||||||
|
|
||||||
|
namespace Migration.Services
|
||||||
|
{
|
||||||
|
public interface IDataverseProvider
|
||||||
|
{
|
||||||
|
Uri Url { get; }
|
||||||
|
string Token { get; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
using System.Net.Http.Headers;
|
||||||
|
using System.Net.Mime;
|
||||||
|
using Migration.Settings;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using Newtonsoft.Json.Serialization;
|
||||||
|
|
||||||
|
namespace Migration.Services;
|
||||||
|
|
||||||
|
public class DataverseProvider : IDataverseProvider
|
||||||
|
{
|
||||||
|
public const string HTTP_CLIENT = "FeedDataverseBQ";
|
||||||
|
|
||||||
|
private static readonly ProductInfoHeaderValue USER_AGENT = ProductInfoHeaderValue.Parse("FeedDataverseBQ");
|
||||||
|
private static readonly MediaTypeWithQualityHeaderValue JSON_MEDIA_TYPE = new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json);
|
||||||
|
public Uri Url { get; init; }
|
||||||
|
public string Token { get; init; }
|
||||||
|
|
||||||
|
public DataverseProvider(IOptions<MainSettings> mainSettingsOptions, IHttpClientFactory httpClientFactory)
|
||||||
|
{
|
||||||
|
var DataverseSettings = mainSettingsOptions.Value;
|
||||||
|
Url = new Uri(DataverseSettings.Url);
|
||||||
|
Token = DataverseSettings.Token;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void Setup(IServiceProvider serviceProvider, HttpClient httpClient)
|
||||||
|
{
|
||||||
|
var dataverseConnectionsProvider = serviceProvider.GetRequiredService<IDataverseProvider>();
|
||||||
|
httpClient.BaseAddress = dataverseConnectionsProvider.Url;
|
||||||
|
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", dataverseConnectionsProvider.Token);
|
||||||
|
httpClient.DefaultRequestHeaders.UserAgent.Add(USER_AGENT);
|
||||||
|
httpClient.DefaultRequestHeaders.Accept.Add(JSON_MEDIA_TYPE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class DataverseDelegatingHandler : DelegatingHandler
|
||||||
|
{
|
||||||
|
protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return SendAsync(request, cancellationToken).GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return await base.SendAsync(request, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
namespace Migration.Settings
|
||||||
|
{
|
||||||
|
public class MainSettings
|
||||||
|
{
|
||||||
|
internal const string ConfigName = "Dataverse";
|
||||||
|
public string Url { get; set; } = null!;
|
||||||
|
public string Token { get; set; } = null!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"Logging": {
|
||||||
|
"LogLevel": {
|
||||||
|
"Default": "Information",
|
||||||
|
"Microsoft.AspNetCore": "Warning"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"AllowedHosts": "*",
|
||||||
|
"GoogleCredentialFile": "osadkowski-hd-2f3bd9b0ab5d.json",
|
||||||
|
"Dataverse": {
|
||||||
|
"Url": "",
|
||||||
|
"Token": ""
|
||||||
|
},
|
||||||
|
"BigQuery": {
|
||||||
|
"ProjectId": "",
|
||||||
|
"DbName": ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.14" />
|
||||||
|
<PackageReference Include="Swashbuckle.AspNetCore" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Google.Cloud.BigQuery.V2" Version="3.11.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
Loading…
Reference in New Issue