Skip to content

Commit 0bb0857

Browse files
michaellwestmichaellwest
authored
Merge pull request #1 from bic742/master
DataSync Module Updates
2 parents 9d9e19a + 2e2bb2e commit 0bb0857

File tree

22 files changed

+746
-261
lines changed

22 files changed

+746
-261
lines changed

‎App_Data/Unicorn/SPE/Extensions/DataSync/DataSync/Content Editor/Context Menu.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
ID: "87e625d0-b074-4e07-bb30-f319bc4609d6"
3+
Parent: "d8394784-a396-48bb-b588-2eabce3d2bff"
4+
Template: "ab154d3d-1126-4ab4-ac21-8b86e6bd70ea"
5+
Path: /sitecore/system/Modules/PowerShell/Script Library/SPE/Extensions/DataSync/Content Editor/Context Menu
6+
DB: master
7+
Languages:
8+
- Language: en
9+
Versions:
10+
- Version: 1
11+
Fields:
12+
- ID: "25bed78c-4957-4165-998a-ca1b52f67497"
13+
Hint: __Created
14+
Value: 20190307T181845Z

‎App_Data/Unicorn/SPE/Extensions/DataSync/DataSync/Development/Import Runner.yml‎

Lines changed: 75 additions & 161 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,17 @@ SharedFields:
1313
$Watch = [System.Diagnostics.Stopwatch]::StartNew()
1414
)
1515
16+
Import-Function -Name Invoke-SqlCommand
1617
Import-Function -Name New-ItemName
17-
Import-Function -Name Invoke-SourceRequest
18+
Import-Function -Name Get-UriContent
19+
Import-Function -Name Invoke-ProcessSourceItems
1820
1921
class FieldIds {
2022
static [ID]$SourcePath = "{07476EF8-B3B6-46A1-82D7-0E793E7BA658}"
2123
static [ID]$SqlScript = "{53B07925-8E6B-49E5-B58D-F8E2B0489EFE}"
2224
static [ID]$BeforeImportScript = "{2B47ED82-A477-4D3B-AE17-FC1B64D54440}"
2325
static [ID]$ImportTemplate = "{66103474-141A-4CD7-BC64-3E7CF0858441}"
26+
static [ID]$ForceIdScript = "{B0A60B70-8A87-4526-ABF4-62A721961217}"
2427
static [ID]$ItemName = "{FAC6C69F-07F1-4801-9CB8-2DCB77AFC71D}"
2528
static [ID]$ItemDisplayName = "{6D7F51CA-56E5-4E30-8735-CDF77BEABEDE}"
2629
static [ID]$FieldMapping = "{C1561963-9C44-444B-9002-732147905DA2}"
@@ -30,7 +33,9 @@ SharedFields:
3033
static [ID]$DestinationPath = "{CFBA481F-6001-4C0F-A1FC-2FB0CE049F9C}"
3134
static [ID]$ItemParentExpression = "{35DE0F36-FC21-4F02-BC7F-9B3D2BCAA6EC}"
3235
static [ID]$AfterImportScript = "{5721604B-75FF-46A7-984D-6A6F8E8BF4AB}"
33-
static [ID]$PaginationScript = "{E7D5151C-808C-4DB2-A0AE-EBCB662C15C7}"
36+
static [ID]$NextPageScript = "{F23573B3-33F2-4448-B213-93B8607A9612}"
37+
static [ID]$LanguageScript = "{EDD8DCB0-7EB3-48A6-9CB6-369DE0CFE6B2}"
38+
static [ID]$GenerateHeadersScript = "{3E3448F8-5D08-45D9-BC45-73040EA1711C}"
3439
}
3540
3641
class TemplateIds {
@@ -68,7 +73,8 @@ SharedFields:
6873
foreach($importFieldMapping in $importFieldMappings) {
6974
$split = $importFieldMapping -split "="
7075
if($split -and $split.Length -le 2) {
71-
$importFieldMappingsLookup[$split[0]] = $split[1]
76+
$fieldName = $split[0].Replace("_", " ")
77+
$importFieldMappingsLookup[$fieldName] = $split[1]
7278
}
7379
}
7480
@@ -80,181 +86,89 @@ SharedFields:
8086
Write-Host "$($timestamp) Selecting import script $($importScriptItem.TemplateId)"
8187
$sourceItems = @()
8288
$loadFromSource = $true
83-
if($lastImportScriptId -ne $importScriptItem.ID -and $lastSourcePath -and $lastSourcePath -eq $sourcePath) {
84-
if($lastImport) {
85-
Write-Host "$($timestamp) Reusing last items loaded from source"
86-
$sourceItems = $lastImport
87-
$loadFromSource = $false
88-
}
89+
#if($lastSourcePath -and $lastSourcePath -eq $sourcePath) {
90+
# if($lastImport) {
91+
# Write-Host "$($timestamp) Reusing last items loaded from source"
92+
# $sourceItems = $lastImport
93+
# $loadFromSource = $false
94+
# }
95+
#}
96+
97+
$statTracker = @{
98+
"Import items processed" = 0
99+
"New items created" = 0
100+
"Existing items skipped" = 0
101+
"Existing items updated" = 0
102+
"Execution time in seconds" = 0
89103
}
90104
91-
$beforeImportScript = $importScriptItem.Fields[[FieldIds]::BeforeImportScript].Value
92-
$paginationScript = $importScriptItem.Fields[[FieldIds]::PaginationScript].Value
93105
if($loadFromSource) {
94-
Write-Host "$($timestamp) Loading items from source"
95-
Write-Host "$($timestamp) Source $($sourcePath)"
96-
$lastResult = Invoke-SourceRequest -ImportItem $importScriptItem -SourcePath $sourcePath
97-
98-
if($lastResult) {
99-
$sourceItems += $lastResult
100-
}
101-
102-
if($lastResult -and ![string]::IsNullOrEmpty($paginationScript)) {
103-
104-
while($true) {
105-
Write-Progress -Activity "Running pagination script" -Status $status
106-
$pagedSourcePath = Invoke-Expression -Command $paginationScript
107-
if([string]::IsNullOrEmpty($pagedSourcePath)) { break }
108-
Write-Host "$($timestamp) Pagination source $($pagedSourcePath)"
109-
$lastResult = Invoke-SourceRequest -ImportItem $importScriptItem -SourcePath $pagedSourcePath
110-
if($lastResult) {
111-
$sourceItems += $lastResult
106+
switch($importScriptItem.TemplateId) {
107+
([TemplateIds]::ImportFlatFileScript) {
108+
# Flat File
109+
$extension = [System.IO.Path]::GetExtension($sourcePath)
110+
if([string]::IsNullOrEmpty($extension)) { exit }
111+
if($extension -eq ".csv") {
112+
$sourceItems = Import-Csv -Path $sourcePath
113+
} elseif ($extension -eq ".xml") {
114+
$sourceItems = [xml](Get-Content -Path $sourcePath -Raw)
115+
} elseif ($extension -eq ".json") {
116+
$sourceItems = (Get-Content -Path $sourcePath -Raw) | ConvertFrom-Json
112117
} else {
113-
break
118+
$sourceItems = Get-Content -Path $sourcePath -Raw
114119
}
120+
121+
Invoke-ProcessSourceItems -SourceItems $sourceItems -ScriptItem $importScriptItem -FieldMappings $importFieldMappingsLookup -StatTracker ([ref]$statTracker)
115122
}
116-
}
117-
118-
if($sourceItems -and $sourceItems -isnot [array]) {
119-
$sourceItemsTemp = $sourceItems
120-
$sourceItems = [System.Collections.ArrayList]@()
121-
$sourceItems.Add($sourceItemsTemp) > $null
122-
}
123-
124-
if(![string]::IsNullOrEmpty($beforeImportScript)) {
125-
Write-Progress -Activity "Running pre import script" -Status $status
126-
Invoke-Expression -Command $beforeImportScript
127-
}
128-
129-
$lastImportScriptId = $importScriptItem.ID
130-
$lastSourcePath = $sourcePath
131-
$lastImport = $sourceItems
132-
} else {
133-
if(![string]::IsNullOrEmpty($beforeImportScript)) {
134-
Write-Progress -Activity "Running pre import script" -Status $status
135-
Invoke-Expression -Command $beforeImportScript
136-
}
137-
}
138-
139-
if($sourceItems -isnot [array]) {
140-
$sourceItemsTemp = $sourceItems
141-
$sourceItems = [System.Collections.ArrayList]@()
142-
$sourceItems.Add($sourceItemsTemp) > $null
143-
}
144-
145-
Write-Progress -Activity "Configuring import settings" -Status $status
146-
$itemSyncFieldName = $importScriptItem.Fields[[FieldIds]::ItemSyncField].Value
147-
$checkSyncField = $false
148-
$skipExisting = [Sitecore.MainUtil]::GetBool($importScriptItem.Fields[[FieldIds]::SkipExisting].Value, $false)
149-
if(![string]::IsNullOrEmpty($itemSyncFieldName)) {
150-
$checkSyncField = $true
151-
}
152-
153-
$destinationItemId = $importScriptItem.Fields[[FieldIds]::DestinationPath].Value
154-
$destinationRoot = Get-Item -Path "master:" -ID $destinationItemId
155-
$syncFieldLookup = @{}
156-
if($checkSyncField) {
157-
$existingItems = $destinationRoot.Axes.GetDescendants() | Where-Object { $_.TemplateId -eq $importTemplateId } | Initialize-Item
158-
foreach($existingItem in $existingItems) {
159-
$existingItemSyncFieldValue = $existingItem.Fields[$itemSyncFieldName].Value
160-
if(!$existingItemSyncFieldValue) {
161-
$member = Get-Member -InputObject $existingItem -Name $itemSyncFieldName
162-
if($member) {
163-
$existingItemSyncFieldValue = $existingItem.($member.Name)
164-
}
123+
([TemplateIds]::ImportSqlScript) {
124+
# Sql Script
125+
$query = $importScriptItem.Fields[[FieldIds]::SqlScript].Value
126+
$sourceItems = Invoke-SqlCommand -Query $query -Connection $sourcePath
127+
128+
Invoke-ProcessSourceItems -SourceItems $sourceItems -ScriptItem $importScriptItem -FieldMappings $importFieldMappingsLookup -StatTracker ([ref]$statTracker)
165129
}
166-
if([string]::IsNullOrEmpty($existingItemSyncFieldValue)) { continue }
167-
$syncFieldLookup[$existingItemSyncFieldValue] = $existingItem
168-
}
169-
}
130+
([TemplateIds]::ImportWebServiceScript) {
131+
# Web service
132+
if (![Net.ServicePointManager]::SecurityProtocol.HasFlag([Net.SecurityProtocolType]::Tls12)) {
133+
[Net.ServicePointManager]::SecurityProtocol += [Net.SecurityProtocolType]::Tls12
134+
}
170135
171-
$destinationItems = [System.Collections.ArrayList]@()
172-
$count = 0
173-
$countTotal = $sourceItems.Count
174-
$skippedItemsCount = 0
175-
$newItemsCount = 0
176-
$existingItemsUpdatedCount = 0
177-
$parentPathExpression = $importScriptItem.Fields[[FieldIds]::ItemParentExpression].Value
178-
foreach($sourceItem in $sourceItems) {
179-
$count++
180-
Write-Host "$($timestamp) Processing source item $($count) of $($countTotal)"
181-
$itemName = Invoke-Expression -Command $importScriptItem.Fields[[FieldIds]::ItemName].Value
182-
Write-Progress -Activity "Creating or updating destination items" -Status $status -PercentComplete ($count / $countTotal * 100) -CurrentOperation "$($itemName)"
183-
$destinationItem = $null
184-
if($checkSyncField -and $importFieldMappingsLookup.ContainsKey($itemSyncFieldName)) {
185-
$lookupValue = Invoke-Expression -Command $importFieldMappingsLookup[$itemSyncFieldName]
186-
if($lookupValue -and $syncFieldLookup.ContainsKey($lookupValue)) {
187-
if($skipExisting) {
188-
Write-Host "- [S] Item found by matching field $($itemSyncFieldName)"
189-
$skippedItemsCount++
190-
continue
136+
$generateHeadersScript = $importScriptItem.Fields[[FieldIds]::GenerateHeadersScript].Value
137+
if (![string]::IsNullOrEmpty($generateHeadersScript)) {
138+
$PSDefaultParameterValues = Invoke-Expression -Command $generateHeadersScript
139+
}
140+
141+
while (![string]::IsNullOrEmpty($sourcePath)) {
142+
$sourceItems = Get-UriContent -sourcePath $sourcePath
143+
144+
$nextPageSourcePath = "";
145+
146+
Write-Progress -Activity "Running next page script" -Status $status
147+
$nextPageScript = $importScriptItem.Fields[[FieldIds]::NextPageScript].Value
148+
if(![string]::IsNullOrEmpty($nextPageScript)) {
149+
$nextPageSourcePath = Invoke-Expression -Command $nextPageScript
150+
}
151+
152+
$sourcePath = $nextPageSourcePath
153+
154+
Invoke-ProcessSourceItems -SourceItems $sourceItems -ScriptItem $importScriptItem -FieldMappings $importFieldMappingsLookup -StatTracker ([ref]$statTracker)
191155
}
192-
$destinationItem = $syncFieldLookup[$lookupValue]
193156
}
194-
}
195-
196-
$path = "$($destinationRoot.ProviderPath)\$($itemName)"
197-
if(![string]::IsNullOrEmpty($parentPathExpression)) {
198-
$parentPath = Invoke-Expression -Command $parentPathExpression
199-
$parentPath = "$($destinationRoot.ProviderPath)\$($parentPath.Trim('\','/'))"
200-
$path = "$($parentPath)\$($itemName)"
201-
$parentItem = Get-Item -Path $parentPath
202-
} else {
203-
$parentItem = $destinationRoot
204-
}
205-
206-
if((Test-Path -Path $path)) {
207-
Write-Host "- [!] Item exists at path $($path)"
208-
$destinationItem = Get-Item -Path $path
209-
}
210-
if(!$destinationItem) {
211-
Write-Host "- [A] $($path)"
212-
$newItemsCount++
213-
$destinationItem = New-Item -Parent $parentItem -Name $itemName -ItemType $importTemplateId
214-
} else {
215-
Write-Host "- [S] Item with the same name exists"
216-
$existingItemsUpdatedCount++
217-
}
218-
219-
$displayName = $importScriptItem.Fields[[FieldIds]::ItemDisplayName].Value
220-
if(![string]::IsNullOrEmpty($displayName)) {
221-
$displayName = Invoke-Expression -Command $displayName
222-
}
223-
$destinationItem.Editing.BeginEdit()
224-
if(![string]::IsNullOrEmpty($displayName)) {
225-
$destinationItem[[Sitecore.FieldIDs]::DisplayName] = $displayName
226-
}
227-
foreach($fieldName in $importFieldMappingsLookup.Keys) {
228-
if($destinationItem.Fields[$fieldName]) {
229-
$destinationItem.Fields[$fieldName].Value = Invoke-Expression -Command $importFieldMappingsLookup[$fieldName]
230-
} else {
231-
Write-Host " - [S] $($fieldName) does not exist on the item"
157+
default {
158+
Write-Host "Unable to complete because the import script is unrecognized." -ForegroundColor White -BackgroundColor Red
159+
exit
232160
}
233161
}
234162
235-
if(![string]::IsNullOrEmpty($importScriptItem.Fields[[FieldIds]::ForEachItemScript].Value)) {
236-
Invoke-Expression -Command $importScriptItem.Fields[[FieldIds]::ForEachItemScript].Value
237-
}
238-
239-
$destinationItem.Editing.EndEdit() > $null
240-
$destinationItems.Add($destinationItem) > $null
241-
}
242-
243-
Write-Progress -Activity "Running post import script" -Status $status
244-
$afterImportScript = $importScriptItem.Fields[[FieldIds]::AfterImportScript].Value
245-
if(![string]::IsNullOrEmpty($afterImportScript)) {
246-
Invoke-Expression -Command $afterImportScript
163+
$lastSourcePath = $sourcePath
164+
$lastImport = $sourceItems
247165
}
248166
249167
Write-Host "$($timestamp) $($importScriptItem.Name) completed" -ForegroundColor Green
250168
$time = $watch.ElapsedMilliseconds / 1000
251-
[PSCustomObject]@{
252-
"Import items processed" = $sourceItems.Count
253-
"New items created" = $newItemsCount
254-
"Existing items skipped" = $skippedItemsCount
255-
"Existing items updated" = $existingItemsUpdatedCount
256-
"Execution time in seconds" = $time
257-
} | Format-List
169+
170+
$statTracker["Execution time in seconds"] = $time
171+
$statTracker | Format-List
258172
259173
Write-Progress -Activity "Completed importing data for $($importScriptItem.Name)" -Completed -Status "Import Complete"
260174

‎App_Data/Unicorn/SPE/Extensions/DataSync/DataSync/Functions/Custom Functions.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
ID: "3950ab30-fdf8-4149-9c2e-59c43ca9ad6f"
3+
Parent: "d3c93968-b343-4fa5-be29-0b7138359988"
4+
Template: "ab154d3d-1126-4ab4-ac21-8b86e6bd70ea"
5+
Path: /sitecore/system/Modules/PowerShell/Script Library/SPE/Extensions/DataSync/Functions/Custom Functions
6+
DB: master
7+
SharedFields:
8+
- ID: "ba3f86a2-4a1c-4d78-b63d-91c2779c1b5e"
9+
Hint: __Sortorder
10+
Value: 100
11+
Languages:
12+
- Language: en
13+
Versions:
14+
- Version: 1
15+
Fields:
16+
- ID: "25bed78c-4957-4165-998a-ca1b52f67497"
17+
Hint: __Created
18+
Value: 20190327T190416Z

‎App_Data/Unicorn/SPE/Extensions/DataSync/DataSync/Functions/Invoke-SourceRequest.yml‎

Lines changed: 0 additions & 77 deletions
This file was deleted.

0 commit comments

Comments
 (0)