Hello,
I am developing an ADF pipeline with a Dataflow at the end. In the last test I made I got the error 'This resource references resources which could not be loaded'.
I was able to run it fine before, the only thing I adjusted before the last run was the Dataflow. In the last run I made in debug mode It took me out of the design mode into a JSON script view with the error 'This resource references resources which could not be loaded'. With no option of returning back to the design mode.
Pipeline JSON containing Dataflow below:
{
"name": "PL_COPY_ANAPLAN_FCST_WAREHOUSE_WFL",
"properties": {
"activities": [
{
"name": "ForEachFolder",
"type": "ForEach",
"dependsOn": [
{
"activity": "Warehouse 0 Activity",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Warehouse 0 Activity').output.value",
"type": "Expression"
},
"isSequential": false,
"batchCount": 10,
"activities": [
{
"name": "COPY_RAW_INTO_PROJECT_CHILD",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "PL_COPY_WAREHOUSE_ARCHIVE",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "PL_COPY_ANAPLAN_FCST_WAREHOUSE_LANDING-PROJECT_STL",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Source_Container_Name": {
"value": "@item().Source_Container_Name",
"type": "Expression"
},
"Source_Folder_Name": {
"value": "@item().Source_Folder_Name",
"type": "Expression"
},
"Destination_Container_Name": {
"value": "@item().Destination_Container_Name",
"type": "Expression"
},
"Destination_Folder_Name": {
"value": "@item().Destination_Folder_Name",
"type": "Expression"
},
"Pipeline_Activity": {
"value": "@item().PIPELINE_JOB_REF",
"type": "Expression"
}
}
}
},
{
"name": "PL_COPY_WAREHOUSE_ARCHIVE",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "PL_COPY_ANAPLAN_FCST_WAREHOUSE_ARCHIVE_STL",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Destination_Container": {
"value": "@item().Destination_Container_Name",
"type": "Expression"
},
"Destination_Folder": {
"value": "@item().Destination_Folder_Name",
"type": "Expression"
},
"Archive_Folder": {
"value": "@item().Archive_Destination_Folder",
"type": "Expression"
}
}
}
}
]
}
},
{
"name": "LookupFolders",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": "SELECT * FROM SYS_NEBULA.ADFPipeline_Exports_ANAPLAN",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "SDS_SQLDB_GENERIC",
"type": "DatasetReference",
"parameters": {
"SchemaName": "SYS_NEBULA",
"TableName": "ADFPipeline_Exports_ANAPLAN"
}
},
"firstRowOnly": false
}
},
{
"name": "Warehouse 0 Activity",
"type": "Filter",
"dependsOn": [
{
"activity": "LookupFolders",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('LookupFolders').output.value",
"type": "Expression"
},
"condition": {
"value": "@equals(item().PIPELINE_JOB_REF,'Export Warehouse 0')",
"type": "Expression"
}
}
},
{
"name": "Warehouse Neo Output Activity",
"type": "Filter",
"dependsOn": [
{
"activity": "LookupFolders",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('LookupFolders').output.value",
"type": "Expression"
},
"condition": {
"value": "@equals(item().PIPELINE_JOB_REF,'Export Fcst Neo Output')",
"type": "Expression"
}
}
},
{
"name": "ForEachFolder_copy1",
"type": "ForEach",
"dependsOn": [
{
"activity": "Warehouse Neo Output Activity",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Warehouse Neo Output Activity').output.value",
"type": "Expression"
},
"isSequential": false,
"batchCount": 10,
"activities": [
{
"name": "COPY_RAW_INTO_PROJECT_CHILD_copy1",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "PL_COPY_WAREHOUSE_ARCHIVE_copy1",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "PL_COPY_ANAPLAN_FCST_WAREHOUSE_LANDING-PROJECT_STL",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Source_Container_Name": {
"value": "@item().Source_Container_Name",
"type": "Expression"
},
"Source_Folder_Name": {
"value": "@item().Source_Folder_Name",
"type": "Expression"
},
"Destination_Container_Name": {
"value": "@item().Destination_Container_Name",
"type": "Expression"
},
"Destination_Folder_Name": {
"value": "@item().Destination_Folder_Name",
"type": "Expression"
},
"Pipeline_Activity": {
"value": "@item().PIPELINE_JOB_REF",
"type": "Expression"
}
}
}
},
{
"name": "PL_COPY_WAREHOUSE_ARCHIVE_copy1",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "PL_COPY_ANAPLAN_FCST_WAREHOUSE_ARCHIVE_STL",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Destination_Container": {
"value": "@item().Destination_Container_Name",
"type": "Expression"
},
"Destination_Folder": {
"value": "@item().Destination_Folder_Name",
"type": "Expression"
},
"Archive_Folder": {
"value": "@item().Archive_Destination_Folder",
"type": "Expression"
}
}
}
}
]
}
},
{
"name": "DF_COPY_ANAPLAN_FCST_STL-STL",
"type": "ExecuteDataFlow",
"dependsOn": [
{
"activity": "DeleteAfterArchive",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "1.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "DF_COPY_ANAPLAN_FCST_STL-STL",
"type": "DataFlowReference"
},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine"
}
},
{
"name": "CopytoArchive",
"type": "Copy",
"dependsOn": [
{
"activity": "ForEachFolder_copy1",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "AzureBlobFSReadSettings",
"recursive": true,
"wildcardFolderPath": "prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/zzDataflow Output/",
"wildcardFileName": ".csv",
"enablePartitionDiscovery": false
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "SDS_STL_CSV_GENERIC",
"type": "DatasetReference",
"parameters": {
"Container": {
"value": "@pipeline().parameters.Container_Name",
"type": "Expression"
},
"FilePath": "prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/zzDataflow Output/",
"FileName": ".csv"
}
}
],
"outputs": [
{
"referenceName": "SDS_STL_CSV_GENERIC",
"type": "DatasetReference",
"parameters": {
"Container": {
"value": "@pipeline().parameters.Container_Name",
"type": "Expression"
},
"FilePath": "prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/zzDataflow Output/Archive/",
"FileName": {
"value": "@markus.bohland@hotmail.de ('FcstW_Current_Output',utcnow('ddMMyyyy'))",
"type": "Expression"
}
}
}
]
},
{
"name": "DeleteAfterArchive",
"type": "Delete",
"dependsOn": [
{
"activity": "CopytoArchive",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "SDS_STL_CSV_GENERIC",
"type": "DatasetReference",
"parameters": {
"Container": {
"value": "@pipeline().parameters.Container_Name",
"type": "Expression"
},
"FilePath": "prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/zzDataflow Output/",
"FileName": ".csv"
}
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobFSReadSettings",
"recursive": true,
"wildcardFileName": ".csv",
"enablePartitionDiscovery": false
}
}
}
],
"parameters": {
"Container_Name": {
"type": "string",
"defaultValue": "prw-dev-datalake-project-production"
}
},
"variables": {
"FolderName": {
"type": "String"
},
"filenamearray": {
"type": "Array"
},
"filenames": {
"type": "String"
},
"SetLastModifiedDate": {
"type": "String"
},
"varRefDateTaime": {
"type": "String"
},
"varLatestFileName": {
"type": "String"
},
"varReferenceDateTime": {
"type": "String",
"defaultValue": "1900-01-01 00:00:00"
}
},
"folder": {
"name": "WORKFLOWS/ANAPLAN"
},
"annotations": []
}
}
DataFlow JSON Below:
{
"name": "DF_COPY_ANAPLAN_FCST_STL-STL",
"properties": {
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"linkedService": {
"referenceName": "STL",
"type": "LinkedServiceReference"
},
"name": "Australia"
},
{
"linkedService": {
"referenceName": "STL",
"type": "LinkedServiceReference"
},
"name": "NewZealand"
},
{
"linkedService": {
"referenceName": "STL",
"type": "LinkedServiceReference"
},
"name": "PacificTR"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "STL",
"type": "LinkedServiceReference"
},
"name": "CopyCurrentFile",
"rejectedDataLinkedService": {
"referenceName": "STL",
"type": "LinkedServiceReference"
}
}
],
"transformations": [
{
"name": "AddCustomColumnsAU"
},
{
"name": "AddCustomColumnsNZ"
},
{
"name": "AddCustomColumnsTR"
},
{
"name": "AppendFiles"
},
{
"name": "FormatColumns"
}
],
"scriptLines": [
"source(output(",
" Warehouses as string,",
" {Products: ProductName} as string,",
" {CalWeeks: datDeb} as string,",
" SKUCode as string,",
" QtyFcst as float",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: true,",
" format: 'delimited',",
" fileSystem: 'prw-dev-datalake-project-production',",
" columnDelimiter: '\t',",
" escapeChar: '\\',",
" quoteChar: '\\"',",
" columnNamesAsHeader: true,",
" wildcardPaths:['prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/Australia/.txt']) ~> Australia",
"source(output(",
" Warehouses as string,",
" {Products: ProductName} as string,",
" {CalWeeks: datDeb} as string,",
" SKUCode as string,",
" QtyFcst as float",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false,",
" format: 'delimited',",
" fileSystem: 'prw-dev-datalake-project-production',",
" columnDelimiter: '\t',",
" escapeChar: '\\',",
" quoteChar: '\\"',",
" columnNamesAsHeader: true,",
" wildcardPaths:['prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/NewZealand/.txt']) ~> NewZealand",
"source(output(",
" Warehouses as string,",
" {Products: ProductName} as string,",
" {CalWeeks: datDeb} as string,",
" SKUCode as string,",
" QtyFcst as float",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false,",
" format: 'delimited',",
" fileSystem: 'prw-dev-datalake-project-production',",
" columnDelimiter: '\t',",
" escapeChar: '\\',",
" quoteChar: '\\"',",
" columnNamesAsHeader: true,",
" wildcardPaths:['prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/PacificTR/**.txt']) ~> PacificTR",
"Australia derive({Units/Case} = '',",
" {Stamp Date} = currentDate(),",
" Channel = case(equals(left(Warehouses,1),'A'),'PRA','')) ~> AddCustomColumnsAU",
"NewZealand derive({Units/Case} = '',",
" {Stamp Date} = currentDate(),",
" Channel = case(equals(left(Warehouses,1),'A'),'PNZ','')) ~> AddCustomColumnsNZ",
"PacificTR derive({Units/Case} = '',",
" {Stamp Date} = currentDate(),",
" Channel = case(equals(left(Warehouses,1),'A'),'AUS', case(equals(left(Warehouses,1),'N'),'NZT',''))) ~> AddCustomColumnsTR",
"AddCustomColumnsAU, AddCustomColumnsNZ, AddCustomColumnsTR union(byName: true)~> AppendFiles",
"AppendFiles derive(SKUCode = substringIndex(SKUCode,'_',-1),",
" Warehouses = substringIndex(Warehouses,'-',1),",
" {CalWeeks: datDeb} = toDate({CalWeeks: datDeb}, 'yyyy-MM-dd')) ~> FormatColumns",
"FormatColumns sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'delimited',",
" fileSystem: 'prw-dev-datalake-project-production',",
" folderPath: (prw/Anaplan/Anaplan_Exports/FcstW txt Export - Neo Output/zzDataflow Output),",
" columnDelimiter: ',',",
" escapeChar: '\\',",
" quoteChar: '\\"',",
" columnNamesAsHeader: true,",
" partitionFileNames:[(concat('FcstW_Current_Output',toString(currentDate(),'ddMMyyyy')))],",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true,",
" mapColumn(",
" {Stamp Date},",
" {Product Code} = SKUCode,",
" {Product Description} = {Products: ProductName},",
" {Units/Case},",
" Channel,",
" Date = {CalWeeks: datDeb},",
" Branch = Warehouses,",
" {Volume (CA)} = QtyFcst",
" ),",
" partitionBy('hash', 1)) ~> CopyCurrentFile"
]
}
}
}
