Azure Datafactory unable to load encrypted HDFS data from to ADLS Gen 2, successully loading unencrypted data though using SHIR
Hi team,
*ADF pipeline created in Azure is able to copy HDFS structured data which is not encrypted with keys,
However it fails when trying to copy datasets encrypted with keys. We have set up SHIR and having good performance using copy activity, however below is error when it comes to databases encrypted with keys.
Source - Microsoft.DataTRansfer.Common, 'Type 'Microsoft.DataTRansfer,Common.Shared.HybridDelivery Exception, Message=Failed to read a HDFS file', File path=asa/dsfsfs/sdsdsd
Here is JSON for source.
name": "CopyHDFSToAzure",
"properties": {
"activities": [
{
"name": "Copy data1",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HdfsReadSettings",
"recursive": true
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false,
"skipErrorFile": {
"dataInconsistency": false
},
"validateDataConsistency": true,
"logSettings": {
"enableCopyActivityLog": true,
"copyActivityLogSettings": {
"logLevel": "Info",
"enableReliableLogging": false
},
"logLocationSettings": {
"linkedServiceName": {
"referenceName": "LS_ADLS_logs",
"type": "LinkedServiceReference"
},
"path": "dsdsd/sdsds/sdsds"
}
}
},
"inputs": [
{
"referenceName": "DS_HDFS_Test",
"type": "DatasetReference",
"parameters": {
"SourcePath": {
"value": "@pipeline().parameters.SourcePath",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "DS_ADLS_foundation",
"type": "DatasetReference",
"parameters": {
"DestinationPath": {
"value": "@pipeline().parameters.DestinationPath",
"type": "Expression"
}
}
}
]
}
],
"parameters": {
"DestinationPath": {
"type": "String",
"defaultValue": "DestinationPath"
},
"SourcePath": {
"type": "String",
"defaultValue": "SourcePath"
}
},
"annotations": [],
"lastPublishTime": "2022-10-31T12:37:28Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}