with Data Factory V2 I'm trying to implement a stream of data copy from one Azure SQL database to another. I have to merge multiple tables into one target table.
To do this I created a lookup activity that creates the name of the three tables to be copied. The output JSON file is passed to a foreach activity that should copy the data of each table into the destination table, but the pipeline execution is not successful.
I report the code of my pipeline below:
{
"name": "FLD_Item_base",
"properties": {
"activities": [
{
"name": "myLookup",
"type": "Lookup",
"dependsOn": [
{
"activity": "myStoredProcedure",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"typeProperties": {
"source": {
"type": "SqlSource",
"sqlReaderQuery": "SELECT tsy_company_desc, tsy_tabella_desc, tsy_company_id, \ncase when tsy_company_desc = 'all' \nthen '['+ tsy_tabella_desc + ']' else '['+tsy_company_desc + '$' + tsy_tabella_desc + ']' end as nome_tabella_completo \nfrom dbo.TSY_FLUSSI_LOAD_DWH \nwhere tsy_flusso_desc = 'item_base' \nand tsy_tabella_desc='Item' \n"
},
"dataset": {
"referenceName": "TLD_Item",
"type": "DatasetReference"
},
"firstRowOnly": false
}
},
{
"name": "myStoredProcedure",
"type": "SqlServerStoredProcedure",
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"typeProperties": {
"storedProcedureName": "[dbo].[myStoredProcedure]"
},
"linkedServiceName": {
"referenceName": "Sink",
"type": "LinkedServiceReference"
}
},
{
"name": "IterateSQLTables",
"type": "ForEach",
"dependsOn": [
{
"activity": "myLookup",
"dependencyConditions": [
"Succeeded"
]
}
],
"typeProperties": {
"items": {
"value": "@activity('myLookup').output",
"type": "Expression"
},
"activities": [
{
"name": "FullCopyActivity",
"type": "Copy",
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"typeProperties": {
"source": {
"type": "SqlSource",
"sqlReaderQuery": {
"value": "SELECT * FROM @{item().value.name_tab}",
"type": "Expression"
}
},
"sink": {
"type": "SqlSink",
"writeBatchSize": 10000
},
"enableStaging": false,
"dataIntegrationUnits": 0
},
"inputs": [
{
"referenceName": "Source",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "TLD_Item",
"type": "DatasetReference"
}
]
}
]
}
}
Pipeline execution returns the following error:
{
"errorCode": "400",
"message": "Activity failed because an inner activity failed",
"failureType": "UserError",
"target": "IterateSQLTables"
}
Does anyone know how to solve the problem? Thank you
