Filter Array value in Azure Data Factory

Question

Hello All,
I have an Array variable with the below value :
[

        {
            "expectation_config": {
                "meta": {},
                "kwargs": {
                    "column": "Resource Personnel Number",
                    "result_format": {
                        "result_format": "SUMMARY",
                        "partial_unexpected_count": 20
                    }
                },
                "expectation_type": "expect_column_values_to_not_be_null"
            },
            "meta": {},
            "exception_info": {
                "raised_exception": false,
                "exception_message": null,
                "exception_traceback": null
            },
            "result": {
                "element_count": 999,
                "unexpected_count": 0,
                "unexpected_percent": 0.0,
                "unexpected_percent_total": 0.0,
                "partial_unexpected_list": []
            },
            "success": true
        },

        {
            "expectation_config": {
                "meta": {},
                "kwargs": {
                    "column": "Resource Previous Period Chargeability",
                    "min_value": 0.0,
                    "result_format": {
                        "result_format": "SUMMARY",
                        "partial_unexpected_count": 20
                    }
                },
                "expectation_type": "expect_column_values_to_be_between"
            },
            "meta": {},
            "exception_info": {
                "raised_exception": false,
                "exception_message": null,
                "exception_traceback": null
            },
            "result": {
                "element_count": 999,
                "missing_count": 0,
                "missing_percent": 0.0,
                "unexpected_count": 0,
                "unexpected_percent": 0.0,
                "unexpected_percent_total": 0.0,
                "unexpected_percent_nonmissing": 0.0,
                "partial_unexpected_list": [],
                "partial_unexpected_index_list": [],
                "partial_unexpected_counts": []
            },
            "success": true
        },
        {
            "expectation_config": {
                "meta": {},
                "kwargs": {
                    "column": "Resource YTD Chargeability",
                    "max_value": 1.0,
                    "min_value": 0.0,
                    "result_format": {
                        "result_format": "SUMMARY",
                        "partial_unexpected_count": 20
                    }
                },
                "expectation_type": "expect_column_min_to_be_between"
            },
            "meta": {},
            "exception_info": {
                "raised_exception": false,
                "exception_message": null,
                "exception_traceback": null
            },
            "result": {
                "observed_value": -0.0106,
                "element_count": 999,
                "missing_count": null,
                "missing_percent": null
            },
            "success": false
        }
    ]

I need to filter out the section wherein success :false

Currently I am using a for activity and within it for each iteration checking if the value is false or not.

Is there any way to extract the entire sub section wherein success:false via filter activity

Answer 1

Hello again @Nandan Hegde and welcome back.

to keep only the entries that had success = true, I used:

@bool(item().success)  

pipeline repro code:

{  
    "name": "pipeline9",  
    "properties": {  
        "activities": [  
            {  
                "name": "Set variable1",  
                "type": "SetVariable",  
                "dependsOn": [],  
                "userProperties": [],  
                "typeProperties": {  
                    "variableName": "from",  
                    "value": {  
                        "value": "@json('[\n\t{\n\t\t\"expectation_config\": {\n\t\t\t\"meta\": {},\n\t\t\t\"kwargs\": {\n\t\t\t\t\"column\": \"Resource Personnel Number\",\n\t\t\t\t\"result_format\": {\n\t\t\t\t\t\"result_format\": \"SUMMARY\",\n\t\t\t\t\t\"partial_unexpected_count\": 20\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"expectation_type\": \"expect_column_values_to_not_be_null\"\n\t\t},\n\t\t\"meta\": {},\n\t\t\"exception_info\": {\n\t\t\t\"raised_exception\": false,\n\t\t\t\"exception_message\": null,\n\t\t\t\"exception_traceback\": null\n\t\t},\n\t\t\"result\": {\n\t\t\t\"element_count\": 999,\n\t\t\t\"unexpected_count\": 0,\n\t\t\t\"unexpected_percent\": 0.0,\n\t\t\t\"unexpected_percent_total\": 0.0,\n\t\t\t\"partial_unexpected_list\": []\n\t\t},\n\t\t\"success\": true\n\t},\n\t{\n\t\t\"expectation_config\": {\n\t\t\t\"meta\": {},\n\t\t\t\"kwargs\": {\n\t\t\t\t\"column\": \"Resource Previous Period Chargeability\",\n\t\t\t\t\"min_value\": 0.0,\n\t\t\t\t\"result_format\": {\n\t\t\t\t\t\"result_format\": \"SUMMARY\",\n\t\t\t\t\t\"partial_unexpected_count\": 20\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"expectation_type\": \"expect_column_values_to_be_between\"\n\t\t},\n\t\t\"meta\": {},\n\t\t\"exception_info\": {\n\t\t\t\"raised_exception\": false,\n\t\t\t\"exception_message\": null,\n\t\t\t\"exception_traceback\": null\n\t\t},\n\t\t\"result\": {\n\t\t\t\"element_count\": 999,\n\t\t\t\"missing_count\": 0,\n\t\t\t\"missing_percent\": 0.0,\n\t\t\t\"unexpected_count\": 0,\n\t\t\t\"unexpected_percent\": 0.0,\n\t\t\t\"unexpected_percent_total\": 0.0,\n\t\t\t\"unexpected_percent_nonmissing\": 0.0,\n\t\t\t\"partial_unexpected_list\": [],\n\t\t\t\"partial_unexpected_index_list\": [],\n\t\t\t\"partial_unexpected_counts\": []\n\t\t},\n\t\t\"success\": true\n\t},\n\t{\n\t\t\"expectation_config\": {\n\t\t\t\"meta\": {},\n\t\t\t\"kwargs\": {\n\t\t\t\t\"column\": \"Resource YTD Chargeability\",\n\t\t\t\t\"max_value\": 1.0,\n\t\t\t\t\"min_value\": 0.0,\n\t\t\t\t\"result_format\": {\n\t\t\t\t\t\"result_format\": \"SUMMARY\",\n\t\t\t\t\t\"partial_unexpected_count\": 20\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"expectation_type\": \"expect_column_min_to_be_between\"\n\t\t},\n\t\t\"meta\": {},\n\t\t\"exception_info\": {\n\t\t\t\"raised_exception\": false,\n\t\t\t\"exception_message\": null,\n\t\t\t\"exception_traceback\": null\n\t\t},\n\t\t\"result\": {\n\t\t\t\"observed_value\": -0.0106,\n\t\t\t\"element_count\": 999,\n\t\t\t\"missing_count\": null,\n\t\t\t\"missing_percent\": null\n\t\t},\n\t\t\"success\": false\n\t}\n]')",  
                        "type": "Expression"  
                    }  
                }  
            },  
            {  
                "name": "Filter1",  
                "type": "Filter",  
                "dependsOn": [  
                    {  
                        "activity": "Set variable1",  
                        "dependencyConditions": [  
                            "Succeeded"  
                        ]  
                    }  
                ],  
                "userProperties": [],  
                "typeProperties": {  
                    "items": {  
                        "value": "@variables('from')",  
                        "type": "Expression"  
                    },  
                    "condition": {  
                        "value": "@bool(item().success)",  
                        "type": "Expression"  
                    }  
                }  
            }  
        ],  
        "variables": {  
            "from": {  
                "type": "Array"  
            }  
        },  
        "annotations": []  
    }  
}