Permission issue while creating a ML pipeline job using Azure CLI 2.46.0 through Azure DevOps build pipeline

Question

I am calling a pythons Script from azure CLI task,

python Test-Deployment.py

Inside this script I am authenticating using a service principal with contributor access on resource group level to authorize the command. The script is able to find the workspace and associated cluster as below,

2023-03-21T01:58:05.5062560Z Found existing compute target
2023-03-21T01:58:05.5063475Z Azure Machine Learning Compute Attached
2023-03-21T01:58:05.5064101Z Step WrapperFunction is ready to be created [444dce21]

But on publishing the pipeline step,

published_pipeline = Tests_Pipeline.publish( name="BaseOrderData_Pipeline_Notebook", description="BaseOrderData Published Pipeline Description", version="1.0")

I am getting following error ,

2023-03-21T01:58:05.5161488Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/pipeline.py", line 309, in publish
2023-03-21T01:58:05.5162454Z     return self.graph._save(name=name, description=description, version=version,
2023-03-21T01:58:05.5163574Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/graph.py", line 3844, in _save
2023-03-21T01:58:05.5164547Z     self._validate_and_finalize(pipeline_parameters=None, regenerate_outputs=regenerate_outputs)
2023-03-21T01:58:05.5165682Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/graph.py", line 3863, in _validate_and_finalize
2023-03-21T01:58:05.5166630Z     self.finalize(dry_run=False, regenerate_outputs=regenerate_outputs)
2023-03-21T01:58:05.5167689Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/graph.py", line 3609, in finalize
2023-03-21T01:58:05.5168566Z     Graph._check_threadpool_exceptions(done, not_done)
2023-03-21T01:58:05.5169661Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/graph.py", line 3734, in _check_threadpool_exceptions
2023-03-21T01:58:05.5170496Z     raise ex
2023-03-21T01:58:05.5171214Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/concurrent/futures/thread.py", line 57, in run
2023-03-21T01:58:05.5171999Z     result = self.fn(*self.args, **self.kwargs)
2023-03-21T01:58:05.5173083Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/graph.py", line 3596, in get_or_create_module_for_fingerprint
2023-03-21T01:58:05.5173910Z     raise ex
2023-03-21T01:58:05.5174933Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/graph.py", line 3594, in get_or_create_module_for_fingerprint
2023-03-21T01:58:05.5175844Z     module_id = node._module_builder.build()
2023-03-21T01:58:05.5176863Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/_module_builder.py", line 62, in build
2023-03-21T01:58:05.5177744Z     module_id = self._module_provider.create_module(
2023-03-21T01:58:05.5178781Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/_aeva_provider.py", line 327, in create_module
2023-03-21T01:58:05.5180050Z     module_entity = _AevaModuleProvider.module_creation(module_def, content_path,
2023-03-21T01:58:05.5181204Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/_aeva_provider.py", line 293, in module_creation
2023-03-21T01:58:05.5187263Z     storage_id = module_uploader.upload(directory=content_path)
2023-03-21T01:58:05.5188102Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/pipeline/core/_aeva_provider.py", line 81, in upload
2023-03-21T01:58:05.5188726Z     storage_id = snapshots_client.create_snapshot(directory)
2023-03-21T01:58:05.5189496Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/snapshots_client.py", line 121, in create_snapshot
2023-03-21T01:58:05.5190210Z     revision_list = self._upload_snapshot_files(entries_to_send, file_or_folder_path, exclude_function)
2023-03-21T01:58:05.5191106Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/snapshots_client.py", line 245, in _upload_snapshot_files
2023-03-21T01:58:05.5191781Z     revision_list = self._upload_files_batch(file_nodes, file_or_folder_path, session)
2023-03-21T01:58:05.5192624Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/snapshots_client.py", line 210, in _upload_files_batch
2023-03-21T01:58:05.5193214Z     revision_list.append(file_revision)
2023-03-21T01:58:05.5193920Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_common/async_utils/task_queue.py", line 55, in __exit__
2023-03-21T01:58:05.5194715Z     self.flush(self.identity)
2023-03-21T01:58:05.5195412Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_common/async_utils/task_queue.py", line 118, in flush
2023-03-21T01:58:05.5196088Z     self._results.extend((task.wait(awaiter_name=self.identity) for task in completed_tasks))
2023-03-21T01:58:05.5196929Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_common/async_utils/task_queue.py", line 118, in <genexpr>
2023-03-21T01:58:05.5197590Z     self._results.extend((task.wait(awaiter_name=self.identity) for task in completed_tasks))
2023-03-21T01:58:05.5198725Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_common/async_utils/async_task.py", line 58, in wait
2023-03-21T01:58:05.5199344Z     res = self._handler(self._future, self._logger)
2023-03-21T01:58:05.5200157Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_common/async_utils/async_task.py", line 16, in basic_handler
2023-03-21T01:58:05.5200726Z     return future.result()
2023-03-21T01:58:05.5201196Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/concurrent/futures/_base.py", line 437, in result
2023-03-21T01:58:05.5201681Z     return self.__get_result()
2023-03-21T01:58:05.5202177Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
2023-03-21T01:58:05.5202667Z     raise self._exception
2023-03-21T01:58:05.5203141Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/concurrent/futures/thread.py", line 57, in run
2023-03-21T01:58:05.5203639Z     result = self.fn(*self.args, **self.kwargs)
2023-03-21T01:58:05.5204396Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/snapshots_client.py", line 199, in perform_upload
2023-03-21T01:58:05.5205068Z     return upload_blob_from_stream(data, upload_url, session=session, timeout=TIMEOUT,
2023-03-21T01:58:05.5205887Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_file_utils/upload.py", line 62, in upload_blob_from_stream
2023-03-21T01:58:05.5206440Z     execute_func_with_reset(
2023-03-21T01:58:05.5207150Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/clientbase.py", line 93, in execute_func_with_reset
2023-03-21T01:58:05.5208069Z     return ClientBase._execute_func_internal(backoff, retries, module_logger, func, reset_func, *args, **kwargs)
2023-03-21T01:58:05.5208969Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/clientbase.py", line 367, in _execute_func_internal
2023-03-21T01:58:05.5209635Z     left_retry = cls._handle_retry(back_off, left_retry, total_retry, error, logger, func)
2023-03-21T01:58:05.5210449Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/clientbase.py", line 433, in _handle_retry
2023-03-21T01:58:05.5210954Z     raise error
2023-03-21T01:58:05.5211648Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_restclient/clientbase.py", line 358, in _execute_func_internal
2023-03-21T01:58:05.5212222Z     response = func(*args, **kwargs)
2023-03-21T01:58:05.5212916Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azure/core/tracing/decorator.py", line 78, in wrapper_use_tracer
2023-03-21T01:58:05.5213469Z     return func(*args, **kwargs)
2023-03-21T01:58:05.5214201Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_vendor/azure_storage/blob/_blob_client.py", line 731, in upload_blob
2023-03-21T01:58:05.5214793Z     return upload_block_blob(**options)
2023-03-21T01:58:05.5215581Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_vendor/azure_storage/blob/_upload_helpers.py", line 197, in upload_block_blob
2023-03-21T01:58:05.5216289Z     process_storage_error(error)
2023-03-21T01:58:05.5217121Z   File "/opt/hostedtoolcache/Python/3.8.16/x64/lib/python3.8/site-packages/azureml/_vendor/azure_storage/blob/_shared/response_handlers.py", line 181, in process_storage_error
2023-03-21T01:58:05.5217901Z     exec("raise error from None")   # pylint: disable=exec-used # nosec
2023-03-21T01:58:05.5218311Z   File "<string>", line 1, in <module>
2023-03-21T01:58:05.5218795Z azure.core.exceptions.HttpResponseError: This request is not authorized to perform this operation.
2023-03-21T01:58:05.5219394Z RequestId:c8b26985-201e-0022-3398-5b10a7000000
2023-03-21T01:58:05.5219884Z Time:2023-03-21T01:58:05.3445101Z
2023-03-21T01:58:05.5220252Z ErrorCode:AuthorizationFailure
2023-03-21T01:58:05.5220999Z Content: <?xml version="1.0" encoding="utf-8"?><Error><Code>AuthorizationFailure</Code><Message>This request is not authorized to perform this operation.
2023-03-21T01:58:05.5221693Z RequestId:c8b26985-201e-0022-3398-5b10a7000000
2023-03-21T01:58:05.5222204Z Time:2023-03-21T01:58:05.3445101Z</Message></Error>
2023-03-21T01:58:05.8088397Z ##[error]Script failed with exit code: 1
2023-03-21T01:58:05.8105983Z [command]/usr/bin/az account clear
2023-03-21T01:58:06.7588979Z ##[section]Finishing: Azure CLI 

This works fine when I execute it from inside the Azure ML Studio,

ws = Workspace(
    subscription_id="**********************",
    resource_group="************",
    workspace_name="**************",
    auth=svc_pr
    )

Source_directory="./"


aml_compute_target="xxxxxx"
try:
    aml_compute=AmlCompute(ws,aml_compute_target)
    print("Found existing compute target")
except ComputeTargetException:
    provisoning_config=AmlCompute.provisioning_configuration(vm_size='Standard_F32s_v2',
                                                             min_nodes=1,
                                                             max_nodes=4)
    aml_compute=ComputeTarget.create(ws,aml_compute_target,provisoning_config)
    aml_compute.wait_for_completion(show_output=True,min_node_count=None,timeout_in_minutes=20)

print("Azure Machine Learning Compute Attached")


from azureml.core import Environment

env = Environment.from_conda_specification(name='myCustomEnv', file_path='../parameters/conda_dependencies.yml')
DockerConfiguration(use_docker=True)
env.docker.base_image = 'mcr.microsoft.com/azureml/minimal-ubuntu20.04-py38-cpu-inference:latest'


# # Create Docker Image 


aml_run_config=RunConfiguration()
aml_run_config.target=aml_compute
aml_run_config.environment=env


# # Create steps in Experiement

step1=PythonScriptStep(name="WrapperFunction",
                        script_name="../src/xxxx.py",
                        compute_target=aml_compute,
                        runconfig=aml_run_config,
                        source_directory=Source_directory)



steps_=[step1]


# # Create Pipeline

Tests_Pipeline=Pipeline(workspace=ws,steps=steps_)



# # Validate pipeline



Tests_Pipeline.validate()


# Run Experiem


Tests_Pipeline=Experiment(ws,'BaseOrderData').submit(Tests_Pipeline,regenerate_outputs=True)


# In[13]:


published_pipeline = Tests_Pipeline.publish( name="BaseOrderData_Pipeline_Notebook", description="BaseOrderData Published Pipeline Description", version="1.0")