Get detailed costs in scope of Azureml workspace, using python sdk

Question

Get detailed costs in scope of Azureml workspace, using python sdk

Aziz 0

Hi, how to get detailed costs in scope of Azureml workspace, using python sdk? I need some kind of json repr of the costs by compute cluster names, online endpoints etc..

Aziz 0

@Amira Bedhiafi I modified code a little bit, bc QueryDefinition class required parameters wasn't been provided. Now code looks like this, but I get error:

from azure.mgmt.costmanagement.models import QueryDefinition, QueryTimePeriod
from azure.mgmt.costmanagement import CostManagementClient
from azure.identity import DefaultAzureCredential

from datetime import datetime, timedelta
from azure.ai.ml import MLClient

credential = DefaultAzureCredential()
cost_management_client = CostManagementClient(credential, 
                                              subscription_id=os.environ["SUBSCRIPTION_ID"])
mlclient = MLClient(credential=credential, 
                    subscription_id=os.environ["SUBSCRIPTION_ID"],
                    resource_group_name=os.environ["RESOURCE_GROUP_NAME"],
                    workspace_name=os.environ["WORKSPACE_NAME"])

workspace = mlclient.workspaces.get(os.environ["WORKSPACE_NAME"])


# Define the query time period

end_date = datetime.today().date()

start_date = end_date - timedelta(days=30)

# Define the query

query_definition = QueryDefinition(
    type='Usage',
    timeframe='Custom',
    time_period=QueryTimePeriod(

        from_property=start_date,

        to=end_date

    ),

    dataset={

        "granularity": "None",

        "aggregation": {

            "totalCost": {

                "name": "PreTaxCost",

                "function": "Sum"

            }

        },

        "grouping": [

            {

                "type": "Dimension",

                "name": "ResourceGroupName"

            },

            {

                "type": "Dimension",

                "name": "ResourceId"

            }

        ]

    }

)
# Execute the query

cost_query = cost_management_client.query.usage(workspace.id, query_definition)

# ResourceNotFoundError: Option returned an invalid status 'Not Found'
# Content: The resource you are looking for has been removed, had its name changed, or is temporarily unavailable.

I got ResourceNotFoundError.

Aziz 0 Reputation points

2024-08-06T07:52:30.86+00:00

workspace.id is defined, correct scope string.
Aziz 0 Reputation points

2024-08-06T09:24:12.9233333+00:00
from docs about scope parameter:
""" The scope associated with query and export operations. This includes '/subscriptions/{subscriptionId}/' for subscription scope, '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}' for resourceGroup scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}' for Billing Account scope and '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/departments/{departmentId}' for Department scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/enrollmentAccounts/{enrollmentAccountId}' for EnrollmentAccount scope, '/providers/Microsoft.Management/managementGroups/{managementGroupId} for Management Group scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/billingProfiles/{billingProfileId}' for billingProfile scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/billingProfiles/{billingProfileId}/invoiceSections/{invoiceSectionId}' for invoiceSection scope, and '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/customers/{customerId}' specific for partners. Required.
"""
Issue is in out scope. But if take like this:

scope_ = f'/subscriptions/{os.environ["SUBSCRIPTION_ID"]}/resourceGroups/{resource_group_name}'

It will return whole AML costs as a sum to the workspace.
Aziz 0 Reputation points

2024-08-06T09:49:48.7966667+00:00

from machinelearningservices there are only workspaces itself

1 answer

Your answer

Aziz 0 Reputation points

2024-08-06T07:50:51.59+00:00

@Amira Bedhiafi I modified code a little bit, bc QueryDefinition class required parameters wasn't been provided. Now code looks like this, but I get error:

from azure.mgmt.costmanagement.models import QueryDefinition, QueryTimePeriod from azure.mgmt.costmanagement import CostManagementClient from azure.identity import DefaultAzureCredential from datetime import datetime, timedelta from azure.ai.ml import MLClient credential = DefaultAzureCredential() cost_management_client = CostManagementClient(credential, subscription_id=os.environ["SUBSCRIPTION_ID"]) mlclient = MLClient(credential=credential, subscription_id=os.environ["SUBSCRIPTION_ID"], resource_group_name=os.environ["RESOURCE_GROUP_NAME"], workspace_name=os.environ["WORKSPACE_NAME"]) workspace = mlclient.workspaces.get(os.environ["WORKSPACE_NAME"]) # Define the query time period end_date = datetime.today().date() start_date = end_date - timedelta(days=30) # Define the query query_definition = QueryDefinition( type='Usage', timeframe='Custom', time_period=QueryTimePeriod( from_property=start_date, to=end_date ), dataset={ "granularity": "None", "aggregation": { "totalCost": { "name": "PreTaxCost", "function": "Sum" } }, "grouping": [ { "type": "Dimension", "name": "ResourceGroupName" }, { "type": "Dimension", "name": "ResourceId" } ] } ) # Execute the query cost_query = cost_management_client.query.usage(workspace.id, query_definition) # ResourceNotFoundError: Option returned an invalid status 'Not Found' # Content: The resource you are looking for has been removed, had its name changed, or is temporarily unavailable.

I got ResourceNotFoundError.
Aziz 0 Reputation points

2024-08-06T07:52:30.86+00:00

workspace.id is defined, correct scope string.
Aziz 0 Reputation points

2024-08-06T09:24:12.9233333+00:00

from docs about scope parameter:
""" The scope associated with query and export operations. This includes '/subscriptions/{subscriptionId}/' for subscription scope, '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}' for resourceGroup scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}' for Billing Account scope and '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/departments/{departmentId}' for Department scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/enrollmentAccounts/{enrollmentAccountId}' for EnrollmentAccount scope, '/providers/Microsoft.Management/managementGroups/{managementGroupId} for Management Group scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/billingProfiles/{billingProfileId}' for billingProfile scope, '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/billingProfiles/{billingProfileId}/invoiceSections/{invoiceSectionId}' for invoiceSection scope, and '/providers/Microsoft.Billing/billingAccounts/{billingAccountId}/customers/{customerId}' specific for partners. Required.
"""
Issue is in out scope. But if take like this:

scope_ = f'/subscriptions/{os.environ["SUBSCRIPTION_ID"]}/resourceGroups/{resource_group_name}'

It will return whole AML costs as a sum to the workspace.
Aziz 0 Reputation points

2024-08-06T09:49:48.7966667+00:00

from machinelearningservices there are only workspaces itself

Answer 1

You need the azure-mgmt-costmanagement library and azureml-core SDK.


pip install azure-mgmt-costmanagement azureml-core

You'll need to authenticate using your Azure credentials and initialize the required clients. This example uses a service principal for authentication.


from azure.identity import ClientSecretCredential

from azure.mgmt.costmanagement import CostManagementClient

from azureml.core import Workspace

import os

# Set your Azure credentials

TENANT_ID = os.getenv("AZURE_TENANT_ID")

CLIENT_ID = os.getenv("AZURE_CLIENT_ID")

CLIENT_SECRET = os.getenv("AZURE_CLIENT_SECRET")

SUBSCRIPTION_ID = os.getenv("AZURE_SUBSCRIPTION_ID")

RESOURCE_GROUP = os.getenv("AZURE_RESOURCE_GROUP")

WORKSPACE_NAME = os.getenv("AZURE_WORKSPACE_NAME")

# Authenticate using service principal

credential = ClientSecretCredential(tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET)

# Initialize Cost Management client

cost_management_client = CostManagementClient(credential, subscription_id=SUBSCRIPTION_ID)

# Initialize AzureML workspace

workspace = Workspace(subscription_id=SUBSCRIPTION_ID, resource_group=RESOURCE_GROUP, workspace_name=WORKSPACE_NAME)

Now, you can query the cost data. You can use the QueryClient from the azure-mgmt-costmanagement library to get cost details.


from azure.mgmt.costmanagement.models import QueryDefinition, QueryTimePeriod

from datetime import datetime, timedelta

# Define the query time period

end_date = datetime.today().date()

start_date = end_date - timedelta(days=30)

# Define the query

query_definition = QueryDefinition(

    time_period=QueryTimePeriod(

        from_property=start_date,

        to=end_date

    ),

    dataset={

        "granularity": "None",

        "aggregation": {

            "totalCost": {

                "name": "PreTaxCost",

                "function": "Sum"

            }

        },

        "grouping": [

            {

                "type": "Dimension",

                "name": "ResourceGroupName"

            },

            {

                "type": "Dimension",

                "name": "ResourceId"

            }

        ]

    }

)

# Execute the query

cost_query = cost_management_client.query.usage(workspace.id, query_definition)

Finally, process the results and convert them to JSON .


import json

# Process the results

cost_details = []

for row in cost_query.rows:

    cost_details.append({

        "ResourceGroupName": row[0],

        "ResourceId": row[1],

        "TotalCost": row[2]

    })

# Convert to JSON

cost_details_json = json.dumps(cost_details, indent=4)

print(cost_details_json)

Share via

Get detailed costs in scope of Azureml workspace, using python sdk

1 answer

Your answer