Gray Agents

# Gets all Gray Agents and finds which are hosted in Azure (IaaS VMs), VMM, or other/Unknown.

# Puts list of Gray agent Name and Comment (Azure VM, VMM Host, Server is Running, Server is Hung, ... etc.)

# into Results.csv. Writes gray agent breakdown of total counts in Report.txt. Sends email of HTML report.

#

# Laura.Park@microsoft.com

# August 2017





### Prep

$starttime = get-date



#load VMM module

cd C:\temp\VMM_Module\bin\psModules

Import-Module .\virtualmachinemanager

Get-SCVMMServer -ComputerName vmm1.contoso.com -setasdefault



#load SCOM module

import-module operationsmanager

$connect = New-SCOMManagementGroupConnection –ComputerName managementserver1.contoso.com

####



# variables

$CSVFilePath = "C:\temp\GrayAgentResults$(get-date -f yyyy-MM-dd_HH-mm).csv"  ### Replace with desired file path

$TXTFilePath = "C:\temp\GrayAgentReport$(get-date -f yyyy-MM-dd_HH-mm).txt"   ### Replace with desired file path

$tempTest = 'C:\temp\winrmjob.log'; #log for WinRM tests



# Email variables

$EmailServer =  "smtp.contoso.com"

$EmailTo = "MONEngineers@contoso.com, Admins@contoso.com"

$EmailFrom = "SCOM-HealthCheckReport@contoso.com"

#

$whoami = whoami #account running this script



$colMonitoringObjects = @()



$WindowsAgentClass = Get-SCOMClass -Name "Microsoft.SystemCenter.Agent"

$UXClass = Get-SCClass -name "Microsoft.Unix.Computer"



$WindowsAgents = Get-SCOMMonitoringObject -Class $WindowsAgentClass

$UnixAgents = Get-SCOMMonitoringObject -Class $UXClass

$AgentlessMonitored = (Get-SCOMAgentlessManagedComputer).Computer



$colMonitoringObjects += $WindowsAgents

$colMonitoringObjects += $UnixAgents

$colMonitoringObjects += $AgentlessMonitored

#

$colVMMmanagedcomputers = Get-SCVMMManagedComputer # VMM hosts

$colVMMmanagedcomputers += Get-SCvirtualmachine    # VMM vms



# all Gray Agents in SCOM

$colGrayAgents = $colMonitoringObjects | where {$_.IsAvailable –eq $false} # Mon Obj type



$colVMMGrayAgents = @()       # VMM obj type

$colVMMGrayAgentsMonObj = @() # Mon Obj type

$colAzureGrayAgents = @()     # Mon Obj type

$colUnknownGrayAgents = @()   # DisplayName



$colVMMRunningTEMP = @() # before we know whether a VM is really Running or Hung

$colVMMRunning = @()     # VMM obj type

$colVMMHung = @()        # VMM obj type

$colVMMOtherStatus = @() # VMM obj type

$colVMMHosts = @()       # VMM obj type



# count variables for final Report

$countVMMGrayAgents = 0



# possible gray state causes

$countAzureGrayAgents = 0 # Azure VM

$countUnknownGrayAgents = 0 # Unknown (not in Azure, not in VMM)

$countVMMHung = 0 # hung

$countVMMRunning = 0 # running

$countVMMOtherStatus = 0 # VM is a state other than "Running" or [empty]

$countVMMHosts = 0 # VMM host  (empty status)

#

# Find source (VMM, Azure, neither/unknown) of gray agents

foreach ($agent in $colGrayAgents)

{

    # Separate Azure-hosted gray agents based on naming convention (if applicable)

    if ($agent.DisplayName -like "AZURE*")

    {

        Write-debug "Skipping AzureVM : $($agent.DisplayName)"

        $countAzureGrayAgents++

        $colAzureGrayAgents += $agent

       

    } else { # VMM-hosted gray agents    

        foreach ($vm in $colVMMmanagedcomputers)

        {

            if ($vm.Name -eq $agent.DisplayName)

            {

                $colVMMGrayAgents += $vm

                $colVMMGrayAgentsMonObj += $agent 

                $countVMMGrayAgents++



                Write-verbose "Found #$countVMMGrayAgents gray agent in VMM : $($vm.name) :: VMM Status: $($vm.status)"



                if ($vm.status -eq "Running")

                {

                    $colVMMRunningTEMP += $vm # "TEMP" because need to check with WinRM if VM is actually Running or Hung

               

                } elseif ($vm.status -eq $null)

                {

                    $colVMMHosts += $vm

                    $countVMMHosts = $colVMMHosts.count

                } else

                {

                    $colVMMOtherStatus += $vm

                    $countVMMOtherStatus = $colVMMOtherStatus.count

                }  

                break #inner foreach loop to move onto searching source of next gray $agent

            }

        }

        # if we get to this point, $agent not found in Azure nor VMM

        #$colUnknownGrayAgents += $agent  # see next statement, more efficient

    }

}



# Gray agents that are not hosted in Azure or VMM

$colUnknownGrayAgents = Compare -Ref $colGrayAgents -Diff ($colVMMGrayAgentsMonObj + $colAzureGrayAgents) -Property DisplayName | foreach DisplayName

#display names only

$countUnknownGrayAgents = $colUnknownGrayAgents.count



# How long did this script take to up to this point?

$CheckTime1=Get-Date

$Check1RunTime=$CheckTime1-$StartTime



write-verbose "Completed finding source (VMM, Azure, unknown) of all gray agents"

write-verbose " - Checkpoint 1 Runtime: $($Check1RunTime.hours) hrs $($Check1RunTime.minutes) min $($Check1RunTime.seconds) seconds"



#### Write Partial Results to .csv File

# - up to this point we have all Gray Agents in various $col* variables

# - still need to check the "Running" VMM VMs

# - write the current results up to this point to file



# write the Azure gray agent VMs to Results.csv

$temp1 = @()

$colAzureGrayAgents | ForEach-Object {

    $row1 = "" | Select "Name","Comment"

    $row1."Name" = $_.DisplayName

    $row1."Comment" = "Azure VM"

    $temp1 += $row1

}

$temp1 | Export-Csv $CSVFilePath -NoTypeInformation



# write the Unknown VMs (not Azure or VMM) to Results.csv

$temp2 = @()

$colUnknownGrayAgents | ForEach-Object {

    $row2 = "" | Select "Name","Comment"

    $row2."Name" = $_# .DisplayName

    $row2."Comment" = "Unknown (not Azure or VMM)"

    $temp2 += $row2

}

$temp2 | Export-Csv $CSVFilePath -Append -NoTypeInformation



# write the VMM Hosts to Results.csv

$temp3 = @()

$colVMMHosts | ForEach-Object {

    $row3 = "" | Select "Name","Comment"

    $row3."Name" = $_.name # VMM obj type

    $row3."Comment" = "VMM Host"

    $temp3 += $row3

}

$temp3 | Export-Csv $CSVFilePath -Append -NoTypeInformation



# write the VMM other status ("Paused", "Saved", "PowerOff", etc.) to Results.csv

$temp4 = @()

$colVMMOtherStatus | ForEach-Object {

    $row4 = "" | Select "Name","Comment"

    $row4."Name" = $_.name # VMM obj type

    $row4."Comment" = $_.status

    $temp4 += $row4

}

$temp4 | Export-Csv $CSVFilePath -Append -NoTypeInformation

#### End of writing partial results



# How long did this script take to up to this point?

$CheckTime2=Get-Date

$Check2RunTime=$CheckTime2-$StartTime



write-verbose "Completed writing partial results to file. Beginning VMM running/hung state checks. This may take a while."

write-verbose " - Checkpoint 2 Runtime: $($Check2RunTime.hours) hrs $($Check2RunTime.minutes) min $($Check2RunTime.seconds) seconds"



####

# WinRM check on "Running" VMM VMs

$originalErrActionPref = $ErrorActionPreference

$ErrorActionPreference = "SilentlyContinue"



$checkcount = 0



#region WinRM Test - Remote Jobs (Batch/Parallel invocation) -- (thank you Arlu!)

try {

    $remoteJobQueue = [System.Collections.Queue]::Synchronized((New-Object System.Collections.Queue));

    $colVMMRunningTEMP | % {$remoteJobQueue.Enqueue($_);}

    $maxAllowed = 30; #number of jobs allowed per batch

    $maxTime = 15; #max time allowed for the whole job process (minutes)

    $allJobs = @();

    $jobArray = @();

    $timeOut = [datetime]::Now.AddMinutes($maxTime);

    $batchNumber = 1;

    while ($remoteJobQueue.Count -gt 0 -and ([datetime]::Now -lt $timeOut)) {

        Write-Verbose "Starting batch $batchNumber. $($remoteJobQueue.Count) remaining in queue." -Verbose;

        $jobArray = @();

        while ($jobArray.Count -lt $maxAllowed) {

            if ($remoteJobQueue.Count -eq 0) {

                Write-Verbose "Queue is empty" -Verbose;

                break;

            }

            $remoteVm = $remoteJobQueue.Dequeue();

            Write-Verbose "Starting test for $($remoteVm.Name). $($remoteJobQueue.Count) remaining in the queue." -Verbose;

            $jobArray += Invoke-Command -ComputerName $remoteVm.Name -ScriptBlock { return $true } -AsJob;

            Write-Verbose "Remote job count: $($jobArray.Count)" -Verbose;

        }

        Write-Verbose "Waiting for batch $batchNumber to complete" -Verbose;

        $null = Wait-Job -Job $jobArray -Timeout 120 -ErrorAction SilentlyContinue;

        Write-Verbose "All jobs in batch $batchNumber are completed." -Verbose;

        $allJobs += $jobArray;   

        $batchNumber++;

    }



    #Get result from all remote jobs.

    foreach ($job in $allJobs) {

        if ($job.state -eq 'Completed') {

            $colVMMRunning += $colVMMRunningTEMP | ? { $_.Name -eq $job.Location };

            Write-Verbose "WinRM connection successful - $($job.Location) server is running";

        }

        else {       

            $colVMMHung += $colVMMRunningTEMP | ? { $_.Name -eq $job.Location };

            Write-Verbose "WinRM connection failed - $($job.Location) server is hung";

        }

    }

    Get-Job | Remove-Job -Force -ErrorAction SilentlyContinue;

}

catch {   

    $errorMsg =  "Error encounter during WinRM test process: $($_.Exception.Message)";

    "{0}: {1}" -f [datetime]::Now.ToString('MMddyyy:hh:mm:ss'),$errorMsg | Out-File $tempTest

    throw "Error encounter during WinRM test process: $($_.Exception.Message)";

}

#endregion



$countVMMRunning = $colVMMRunning.count

$countVMMHung = $colVMMHung.count



$ErrorActionPreference = $originalErrActionPref

### End of WinRM check



Write-verbose "Completed all $checkcount VMM running/hung checks"

$CheckTime3=Get-Date

$Check3RunTime=$CheckTime3-$StartTime

write-verbose " - Checkpoint 3 Runtime: $($Check3RunTime.hours) hrs $($Check3RunTime.minutes) min $($Check3RunTime.seconds) seconds"



### Write final results to .csv file

$temp5 = @()

$colVMMRunning | ForEach-Object {

    $row5 = "" | Select "Name","Comment"

    $row5."Name" = $_.name # VMM obj type

    $row5."Comment" = "Server is running"

    $temp5 += $row5

}

$temp5 | Export-Csv $CSVFilePath -Append -NoTypeInformation



$temp6 = @()

$colVMMHung | ForEach-Object {

    $row6 = "" | Select "Name","Comment"

    $row6."Name" = $_.name # VMM obj type

    $row6."Comment" = "Server is hung"

    $temp6 += $row6

}

$temp6 | Export-Csv $CSVFilePath -Append -NoTypeInformation

### End of writing results to .csv file



### Write .txt Report with final counts

$txt = "Total Agents in SCOM:  $($colMonitoringObjects.count - $AgentlessMonitored.count)"

$txt += "`nTotal Gray Agent count*:  $($colGrayAgents.count)"

$txt += "`n(*may include gray Agentless objects)`n"

$txt += "`nBreakdown of Gray Agents:"



if ($countVMMGrayAgents -ge $countAzureGrayAgents){

    # most gray agents are in VMM

    $txt += "`n`t$countVMMGrayAgents gray agents in VMM. Further breakdown:"

    $txt += "`n`t`t- $countVMMHosts are VMM hosts"

    $txt += "`n`t`t- $countVMMHung are hung"

    $txt += "`n`t`t- $countVMMRunning are Running"

    $txt += "`n`t`t- $countVMMOtherStatus are in a state other than Running (e.g. Saved, Paused, etc.)"

    $txt += "`n`n`t$countAzureGrayAgents gray agents in Azure."

} else { # most gray agents are in Azure

    $txt += "`n`t$countAzureGrayAgents gray agents in Azure."

    $txt += "`n`n`t$countVMMGrayAgents gray agents in VMM. Further breakdown:"

    $txt += "`n`t`t- $countVMMHosts are VMM hosts"

    $txt += "`n`t`t- $countVMMHung are hung"

    $txt += "`n`t`t- $countVMMRunning are Running"

    $txt += "`n`t`t- $countVMMOtherStatus are in a state other than Running (e.g. Saved, Paused, etc.)"

}



$txt += "`n`n`t$countUnknownGrayAgents gray agents are not in Azure, not in VMM"

$txt += "`n`nEnd of report"

$txt | out-file $TXTFilePath

### End of writing .txt Report



### Interactive output



# see matched VMM vms name and status in VMM

write-verbose "`nGray agents in VMM and their Status:`n"

$colVMMGrayAgents | select name, status | Format-Table



write-verbose "`nGray agents in Azure:`n"

$colAzureGrayAgents | select displayname | Format-Table



# including Agent + Agentless

write-verbose "`nFound $($colMonitoringObjects.count) agents/agentless in SCOM ( $($WindowsAgents.count) Windows, $($UnixAgents.count) Unix, $($AgentlessMonitored.count) Agentless)"



# not including Agentless

write-verbose "Found $($colMonitoringObjects.count - $AgentlessMonitored.count) agents in SCOM ( $($WindowsAgents.count) Windows, $($UnixAgents.count) Unix, not including Agentless)"



Write-verbose "Found $($colGrayAgents.count) gray agents in SCOM"

#Write-verbose "Found $($colVMMmanagedcomputers.count) VMs in VMM"

Write-verbose "Found $countVMMGrayAgents gray agents in VMM"

Write-verbose "Found $countAzureGrayAgents agents in Azure"

# count of agents not in VMM or Azure

$countUnknownGrayAgents = $colGrayAgents.count - ($countVMMGrayAgents + $countAzureGrayAgents)

Write-verbose "Found $countUnknownGrayAgents gray agents not in VMM or Azure (unknown)"



###

# How long did this script take to up to this point?

$CheckTime5=Get-Date

$Check5RunTime=$CheckTime5-$StartTime



write-verbose "Completed writing results to file. Beginning email report."

write-verbose " - Checkpoint 5 Runtime: $($Check5RunTime.hours) hrs $($Check5RunTime.minutes) min $($Check5RunTime.seconds) seconds"



# Create header for HTML Report

$Head = "<style>"

$Head +="BODY{background-color:#CCCCCC;font-family:Verdana,sans-serif; font-size: small;}"

$Head +="TABLE{border-width: 1px;border-style: solid;border-color: black;border-collapse: collapse; width: 98%;}"

$Head +="TH{border-width: 1px;padding: 0px;border-style: solid;border-color: black;background-color:#293956;color:white;padding: 5px; font-weight: bold;text-align:left;}"

$Head +="TD{border-width: 1px;padding: 0px;border-style: solid;border-color: black;background-color:#F0F0F0; padding: 2px;}"

$Head +="</style>"



#beginning of HTML content:



#Section to insert the Script Generic info

$ReportOutput += "<h2>Gray Agent Report Info</h2>"

$ReportOutput += "<p>Script Start Time      :  $StartTime</p>"

$ReportOutput += "<p>User Account Used To Produce Report      :  $WhoAmI</p>"



$ReportOutput += "<br>"

$ReportOutput += "<h2>Gray Agents</h2>"



$ReportOutput += "<p>Total Agents in SCOM:  $($colMonitoringObjects.count - $AgentlessMonitored.count)</p>"

$ReportOutput += "<p>Total Gray Agent count:  $($colGrayAgents.count)</p>"

$ReportOutput += "<br>"

$ReportOutput += "<p>$countVMMGrayAgents gray agents in VMM.</p>"

$ReportOutput += "<p>-  $countVMMHosts are VMM hosts</p>"

$ReportOutput += "<p>-  $countVMMHung are hung</p>"

$ReportOutput += "<p>- $countVMMRunning are Running</p>"

$ReportOutput += "<p>- $countVMMOtherStatus are in a state other than Running (e.g. Saved, Paused, etc.)</p>"

$ReportOutput += "<p>$countAzureGrayAgents gray agents in Azure.</p>"

$ReportOutput += "<p>$countUnknownGrayAgents gray agents are not in Azure, not in VMM</p>"

$ReportOutput += "<br>"





# Gray Agent Table

$ReportOutput += ($temp1 + $temp2 + $temp3 + $temp4 + $temp5 + $temp6) | Sort-Object Name  | Select Name, Comment| ConvertTo-HTML -fragment



# How long did this script take to run?

$EndTime=Get-Date

$TotalRunTime=$EndTime-$StartTime



# Close the Body of the Report

$ReportOutput += "<br>"

$ReportOutput += "<p>Total Script Run Time: $($TotalRunTime.hours) hrs $($TotalRunTime.minutes) min $($TotalRunTime.seconds) sec</p>"

$ReportOutput += "</body>"



# Save the Final Report to a File (HTML)

#ConvertTo-HTML -head $Head -body "$ReportOutput" | Out-File $ReportPath



# Send Final Report by email...

Write-verbose "Emailing Report"

$SMTPServer = $EmailServer

$SmtpClient = New-Object Net.Mail.SmtpClient($smtpServer)



$Body = ConvertTo-HTML -head $Head -body "$ReportOutput"

$mailmessage = New-Object system.net.mail.mailmessage

$mailmessage.from = $EmailFrom

$mailmessage.To.add($EmailTo)



$mailmessage.Subject = "SCOM Gray Agent Report"

$MailMessage.IsBodyHtml = $true

$mailmessage.Body = $Body

#$smtpclient.Send($mailmessage)

Send-MailMessage -To $mailmessage.To -Subject $mailmessage.Subject -From $mailmessage.From -Body $mailmessage.Body -SmtpServer $EmailServer -BodyAsHtml -Encoding $mailmessage.BodyEncoding



# How long did this script take to up to this point?

$CheckTime6=Get-Date

$Check6RunTime=$CheckTime6-$StartTime



write-verbose "Script complete."

write-verbose " - Total Runtime: $($Check6RunTime.hours) hrs $($Check6RunTime.minutes) min $($Check6RunTime.seconds) seconds"

# end of script

# Script to delete decommissioned servers (where uninstall is not possible)
# Laura.Park@microsoft.com
# May 2017
# Based off of https://www.scom2k7.com/how-to-delete-scom-2012-r2-managed-computers-using-powershell/

[CmdletBinding()] 
param (
    [Parameter(Mandatory=$True, 
    ValueFromPipeline=$True, 
    ValueFromPipelineByPropertyName=$True, 
    HelpMessage='What is the SCOM Management Server name?')] 
    [Alias("MS")] 
    [string[]]$MSserver,

    [Parameter(Mandatory=$True, 
    ValueFromPipeline=$True, 
    ValueFromPipelineByPropertyName=$True, 
    HelpMessage='Allows you to specify a text file containing agent FQDNs')] 
    [string[]]$Path
)

[System.Reflection.Assembly]::Load("Microsoft.EnterpriseManagement.Core, Version=7.0.5000.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35")
[System.Reflection.Assembly]::Load("Microsoft.EnterpriseManagement.OperationsManager, Version=7.0.5000.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35")

function New-Collection ( [type] $type ) 
{
    $typeAssemblyName = $type.AssemblyQualifiedName;
    $collection = new-object "System.Collections.ObjectModel.Collection``1[[$typeAssemblyName]]";
    return ,($collection);
}

$deleteList = get-content -Path $path

write-output "File contains $($deleteList.count) server names"

# Connect to management group
Write-output "Connecting to management group"

$ConnectionSetting = New-Object Microsoft.EnterpriseManagement.ManagementGroup($MSServer)
$admin = $ConnectionSetting.GetAdministration()


Write-output "Getting agent managed computers"
$agentManagedComputers = $admin.GetAllAgentManagedComputers()

# Get list of agents to delete
foreach ($name in $deleteList) 
{
    Write-output "Checking for $name"
    foreach ($agent in $agentManagedComputers)
    {
        if ($deleteCollection -eq $null) 
        {
            $deleteCollection = new-collection $agent.GetType()
        }

        if (@($agent.PrincipalName -eq $name))
        {
            Write-output "- Matched $name"
            $deleteCollection.Add($agent)
            break
        }
    }
}

if ($deleteCollection.Count -gt 0) 
{
    Write-output "Deleting agents"
    $admin.DeleteAgentManagedComputers($deleteCollection)
    if($?){ Write-output "$($deleteCollection.count) Agents deleted" }
}

# See what version(s) of PowerShell are installed, if any
# v2, v3+, and Core can all exist side-by-side

$originalErrorActionPref = $ErrorActionPreference
$ErrorActionPreference = "SilentlyContinue"

#Reg Installation path for v1 and v2 - does NOT mean it's actually available though
$regPathV1or2 = "hklm:\SOFTWARE\Microsoft\PowerShell\1\PowerShellEngine\"

# Reg path for v3 and above
$regPathV3plus = "hklm:\SOFTWARE\Microsoft\PowerShell\3\PowerShellEngine\"

# Reg path for Core
$regPathVCore = "hklm:\SOFTWARE\Microsoft\PowerShellCore\InstalledVersions"

$vOutput = @{ name="";
    v1="";
    v2="";
    v3="";
    v4="";
    v5="";
    v6="";
}

# test for v1 or v2
if (test-path $regPathV1or2)
{
    [string]$versionA = (Get-ItemProperty -path $regPathV1or2).powershellversion

    if ($PSVersionTable.PSVersion.Major -le $versionA)
    { # running v1 or v2 only
        if ($PSVersionTable.PSVersion.Major -eq 1) {
            $vOutput.v1 = $PSVersionTable.PSVersion.ToString()
        } else {
            $vOutput.v2 = $PSVersionTable.PSVersion.ToString()
        }
    } else {
        # machines with v2 only don't have -psversion parameter
        $job = Start-Job {Get-DateTime} -PSVersion 2.0
        if($?) {
             $vOutput.v2 = $versionA
        } else {# no v2 installed
           remove-job $job.id
        }
    }
}

# test for v3 or above
if (test-path $regPathV3plus) 
{
    [string]$versionB = (Get-ItemProperty -path $regPathV3plus).powershellversion

    if($PSVersionTable.PSVersion.Major -eq 3) {
        $vOutput.v3 = $versionB
    } elseif($PSVersionTable.PSVersion.Major -eq 4) {
        $vOutput.v4 = $versionB
    } elseif($PSVersionTable.PSVersion.Major -eq 5) {
        $vOutput.v5 = $versionB
    }
}

# test for Core ----------test-------------------
if (test-path $regPathVCore)
{
    $vOutput.v6 = "Core"
}


$vOutput.name = "$(hostname).$((Get-WmiObject win32_computersystem).Domain)"

#Write-Output $vOutput
write-output "$($vOutput.name);$($vOutput.v1);$($vOutput.v2);$($vOutput.v3);$($vOutput.v4);$($vOutput.v5);$($vOutput.v6)"

$ErrorActionPreference = $originalErrorActionPref

• Monitors based on Availability, Performance, Configuration, Security
  • AD Forest, Domain, Site, SiteLink, Connection object, Domain controller computer
• Key Monitoring Scenarios include:
  • Multi-Forest Monitoring
  • Replication
  • Essential Services
  • Trust Monitoring
  • Directory Service Availability
  • Active Directory Database Monitoring
  • Time Skew Monitoring
  • Active Directory Web Service Monitoring
  • Domain Controller Performance
  • Domain Member Perspective

• Azure Resource Health (for IaaS VM): https://docs.microsoft.com/en-us/azure/service-health/resource-health-overview
• OMS offers a wide variety of solutions and on-demand assessments, including AD Assessment and AD Replication Status, and other features beyond the scope of AD monitoring
  • AD Assessment (solution) will assess the risk and health of your server environments on a regular interval.  This solution provides a prioritized list of recommendations specific to your deployed server infrastructure. The recommendations are categorized across four focus areas, which help you quickly understand the risk and take action.
  • AD Replication Status (solution) regularly monitors your Active Directory environment for any replication failures
  • On-Demand Assessment’s Services Hub portal, Executive Summary presentations, and optional PFE engineer delivery
  • https://aka.ms/assessment_setup_service
• Azure AD Connect Health: https://docs.microsoft.com/en-us/azure/active-directory/connect-health/active-directory-aadconnect-health
  • Monitoring alerts to detect when domain controllers are unhealthy and email notifications for critical alerts
  • The Domain Controllers dashboard, which provides a quick view of the health and operational status of your domain controllers
  • The Replication Status dashboard that has the latest replication information and links to troubleshooting guides when errors are detected
  • Quick anywhere access to performance data graphs of popular performance counters, which are necessary for troubleshooting and monitoring purposes

• Collects:
  • performance counters and events, including (not full list):
    • AD Database size
    • Primary domain controller (PDC) emulator master availability
    • Replication latency or queue size
    • Available disk space for drive hosting the Active Directory database

• OMS can collect:
  • Performance data
  •  Windows event logs
  • Custom logs
  • IIS logs
  • Syslog events
  • AD Assessment (solution):
    • Registry
    • LDAP
    • .NET Framework
    • Event log
    • Active Directory Service interfaces (ADSI)
    • Windows PowerShell
    • File data
    • Windows Management Instrumentation (WMI)
    • DCDIAG tool API
    • File Replication Service (NTFRS) API
    • Custom C# code
• Azure AD Connect Health can collect:
  • Disk Space Usage
  • Domain Services Threads in Use
  • Kerberos Authentications/sec
  • LDAP Active Threads
  • LDAP Bind Time
  • LDAP Searches/sec
  • LDAP Successful Binds/sec
  • LSASS Performance
  • NTML Authentication/sec
  • Replication Queue
  • TCP Connections Established
  • Used Memory (%)
  • Used Processor (%)

• Monitor-based alerts
• Event-based alerts
• Performance (metric)-based alerts
• Typically <1 minute alerting latency

• Metric Alerts (new) by Azure Monitor:
  • includes OMS logs
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-unified-alerts
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-near-real-time-metric-alerts
  • 1-5 minute current latency
• Azure Monitor classic alerts
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/insights-alerts-portal
• OMS:
  • Includes routed Diagnostics logs
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-alerts
  • SLA of 6hrs; usually 10-15 min latency
  • Log Analytics alerts will automatically be extended into the new unifed alerts (see above) on May 14, 2018 https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-alerts-extend
  • AD Assessment (solution) collects data on the domain controller and forwards data to Log Analytics every 7 days
  • AD Replication Status (solution) collects data on the agent and forward data to Log Analytics every 5 days

• Use the SCOM console or web console
• Built-in management pack dashboards, views:
  • Active Alerts, Events, All Performance Data, Health State views
  • Topology diagram
  • Inter-Site Replication Traffic
  • Replication performance
  • ATQ Thread Pool Metrics
  • Database and Log Overview
  • Database Size
  • DC/GC Response
  • DC OS Metrics Overview
  • DC Response Time
  • AD Database and Log Disk Space
  • Global Catalog Response/Search Time
  • PDC Response Times
  • LSASS Processor Time
  • Memory Metrics
  • OpMaster Performance
• Built-in management pack reports:
  • AD DC Replication Bandwidth
  • DC Disk Space Chart
  • AD Domain Changes
  • Event Analysis by Event ID
• Ability to custom author dashboards, views, reports
• Ability to directly query OpsMgr database and datawarehouse

• Azure AD Connect Health:
  • Domain Controllers Dashboard
  • Replication Status Dashboard
  • Performance Monitors List
• OMS:
  • AD Assessment (solution)
    • Summary information for focus areas is shown on the AD Health Check dashboard for the infrastructure in your environment
  • AD Replication Status (solution)
    • Replication Errors,  Destination Server Status and Source Server Status, Tombstone Lifetime
  • Assessment solutions:
    • Services Hub portal
    • Executive Summary presentation (PowerPoint)
    • Recommendations Plan report (Excel)
  • Log Analytics
  • View Designer
  • PowerBI
  • Service Map

• Installing and tuning the AD management pack
  • With both SCOM and Azure, a key part of monitoring is knowing what you want to monitor/collect

• Install agent for Azure AD Connect Health for AD DS
  • https://docs.microsoft.com/en-us/azure/active-directory/connect-health/active-directory-aadconnect-health-agent-install#installing-the-azure-ad-connect-health-agent-for-ad-ds
  • Requirements:  https://docs.microsoft.com/en-us/azure/active-directory/connect-health/active-directory-aadconnect-health-agent-install#requirements
• Enabling OMS
  • Perform health check using AD Assessment (Solution): https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-ad-assessment
  • Install and configure the AD Replication Status (solution), install agents to send the replication data to Log Analytics:  https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-ad-replication-status#ad-replication-status-data-collection-details
  • Assessment solutions are automatically refreshed with new data and recommendations every week
  • For Windows VMs, add the extension to the VM Resource Manager template or use PowerShell: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/extensions-oms
  • For Linux VMs, add the extension to the VM Resource Manager template or use Azure CLI: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/extensions-oms
  • Configure data sources from the Data menu in Log Analytics Advanced Settings (any configuration is delivered to all connected sources in your workspace)

• Two databases: OpsMgr database and Data Warehouse
• OpsMgr database holds 7 days by default
• Data Warehouse holds 400 days by default

• Azure AD Connect Health does not generate reports, perform analytics, or provide insights beyond 30 days. Therefore, Azure AD Connect Health does not store, process, or retain any data beyond 30 days. This design is compliant with the GDPR regulations, Microsoft privacy compliance regulations, and Azure AD data retention policies.
• Log Analytics has a retention age of 31 to 730 days

• Determine the seed discovery class for the management pack. This is the first class that gets populated after the MP is imported.
  • To do this, look up all the discovery rules in the MP. You can find these discovery rules by finding your MP on SystemCenterCore.com or use the MPViewer tool. In this example, I'll use SystemCenterCore.com.
  • We're looking for the seed discovery rule. The seed discovery rule usually says “seed” somewhere in the name. But if this is missing, you can also find it because it’s typically the only discovery that targets a more common class:
    • Here's a screenshot from SystemCenterCore.com (note, this is a non-Microsoft website) showing all the discoveries in the Microsoft BizTalk Server 2013 MP
    • Look at the "Target" column for all these discoveries:
      • 
    • We see one discovery rule that targets Windows Computers, a 'common' class. This is our seed discovery; the one that all the other discovery rules depend on.
• Find the Class name that the seed discovery rule discovers/populates (this is NOT the same as the Target)
  • 
  • In this example, the class name is Microsoft.BizTalk.Server.2013.BizTalkInstallation

• • In the SCOM console, Monitoring Pane, Discovered Inventory view – Change Target Type (see right-side Task pane) of this view to the class name you just found. This will show us if the seed class has populated.
    • 
  • If there are no results, the seed discovery is not working. If there are results, the seed discovery IS working, but perhaps a subsequent discovery rule is not, or the other view you were using is not populating correctly.
    • Check that the discovery rules are enabled
    • Check if the MP uses a Run As account: check the Distribution of the account and all appropriate agents have this account distributed to them
    • Look in the OpsMgr event log on an agent and verify it has downloaded the MP (filter for event ID 1201 - the event description includes MP name and version)
    • Check the frequency/interval of the discovery rule – has enough time passed?
    • Finally, after confirming everything above – look at the actual configuration of the discovery rule. You can see this in the Authoring pane, MPViewer, or SystemCenterCore.com. The raw XML will show you what the rule looks for on the agent. Usually it’s a registry key. Verify the agent has this.
      • <Continuing the example with screenshots from SystemCenterCore.com:>
      • 
    • At this point, if everything is verified – Run As Account, healthy Agent, existence of registry key…you’re best off opening a support case because something is going wrong! Possibly the management group has stopped processing configuration. But in my experience, 98% of the time, some step above has been missed. Double check everything.

• Monitors based on Availability, Performance, Configuration, Security

• Azure Resource Health: https://docs.microsoft.com/en-us/azure/service-health/resource-health-overview
• SCOM vs OMS Log Analytics: https://docs.microsoft.com/en-us/azure/operations-management-suite/operations-management-suite-monitoring-product-comparison
• OMS offers a wide variety of other solutions and assessments, including Service Map, and other features beyond the scope of OS monitoring

• Collects:
  • performance counters,
  • event logs,
  • probe/script-based rules

• Use Azure Monitor for host-level monitoring: https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-metrics
• Use Azure Diagnostics or OMS for guest VM operating system monitoring
• Azure Diagnostics can collect:
  • Performance data
  • Application logs
  • Windows Event logs
  • Custom error logs created by app or service
  • .NET Event source
  • IIS logs
  • ETW traces
  • Crash dumps
  • Azure diagnostics infrastructure logs
• OMS can collect:
  • Performance data
  • Windows event logs
  • Custom logs
  • IIS logs
  • Syslog events

• Monitor-based alerts
• Event-based alerts
• Performance (metric)-based alerts
• Typically <1 minute alerting latency

• Metric Alerts (new) by Azure Monitor:
  • includes OMS logs
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-unified-alerts
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-near-real-time-metric-alerts
  • 1-5 minute current latency
• Azure Monitor classic alerts
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/insights-alerts-portal
• OMS:
  • Includes routed Diagnostics logs
  • https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-alerts
  • SLA of 6hrs; usually 10-15 min latency
  • Log Analytics alerts will automatically be extended into the new unifed alerts (see above) on May 14, 2018 https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-alerts-extend

• Use the SCOM console or web console
• Built-in dasbhoards, views, reports
• Ability to custom author dashboards, views, reports
• Queries OpsMgr database and datawarehouse

• Azure Monitor:
  • Azure portal
  • Ability to custom author views, charts
  • Azure Monitor Metrics Explorer (preview)
• OMS:
  • Log Analytics
  • View Designer
  • PowerBI
  • Service Map
• Azure Diagnostics:
  • Route data to Azure Application Insights
  • Route data to Microsoft PowerBI
  • Route the data to a third-party visualization tool using either live streaming or by having the tool read from an archive in Azure storage

• Installing and tuning the management pack
  • With both SCOM and Azure, a key part of monitoring is knowing what you want to monitor/collect

• Enabling Azure Diagnostics
  • Add the extension to the VM Resource Manager template: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/extensions-diagnostics-template?toc=%2fazure%2fvirtual-machines%2fwindows%2ftoc.json
  • Use PowerShell to enable the extension: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/ps-extensions-diagnostics
  • Use Visual Studio to enable the extension: https://docs.microsoft.com/en-us/azure/vs-azure-tools-diagnostics-for-cloud-services-and-virtual-machines#turn-on-diagnostics-on-azure-virtual-machines
• Enabling OMS
  • For Windows VMs, add the extension to the VM Resource Manager template or use PowerShell: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/extensions-oms
  • For Linux VMs, add the extension to the VM Resource Manager template or use Azure CLI: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/extensions-oms
  • Configure data sources from the Data menu in Log Analytics Advanced Settings (any configuration is delivered to all connected sources in your workspace)

• Two databases: OpsMgr database and Data Warehouse
• OpsMgr database holds 7 days by default
• Data Warehouse holds 400 days by default

• Archive Azure Diagnostics Logs or Azure Monitor data to storage account to retain forever (or specify retention age)
• Log Analytics has a retention age of 31 to 730 days