I am having a problem that I cannot seem to figure out what is going on. I have a set of 4 scripts that run every night in a schedule task as the System account. These 4 scripts work exactly the same way. They generate a list of server from a query and turn on Alert Suppression on those nodes for 10 hours. The problem is the scripts are randomly failing. The script that runs at 8pm will be successfully, but the one at 9pm, 10pm, and 11pm will fail. I have error handling set up in the script but the error that is generated during the Alert Suppression task is "one or more errors occurred" Can someone point me to the log file that SWIS SDK powershell cmdlets use so I can try and see what the one or more errors are to attempt to fix the problem? I have added the script that I am using.
Function Create-EventSource
{
param(
[string]$logfileName,
[string]$sourceName
)
If([System.Diagnostics.EventLog]::Exists($logfileName))
{
write-output "$logFileName Exists"
If([System.Diagnostics.EventLog]::SourceExists($sourceName))
{
}
else
{
New-EventLog -LogName $logfilename -Source $sourceName
}
}
else
{
$event = "Creating $logfilename EventLog"
New-EventLog -LogName $logfileName -Source $sourceName
Limit-EventLog -LogName $logFileName -OverflowAction OverwriteAsNeeded -MaximumSize 50MB
Write-EventLog -LogName Application -Source OVO -EntryType Information -Category 0 -EventId 200 -Message $event
}
}
Function Send-Email
{
param(
[string]$body
)
$subject = "Script Failed to Connect to SolarWinds"
$smtpTo = "some email address"
$smtpFrom = "some other address"
$smtpServer = "smtp server"
Send-MailMessage -From $smtpFrom -To $smtpTo -Body $body -Subject $subject -SmtpServer $smtpServer
}
Function Validate-MaintenanceMode
{
Param(
[string]$nodeName,
[string]$uri,
[object]$swisConnection
)
$tmpquery1 = "select entityuri, suppressfrom, suppressuntil from orion.AlertSuppression where entityuri = '$uri'"
$AlertSuppressionInfo = Get-SwisData -SwisConnection $swisConnection -Query $tmpQuery1
If(([string]::IsNullOrEmpty($AlertSuppressionInfo.suppressfrom)) -and ([string]::IsNullOrEmpty($AlertSuppressionInfo.suppressuntil)))
{
$IsSet = "No"
}
elseif([string]::IsNullOrEmpty($AlertSuppressionInfo.suppressuntil))
{
$IsSet = "Manual"
}
else
{
$IsSet = "Yes"
}
$obj = New-Object PSObject -Property @{
NodeName = $nodeName
MMStartTime = $AlertSuppressionInfo.suppressfrom
MMEndTime = $AlertSuppressionInfo.suppressUntil
URI = "$uri"
isSet = $IsSet
}
return $obj
}
######################### Main #############################################################
$source = "ArgoMaintenanceModeMountain"
$eventLogMessage = ""
$eventLogName = "EventMonitoring"
Create-EventSource -logfileName $eventLogName -sourceName $source
#Add the snapin | import the module for SolarWinds API
#Add-PSSnapin SwisSnapin
Import-Module C:\SLW\PSModules\SwisPowerShell\2.3.0.108\SwisPowerShell.psd1
#create a connection to solarwinds
$hostname = 'orion server ip address'
$username = 'username'
$password = 'password'
#Set DateTime Parameters
$startTime = get-date
$endtime = $startTime.AddHours(10)
$eventLogMessage += "Getting the starting time and end time for Alert Suppression for the Argo Servers in Eastern Time Zone`r`n"
$eventLogMessage += "Start Time: $startTime`r`n"
$eventLogMessage += "End Time: $endTime`r`n"
$eventLogMessage += "Creating Connection to SolarWinds Application`r`n"
try{
$swis = Connect-Swis -Hostname $hostname -Usernam $UserName -Password $password -ErrorAction stop
}
catch
{
$body = "$($MyInvocation.InvocationName) was unable to connect to Production SolarWinds through API. Please Validate the username and password are correct"
$eventLogMessage += $body
Send-Email -body $body
Write-EventLog -LogName $eventLogName -Source $source -EntryType Error -Category 0 -EventId 3245 -Message $eventLogMessage
exit
}
$eventLogMessage += "Generating the List of Argo Servers to be placed into Maintenance Mode`r`n"
$query = "select caption,uri from orion.nodes where caption like '%azargmtw%'"
try
{
$uri = Get-SwisData -SwisConnection $swis -query $query -ErrorAction Stop
}
catch
{
$body = "$($MyInvocation.InvocationName) hit an exception during the query to get server information. The Error:`r`n $($error[0])"
$eventLogMessage += "$body"
send-email -body $body
Write-EventLog -LogName $eventLogName -Source $source -EntryType Error -Category 0 -EventId 3245 -Message $eventLogMessage
Exit
}
if([string]::IsNullOrEmpty($uri))
{
$body = "$($MyInvocation.InvocationName): SolarWinds Query did not return any values. Please Validate Query in the SWQL Studio`r`n The query was '$query'"
$eventLogMessage += $body
Send-Email -body $body
}
else
{
$eventlogMessage += "Found $($uri.count) servers to put into maintenance mode`r`n"
$eventlogMessage += "Placing Servers into maintenance mode`r`n"
try
{
Invoke-SwisVerb -SwisConnection $swis -EntityName Orion.AlertSuppression -Verb SuppressAlerts -Arguments @([string[]] $uri.uri, $startTime, $endtime) -ErrorAction Stop | Out-Null
}
catch
{
$body = "$($MyInvocation.InvocationName): Exception when attempting to Set AlertSuppression on the Node. The Error:`r`n $($error[0])"
$eventLogMessage += $body
Send-Email -body $body
}
$eventLogMessage += "Validating Maintenance Mode was Set on the Servers`r`n"
foreach($info in $uri)
{
$result = Validate-MaintenanceMode -nodeName $info.caption -uri $info.uri -swisConnection $swis
Switch($result.IsSet)
{
{$_ -eq "No"} {$eventLogMessage += "$($result.nodeName): Maintenance Mode was not set`r`n"}
{$_ -eq "Manual"} {$eventLogMessage += "$($result.nodeName): Maintenance Mode was set at $($result.MMStartTime.ToLocalTime()) but needs to be manually reset`r`n"}
{$_ -eq "Yes"} {$eventLogMessage += "$($result.nodeName): Maintenance Mode was set at $($result.MMStartTime.ToLocalTime()) to $($result.MMEndTime.ToLocalTime())`r`n"}
}
}
}
$eventLogMessage
Write-EventLog -LogName $eventLogName -Source $source -EntryType Information -Category 0 -EventId 200 -Message $eventLogMessage