How to Set Up Effective Server Monitoring and Alerting with ChatOps

# SNS topic  -  receives CloudWatch alarm state changes
resource "aws_sns_topic" "ops_alerts" {
  name = "ops-alerts"
}

# CPU alarm  -  fires when CPU > 85% sustained across 2 consecutive 5-minute periods (10 min total)
resource "aws_cloudwatch_metric_alarm" "cpu_high" {
  alarm_name          = "HighCPU"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 2        # 2 periods must breach before alarm fires
  metric_name         = "CPUUtilization"
  namespace           = "AWS/EC2"
  period              = 300      # 5-minute periods; alarm fires after 10 minutes sustained
  statistic           = "Average"
  threshold           = 85
  alarm_description   = "CPU above 85% for 10 consecutive minutes"
  alarm_actions       = [aws_sns_topic.ops_alerts.arn]
  ok_actions          = [aws_sns_topic.ops_alerts.arn] # also notify on recovery

  dimensions = {
    InstanceId = "<YOUR_INSTANCE_ID>" # replace with your aws_instance resource or literal ID
  }
}

# Disk alarm  -  requires CloudWatch Agent publishing to the CWAgent namespace
resource "aws_cloudwatch_metric_alarm" "disk_high" {
  alarm_name          = "HighDiskUsage"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "disk_used_percent"
  namespace           = "CWAgent"
  period              = 300
  statistic           = "Average"
  threshold           = 80
  alarm_description   = "Disk usage above 80%"
  alarm_actions       = [aws_sns_topic.ops_alerts.arn]
  ok_actions          = [aws_sns_topic.ops_alerts.arn]

  dimensions = {
    path   = "/"
    device = "xvda1"
    fstype = "ext4"
  }
}

# Subscribe the Lambda to the SNS topic
resource "aws_sns_topic_subscription" "teams" {
  topic_arn = aws_sns_topic.ops_alerts.arn
  protocol  = "lambda"
  endpoint  = aws_lambda_function.teams_notifier.arn
}

# Grant SNS permission to invoke the Lambda  -  required, or invocations will be denied
resource "aws_lambda_permission" "allow_sns" {
  statement_id  = "AllowSNSInvoke"
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.teams_notifier.function_name
  principal     = "sns.amazonaws.com"
  source_arn    = aws_sns_topic.ops_alerts.arn
}

import json
import os
import urllib.parse   # quote() lives here  -  not in urllib.request
import urllib.request

TEAMS_WEBHOOK = os.environ["TEAMS_WEBHOOK_URL"]

# Adaptive Card colour tokens (Teams Workflow format)
COLORS = {
    "ALARM": "Attention",  # red
    "OK":    "Good",       # green
}


def lambda_handler(event, context):
    for record in event["Records"]:
        message = json.loads(record["Sns"]["Message"])
        _post_to_teams(message)


def _post_to_teams(msg: dict):
    state       = msg.get("NewStateValue", "UNKNOWN")
    alarm_name  = msg.get("AlarmName", "Unknown Alarm")
    reason      = msg.get("NewStateReason", "")
    region      = msg.get("Region", "")
    account     = msg.get("AWSAccountId", "")
    description = msg.get("AlarmDescription", "")

    icon  = "[ALARM]" if state == "ALARM" else "[OK]" if state == "OK" else "[WARN]"
    color = COLORS.get(state, "Default")

    # Build CloudWatch deep-link (urllib.parse.quote handles spaces and special chars)
    encoded_name = urllib.parse.quote(alarm_name)
    cw_url = (
        f"https://console.aws.amazon.com/cloudwatch/home"
        f"?region={region}#alarmsV2:alarm/{encoded_name}"
    )

    # Adaptive Card payload  -  compatible with Teams Workflows (Power Automate)
    card = {
        "type": "message",
        "attachments": [
            {
                "contentType": "application/vnd.microsoft.card.adaptive",
                "content": {
                    "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
                    "type": "AdaptiveCard",
                    "version": "1.4",
                    "body": [
                        {
                            "type": "TextBlock",
                            "text": f"{icon} {alarm_name}  -  {state}",
                            "size": "Large",
                            "weight": "Bolder",
                            "color": color,
                            "wrap": True,
                        },
                        {
                            "type": "TextBlock",
                            "text": description or "CloudWatch Alarm",
                            "isSubtle": True,
                            "wrap": True,
                        },
                        {
                            "type": "FactSet",
                            "facts": [
                                {"title": "State",   "value": state},
                                {"title": "Reason",  "value": reason},
                                {"title": "Region",  "value": region},
                                {"title": "Account", "value": account},
                            ],
                        },
                    ],
                    "actions": [
                        {
                            "type": "Action.OpenUrl",
                            "title": "View in CloudWatch",
                            "url": cw_url,
                        }
                    ],
                },
            }
        ],
    }

    req = urllib.request.Request(
        TEAMS_WEBHOOK,
        data=json.dumps(card).encode(),
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    with urllib.request.urlopen(req, timeout=5) as resp:
        return resp.status

# Create execution role
aws iam create-role \
  --role-name lambda-teams-notifier \
  --assume-role-policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}'

aws iam attach-role-policy \
  --role-name lambda-teams-notifier \
  --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole

# Package and deploy
zip teams-notifier.zip handler.py

aws lambda create-function \
  --function-name ops-teams-notifier \
  --runtime python3.12 \
  --handler handler.lambda_handler \
  --zip-file fileb://teams-notifier.zip \
  --role arn:aws:iam::ACCOUNT_ID:role/lambda-teams-notifier \
  --environment "Variables={TEAMS_WEBHOOK_URL=https://prod-XX.westus.logic.azure.com/workflows/...}"