Monitoring & Alerting

# Key monitoring endpoints
curl -s "http://localhost:9200/_cluster/health?pretty"
curl -s "http://localhost:9200/_cat/nodes?v&h=name,role,heap.percent,disk.used_percent,cpu,load_1m"
curl -s "http://localhost:9200/_cat/indices?v&h=index,health,pri,rep,docs.count,store.size&s=store.size:desc"
curl -s "http://localhost:9200/_cat/thread_pool?v&h=node_name,name,active,queue,rejected&s=rejected:desc"
curl -s "http://localhost:9200/_nodes/stats/jvm,os,process?pretty"

# Pending tasks (cluster stability)
curl -s "http://localhost:9200/_cluster/pending_tasks?pretty"

# Hot threads (CPU debug)
curl -s "http://localhost:9200/_nodes/hot_threads"

// Health check for ASP.NET
public class ElasticsearchHealthCheck : IHealthCheck
{
    private readonly ElasticsearchClient _client;

    public ElasticsearchHealthCheck(ElasticsearchClient client) => _client = client;

    public async Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context, CancellationToken ct = default)
    {
        try
        {
            var response = await _client.Cluster.HealthAsync(h => h
                .Timeout(TimeSpan.FromSeconds(5)), ct);

            if (!response.IsValidResponse)
                return HealthCheckResult.Unhealthy("ES unreachable");

            return response.Status switch
            {
                HealthStatus.Green => HealthCheckResult.Healthy("Cluster green"),
                HealthStatus.Yellow => HealthCheckResult.Degraded(
                    "Cluster yellow: " + response.UnassignedShards + " unassigned"),
                _ => HealthCheckResult.Unhealthy("Cluster RED!")
            };
        }
        catch (Exception ex)
        {
            return HealthCheckResult.Unhealthy("ES connection failed", ex);
        }
    }
}

// Register in DI
builder.Services.AddHealthChecks()
    .AddCheck<ElasticsearchHealthCheck>("elasticsearch");

{
  "dashboard": {
    "title": "Elasticsearch Cluster Overview",
    "tags": ["elasticsearch", "production"],
    "timezone": "browser",
    "panels": [
      {
        "title": "Cluster Health Status",
        "type": "stat",
        "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
        "targets": [{
          "expr": "elasticsearch_cluster_health_status{color="green"}",
          "legendFormat": "Green"
        }],
        "fieldConfig": {
          "defaults": {
            "thresholds": {
              "steps": [
                { "color": "red", "value": 0 },
                { "color": "green", "value": 1 }
              ]
            },
            "mappings": [
              { "type": "value", "options": { "0": { "text": "RED/YELLOW" }, "1": { "text": "GREEN" } } }
            ]
          }
        },
        "description": "Cluster health. GREEN=all shards assigned. YELLOW=replicas missing. RED=primaries missing."
      },
      {
        "title": "JVM Heap Usage (%)",
        "type": "timeseries",
        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
        "targets": [{
          "expr": "elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} * 100",
          "legendFormat": "{{name}}"
        }],
        "fieldConfig": {
          "defaults": {
            "unit": "percent",
            "thresholds": {
              "steps": [
                { "color": "green", "value": 0 },
                { "color": "yellow", "value": 75 },
                { "color": "red", "value": 85 }
              ]
            }
          }
        },
        "description": "JVM heap per node. Alert threshold: >85% sustained = GC pressure. >90% = OOM risk. Max 30GB heap."
      },
      {
        "title": "Indexing & Search Rate",
        "type": "timeseries",
        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
        "targets": [
          {
            "expr": "rate(elasticsearch_indices_indexing_index_total[5m])",
            "legendFormat": "Indexing/s {{name}}"
          },
          {
            "expr": "rate(elasticsearch_indices_search_query_total[5m])",
            "legendFormat": "Search/s {{name}}"
          }
        ],
        "fieldConfig": { "defaults": { "unit": "ops" } },
        "description": "Indexing and search operations per second. Sudden drops indicate bulk rejections or circuit breaker trips."
      },
      {
        "title": "Thread Pool Rejections",
        "type": "timeseries",
        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
        "targets": [
          {
            "expr": "rate(elasticsearch_thread_pool_rejected_count{name=~"search|write|bulk"}[5m])",
            "legendFormat": "{{name}} rejected {{node}}"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "ops",
            "thresholds": {
              "steps": [
                { "color": "green", "value": 0 },
                { "color": "red", "value": 1 }
              ]
            }
          }
        },
        "description": "Thread pool rejections indicate overload. Any rejection >0 = capacity issue. Scale up or reduce load."
      }
    ],
    "time": { "from": "now-1h", "to": "now" },
    "refresh": "30s"
  }
}

Alert thresholds (Prometheus rules):