ELK Stack

---

1 - Architecture

---

2 - Elasticsearch

2.1 Installation Docker

# docker-compose.yml
version: '3.8'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ports:
      - "9200:9200"
    volumes:
      - es_data:/usr/share/elasticsearch/data

volumes:
  es_data:

2.2 Concepts clés

Concept	Description
Index	Collection de documents
Document	Unité de données (JSON)
Mapping	Schéma des champs
Shard	Partition horizontale
Replica	Copie pour HA

2.3 Index Template

PUT _index_template/logs-template
{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1,
      "index.lifecycle.name": "logs-policy"
    },
    "mappings": {
      "properties": {
        "@timestamp": { "type": "date" },
        "message": { "type": "text" },
        "level": { "type": "keyword" },
        "service": { "type": "keyword" },
        "trace_id": { "type": "keyword" },
        "host": {
          "properties": {
            "name": { "type": "keyword" },
            "ip": { "type": "ip" }
          }
        }
      }
    }
  }
}

2.4 Queries

// Match query
GET logs-*/_search
{
  "query": {
    "match": {
      "message": "error connection"
    }
  }
}

// Bool query
GET logs-*/_search
{
  "query": {
    "bool": {
      "must": [
        { "match": { "level": "error" } }
      ],
      "filter": [
        { "range": { "@timestamp": { "gte": "now-1h" } } },
        { "term": { "service": "api-gateway" } }
      ]
    }
  }
}

// Aggregations
GET logs-*/_search
{
  "size": 0,
  "aggs": {
    "errors_per_service": {
      "terms": { "field": "service" },
      "aggs": {
        "error_count": {
          "filter": { "term": { "level": "error" } }
        }
      }
    }
  }
}

---

3 - Logstash

3.1 Pipeline

# logstash.conf
input {
  beats {
    port => 5044
  }
  
  tcp {
    port => 5000
    codec => json
  }
}

filter {
  # Parse JSON si nécessaire
  if [message] =~ /^\{/ {
    json {
      source => "message"
    }
  }
  
  # Grok pour logs non structurés
  grok {
    match => { 
      "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:log_message}" 
    }
  }
  
  # Date parsing
  date {
    match => ["timestamp", "ISO8601"]
    target => "@timestamp"
  }
  
  # Add fields
  mutate {
    add_field => { "environment" => "production" }
    remove_field => ["timestamp"]
  }
  
  # GeoIP
  if [client_ip] {
    geoip {
      source => "client_ip"
    }
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "logs-%{+YYYY.MM.dd}"
  }
  
  # Debug
  stdout { codec => rubydebug }
}

3.2 Grok Patterns

# Patterns courants
%{IP:client_ip}
%{TIMESTAMP_ISO8601:timestamp}
%{LOGLEVEL:level}
%{NUMBER:response_time}
%{WORD:http_method}
%{URIPATH:uri_path}
%{NUMBER:status_code}

# Pattern custom
(?<custom_field>pattern)

# Exemple: Apache Combined Log
%{COMBINEDAPACHELOG}

3.3 Conditionals

filter {
  if [type] == "apache" {
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
  }
  
  if [status] >= 500 {
    mutate {
      add_tag => ["error"]
    }
  }
  
  if "error" in [tags] {
    # Notification
  }
}

---

4 - Beats

4.1 Filebeat

# filebeat.yml
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/log/app/*.log
    fields:
      service: my-app
    multiline:
      pattern: '^\d{4}-\d{2}-\d{2}'
      negate: true
      match: after

  - type: container
    paths:
      - /var/lib/docker/containers/*/*.log
    processors:
      - add_docker_metadata: ~
      - add_kubernetes_metadata: ~

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  index: "logs-%{+yyyy.MM.dd}"

# Ou vers Logstash
output.logstash:
  hosts: ["logstash:5044"]

4.2 Metricbeat

# metricbeat.yml
metricbeat.modules:
  - module: system
    metricsets:
      - cpu
      - memory
      - network
      - filesystem
    period: 10s

  - module: docker
    metricsets:
      - container
      - cpu
      - memory
    hosts: ["unix:///var/run/docker.sock"]
    period: 10s

  - module: kubernetes
    metricsets:
      - pod
      - container
      - node
    period: 10s
    hosts: ["kube-state-metrics:8080"]

output.elasticsearch:
  hosts: ["elasticsearch:9200"]

---

5 - Kibana

5.1 Installation

# docker-compose.yml
kibana:
  image: docker.elastic.co/kibana/kibana:8.11.0
  ports:
    - "5601:5601"
  environment:
    - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
  depends_on:
    - elasticsearch

5.2 Data Views (Index Patterns)

POST /api/data_views/data_view
{
  "data_view": {
    "title": "logs-*",
    "timeFieldName": "@timestamp"
  }
}

5.3 Discover Queries

// KQL (Kibana Query Language)
level: error
service: api-gateway AND level: (error OR warn)
message: "connection" AND NOT message: "success"
response_time > 1000
@timestamp >= "2024-01-01" AND @timestamp < "2024-02-01"

5.4 Visualizations

Type	Usage
Line	Tendances temporelles
Bar	Comparaisons
Pie	Répartition
Metric	Valeur unique
Data Table	Données détaillées
Heat Map	Distribution 2D
Lens	Exploration rapide

---

6 - Index Lifecycle Management

PUT _ilm/policy/logs-policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50GB",
            "max_age": "1d"
          }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "shrink": { "number_of_shards": 1 },
          "forcemerge": { "max_num_segments": 1 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "freeze": {}
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

---

7 - Alerting

PUT _watcher/watch/high_error_rate
{
  "trigger": {
    "schedule": { "interval": "5m" }
  },
  "input": {
    "search": {
      "request": {
        "indices": ["logs-*"],
        "body": {
          "query": {
            "bool": {
              "must": [
                { "term": { "level": "error" } },
                { "range": { "@timestamp": { "gte": "now-5m" } } }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": { "ctx.payload.hits.total.value": { "gte": 100 } }
  },
  "actions": {
    "slack_notification": {
      "slack": {
        "message": {
          "from": "watcher",
          "to": ["#alerts"],
          "text": "High error rate: {{ctx.payload.hits.total.value}} errors"
        }
      }
    }
  }
}

---

Résumé

Dans ce chapitre, nous avons appris :

• L'architecture ELK
• Elasticsearch : indexation et queries
• Logstash : pipelines et transformations
• Beats : collecte de données
• Kibana : visualisation
• ILM : gestion du cycle de vie
• Alerting avec Watcher

---

Prochaine étape

Dans le prochain chapitre, nous verrons Grafana Loki.

→ Chapitre suivant : Grafana Loki

---

← Retour à la table des matières