fluentd.conf

# Fluentd configuration for GistPin log aggregation

# Input: collect logs from all containers
<source>
  @type tail
  path /var/log/containers/*.log
  pos_file /var/log/fluentd-containers.log.pos
  tag kubernetes.*
  read_from_head true
  <parse>
    @type json
    time_format %Y-%m-%dT%H:%M:%S.%NZ
  </parse>
</source>

# Input: system logs
<source>
  @type tail
  path /var/log/syslog
  pos_file /var/log/fluentd-syslog.pos
  tag system.syslog
  <parse>
    @type syslog
  </parse>
</source>

# Filter: parse Kubernetes metadata
<filter kubernetes.**>
  @type kubernetes_metadata
  @id filter_kube_metadata
</filter>

# Filter: parse GistPin backend JSON logs
<filter kubernetes.**gistpin-backend**>
  @type parser
  key_name log
  reserve_data true
  <parse>
    @type json
  </parse>
</filter>

# Filter: enrich logs with environment info
<filter **>
  @type record_transformer
  <record>
    environment "#{ENV['ENVIRONMENT'] || 'production'}"
    cluster "#{ENV['CLUSTER_NAME'] || 'gistpin-cluster'}"
    hostname "#{Socket.gethostname}"
  </record>
</filter>

# Filter: drop health check noise
<filter kubernetes.**>
  @type grep
  <exclude>
    key $.request_path
    pattern ^/health
  </exclude>
</filter>

# Output: Elasticsearch for search/analysis
<match kubernetes.**>
  @type elasticsearch
  host "#{ENV['ELASTICSEARCH_HOST'] || 'elasticsearch'}"
  port "#{ENV['ELASTICSEARCH_PORT'] || 9200}"
  logstash_format true
  logstash_prefix gistpin
  include_tag_key true
  tag_key @log_name
  flush_interval 5s
  <buffer>
    @type file
    path /var/log/fluentd-buffers/kubernetes.system.buffer
    flush_mode interval
    retry_type exponential_backoff
    flush_interval 5s
    retry_forever false
    retry_max_interval 30
    chunk_limit_size 2M
    queue_limit_length 8
    overflow_action block
  </buffer>
</match>

# Output: system logs to Elasticsearch
<match system.**>
  @type elasticsearch
  host "#{ENV['ELASTICSEARCH_HOST'] || 'elasticsearch'}"
  port "#{ENV['ELASTICSEARCH_PORT'] || 9200}"
  logstash_format true
  logstash_prefix gistpin-system
  flush_interval 10s
</match>