本文档只展示搭建LGP轻量级日志系统docker-compose相关配置见文件,如果需要学习或者了解具体搭建细节和搭建过程中可能遇到的问题,请前往LPG日志搭建

主要docker-compose文件

docker-compose.yml

version: "3"

services:
  loki:
    image: grafana/loki:2.9.4
    container_name: log-loki
    ports:
      - 3100:3100
    volumes:
      - ./loki-config.yaml:/etc/loki/local-config.yaml
    command: -config.file=/etc/loki/local-config.yaml
    depends_on:
      - minio

  promtail:
    image: grafana/promtail:2.9.4
    container_name: log-promtail
    volumes:
      - ./promtail-config.yaml:/etc/promtail/promtail-config.yaml
      - /var/run/docker.sock:/var/run/docker.sock # 直接把本机的docker套接字映射进去，这样就可以直接抓取所有容器的日志了
    command: -config.file=/etc/promtail/promtail-config.yaml

  grafana:
    image: grafana/grafana:10.4.0
    container_name: log-grafana
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
      - ./grafana/data:/var/lib/grafana
    environment:
      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    ports:
      - 3000:3000
    command: -config /etc/grafana/grafana.ini


  minio:
    image: minio/minio
    container_name: log-minio
    entrypoint:
      - sh
      - -euc
      - |
        mkdir -p /data/loki-data && \
        mkdir -p /data/loki-ruler && \
        minio server --console-address ":9001" /data
    environment:
      - MINIO_ACCESS_KEY=loki
      - MINIO_SECRET_KEY=supersecret
      - MINIO_PROMETHEUS_AUTH_TYPE=public
      - MINIO_UPDATE=off
    ports:
      - 9000:9000
      - 9001:9001 
    volumes:
      - ./minio/data:/data

  flog:
    image: mingrammer/flog
    container_name: flog
    command: -f json -d 1s -l

Loki配置文件

loki-config.yaml

---
auth_enabled: false
server:
  http_listen_port: 3100
memberlist:
  join_members:
    - log-loki:7946
schema_config:
  configs:
    - from: 2021-08-01
      store: boltdb-shipper
      object_store: s3
      schema: v11
      index:
        prefix: index_
        period: 24h
common:
  path_prefix: /loki
  replication_factor: 1
  storage:
    s3:
      endpoint: http://log-minio:9000
      insecure: true
      bucketnames: loki-data
      access_key_id: loki
      secret_access_key: supersecret
      s3forcepathstyle: true
  ring:
    kvstore:
      store: memberlist
ruler:
  storage:
    s3:
      bucketnames: loki-ruler

Promtail配置文件

promtail-config.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://log-loki:3100/loki/api/v1/push
    tenant_id: sidss

scrape_configs:
  - job_name: flog_scrape
    docker_sd_configs: # 对于promtail来说，又可以定义各种日志来源，请见 https://grafana.com/docs/loki/latest/clients/promtail/configuration/#scrape_configs
      - host: unix:///var/run/docker.sock # 这里直接从docker套接字拿日志
        refresh_interval: 5s
    relabel_configs:
      - source_labels: ['__meta_docker_container_name'] # 根据容器名打标签分类
        regex: '/(.*)'
        target_label: 'container' # 标签名为 container
    pipeline_stages:
      - match:
          selector: '{container="log-promtail"}'
          stages:
            - regex: 
                expression: '.*level=(?P<level>[a-zA-Z]+).*ts=(?P<timestamp>[T\d-:.Z]*).*component=(?P<component>[a-zA-Z]+)'

            - labels:
                level:
                component:

            - timestamp:
                format: RFC3339Nano
                source: timestamp

      - match:
          selector: '{container="log-grafana"}' 
          stages:
            # regex 阶段将提取一个 level 合 componet 值，供后面的阶段使用，允许 level 被定义为 lvl=<level> 或 level=<level>，组件被定义为 logger=<component> 或 component=<component>
            - regex:
                expression: ".*(lvl|level)=(?P<level>[a-zA-Z]+).*(logger|component)=(?P<component>[a-zA-Z]+)"

            # 然后标签阶段将从上面 regex 阶段提取的 level 和 component 变为标签。
            - labels:
                level:
                component: