Spring Boot Tutorial: Observability using Prometheus, Loki, Grafana

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
 
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>3.4.5</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
 
    <groupId>top.cxhello</groupId>
    <artifactId>spring-boot-monitor</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>spring-boot-monitor</name>
    <description>spring-boot-monitor</description>
    <url/>
 
    <licenses>
        <license/>
    </licenses>
 
    <developers>
        <developer/>
    </developers>
 
    <scm>
        <connection/>
        <developerConnection/>
        <tag/>
        <url/>
    </scm>
 
    <properties>
        <java.version>17</java.version>
    </properties>
 
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-actuator</artifactId>
        </dependency>
 
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
 
        <dependency>
            <groupId>io.micrometer</groupId>
            <artifactId>micrometer-registry-prometheus</artifactId>
            <scope>runtime</scope>
        </dependency>
 
        <dependency>
            <groupId>org.codehaus.janino</groupId>
            <artifactId>janino</artifactId>
        </dependency>
 
        <dependency>
            <groupId>com.github.loki4j</groupId>
            <artifactId>loki-logback-appender</artifactId>
            <version>1.6.0</version>
        </dependency>
 
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>
 
    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
 
</project>

<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="true" scanPeriod="1 seconds">
 
    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
    <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
    <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
    <property name="log.path" value="./logs"/>
    <!--  应用名称  -->
    <springProperty scope="context" name="log.name" source="spring.application.name"/>
    <springProperty scope="context" name="lokiUrl"  source="loki.url"/>
    <springProperty scope="context" name="lokiEnable"  source="loki.enable"/>
    <property name="log.pattern" value="%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %X{traceId} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
 
    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${log.pattern}</pattern>
        </encoder>
    </appender>
 
    <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${log.path}/${log.name}.log</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}/%d{yyyy-MM-dd}/${log.name}-%d{yyyy-MM-dd}.log.gz</fileNamePattern>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %X{traceId} %logger{36} [%file : %line] %msg%n</pattern>
        </encoder>
    </appender>
 
    <appender name="file_info" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${log.path}/${log.name}-info.log</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}/%d{yyyy-MM-dd}/${log.name}-info-%d{yyyy-MM-dd}.log.gz</fileNamePattern>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %X{traceId} %logger{36} [%file : %line] %msg%n</pattern>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>INFO</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>
 
    <appender name="file_error" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${log.path}/${log.name}-error.log</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}/%d{yyyy-MM-dd}/${log.name}-error-%d{yyyy-MM-dd}.log.gz</fileNamePattern>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %X{traceId} %logger{36} [%file : %line] %msg%n</pattern>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>
 
    <if condition='Boolean.valueOf(property("lokiEnable"))'>
        <then>
            <appender name="LOKI" class="com.github.loki4j.logback.Loki4jAppender">
                <http>
                    <url>${lokiUrl}/loki/api/v1/push</url>
                </http>
                <format>
                    <label>
                        <!-- Labels -->
                        <pattern>
                            app = ${log.name},
                            host = ${HOSTNAME}
                        </pattern>
                        <!-- Structured metadata (since Loki v2.9.0) -->
                        <structuredMetadataPattern>
                            level = %level,
                            thread = %thread,
                            class = %logger,
                            traceId = %mdc{traceId:-none}
                        </structuredMetadataPattern>
                    </label>
                    <message>
<!--                        <pattern>%-5level %logger{20} %msg %ex</pattern>-->
                        <pattern>%date %level [%thread] %X{traceId} %logger{36} [%file : %line] %msg%n</pattern>
                    </message>
                </format>
            </appender>
        </then>
    </if>
 
    <root level="info">
        <appender-ref ref="console"/>
        <appender-ref ref="file"/>
        <appender-ref ref="file_info"/>
        <appender-ref ref="file_error"/>
        <appender-ref ref="LOKI"/>
        <if condition='Boolean.valueOf(property("lokiEnable"))'>
            <then>
                <appender-ref ref="LOKI"/>
            </then>
        </if>
    </root>
 
</configuration>

version: '3.3'
 
networks:
  loki:
 
services:
  loki:
    image: grafana/loki:latest
    ports:
      - '3100:3100'
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - loki
 
  promtail:
    image: grafana/promtail:latest
    #    volumes:
    #      - /var/log:/var/log
    command: -config.file=/etc/promtail/config.yml
    networks:
      - loki
 
  grafana:
    environment:
      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
      - GF_FEATURE_TOGGLES_ENABLE=alertingSimplifiedRouting,alertingQueryAndExpressionsStepMode
    entrypoint:
      - sh
      - -euc
      - |
        mkdir -p /etc/grafana/provisioning/datasources
        cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml
        apiVersion: 1
        datasources:
        - name: Loki
          type: loki
          access: proxy 
          orgId: 1
          url: http://loki:3100
          basicAuth: false
          isDefault: true
          version: 1
          editable: false
        EOF
        /run.sh
    image: grafana/grafana:latest
    ports:
      - '3000:3000'
    networks:
      - loki
 
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    hostname: prometheus
    restart: always
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - '9090:9090'
    networks:
      - loki

global:
  scrape_interval: 15s # By default, scrape targets every 15 seconds.
 
  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    monitor: 'codelab-monitor'
 
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'
 
    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s
 
    static_configs:
      - targets: ['localhost:9090']
 
  - job_name: 'spring-boot-monitor'
    metrics_path: '/actuator/prometheus'
    static_configs:
      - targets: ['host.docker.internal:8080']