Artifactory日志分析fluentd+Prometheus+Grafana

使用Grafana在Prometheus中查看JFrog日志数据

GitHub地址：https://github.com/jfrog/log-analytics-prometheus

创建fluentd镜像

1.下载ubuntu镜像

docker pull ubuntu:16.04

2.启动容器查看发行版本信息

使用ubuntu下载Fluentd，需要先查看系统代号，使用命令：

root@ubuntu:~# lsb_release -c
Codename:	xenial

如果没有lsb_release命令：apt-get install -y lsb-release

3.安装fluentd

Fluentd官网下载：https://docs.fluentd.org/installation/install-by-deb

进去找到相应的版本信息复制

curl -fsSL https://toolbelt.treasuredata.com/sh/install-ubuntu-xenial-td-agent4.sh | sh

4.安装完成后将/etc/td-agent/文件全部拷贝到宿主机

docker cp ubuntu:/etc/td-agent /app/dockerdata/fluentd/conf

5.保存容器镜像

docker commit -m "install fluentd"  ubuntu fluentd:1.0

6.启动新的容器

docker run -di --name fluentd -p 24231:24231 -v /app/dockerdata/8082/artifactory/var/log/:/jfrog/log -v /app/dockerdata/8084/artifactory/var/log/:/apkjfrog/log -v /app/dockerdata/fluentd/conf/:/etc/td-agent/ 10.30.4.50:8082/soimt/fluentd/soimt-td-agent:v1.0

docker exec -it fluentd bash

修改配置文件

1.在root目录下 vim .bashrc

添加

export JF_PRODUCT_DATA_INTERNAL=/apkjfrog
export JF_PRODUCT_DATA=/jfrog

在/lib/systemd/system/td-agent.service

添加

User=root
Group=root
Environment=JF_PRODUCT_DATA_INTERNAL=/apkjfrog
Environment=JF_PRODUCT_DATA=/jfrog

2.修改配置文件

vim /app/dockerdata/fluentd/conf/td-agent.conf
<source>
  @type prometheus
</source>

@include config.d/*.conf

mkdir config.d
vim config.d/td-agent_82.conf
<source>
  @type tail
  @id access_service_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/access-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/access-service.log.pos"
  tag jfrog.rt.access.service.82
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id artifactory_service_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/artifactory-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/artifactory-service.log.pos"
  tag jfrog.rt.artifactory.service.82
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id frontend_service_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/frontend-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/frontend-service.log.pos"
  tag jfrog.rt.frontend.service.82
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_service_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/metadata-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/metadata-service.log.pos"
  tag jfrog.rt.metadata.service.82
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_service_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/router-service.log.pos"
  tag jfrog.rt.router.service.82
  <parse>
    @type none
  </parse>
</source>
# Strip out color codes then field extract the service fields
<filter jfrog.rt.**.service.82>
  @type record_transformer
  enable_ruby true
  <record>
    message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
  </record>
</filter>
<filter jfrog.rt.**.service.82>
  @type parser
  key_name message
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
      time_key timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
  emit_invalid_record_to_error false
</filter>
# End Service Fields Extraction
<source>
  @type tail
  @id router_traefik_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/router-traefik.log.pos"
  tag jfrog.rt.router.traefik.82
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] - (?<message>.+)$/
      time_key timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id access_request_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/access-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/access-request.log.pos"
  tag jfrog.rt.access.request.82
  <parse>
    @type regexp
    expression /^(?<timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$/m
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id artifactory_request_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/artifactory-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/artifactory-request.log.pos"
  tag jfrog.rt.artifactory.request.82
  <parse>
    @type regexp
    expression /^(?<timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$/m
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
    types response_content_length:integer, request_content_length:integer, return_status_code:integer
  </parse>
</source>
<filter jfrog.rt.artifactory.request.82>
  @type record_transformer
  enable_ruby true
  <record>
    user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")}
    repo ${record["request_url"].strip().start_with?("/Components") | record["request_url"].strip().start_with?("/Packages") | record["request_url"].strip().start_with?("/ToolChains") ? (record["request_url"].strip().split('/')[1]) : ("")}
    artifact ${record["request_url"].strip().start_with?("/Integrated") | record["request_url"].strip().start_with?("/Devolop") | record["request_url"].strip().start_with?("/Product") ? (val = record["request_url"].strip().split('/'); val[val.length()-1]) : ("")}
    data_download ${record["response_content_length"] == -1 ? 0 : record["response_content_length"]}
    data_upload ${record["request_content_length"] == -1 ? 0 : record["request_content_length"]}
  </record>
</filter>
<source>
  @type tail
  @id frontend_request_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/frontend-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/frontend-request.log.pos"
  tag jfrog.rt.frontend.request.82
  <parse>
    @type regexp
    expression /^(?<timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$/m
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<filter jfrog.rt.frontend.request.82>
  @type record_transformer
  enable_ruby true
  <record>
    user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")}
  </record>
</filter>
<source>
  @type tail
  @id metadata_request_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/metadata-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/metadata-request.log.pos"
  tag jfrog.rt.metadata.request.82
  <parse>
    @type regexp
    expression /^(?<timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$/m
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<filter jfrog.rt.metadata.request.82>
  @type record_transformer
  enable_ruby true
  <record>
    user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")}
  </record>
</filter>
<source>
  @type tail
  @id router_request_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/router-request.log.pos"
  tag jfrog.rt.router.request.82
  <parse>
    @type json
    time_key time
    time_format %Y-%m-%dT%H:%M:%SZ+HH:MM
  </parse>
</source>
<source>
  @type tail
  @id artifactory_access_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/artifactory-access.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/artifactory-access.log.pos"
  tag jfrog.rt.artifactory.access.82
  <parse>
    @type regexp
    expression /^(?<timestamp>[^ ]*) \[(?<trace_id>[^\]]*)\] \[(?<action_response>[^\]]*)\] (?<repo_path>.*) for client : (?<username>.+)/(?<ip>.+)\.$/m
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id access_security_audit_tail_82
  path "#{ENV['JF_PRODUCT_DATA']}/log/access-security-audit.log"
  pos_file "#{ENV['JF_PRODUCT_DATA']}/log/access-security-audit.log.pos"
  tag jfrog.rt.access.audit.82
  <parse>
    @type regexp
    expression /^(?<timestamp>[^\|]*)\|(?<token_id>[^\|]*)\|(?<user_ip>[^\|]*)\|(?<user>[^\|]*)\|(?<logged_principal>[^\|]*)\|(?<entity_name>[^\|]*)\|(?<event_type>[^\|]*)\|(?<event>[^\|]*)\|(?<data_changed>.*)$/m
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>

# WHAT LOG IT WAS INTO THE JSON
<filter jfrog.**>
  @type record_transformer
  <record>
    hostname "#{Socket.gethostname}"
    log_source ${tag}
  </record>
</filter>

<filter jfrog.rt.artifactory.request.82>
  @type prometheus
  <metric>
    name jfrog_rt_data_download_total
    type counter
    desc artifactory data download
    key data_download
    <labels>
      NAME 8082
      remote_address ${remote_address}
      artifact ${artifact}
      repo ${repo}
      response_content_length ${response_content_length}
      data_download ${data_download}
      method ${request_method}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_data_upload_total
    type counter
    desc artifactory data upload
    key data_upload
    <labels>
      NAME 8082
      remote_address ${remote_address}
      artifact ${artifact}
      repo ${repo}
      request_content_length ${request_content_length}
      data_upload ${data_upload}
      method ${request_method}
    </labels>
  </metric>

  <metric>
    name jfrog_rt_req_total
    type counter
    desc artifactory requests
    <labels>
      user ${user}
      NAME 8082
      request_url ${request_url}
      repo ${repo}
      artifact ${artifact}
      return_status ${return_status}
      remote_address ${remote_address}
      method ${request_method}
    </labels>
  </metric>
</filter>

<filter jfrog.rt.artifactory.service.82>
@type prometheus
  <metric>
    name jfrog_rt_log_level_total
    type counter
    desc artifactory log_levels
    <labels>
      NAME 8082
      log_level ${log_level}
    </labels>
  </metric>

  <metric>
      name jfrog_rt_service_message_total
      type counter
      desc artifactory service message
      key repoupload_length
      <labels>
        NAME 8082
        message ${message}
      </labels>
    </metric>
</filter>

<filter jfrog.rt.artifactory.access.82>
@type prometheus
<metric>
  name jfrog_rt_access_total
  type counter
  desc artifactory access
  <labels>
    NAME 8082
    user ${username}
    repo_path ${repo_path}
    action_response ${action_response}
    ip ${ip}
  </labels>
</metric>

</filter>
<filter jfrog.rt.access.audit.82>
@type prometheus
<metric>
  name jfrog_rt_access_audit_total
  type counter
  desc artifactory access audit
  <labels>
    NAME 8082
    user ${user}
    event_type ${event_type}
    event ${event}
  </labels>
</metric>
</filter>

3.启动fluentd

nohup /opt/td-agent/bin/ruby /usr/sbin/td-agent &

可以在http://localhost:24231查看采集到的数据

4.在Prometheus中vim prometheus.yml

-job_name: "fluentd"

​ static_configs:

​ -targets: ["fluentd服务器IP:24231"]