记一次内网测试服 td-agent 升级

记一次内网测试服 td-agent 升级

六月 22, 2018 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# curl -L [https://toolbelt.treasuredata.com/sh/install-amazon1-td-agent3.sh](https://toolbelt.treasuredata.com/sh/install-amazon1-td-agent3.sh) | sh

# cd /etc/td-agent/

[root@ip td-agent]# cat td-agent.conf

@include /etc/td-agent/conf.d/*.conf

/etc/td-agent/conf.d

[root@ip conf.d]# cat td-gamex.conf

<source>

  @type forward

  port 24224

</source>

##########################################################################################

<source>

  @type tail

  format json

  time_key time

  time_format %N

  pos_file /var/log/td-agent/a.log.pos

  path /opt/supervisor/log/log*.%d.%m.%Y.log

  tag log.106

</source>

##########################################################################################

<match debug.**>

  @type stdout

</match>

##########################################################################################

<match log.**>

  @type forward # forward模式，转发给其他服务器处理

  send_timeout 60s

  recover_wait 10s

  #transport

  phi_threshold 16

  hard_timeout 60s

  <server>

     name log.shard

     host 【IP】

     port 24224

     weight 1

  </server>

  <secondary>

     @type file

     path /var/log/td-agent/log-forward-failed

  </secondary>

 </match>

例如添加一个 client log

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
[root@ip conf.d]# cat client.conf

<source>

  @type tail

  format json

  time_key time

  time_format %N

  pos_file /var/log/td-agent/client_150.log.pos

  path /opt/supervisor/log/client*.%d.%m.%Y.log

  tag client.150

</source>

##########################################################################################

<match debug.**>

  @type stdout

</match>

##########################################################################################

<match client_log.**>

  @type forward # forward模式，转发给其他服务器处理

  send_timeout 60s

  recover_wait 10s

  #transport

  phi_threshold 16

  hard_timeout 60s

  <server>

     name client_log

     host 【IP】

     port 24224

     weight 1

  </server>

  <secondary>

     @type file

     path /var/log/td-agent/logics-forward-failed

  </secondary>

</match>

中转机器上添加配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
[root@ip conf.d]# cat td-client.conf

<match debug.**>

  @type stdout

</match>

##################################################################################

<match client_log.**>

  @type copy

  <store>

    @type elasticsearch_dynamic

    host  【IP】

    port 9200

    request_timeout 15s #defaults to 5s

    reload_connections false

    reload_on_failure true # defaults to false

    logstash_format true

    logstash_prefix client_logiclog

    logstash_dateformat %Y.%m.%d

    time_key time

    <buffer>

      @type file

      path /var/log/td-agent/buffer/td-client_log-buffer

      chunk_limit_size 512MB #Default: 8MB (memory) / 256MB (file)

      total_limit_size 32GB #Default: 512MB (memory) / 64GB (file)

      chunk_full_threshold 0.9 #output plugin will flush the chunk when actual size reaches chunk_limit_size *
      chunk_full_threshold

      compress text #The option to specify compression of each chunks, during events are buffered

      flush_mode default

      flush_interval 15s #Default: 60s

      flush_thread_count 1 #Default: 1 The number of threads of output plugins, which is used to write chunks in parallel

      delayed_commit_timeout 60 #The timeout seconds until output plugin decides that async write operation fails

      overflow_action throw_exception

      retry_timeout 10m

    </buffer>

  </store>

</match>