elastic / elastic/logstash

Memory leak (rabbitmq-input plugin)

Open
#3,992 4 comments 0 reactions 1 assignee Claimed by @suyograo View on GitHub
bug
Dominant language
Java
Stars
14.9k
Forks
3.5k
Avg merge
19h 14m
Merged PRs (30d)
63

Description

I have memory leak for logstash (try 1.5.4 and 1.4.4).
Supose it's elasticsearch output plugin.

I using [phusion/baseimage](https://github.com/phusion/baseimage-docker) (special Ubuntu Docker image)

``` bash
> java -version
java version "1.8.0_45"
Java(TM) SE Runtime Environment (build 1.8.0_45-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)
```

My filter and input:

``` javascript
input {
rabbitmq {
type => "logs"
codec => "json"
durable => true
ack => false
queue => "logs"
threads => 1
user => "guest1"
password => "***"
host => "10.**.**.***"
}

rabbitmq {
type => "collectors"
codec => "json"
durable => true
ack => false
queue => "collectors"
threads => 1
user => "guest1"
password => "***"
host => "10.**.**.***"
}
}

filter {
json {
source => "@message"
}

mutate {
remove_field => [ "kafka" ]
}

if [type] == 'logs' {
mutate {
add_tag => [ "_logs" ]
}
}

if [type] == 'collector_users' {
mutate {
add_tag => [ "_users", "supermap", "output" ]
}
}

if [type] == 'collector_internal' {
mutate {
add_tag => [ "_internal", "output" ]
}
}

if [type] == 'collector_errors' {
mutate {
add_tag => [ "_errors", "output" ]
}
}

if [type] == 'bughunt' {
mutate {
add_tag => [ "_bughunt" ]
}
}

if ('supermap' in [tags]) and ('prod' in [env]) and ([marker] in [
'topic.some',
'topic.other'
]){
mutate {
remove_tag => [ "supermap" ]
}
clone {
remove_tag => [ "_users", "output" ]
add_tag => [ "supermap" ]
}
} else {
mutate {
remove_tag => [ "supermap" ]
}
}

if 'supermap' in [tags] {
anonymize {
key => "not used"
algorithm => "MD5"
fields => [ "fields.ID", "fields.service_sessionID" ]
}
}
}
```
- With this output section logstash was't leaked:
(**OUTPUT_1**)

``` javascript
output {
if 'output' in [tags] {
rabbitmq {
workers => 1
durable => true
persistent => true
codec => "json"
exchange => "output-logs-exchange"
exchange_type => "direct"
host => "10.**.**.***"
vhost => "outputq"
key => "output-logs"
user => "guest1"
password => "***"
}
}

if 'es_users' in [tags] {
elasticsearch {
codec => "json"
cluster => "cluster_name"
host => "elasticsearch-cpu"
index => "collector_users-%{+YYYY.MM.dd}"
workers => 1
}
}
}
```
- With this outputs section logstash was leaked:
(**OUTPUT_2**)

``` javascript
output {
if 'output' in [tags] {
rabbitmq {
workers => 1
durable => true
persistent => true
codec => "json"
exchange => "output-logs-exchange"
exchange_type => "direct"
host => "10.**.**.***"
vhost => "outputq"
key => "output-logs"
user => "guest1"
password => "***"
}
}

if 'es_users' in [tags] {
elasticsearch {
codec => "json"
cluster => "cluster_name"
host => "elasticsearch-cpu"
index => "collector_users-%{+YYYY.MM.dd}"
workers => 1
}
}
if 'es_users' in [tags] {
elasticsearch {
codec => "json"
cluster => "cluster_name"
host => "elasticsearch-cpu"
index => "collector_users-%{+YYYY.MM.dd}"
workers => 1
}
}
}
```

(**OUTPUT_3**):

``` javascript
output {
if 'output' in [tags] {
rabbitmq {
workers => 1
durable => true
persistent => true
codec => "json"
exchange => "output-logs-exchange"
exchange_type => "direct"
host => "10.**.**.***"
vhost => "outputq"
key => "output-logs"
user => "guest1"
password => "***"
}
}

if 'es_users' in [tags] {
elasticsearch {
codec => "json"
cluster => "cluster_name"
host => "elasticsearch-cpu"
index => "collector_users-%{+YYYY.MM.dd}"
workers => 1
}
}
if 'es_users' in [tags] {
elasticsearch {
codec => "json"
cluster => "cluster_name"
host => "elasticsearch-cpu"
index => "collector_users-%{+YYYY.MM.dd}"
workers => 1
}
}
if 'es_internal' in [tags] {
elasticsearch {
codec => "json"
cluster => "cluster_name"
host => "elasticsearch-cpu"
index => "collector_internal-%{+YYYY.MM.dd}"
workers => 1
}
}
}
```
- GC plot:
![screenshot_14 - copy](https://cloud.githubusercontent.com/assets/2276472/10287291/cb2f994c-6b9a-11e5-9f1d-98fa5b111b46.png)
- Memory stats:
![screenshot_15](https://cloud.githubusercontent.com/assets/2276472/10287272/b9d7c4c6-6b9a-11e5-9b85-3b675e0065ac.png)
![screenshot_16](https://cloud.githubusercontent.com/assets/2276472/10287277/bb1ed3f6-6b9a-11e5-8d29-c818c413cd29.png)

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.