elasticsearch - Logstash/Graylog won't write to ES with data node down -
we ran issue 1 of our elasticsearch data nodes died in aws. second node had trouble getting cluster online. came in yellow state , retrieve data not write it.
elasticsearch error
[2016-09-12 13:16:42,312][warn ][discovery.zen.ping.unicast] [witness.domain.net] failed send ping [{#cloud-i-cb397257-0}{1.1.1.1}{1.1.1.1:9300}] remotetransportexception[[datanode.domain.net][1.1.1.1:9300][internal:discovery/zen/unicast]]; nested: illegalstateexception[received ping request while not started]; caused by: java.lang.illegalstateexception: received ping request while not started
logstash error
{:timestamp=>"2016-09-12t15:48:15.898000-0400", :message=>#900592725, :events_consumed=>900732955, :worker_count=>4, :inflight_count=>0, :worker_states=>[{:status=>"sleep", :alive=>true, :index=>0, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>1, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>2, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>3, :inflight_count=>0}], :output_info=>[{:type=>"gelf", :config=>{"host"=>"127.0.0.1", "allow_env"=>false}, :is_multi_worker=>false, :events_received=>900592725, :workers=>"127.0.0.1", codec=>"utf-8">, workers=>1, port=>12201, chunksize=>1420, sender=>"%{host}", level=>["%{severity}", "info"], ship_metadata=>true, ship_tags=>true, ignore_metadata=>["@timestamp", "@version", "severity", "host", "source_host", "source_path", "short_message"], full_message=>"%{message}", short_message=>"short_message">]>, :busy_workers=>0}], :thread_info=>[{"thread_id"=>17, "name"=>"[main]nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in accept'", "[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:inaccept'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:112:in run_server'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:84:inrun'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:342:in inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:336:instart_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in accept'"}, {"thread_id"=>18, "name"=>"[main]>worker0", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:inworker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>19, "name"=>"[main]>worker1", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:instart_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>20, "name"=>"[main]>worker2", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:inworker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>21, "name"=>"[main]>worker3", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:instart_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}], :stalling_threads_info=>[{"thread_id"=>17, "name"=>"[main]<tcp", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:inaccept'"}, {"thread_id"=>18, "name"=>"[main]>worker0", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>19, "name"=>"[main]>worker1", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>20, "name"=>"[main]>worker2", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>21, "name"=>"[main]>worker3", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}]}>, :level=>:warn}
the problem ended being using aws discovery es cluster. though removed dead instance (still visible in aws, no accessible) security group using did not restart of remaining es nodes. once did started working again.
Comments
Post a Comment