elasticsearch - Logstash/Graylog won't write to ES with data node down -


we ran issue 1 of our elasticsearch data nodes died in aws. second node had trouble getting cluster online. came in yellow state , retrieve data not write it.

elasticsearch error

[2016-09-12 13:16:42,312][warn ][discovery.zen.ping.unicast] [witness.domain.net] failed send ping [{#cloud-i-cb397257-0}{1.1.1.1}{1.1.1.1:9300}] remotetransportexception[[datanode.domain.net][1.1.1.1:9300][internal:discovery/zen/unicast]]; nested: illegalstateexception[received ping request while not started]; caused by: java.lang.illegalstateexception: received ping request while not started

logstash error

{:timestamp=>"2016-09-12t15:48:15.898000-0400", :message=>#900592725, :events_consumed=>900732955, :worker_count=>4, :inflight_count=>0, :worker_states=>[{:status=>"sleep", :alive=>true, :index=>0, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>1, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>2, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>3, :inflight_count=>0}], :output_info=>[{:type=>"gelf", :config=>{"host"=>"127.0.0.1", "allow_env"=>false}, :is_multi_worker=>false, :events_received=>900592725, :workers=>"127.0.0.1", codec=>"utf-8">, workers=>1, port=>12201, chunksize=>1420, sender=>"%{host}", level=>["%{severity}", "info"], ship_metadata=>true, ship_tags=>true, ignore_metadata=>["@timestamp", "@version", "severity", "host", "source_host", "source_path", "short_message"], full_message=>"%{message}", short_message=>"short_message">]>, :busy_workers=>0}], :thread_info=>[{"thread_id"=>17, "name"=>"[main]nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in accept'", "[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:inaccept'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:112:in run_server'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:84:inrun'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:342:in inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:336:instart_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in accept'"}, {"thread_id"=>18, "name"=>"[main]>worker0", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:inworker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>19, "name"=>"[main]>worker1", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:instart_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>20, "name"=>"[main]>worker2", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:inworker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>21, "name"=>"[main]>worker3", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:instart_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}], :stalling_threads_info=>[{"thread_id"=>17, "name"=>"[main]<tcp", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:inaccept'"}, {"thread_id"=>18, "name"=>"[main]>worker0", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>19, "name"=>"[main]>worker1", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>20, "name"=>"[main]>worker2", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>21, "name"=>"[main]>worker3", "plugin"=>["logstash::filters::grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{syslogbase2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:insynchronize'"}]}>, :level=>:warn}

the problem ended being using aws discovery es cluster. though removed dead instance (still visible in aws, no accessible) security group using did not restart of remaining es nodes. once did started working again.


-

Comments

Post a Comment

Popular posts from this blog

javascript - Thinglink image not visible until browser resize -

Image
i trying repurpose pre-existing theme, includes drop down menu displays image. trying embed thinglink drop down menu's content section, not visible until resize browser window. here menu looks like: then when clicked, can see empty space image should show: then when browser resized, image appears: this code content of menu: <div class="ajax_accordion_content" style="display: none;"> <div class="report-detail"> <table border="0" cellpadding="0" cellspacing="0" width="100%" class="table_basic"> <tbody> <!-- thinglink image embedded --> <img style="max-width: 100%;" src="http://cdn.thinglink.me/api/image/832705713204625409/1024/10/scaletowidth#tl-832705713204625409;1043138249'" class="alwaysthinglink"/> <script async charset="utf-8" src="http://cdn.thinglink.me/jse/embed.j...
Read more

firebird - Error "invalid transaction handle (expecting explicit transaction start)" executing script from Delphi -

i'm developing update app delphi 10. i'm running windows 7 64x, firebird 2.5.1.26351 32x. execution order: checks current version(select statement firebird database) downloads update(via ftp) apply it(.exe files, , execute .sql) error code -901 pops out when try execute .sql files. know there incompatibility issue w7 64x , fdb 32x, but, can everything: backup , restore(gbak), db commands(all). tried fdb 64x instead, , not working. code: //components fdwaitcursor: tfdguixwaitcursor; fddriverlink: tfdphysfbdriverlink; fdconnection:tfdconnection; fdscript: tfdscript; function tthr_script.createcomponents:boolean; begin try fddriverlink := tfdphysfbdriverlink.create(fddriverlink); fdwaitcursor := tfdguixwaitcursor.create(fdwaitcursor); fdconnection := tfdconnection.create(fdconnection); fdconnection begin drivername := 'fb'; params.username := thrbanco.bduser; params.password := thrbanco.bdpass; params.database := ...
Read more

Sound is not coming out while implementing Text-to-speech in Android activity -

i using texttospeak in 1 of app. while adding speech string texttospeech object instance, doesn't produce sound. though not getting error in log still sound not coming out. want device speak value of textview gets set resultmsg.settext() method. public class resultactivity extends activity { @override protected void oncreate(bundle savedinstancestate){ ............ resultmsg.settext(resultmsg_str); texttospeech = new texttospeech(getapplicationcontext(),newtexttospeech.oninitlistener(){ @override public void oninit(int status) { system.out.println("txt speech status = "+status); system.out.println("txt speech error status = "+texttospeech.error); system.out.println("txt speech success status = "+texttospeech.success); if(status == texttospeech.success){ texttospeech.setlanguage(locale.english); ...
Read more