|
This post has NOT been accepted by the mailing list yet.
This post was updated on .
Elasticserach cluster 4 nodes 5 shards, 2 Replicas
total documents 40000000
25GB ram
ES_MIN_MEM = ES_MAX_MEM and set to 10GB
32 core CUP
::: {node-3}{gp2dtjDKTIuJATdnWjd2jw}{iLc2fVQ0Rz-Y51Q23_8Zjg}{192.168.150.123}{192.168.150.123:9300}{rack=r3}
Hot threads at 2017-03-23T02:52:28.396Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {node-2}{FhHZDylBT9mcvFYKstRSlg}{PIcJaMHJSAab0QhvtlqPsw}{192.168.150.122}{192.168.150.122:9300}{rack=r2}
Hot threads at 2017-03-23T02:52:30.183Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
74.2% (371.1ms out of 500ms) cpu usage by thread 'elasticsearch[node-2][search][T#18]'
3/10 snapshots sharing following 32 elements
java.lang.ThreadLocal$ThreadLocalMap.getEntryAfterMiss(ThreadLocal.java:444)
java.lang.ThreadLocal$ThreadLocalMap.getEntry(ThreadLocal.java:419)
java.lang.ThreadLocal$ThreadLocalMap.access$000(ThreadLocal.java:298)
java.lang.ThreadLocal.get(ThreadLocal.java:163)
org.apache.lucene.util.CloseableThreadLocal.get(CloseableThreadLocal.java:78)
org.apache.lucene.index.CodecReader.getSortedSetDocValues(CodecReader.java:250)
org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:460)
org.apache.lucene.index.DocValues.getSortedSet(DocValues.java:302)
org.elasticsearch.index.fielddata.plain.SortedSetDVBytesAtomicFieldData.getOrdinalsValues(SortedSetDVBytesAtomicFieldData.java:49)
org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsIndexFieldData$Atomic.getOrdinalsValues(InternalGlobalOrdinalsIndexFieldData.java:72)
org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:152)
org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator.getLeafCollector(GlobalOrdinalsStringTermsAggregator.java:92)
org.elasticsearch.search.aggregations.AggregatorBase.getLeafCollector(AggregatorBase.java:149)
org.elasticsearch.search.aggregations.BucketCollector$2.getLeafCollector(BucketCollector.java:75)
org.elasticsearch.search.aggregations.BucketCollector$2.getLeafCollector(BucketCollector.java:69)
org.apache.lucene.search.MultiCollector.getLeafCollector(MultiCollector.java:121)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:473)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:370)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:106)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:238)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:251)
org.elasticsearch.action.search.SearchTransportService.lambda$registerRequestHandler$6(SearchTransportService.java:276)
org.elasticsearch.action.search.SearchTransportService$$Lambda$1041/1681629677.messageReceived(Unknown Source)
org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1348)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:527)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
4/10 snapshots sharing following 28 elements
java.lang.ThreadLocal$ThreadLocalMap.getEntryAfterMiss(ThreadLocal.java:444)
java.lang.ThreadLocal$ThreadLocalMap.getEntry(ThreadLocal.java:419)
java.lang.ThreadLocal$ThreadLocalMap.access$000(ThreadLocal.java:298)
java.lang.ThreadLocal.get(ThreadLocal.java:163)
org.elasticsearch.search.internal.SearchContext.current(SearchContext.java:96)
org.elasticsearch.search.fetch.subphase.MatchedQueriesFetchSubPhase.hitsExecute(MatchedQueriesFetchSubPhase.java:95)
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:166)
org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregator.buildAggregation(TopHitsAggregator.java:146)
org.elasticsearch.search.aggregations.bucket.BestBucketsDeferringCollector$2.buildAggregation(BestBucketsDeferringCollector.java:204)
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.bucketAggregations(BucketsAggregator.java:116)
org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator.buildAggregation(GlobalOrdinalsStringTermsAggregator.java:199)
org.elasticsearch.search.aggregations.AggregatorFactory$MultiBucketAggregatorWrapper.buildAggregation(AggregatorFactory.java:147)
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.bucketAggregations(BucketsAggregator.java:116)
org.elasticsearch.search.aggregations.bucket.filter.FilterAggregator.buildAggregation(FilterAggregator.java:72)
org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:143)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:238)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:251)
org.elasticsearch.action.search.SearchTransportService.lambda$registerRequestHandler$6(SearchTransportService.java:276)
org.elasticsearch.action.search.SearchTransportService$$Lambda$1041/1681629677.messageReceived(Unknown Source)
org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1348)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:527)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3/10 snapshots sharing following 2 elements
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: {node-1}{FPGIunVATT-VduR1NaFvAw}{5-ysd-KjSEGnbIIj14p6YA}{192.168.150.121}{192.168.150.121:9300}{rack=r1}
Hot threads at 2017-03-23T02:50:49.801Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {node-4}{T9pNc6mESvyvBnGGSzL9xQ}{mAesddWETp-73jRm0ChD4g}{192.168.150.124}{192.168.150.124:9300}{rack=r1}
Hot threads at 2017-03-23T02:47:44.629Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
|