Elasticserach High CPU

[xiangronghua]

 Elasticserach cluster 4 nodes 5 shards, 2 Replicas
total documents 40000000
 25GB ram
      ES_MIN_MEM = ES_MAX_MEM  and set to 10GB
      32 core CUP

::: {node-3}{gp2dtjDKTIuJATdnWjd2jw}{iLc2fVQ0Rz-Y51Q23_8Zjg}{192.168.150.123}{192.168.150.123:9300}{rack=r3}
   Hot threads at 2017-03-23T02:52:28.396Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {node-2}{FhHZDylBT9mcvFYKstRSlg}{PIcJaMHJSAab0QhvtlqPsw}{192.168.150.122}{192.168.150.122:9300}{rack=r2}
   Hot threads at 2017-03-23T02:52:30.183Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
   
   74.2% (371.1ms out of 500ms) cpu usage by thread 'elasticsearch[node-2][search][T#18]'
     3/10 snapshots sharing following 32 elements
       java.lang.ThreadLocal$ThreadLocalMap.getEntryAfterMiss(ThreadLocal.java:444)
       java.lang.ThreadLocal$ThreadLocalMap.getEntry(ThreadLocal.java:419)
       java.lang.ThreadLocal$ThreadLocalMap.access$000(ThreadLocal.java:298)
       java.lang.ThreadLocal.get(ThreadLocal.java:163)
       org.apache.lucene.util.CloseableThreadLocal.get(CloseableThreadLocal.java:78)
       org.apache.lucene.index.CodecReader.getSortedSetDocValues(CodecReader.java:250)
       org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:460)
       org.apache.lucene.index.DocValues.getSortedSet(DocValues.java:302)
       org.elasticsearch.index.fielddata.plain.SortedSetDVBytesAtomicFieldData.getOrdinalsValues(SortedSetDVBytesAtomicFieldData.java:49)
       org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsIndexFieldData$Atomic.getOrdinalsValues(InternalGlobalOrdinalsIndexFieldData.java:72)
       org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:152)
       org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator.getLeafCollector(GlobalOrdinalsStringTermsAggregator.java:92)
       org.elasticsearch.search.aggregations.AggregatorBase.getLeafCollector(AggregatorBase.java:149)
       org.elasticsearch.search.aggregations.BucketCollector$2.getLeafCollector(BucketCollector.java:75)
       org.elasticsearch.search.aggregations.BucketCollector$2.getLeafCollector(BucketCollector.java:69)
       org.apache.lucene.search.MultiCollector.getLeafCollector(MultiCollector.java:121)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:473)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:370)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:106)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:238)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:251)
       org.elasticsearch.action.search.SearchTransportService.lambda$registerRequestHandler$6(SearchTransportService.java:276)
       org.elasticsearch.action.search.SearchTransportService$$Lambda$1041/1681629677.messageReceived(Unknown Source)
       org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
       org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
       org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1348)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:527)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
       java.lang.Thread.run(Thread.java:745)
     4/10 snapshots sharing following 28 elements
       java.lang.ThreadLocal$ThreadLocalMap.getEntryAfterMiss(ThreadLocal.java:444)
       java.lang.ThreadLocal$ThreadLocalMap.getEntry(ThreadLocal.java:419)
       java.lang.ThreadLocal$ThreadLocalMap.access$000(ThreadLocal.java:298)
       java.lang.ThreadLocal.get(ThreadLocal.java:163)
       org.elasticsearch.search.internal.SearchContext.current(SearchContext.java:96)
       org.elasticsearch.search.fetch.subphase.MatchedQueriesFetchSubPhase.hitsExecute(MatchedQueriesFetchSubPhase.java:95)
       org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:166)
       org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregator.buildAggregation(TopHitsAggregator.java:146)
       org.elasticsearch.search.aggregations.bucket.BestBucketsDeferringCollector$2.buildAggregation(BestBucketsDeferringCollector.java:204)
       org.elasticsearch.search.aggregations.bucket.BucketsAggregator.bucketAggregations(BucketsAggregator.java:116)
       org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator.buildAggregation(GlobalOrdinalsStringTermsAggregator.java:199)
       org.elasticsearch.search.aggregations.AggregatorFactory$MultiBucketAggregatorWrapper.buildAggregation(AggregatorFactory.java:147)
       org.elasticsearch.search.aggregations.bucket.BucketsAggregator.bucketAggregations(BucketsAggregator.java:116)
       org.elasticsearch.search.aggregations.bucket.filter.FilterAggregator.buildAggregation(FilterAggregator.java:72)
       org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:143)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:238)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:251)
       org.elasticsearch.action.search.SearchTransportService.lambda$registerRequestHandler$6(SearchTransportService.java:276)
       org.elasticsearch.action.search.SearchTransportService$$Lambda$1041/1681629677.messageReceived(Unknown Source)
       org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
       org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
       org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1348)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:527)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
       java.lang.Thread.run(Thread.java:745)
     3/10 snapshots sharing following 2 elements
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
       java.lang.Thread.run(Thread.java:745)

::: {node-1}{FPGIunVATT-VduR1NaFvAw}{5-ysd-KjSEGnbIIj14p6YA}{192.168.150.121}{192.168.150.121:9300}{rack=r1}
   Hot threads at 2017-03-23T02:50:49.801Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {node-4}{T9pNc6mESvyvBnGGSzL9xQ}{mAesddWETp-73jRm0ChD4g}{192.168.150.124}{192.168.150.124:9300}{rack=r1}
   Hot threads at 2017-03-23T02:47:44.629Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: