Elasticsearch highlight 异常

<div>hi, @lionsoul 大大，在 Elasticsearch 中使用 Jcseg 遇到一些问题

### 问题描述

搜索到某篇文档时，出现 `[IllegalStateException] last() should not be called in this context` 的异常，使用 standard 或其它分词器暂未出现此异常

### 相关数据

Elasticsearch Version: 6.3.0

Jcseg Version: 2.4.0

Index settings:

```json
{
  "code_index" : {
    "aliases" : { },
    "mappings" : {
      "code" : {
        "properties" : {
          "category_id" : {
            "type" : "integer"
          },
          "code_pieces" : {
            "properties" : {
              "content" : {
                "type" : "text",
                "index_options" : "offsets",
                "analyzer" : "jcseg"
              },
              "content_type" : {
                "type" : "integer"
              },
              "id" : {
                "type" : "integer"
              },
              "title" : {
                "type" : "text",
                "index_options" : "offsets",
                "analyzer" : "jcseg_complex"
              }
            }
          },
          "created_at" : {
            "type" : "date"
          },
          "enterprise_id" : {
            "type" : "integer"
          },
          "forks_count" : {
            "type" : "integer"
          },
          "id" : {
            "type" : "integer"
          },
          "in_square" : {
            "type" : "boolean"
          },
          "lang_id" : {
            "type" : "integer"
          },
          "lang_name" : {
            "type" : "keyword"
          },
          "name_with_summary" : {
            "type" : "text",
            "index_options" : "offsets",
            "analyzer" : "jcseg_complex"
          },
          "notes_count" : {
            "type" : "integer"
          },
          "pieces_count" : {
            "type" : "integer"
          },
          "public" : {
            "type" : "integer"
          },
          "root_id" : {
            "type" : "integer"
          },
          "stars_count" : {
            "type" : "integer"
          },
          "summary" : {
            "type" : "text",
            "index_options" : "offsets",
            "analyzer" : "jcseg_complex"
          },
          "updated_at" : {
            "type" : "date"
          },
          "user_id" : {
            "type" : "integer"
          }
        }
      }
    },
    "settings" : {
      "index" : {
        "number_of_shards" : "1",
        "provided_name" : "code_index",
        "creation_date" : "1533553788519",
        "number_of_replicas" : "1",
        "uuid" : "3f-KmuKSTeC4MuCy14tkiw",
        "version" : {
          "created" : "6030199"
        }
      }
    }
  }
}
```

Query:

```json
{  
   "from":0,
   "size":15,
   "query":{  
      "bool":{  
         "must":[  
            {  
               "multi_match":{  
                  "query":"layui",
                  "fields":[  
                     "code_pieces.content^1.0",
                     "code_pieces.title^1.0",
                     "name_with_summary^100.0"
                  ]
               }
            }
         ],
         "filter": [  
            {  
               "term":{  
                  "in_square":{  
                     "value":true
                  }
               }
            },
            {  
               "term":{  
                  "public":{  
                     "value":1
                  }
               }
            }
         ]
      }
   },
   "sort":[  
      {  
         "_score":{  
            "order":"desc"
         }
      }
   ],
   "highlight":{  
      "pre_tags":[  
         "<em>"
      ],
      "post_tags":[  
         "</em>"
      ],
      "fragment_size":200,
      "number_of_fragments":3,
      "fields":{  
         "name_with_summary": {},
         "code_pieces.content": {},
         "code_pieces.title": {}
      }
   }
}
```

相关文档:

```json
        "_source" : {
          "id" : 46867,
          "public" : 1,
          "user_id" : 1619994,
          "lang_id" : 3,
          "root_id" : 0,
          "category_id" : 89,
          "enterprise_id" : 0,
          "stars_count" : 0,
          "forks_count" : 0,
          "notes_count" : 0,
          "pieces_count" : 3,
          "in_square" : true,
          "lang_name" : "JavaScript",
          "summary" : "Layui上传多附件材料代码",
          "name_with_summary" : "LKBBD / Layui上传多附件材料代码",
          "created_at" : "2017-12-03T13:56:06+08:00",
          "updated_at" : "2017-12-03T13:59:33+08:00",
          "code_pieces" : [
            {
              "id" : 62466,
              "content_type" : 0,
              "content" : "基于layui,引入layui.css,layui.js及jquery.js等\r\n<link rel=\"stylesheet\" href=\"/style/layui/css/layui.css\" type=\"text/css\" media=\"all\"></link>\r\n<script type=\"text/javascript\" src=\"/style/layui/layui.js\"></script>",
              "title" : "js静态资源部分(主要)"
            },
            {
              "id" : 62467,
              "content_type" : 0,
              "content" : "<div class=\"layui-upload\">\r\n    <button type=\"button\" class=\"layui-btn layui-btn-normal\" id=\"testList\">选择多文件</button> \r\n    <div class=\"layui-upload-list\">\r\n    <table class=\"layui-table\">\r\n        <thead>\r\n        <tr>\r\n        <th style=\"width:40%;text-align:left;padding:10px 20px;\">文件名</th>\r\n        <th style=\"width:20%;text-align:left;padding:10px 20px;\">大小</th>\r\n        <th style=\"width:20%;text-align:left;padding:10px 20px;\">状态</th>\r\n        <th style=\"width:20%;text-align:left;padding:10px 20px;\">操作</th>\r\n        </tr>\r\n        </thead>\r\n        <tbody id=\"demoList\"></tbody>\r\n    </table>\r\n    </div>\r\n    <button type=\"button\" class=\"layui-btn\" id=\"testListAction\">开始上传</button>\r\n</div> ",
              "title" : "html部分"
            },
            {
              "id" : 62468,
              "content_type" : 0,
              "content" : "layui.use('upload', function(){\r\n    var $ = layui.jquery,\r\n    upload = layui.upload;\r\n    var files = [];\r\n    var demoListView = $('#demoList'),\r\n    uploadListIns = upload.render({\r\n    elem: '#testList',\r\n    url: '/file/ftp/upload/',//FTP上传 \r\n    accept: 'file',\r\n    multiple: true,\r\n    auto: false,\r\n    size: 6000, //限制文件大小，单位 KB\r\n    exts: 'jpg|png|xlsx|docx|pdf',//只允许上传文件类型\r\n    bindAction: '#testListAction',\r\n    choose: function(obj){   \r\n        files = obj.pushFile(); //将每次选择的文件追加到文件队列\r\n        //读取本地文件\r\n        obj.preview(function(index, file, result){\r\n        var tr = $(['<tr id=\"upload-'+ index +'\">',\r\n            '<td style=\\\"width:40%;padding:10px;text-align:left;font-size:14px;\\\">'+ file.name +'</td>',\r\n            '<td style=\\\"width:20%;padding:10px;text-align:left;font-size:14px;\\\">'+ (file.size/1014).toFixed(1) +'kb</td>',\r\n            '<td style=\\\"width:20%;padding:10px;text-align:left;font-size:14px;\\\">等待上传</td>',\r\n            '<td style=\\\"width:20%;padding:10px;text-align:left;font-size:14px;\\\">',\r\n            '<button class=\"layui-btn layui-btn-mini demo-reload layui-hide\">重传</button>',\r\n            '<button class=\"layui-btn layui-btn-mini layui-btn-danger demo-delete\">删除</button>',\r\n            '</td>',\r\n        '</tr>'].join(''));\r\n        //单个重传\r\n        tr.find('.demo-reload').on('click', function(){\r\n            obj.upload(index, file);\r\n        });\r\n        //删除\r\n        tr.find('.demo-delete').on('click', function(){\r\n            delete files[index]; //删除对应的文件\r\n            tr.remove();\r\n        });\r\n        demoListView.append(tr);\r\n        });\r\n    },\r\n    done: function(res, index, upload){\r\n        if(res.code == 0){ //上传成功\r\n        var tr = demoListView.find('tr#upload-'+ index),\r\n        tds = tr.children();\r\n        tds.eq(2).html('<span style=\"color: #5FB878;\">上传成功</span>');\r\n        tds.eq(3).html(''); //清空操作\r\n        var file_ids = $(\"input[name='dwyyzzsmj']\").val();\r\n        file_ids = file_ids + \"//\" + res.fileid;\r\n        $(\"input[name='dwyyzzsmj']\").val(\"\");\r\n        $(\"input[name='dwyyzzsmj']\").val(file_ids);\r\n        delete files[index]; //删除文件队列已经上传成功的文件\r\n        return;\r\n        }\r\n        this.error(index, upload);\r\n    },\r\n    error: function(index, upload){\r\n        var tr = demoListView.find('tr#upload-'+ index),\r\n        tds = tr.children();\r\n        tds.eq(2).html('<span style=\"color: #FF5722;\">上传失败</span>');\r\n        tds.eq(3).find('.demo-reload').removeClass('layui-hide'); //显示重传\r\n    }\r\n  });\r\n});",
              "title" : "js实现部分"
            }
          ]
        }
```

### 错误日志

```
018-08-14T18:13:49,947][DEBUG][o.e.a.s.TransportSearchAction] [elasticsearch_node_001] [code_index][0], node[pPuDMK87ROSebXfQOmgmuw], [P], s[STARTED], a[id=0wGOjF3OS6mXGyMKgPYi5A]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[code_index], indicesOptions=IndicesOptions[id=38, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false], types=[code], routing='null', preference='null', requestCache=null, scroll=null, maxConcurrentShardRequests=5, batchedReduceSize=512, preFilterShardSize=128, allowPartialSearchResults=true, source={"from":0,"size":15,"query":{"bool":{"must":[{"multi_match":{"query":"layui","fields":["code_pieces.content^1.0","code_pieces.title^1.0","name_with_summary^100.0"],"type":"best_fields","operator":"OR","slop":0,"prefix_length":0,"max_expansions":50,"zero_terms_query":"NONE","auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"filter":[{"term":{"in_square":{"value":true,"boost":1.0}}},{"term":{"public":{"value":1,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"sort":[{"_score":{"order":"desc"}}],"highlight":{"pre_tags":["<em>"],"post_tags":["</em>"],"fragment_size":200,"number_of_fragments":3,"fields":{"name_with_summary":{},"code_pieces.content":{},"code_pieces.title":{}}}}}]
org.elasticsearch.transport.RemoteTransportException: [elasticsearch_node_001][192.168.1.111:9300][indices:data/read/search[phase/query]]
Caused by: java.lang.IllegalStateException: last() should not be called in this context
	at org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner.last(BoundedBreakIteratorScanner.java:175) ~[elasticsearch-6.3.1.jar:7.3.1 ae0705edb59eaa567fe13ed3a222fdadc7153680 - caomanhdat - 2018-05-09 09:28:05]
	at org.apache.lucene.search.uhighlight.SplittingBreakIterator.preceding(SplittingBreakIterator.java:218) ~[lucene-highlighter-7.3.1.jar:7.3.1 ae0705edb59eaa567fe13ed3a222fdadc7153680 - caomanhdat - 2018-05-09 09:28:05]
	at org.apache.lucene.search.uhighlight.FieldHighlighter.highlightOffsetsEnums(FieldHighlighter.java:163) ~[lucene-highlighter-7.3.1.jar:7.3.1 ae0705edb59eaa567fe13ed3a222fdadc7153680 - caomanhdat - 2018-05-09 09:28:05]
	at org.apache.lucene.search.uhighlight.FieldHighlighter.highlightFieldForDoc(FieldHighlighter.java:80) ~[lucene-highlighter-7.3.1.jar:7.3.1 ae0705edb59eaa567fe13ed3a222fdadc7153680 - caomanhdat - 2018-05-09 09:28:05]
	at org.apache.lucene.search.uhighlight.UnifiedHighlighter.highlightFieldsAsObjects(UnifiedHighlighter.java:629) ~[lucene-highlighter-7.3.1.jar:7.3.1 ae0705edb59eaa567fe13ed3a222fdadc7153680 - caomanhdat - 2018-05-09 09:28:05]
	at org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.highlightField(CustomUnifiedHighlighter.java:107) ~[elasticsearch-6.3.1.jar:7.3.1 ae0705edb59eaa567fe13ed3a222fdadc7153680 - caomanhdat - 2018-05-09 09:28:05]
	at org.elasticsearch.search.fetch.subphase.highlight.UnifiedHighlighter.highlight(UnifiedHighlighter.java:131) ~[elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase.hitExecute(HighlightPhase.java:114) ~[elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:162) ~[elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:393) ~[elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:368) ~[elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:333) [elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:329) [elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1019) [elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:725) [elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) [elasticsearch-6.3.1.jar:6.3.1]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.1.jar:6.3.1]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
```

### 一些想法

这个问题应该是因为 highlight `code_pieces.content` 才出现的，当将 hightlight 模式从 `offsets strategy` 改为 `plain highlighter` 后没有出现此异常

希望 Jcseg 发展越来越好

thanks

我们的代码片段有些会过大

我之前的高亮一直用的基于offsets的方式，我这边确认下。

@lionsoul 嗯嗯，我试着关闭了 jcseg lexicon 的功能，再重建 code 索引，当搜索到 `layui` 这篇文档，还是会出现异常

下面是 jcseg 相关配置

```
# jcseg properties file.
# bug report chenxin <chenxin619315@gmail.com>

# Jcseg function
#maximum match length. (5-7)
jcseg.maxlen = 7

#recognized the chinese name.(1 to open and 0 to close it)
jcseg.icnname = 0

#maximum length for pair punctuation text.
jcseg.pptmaxlen = 7

#maximum length for chinese last name andron.
jcseg.cnmaxlnadron = 1

#Wether to clear the stopwords.(set 1 to clear stopwords and 0 to close it)
jcseg.clearstopword = 0

#Wether to convert the chinese numeric to arabic number. (set to 1 open it and 0 to close it)
# like '\u4E09\u4E07' to 30000.
jcseg.cnnumtoarabic = 1

#Wether to convert the chinese fraction to arabic fraction.
#@Note: for lucene,solr,elasticsearch eg.. close it.
jcseg.cnfratoarabic = 0

#Wether to keep the unrecognized word. (set 1 to keep unrecognized word and 0 to clear it)
jcseg.keepunregword = 1

#Wether to start the secondary segmentation for the complex english words.
jcseg.ensencondseg = 1

#min length of the secondary simple token. (better larger than 1)
jcseg.stokenminlen = 2

#thrshold for chinese name recognize.
# better not change it before you know what you are doing.
jcseg.nsthreshold = 1000000

#The punctuations that will be keep in an token.(Not the end of the token).
jcseg.keeppunctuations = @#%.&+

####about the lexicon
#abusolte path of the lexicon file.
#Multiple path support from jcseg 1.9.2, use ';' to split different path.
#example: lexicon.path = /home/chenxin/lex1;/home/chenxin/lex2 (Linux)
#        : lexicon.path = D:/jcseg/lexicon/1;D:/jcseg/lexicon/2 (WinNT)
#lexicon.path=/Code/java/JavaSE/jcseg/lexicon
#lexicon.path = {jar.dir}/lexicon ({jar.dir} means the base directory of jcseg-core-{version}.jar)
#@since 1.9.9 Jcseg default to load the lexicons in the classpath
lexicon.path = {jar.dir}/lexicon

#Wether to load the modified lexicon file auto.
lexicon.autoload = 0

#Poll time for auto load. (seconds)
lexicon.polltime = 300

####lexicon load
#Wether to load the part of speech of the entry.
jcseg.loadpos = 0

#Wether to load the pinyin of the entry.
jcseg.loadpinyin = 0

#Wether to load the synoyms words of the entry.
jcseg.loadsyn = 0

#wether to load the entity of the entry
jcseg.loadentity = 0

```

OK，我这边试下结果！

使用最新master分支的代码试试，今儿给了一个修复提交！

已修复：https://www.oschina.net/news/109718

GVP 狮子的魂/jcseg

内容风险标识

评论 (7)

GVP狮子的魂/jcseg

内容风险标识