中文分词模块

import { Segment, useDefault } from 'segmentit';

const segmentit = useDefault(new Segment());
const result = segmentit.doSegment('工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作。');
console.log(result);

const { Segment, useDefault } = require('segmentit');
const segmentit = useDefault(new Segment());
const result = segmentit.doSegment('工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作。');
console.log(result);

// import Segment, { useDefault, cnPOSTag, enPOSTag } from 'segmentit';
const { Segment, useDefault, cnPOSTag, enPOSTag } = require('segmentit');

const segmentit = useDefault(new Segment());

console.log(segmentit.doSegment('一人得道，鸡犬升天').map(i => `${i.w} <${cnPOSTag(i.p)}> <${enPOSTag(i.p)}>`))
// ↑ ["一人得道 <习语,数词 数语素> <l,m>", "， <标点符号> <w>", "鸡犬升天 <成语> <i>"]

// useDefault
import { Segment, modules, dicts, synonyms, stopwords } from 'segmentit';

const segmentit = new Segment();
segmentit.use(modules);
segmentit.loadDict(dicts);
segmentit.loadSynonymDict(synonyms);
segmentit.loadStopwordDict(stopwords);

// load custom module and dicts
import {
  Segment,
  ChsNameTokenizer,
  DictOptimizer,
  EmailOptimizer,
  PunctuationTokenizer,
  URLTokenizer,
  ChsNameOptimizer,
  DatetimeOptimizer,
  DictTokenizer,
  ForeignTokenizer,
  SingleTokenizer,
  WildcardTokenizer,
  pangu,
  panguExtend1,
  panguExtend2,
  names,
  wildcard,
  synonym,
  stopword,
} from 'segmentit';

const segmentit = new Segment();

// load them one by one, or by array
segmentit.use(ChsNameTokenizer);
segmentit.loadDict(pangu);
segmentit.loadDict([panguExtend1, panguExtend2]);
segmentit.loadSynonymDict(synonym);
segmentit.loadStopwordDict(stopword);

// @flow
import { Tokenizer } from 'segmentit';
import type { SegmentToken, TokenStartPosition } from 'segmentit';
export default class ChsNameTokenizer extends Tokenizer {
  split(words: Array<SegmentToken>): Array<SegmentToken> {
    // 可以获取到 this.segment 里的各种信息
    const POSTAG = this.segment.POSTAG;
    const TABLE = this.segment.getDict('TABLE');
    // ...
  }

// @flow
import { Optimizer } from './BaseModule';
import type { SegmentToken } from './type';
export default class DictOptimizer extends Optimizer {
  doOptimize(words: Array<SegmentToken>, isNotFirst: boolean): Array<SegmentToken> {
    // 可以获取到 this.segment 里的各种信息
    const POSTAG = this.segment.POSTAG;
    const TABLE = this.segment.getDict('TABLE');
    // ...
    // 针对组合数字后无法识别新组合的数字问题，需要重新扫描一次
    return isNotFirst === true ? words : this.doOptimize(words, true);
  }

// @flow
// https://github.com/linonetwo/segmentit/blob/master/src/module/AdjectiveOptimizer.js
import { Optimizer } from './BaseModule';
import type { SegmentToken } from './type';

import { colors } from './COLORS';

// 把一些错认为名词的词标注为形容词，或者对名词作定语的情况
export default class AdjectiveOptimizer extends Optimizer {
  doOptimize(words: Array<SegmentToken>): Array<SegmentToken> {
    const { POSTAG } = this.segment;
    let index = 0;
    while (index < words.length) {
      const word = words[index];
      const nextword = words[index + 1];
      if (nextword) {
        // 对于<颜色>+<的>，直接判断颜色是形容词（字典里颜色都是名词）
        if (nextword.p === POSTAG.D_U && colors.includes(word.w)) {
          word.p = POSTAG.D_A;
        }
        // 如果是连续的两个名词，前一个是颜色，那这个颜色也是形容词
        if (word.p === POSTAG.D_N && nextword.p === POSTAG.D_N && colors.includes(word.w)) {
          word.p = POSTAG.D_A;
        }
      }
      // 移到下一个单词
      index += 1;
    }
    return words;
  }
}