1 Star 0 Fork 0

pidan/chroma

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
iterator.go 1.72 KB
一键复制 编辑 原始数据 按行查看 历史
package chroma
import "strings"
// An Iterator across tokens.
//
// EOF will be returned at the end of the Token stream.
//
// If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
type Iterator func() Token
// Tokens consumes all tokens from the iterator and returns them as a slice.
func (i Iterator) Tokens() []Token {
var out []Token
for t := i(); t != EOF; t = i() {
out = append(out, t)
}
return out
}
// Concaterator concatenates tokens from a series of iterators.
func Concaterator(iterators ...Iterator) Iterator {
return func() Token {
for len(iterators) > 0 {
t := iterators[0]()
if t != EOF {
return t
}
iterators = iterators[1:]
}
return EOF
}
}
// Literator converts a sequence of literal Tokens into an Iterator.
func Literator(tokens ...Token) Iterator {
return func() Token {
if len(tokens) == 0 {
return EOF
}
token := tokens[0]
tokens = tokens[1:]
return token
}
}
// SplitTokensIntoLines splits tokens containing newlines in two.
func SplitTokensIntoLines(tokens []Token) (out [][]Token) {
var line []Token // nolint: prealloc
for _, token := range tokens {
for strings.Contains(token.Value, "\n") {
parts := strings.SplitAfterN(token.Value, "\n", 2)
// Token becomes the tail.
token.Value = parts[1]
// Append the head to the line and flush the line.
clone := token.Clone()
clone.Value = parts[0]
line = append(line, clone)
out = append(out, line)
line = nil
}
line = append(line, token)
}
if len(line) > 0 {
out = append(out, line)
}
// Strip empty trailing token line.
if len(out) > 0 {
last := out[len(out)-1]
if len(last) == 1 && last[0].Value == "" {
out = out[:len(out)-1]
}
}
return
}
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Java
1
https://gitee.com/pidanpidan/chroma.git
git@gitee.com:pidanpidan/chroma.git
pidanpidan
chroma
chroma
v3.1.1

搜索帮助