1 Star 0 Fork 2

王布衣/pkg

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
string.go 2.73 KB
一键复制 编辑 原始数据 按行查看 历史
// Package unistring contains an implementation of a hybrid ASCII/UTF-16 string.
// For ASCII strings the underlying representation is equivalent to a normal Go string.
// For unicode strings the underlying representation is UTF-16 as []uint16 with 0th element set to 0xFEFF.
// unicode.String allows representing malformed UTF-16 values (e.g. stand-alone parts of surrogate pairs)
// which cannot be represented in UTF-8.
// At the same time it is possible to use unicode.String as property keys just as efficiently as simple strings,
// (the leading 0xFEFF ensures there is no clash with ASCII string), and it is possible to convert it
// to valueString without extra allocations.
package unistring
import (
"reflect"
"unicode/utf16"
"unicode/utf8"
"unsafe"
)
const (
BOM = 0xFEFF
)
type String string
// Scan checks if the string contains any unicode characters. If it does, converts to an array suitable for creating
// a String using FromUtf16, otherwise returns nil.
func Scan(s string) []uint16 {
utf16Size := 0
for ; utf16Size < len(s); utf16Size++ {
if s[utf16Size] >= utf8.RuneSelf {
goto unicode
}
}
return nil
unicode:
for _, chr := range s[utf16Size:] {
utf16Size++
if chr > 0xFFFF {
utf16Size++
}
}
buf := make([]uint16, utf16Size+1)
buf[0] = BOM
c := 1
for _, chr := range s {
if chr <= 0xFFFF {
buf[c] = uint16(chr)
} else {
first, second := utf16.EncodeRune(chr)
buf[c] = uint16(first)
c++
buf[c] = uint16(second)
}
c++
}
return buf
}
func NewFromString(s string) String {
if buf := Scan(s); buf != nil {
return FromUtf16(buf)
}
return String(s)
}
func NewFromRunes(s []rune) String {
ascii := true
size := 0
for _, c := range s {
if c >= utf8.RuneSelf {
ascii = false
if c > 0xFFFF {
size++
}
}
size++
}
if ascii {
return String(s)
}
b := make([]uint16, size+1)
b[0] = BOM
i := 1
for _, c := range s {
if c <= 0xFFFF {
b[i] = uint16(c)
} else {
first, second := utf16.EncodeRune(c)
b[i] = uint16(first)
i++
b[i] = uint16(second)
}
i++
}
return FromUtf16(b)
}
func FromUtf16(b []uint16) String {
var str string
hdr := (*reflect.StringHeader)(unsafe.Pointer(&str))
hdr.Data = uintptr(unsafe.Pointer(&b[0]))
hdr.Len = len(b) * 2
return String(str)
}
func (s String) String() string {
if b := s.AsUtf16(); b != nil {
return string(utf16.Decode(b[1:]))
}
return string(s)
}
func (s String) AsUtf16() []uint16 {
if len(s) < 4 || len(s)&1 != 0 {
return nil
}
var a []uint16
raw := string(s)
sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&a))
sliceHeader.Data = (*reflect.StringHeader)(unsafe.Pointer(&raw)).Data
l := len(raw) / 2
sliceHeader.Len = l
sliceHeader.Cap = l
if a[0] == BOM {
return a
}
return nil
}
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
1
https://gitee.com/quant1x/pkg.git
git@gitee.com:quant1x/pkg.git
quant1x
pkg
pkg
v0.3.1

搜索帮助