Ai
1 Star 0 Fork 0

长阳/go-utils

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
encode.go 2.19 KB
一键复制 编辑 原始数据 按行查看 历史
Your Name 提交于 2022-04-27 19:02 +08:00 . add cmd expect 功
package s
import (
"bytes"
"fmt"
"io/ioutil"
"unicode/utf16"
"unicode/utf8"
"github.com/saintfish/chardet"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/encoding/traditionalchinese"
"golang.org/x/text/transform"
)
var (
ERR_UTF16_TO_UTF8 = "Must have even length byte slice"
TP_ISO_8859_1 = "ISO-8859-1"
TP_GB_18030 = "GB-18030"
TP_BIG_5 = "Big5"
TP_UTF16 = "UTF-16LE"
TP_UTF8 = "UTF-8"
)
func AutoDecode(fromBytes []byte) (utfStr Str, err error) {
buf, err := AutoEncodeToUTF8(fromBytes)
if err != nil {
return "", err
}
return Str(buf), nil
}
func AutoEncodeToUTF8(output []byte) ([]byte, error) {
detector := chardet.NewTextDetector()
result, err := detector.DetectBest(output)
if err != nil {
return nil, err
}
switch result.Charset {
case TP_UTF8:
return output, nil
case TP_ISO_8859_1:
return output, nil
case TP_GB_18030:
output, err = GbkToUtf8(output)
case TP_BIG_5:
output, err = Big5ToUtf8(output)
case TP_UTF16:
output, err = UTF16ToUTF8(output)
default:
output, err = GbkToUtf8(output)
if err != nil {
output, err = UTF16ToUTF8(output)
}
}
return output, err
}
func GbkToUtf8(s []byte) ([]byte, error) {
reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
d, e := ioutil.ReadAll(reader)
if e != nil {
return nil, e
}
return d, nil
}
func Utf8ToGbk(s []byte) ([]byte, error) {
reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
d, e := ioutil.ReadAll(reader)
if e != nil {
return nil, e
}
return d, nil
}
//convert BIG5 to UTF-8
func Big5ToUtf8(s []byte) ([]byte, error) {
I := bytes.NewReader(s)
O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder())
d, e := ioutil.ReadAll(O)
if e != nil {
return nil, e
}
return d, nil
}
func UTF16ToUTF8(b []byte) ([]byte, error) {
if len(b)%2 != 0 {
return nil, fmt.Errorf(ERR_UTF16_TO_UTF8)
}
u16s := make([]uint16, 1)
ret := &bytes.Buffer{}
b8buf := make([]byte, 4)
lb := len(b)
for i := 0; i < lb; i += 2 {
u16s[0] = uint16(b[i]) + (uint16(b[i+1]) << 8)
r := utf16.Decode(u16s)
n := utf8.EncodeRune(b8buf, r[0])
ret.Write(b8buf[:n])
}
return ret.Bytes(), nil
}
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Go
1
https://gitee.com/dark.H/go-utils.git
git@gitee.com:dark.H/go-utils.git
dark.H
go-utils
go-utils
v1.3.4

搜索帮助