1 Star 0 Fork 0

h79/goutils

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
sensitive.go 2.13 KB
一键复制 编辑 原始数据 按行查看 历史
huqiuyun 提交于 2022-10-30 21:01 . trie
// 敏感词库
package sensitive
import (
"encoding/json"
"errors"
"gitee.com/h79/goutils/common/logger"
"io"
"net/http"
"sync"
"time"
)
type sensMgr struct {
locker sync.Mutex
regexp bool
filterMap map[string]*Filter
}
var mgr *sensMgr
const KDefGroup = "def"
func Init(reg bool) {
mgr = &sensMgr{
regexp: reg,
filterMap: make(map[string]*Filter),
}
Add(KDefGroup)
}
func Add(group string) *Filter {
mgr.locker.Lock()
defer mgr.locker.Unlock()
if filter, ok := mgr.filterMap[group]; ok {
return filter
}
filter := New(mgr.regexp)
mgr.filterMap[group] = filter
return filter
}
func Get(group string) *Filter {
mgr.locker.Lock()
defer mgr.locker.Unlock()
if filter, ok := mgr.filterMap[group]; ok {
return filter
}
return nil
}
func Load(group string, path string) error {
filter := Add(group)
if filter == nil {
return errors.New("filter not exist")
}
if err := filter.LoadWordDict(path); err != nil {
logger.Error("sensitive load, path= '%s'", path)
return err
}
return nil
}
func LoadNetworkFileByGroup(group string, url string) error {
filter := Add(group)
if filter == nil {
return errors.New("filter not exist")
}
if err := filter.LoadWordDictByNetworkFile(url); err != nil {
logger.Error("sensitive load, url= '%s'", url)
return err
}
return nil
}
// LoadUrl
// {"code":200,"data":{"title":[],"":[]}}
func LoadUrl(url string) error {
c := http.Client{
Timeout: 5 * time.Second,
}
rsp, err := c.Get(url)
if err != nil {
logger.Error("sensitive Loaded url=%s, err=%+v", url, err)
return err
}
defer rsp.Body.Close()
out, er := io.ReadAll(rsp.Body)
if er != nil {
logger.Error("sensitive read url=%s, err=%+v", url, er)
return err
}
logger.Debug("sensitive data=%v ", logger.Byte2(logger.DebugLevel, out))
type Result struct {
Code int `json:"code"`
Data map[string][]string `json:"data"`
}
d := Result{}
if er = json.Unmarshal(out, &d); er != nil {
logger.Error("sensitive read url=%s, err=%+v", url, er)
return err
}
if len(d.Data) == 0 {
return nil
}
for k, v := range d.Data {
filter := Add(k)
filter.AddWord(v...)
}
return nil
}
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Go
1
https://gitee.com/h79/goutils.git
git@gitee.com:h79/goutils.git
h79
goutils
goutils
v1.8.63

搜索帮助