1 Star 0 Fork 0

h79 / goutils

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
sensitive.go 2.44 KB
一键复制 编辑 原始数据 按行查看 历史
huqiuyun 提交于 2024-01-20 15:28 . version
// 敏感词库
package sensitive
import (
"encoding/json"
"errors"
"gitee.com/h79/goutils/common/logger"
"gitee.com/h79/goutils/common/option"
"io"
"net/http"
"sync"
"time"
)
func WithRegexOpt() option.Option {
return option.WithOpt(true, "", 1)
}
func WithDefaultGroupOpt() option.Option {
return option.WithOpt(true, "", 2)
}
type SensMgr struct {
locker sync.RWMutex
filterMap map[string]*Filter
}
const DefaultGroup = "def"
func NewMgr(opts ...option.Option) *SensMgr {
m := &SensMgr{
filterMap: make(map[string]*Filter),
}
if option.Select(2, true, opts...) {
m.Add(DefaultGroup)
}
return m
}
func (m *SensMgr) Add(group string, opts ...option.Option) *Filter {
m.locker.Lock()
if filter, ok := m.filterMap[group]; ok {
m.locker.Unlock()
return filter
}
filter := New(option.Select(1, false, opts...))
m.filterMap[group] = filter
m.locker.Unlock()
return filter
}
func (m *SensMgr) Get(group string) *Filter {
m.locker.RLock()
if filter, ok := m.filterMap[group]; ok {
m.locker.RUnlock()
return filter
}
m.locker.RUnlock()
return nil
}
func (m *SensMgr) Load(group string, path string) error {
filter := m.Add(group)
if filter == nil {
return errors.New("filter not exist")
}
if err := filter.LoadWordDict(path); err != nil {
logger.Error("sensitive load, path= '%s'", path)
return err
}
return nil
}
func (m *SensMgr) LoadNetworkFileByGroup(group string, url string) error {
filter := m.Add(group)
if filter == nil {
return errors.New("filter not exist")
}
if err := filter.LoadWordDictByNetworkFile(url); err != nil {
logger.Error("sensitive load, url= '%s'", url)
return err
}
return nil
}
// LoadUrl
// {"code":200,"data":{"title":[],"":[]}}
func (m *SensMgr) LoadUrl(url string) error {
c := http.Client{
Timeout: 5 * time.Second,
}
rsp, err := c.Get(url)
if err != nil {
logger.E("sensitive", "Loaded url=%s, err=%+v", url, err)
return err
}
defer rsp.Body.Close()
out, er := io.ReadAll(rsp.Body)
if er != nil {
logger.E("sensitive", "read url=%s, err=%+v", url, er)
return err
}
logger.N("sensitive", "data=%v ", logger.Byte2(logger.NDebugLevel, out))
type Result struct {
Code int `json:"code"`
Data map[string][]string `json:"data"`
}
d := Result{}
if er = json.Unmarshal(out, &d); er != nil {
logger.E("sensitive", "read url=%s, err=%+v", url, er)
return err
}
for k, v := range d.Data {
filter := m.Add(k)
filter.AddWord(v...)
}
return nil
}
Go
1
https://gitee.com/h79/goutils.git
git@gitee.com:h79/goutils.git
h79
goutils
goutils
v1.20.57

搜索帮助