From 552fd8b146dffbd50f31498dab8362dbb996744d Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Thu, 23 Jun 2022 11:42:58 +0800 Subject: [PATCH] v1.0.7 --- .github/README.md | 3 +- README.md | 1 + analyzer/engine/archive.go | 46 +++++++--------- analyzer/engine/engine.go | 6 +- analyzer/engine/parse.go | 5 +- analyzer/java/analyzer.go | 10 +++- analyzer/java/ext.go | 7 +-- analyzer/java/gradle.go | 45 ++++++++++++--- analyzer/java/pom.go | 2 +- analyzer/python/analyzer.go | 44 +++++++++++++++ analyzer/python/oss.py | 34 ++++++++++++ analyzer/python/pipfile.go | 56 +++++++++++++++++++ analyzer/python/setup.go | 75 +++++++++++++++++++++++++ cli/main.go | 13 +++-- config.json | 3 +- util/args/args.go | 82 ++++++++++++++++------------ util/args/config.go | 63 --------------------- util/bar/bar.go | 2 +- util/cache/cache.go | 4 +- util/client/client.go | 59 ++++++++++---------- util/enum/language/language.go | 14 +++-- util/filter/file.go | 10 +++- util/model/dependency.go | 4 +- util/model/version.go | 2 +- util/report/format.go | 56 +++++++++++++++---- util/report/html.go | 2 +- util/report/{index.html => html_tpl} | 4 +- util/report/json.go | 4 +- util/temp/temp.go | 37 +++++++++++++ util/vuln/local.go | 8 +-- util/vuln/vuln.go | 8 +-- 31 files changed, 493 insertions(+), 216 deletions(-) create mode 100644 analyzer/python/analyzer.go create mode 100644 analyzer/python/oss.py create mode 100644 analyzer/python/pipfile.go create mode 100644 analyzer/python/setup.go delete mode 100644 util/args/config.go rename util/report/{index.html => html_tpl} (30%) create mode 100644 util/temp/temp.go diff --git a/.github/README.md b/.github/README.md index c8b1046..9548a15 100644 --- a/.github/README.md +++ b/.github/README.md @@ -86,6 +86,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | Set the output file. The result defaults to json format. | `-out output.json` | | `db` | `string` | Set the local vulnerability database file. It helps when you prefer to use your own vulnerability database. The format of the vulnerability database is shown below. If the cloud and local vulnerability databases are both set, the result of detection will merge both. | `-db db.json` | | `progress` | `bool` | Show the progress bar. | `-progress` | +| `dedup` | `bool` | Same result deduplication | `-dedup` | ------ @@ -142,7 +143,7 @@ opensca-cli -db db.json -path ${project_path} OpenSCA is an open source project, we appreciate your help! -To contribute, please read our [Contributing Guideline](./docs/Contributing%20Guideline-en%20v1.0.md). +To contribute, please read our [Contributing Guideline](../docs/Contributing%20Guideline-en%20v1.0.md). diff --git a/README.md b/README.md index bc27480..24a79fe 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | 将检测结果保存到指定文件，根据后缀生成不同格式的文件，默认为 `json` 格式 | `-out output.json` | | `db` | `string` | 指定本地漏洞库文件，希望使用自己漏洞库时可用，漏洞库文件为 `json` 格式，具体格式会在之后给出;若同时使用云端漏洞库与本地漏洞库，漏洞查询结果取并集 | `-db db.json` | | `progress` | `bool` | 显示进度条 | `-progress` | +| `dedup` | `bool` | 相同组件去重 | `-dedup` | --- diff --git a/analyzer/engine/archive.go b/analyzer/engine/archive.go index 11da4c3..88d97a9 100644 --- a/analyzer/engine/archive.go +++ b/analyzer/engine/archive.go @@ -17,6 +17,7 @@ import ( "util/filter" "util/logs" "util/model" + "util/temp" "github.com/axgle/mahonia" "github.com/mholt/archiver" @@ -39,7 +40,6 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { filepath = strings.ReplaceAll(filepath, `\`, `/`) // 目录树根 root = model.NewDirTree() - root.Path = path.Base(filepath) var walker archiver.Walker if filter.Tar(filepath) { walker = archiver.NewTar() @@ -86,35 +86,27 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { // 支持解析的文件 root.AddFile(model.NewFileData(fileName, data)) } else if filter.AllPkg(fileName) { - // 支持检测的压缩包 - rootPath, _ := os.Executable() - rootPath = path.Dir(strings.ReplaceAll(rootPath, `\`, `/`)) - tempPath := path.Join(rootPath, ".temp_path") - // 创建临时文件夹 - os.Mkdir(tempPath, os.ModeDir) - targetPath := path.Join(tempPath, path.Base(fileName)) // 将压缩包解压到本地 - if out, err := os.Create(targetPath); err == nil { - _, err = out.Write(data) - out.Close() - if err != nil { - return errors.WithStack(err) - } - // 获取当前目录树 - dir := root.GetDir(fileName) - name := path.Base(fileName) - if _, ok := dir.SubDir[name]; !ok { - // 将压缩包的内容添加到当前目录树 - dir.DirList = append(dir.DirList, name) - dir.SubDir[name] = e.unArchiveFile(targetPath) - } - // 删除压缩包 - if err = os.Remove(targetPath); err != nil { + temp.DoInTempDir(func(tempdir string) { + targetPath := path.Join(tempdir, path.Base(fileName)) + if out, err := os.Create(targetPath); err == nil { + _, err = out.Write(data) + out.Close() + if err != nil { + logs.Error(err) + } + // 获取当前目录树 + dir := root.GetDir(fileName) + name := path.Base(fileName) + if _, ok := dir.SubDir[name]; !ok { + // 将压缩包的内容添加到当前目录树 + dir.DirList = append(dir.DirList, name) + dir.SubDir[name] = e.unArchiveFile(targetPath) + } + } else { logs.Error(err) } - } else { - logs.Error(err) - } + }) } } return nil diff --git a/analyzer/engine/engine.go b/analyzer/engine/engine.go index ed67841..4a06505 100644 --- a/analyzer/engine/engine.go +++ b/analyzer/engine/engine.go @@ -24,6 +24,7 @@ import ( "analyzer/java" "analyzer/javascript" "analyzer/php" + "analyzer/python" "analyzer/ruby" "analyzer/rust" ) @@ -43,6 +44,9 @@ func NewEngine() Engine { rust.New(), golang.New(), erlang.New(), + // 暂不解析groovy文件 + // groovy.New(), + python.New(), }, } } @@ -97,7 +101,7 @@ func (e Engine) ParseFile(filepath string) (depRoot *model.DepTree, taskInfo rep // 获取漏洞 taskInfo.Error = vuln.SearchVuln(depRoot) // 是否仅保留漏洞组件 - if args.OnlyVuln { + if args.Config.OnlyVuln { root := model.NewDepTree(nil) q := model.NewQueue() q.Push(depRoot) diff --git a/analyzer/engine/parse.go b/analyzer/engine/parse.go index 5f61b5d..9049daa 100644 --- a/analyzer/engine/parse.go +++ b/analyzer/engine/parse.go @@ -7,6 +7,7 @@ package engine import ( "path" + "strings" "util/filter" "util/model" ) @@ -36,7 +37,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for _, d := range analyzer.ParseFiles(files) { depRoot.Children = append(depRoot.Children, d) d.Parent = depRoot - if d.Name != "" && d.Version.Ok() { + if d.Name != "" && !strings.ContainsAny(d.Vendor+d.Name, "${}") && d.Version.Ok() { d.Path = path.Join(d.Path, d.Dependency.String()) } // 标识为直接依赖 @@ -73,7 +74,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for len(q) > 0 { n := q[0] q = append(q[1:], n.Children...) - if n.Name == "" || !n.Version.Ok() { + if n.Name == "" || strings.ContainsAny(n.Vendor+n.Name, "${}") || !n.Version.Ok() { n.Move(n.Parent) } } diff --git a/analyzer/java/analyzer.go b/analyzer/java/analyzer.go index 6087b26..10c058e 100644 --- a/analyzer/java/analyzer.go +++ b/analyzer/java/analyzer.go @@ -35,7 +35,7 @@ func (Analyzer) GetLanguage() language.Type { // CheckFile Check if it is a parsable file func (Analyzer) CheckFile(filename string) bool { - return filter.JavaPom(filename) + return filter.JavaPom(filename) || filter.GroovyGradle(filename) } // pomTree pom文件树 @@ -182,9 +182,15 @@ func (a Analyzer) ParseFiles(files []*model.FileInfo) (deps []*model.DepTree) { p.Path = f.Name poms = append(poms, p) } + if filter.GroovyGradle(f.Name) { + dep := model.NewDepTree(nil) + dep.Path = f.Name + parseGradle(dep, f) + deps = append(deps, dep) + } } // 构建jar树 - deps = buildJarTree(jarMap) + deps = append(deps, buildJarTree(jarMap)...) // 构建pom树 deps = append(deps, buildPomTree(poms).parsePomTree(jarMap)...) return diff --git a/analyzer/java/ext.go b/analyzer/java/ext.go index cc44780..2afd04f 100644 --- a/analyzer/java/ext.go +++ b/analyzer/java/ext.go @@ -18,17 +18,14 @@ import ( "util/enum/language" "util/logs" "util/model" + "util/temp" "github.com/pkg/errors" ) // MvnDepTree 调用mvn解析项目获取依赖树 func MvnDepTree(path string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(path) cmd := exec.Command("mvn", "dependency:tree", "--fail-never") out, _ := cmd.CombinedOutput() diff --git a/analyzer/java/gradle.go b/analyzer/java/gradle.go index 51f5e57..673cae1 100644 --- a/analyzer/java/gradle.go +++ b/analyzer/java/gradle.go @@ -6,9 +6,12 @@ import ( "encoding/json" "os" "os/exec" + "regexp" + "strings" "util/enum/language" "util/logs" "util/model" + "util/temp" ) //go:embed oss.gradle @@ -26,14 +29,10 @@ type gradleDep struct { // GradleDepTree 尝试获取 gradle 依赖树 func GradleDepTree(dirpath string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(dirpath) // 复制 oss.gradle - if err = os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { + if err := os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { logs.Warn(err) return } @@ -52,7 +51,7 @@ func GradleDepTree(dirpath string, root *model.DepTree) { data := out[startIndex+len(startTag) : endIndex] out = out[endIndex+1:] gdep := &gradleDep{MapDep: model.NewDepTree(root)} - err = json.Unmarshal(data, &gdep.Children) + err := json.Unmarshal(data, &gdep.Children) if err != nil { logs.Warn(err) } @@ -78,3 +77,35 @@ func GradleDepTree(dirpath string, root *model.DepTree) { } return } + +// parseGradle parse *.gradle or *.gradle.kts +func parseGradle(root *model.DepTree, file *model.FileInfo) { + regexs := []*regexp.Regexp{ + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?name: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?module: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`['"]([^\s:'"]+):([^\s:'"]+):([^\s:'"]+)['"]`), + } + for _, line := range strings.Split(string(file.Data), "\n") { + for _, re := range regexs { + match := re.FindStringSubmatch(line) + // 有捕获内容 + if len(match) == 4 && + // 不以注释开头 + !strings.HasPrefix(strings.TrimSpace(line), "/") && + // 不是测试组件 + !strings.Contains(strings.ToLower(line), "testimplementation") && + // 去掉非组件内容 + !strings.Contains(line, "//") { + ver := model.NewVersion(match[3]) + // 版本号正常 + if ver.Ok() { + dep := model.NewDepTree(root) + dep.Vendor = match[1] + dep.Name = match[2] + dep.Version = ver + break + } + } + } + } +} diff --git a/analyzer/java/pom.go b/analyzer/java/pom.go index ee4d686..61ce5de 100644 --- a/analyzer/java/pom.go +++ b/analyzer/java/pom.go @@ -129,7 +129,7 @@ func (p *Pom) GetProperty(key string) string { return p.Version case "${project.groupId}", "${groupId}", "${pom.groupId}": return p.GroupId - case "${project.artifactId}": + case "${project.artifactId}", "${artifactId}", "${pom.artifactId}": return p.ArtifactId case "${project.parent.version}", "${parent.version}": return p.Parent.Version diff --git a/analyzer/python/analyzer.go b/analyzer/python/analyzer.go new file mode 100644 index 0000000..047d352 --- /dev/null +++ b/analyzer/python/analyzer.go @@ -0,0 +1,44 @@ +package python + +import ( + "util/enum/language" + "util/filter" + "util/model" +) + +type Analyzer struct { +} + +func New() Analyzer { + return Analyzer{} +} + +// GetLanguage get language of analyzer +func (Analyzer) GetLanguage() language.Type { + return language.Python +} + +// CheckFile check parsable file +func (Analyzer) CheckFile(filename string) bool { + return filter.PythonSetup(filename) || + filter.PythonPipfile(filename) || + filter.PythonPipfileLock(filename) +} + +// ParseFiles parse dependency from file +func (Analyzer) ParseFiles(files []*model.FileInfo) []*model.DepTree { + deps := []*model.DepTree{} + for _, f := range files { + dep := model.NewDepTree(nil) + dep.Path = f.Name + if filter.PythonSetup(f.Name) { + parseSetup(dep, f) + } else if filter.PythonPipfile(f.Name) { + parsePipfile(dep, f) + } else if filter.PythonPipfileLock(f.Name) { + parsePipfileLock(dep, f) + } + deps = append(deps, dep) + } + return deps +} diff --git a/analyzer/python/oss.py b/analyzer/python/oss.py new file mode 100644 index 0000000..4179e66 --- /dev/null +++ b/analyzer/python/oss.py @@ -0,0 +1,34 @@ +import re +import sys +import json + +def parse_setup_py(setup_py_path): + """解析setup.py文件""" + with open(setup_py_path, "r") as f: + pass_func = lambda **x: x + try: + import distutils + distutils.core.setup = pass_func + except Exception: + pass + try: + import setuptools + setuptools.setup = pass_func + except Exception: + pass + # 获取setup参数 + args = {} + code = re.sub('(?>oss_end'.format(json.dumps(info))) + +if __name__ == "__main__": + if len(sys.argv) > 1: + parse_setup_py(sys.argv[1]) \ No newline at end of file diff --git a/analyzer/python/pipfile.go b/analyzer/python/pipfile.go new file mode 100644 index 0000000..4b720c9 --- /dev/null +++ b/analyzer/python/pipfile.go @@ -0,0 +1,56 @@ +package python + +import ( + "encoding/json" + "util/logs" + "util/model" + + "github.com/BurntSushi/toml" +) + +// parsePipfile parse Pipfile file +func parsePipfile(root *model.DepTree, file *model.FileInfo) { + pip := struct { + DevPackages map[string]string `toml:"dev-packages"` + Packages map[string]string `toml:"packages"` + }{} + if err := toml.Unmarshal(file.Data, &pip); err != nil { + logs.Warn(err) + } + for name, version := range pip.Packages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } + for name, version := range pip.DevPackages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } +} + +// parsePipfileLock parse pipfile.lock file +func parsePipfileLock(root *model.DepTree, file *model.FileInfo) { + lock := struct { + Default map[string]struct { + Version string `json:"version"` + } `json:"default"` + }{} + err := json.Unmarshal(file.Data, &lock) + if err != nil { + logs.Warn(err) + } + names := []string{} + for n := range lock.Default { + names = append(names, n) + } + for _, n := range names { + v := lock.Default[n].Version + if v != "" { + dep := model.NewDepTree(root) + dep.Name = n + dep.Version = model.NewVersion(v) + } + } + return +} diff --git a/analyzer/python/setup.go b/analyzer/python/setup.go new file mode 100644 index 0000000..5ce3950 --- /dev/null +++ b/analyzer/python/setup.go @@ -0,0 +1,75 @@ +package python + +import ( + _ "embed" + "encoding/json" + "os" + "os/exec" + "path" + "strings" + "util/logs" + "util/model" + "util/temp" +) + +//go:embed oss.py +var ossPy []byte + +// oss.py 脚本输出的依赖结构 +type setupDep struct { + Name string `json:"name"` + Version string `json:"version"` + License string `json:"license"` + Packages []string `json:"packages"` + InstallRequires []string `json:"install_requires"` + Requires []string `json:"requires"` +} + +// parseSetup 解析 setup.py 文件 +func parseSetup(root *model.DepTree, file *model.FileInfo) { + temp.DoInTempDir(func(tempdir string) { + ossfile := path.Join(tempdir, "oss.py") + setupfile := path.Join(tempdir, "setup.py") + // 创建 oss.py + if err := os.WriteFile(ossfile, ossPy, 0444); err != nil { + logs.Warn(err) + return + } + // 创建 setup.py + if err := os.WriteFile(setupfile, file.Data, 0444); err != nil { + logs.Warn(err) + return + } + // 解析 setup.py + cmd := exec.Command("python", ossfile, setupfile) + out, _ := cmd.CombinedOutput() + startTag, endTag := `oss_start<<`, `>>oss_end` + startIndex, endIndex := strings.Index(string(out), startTag), strings.Index(string(out), endTag) + if startIndex == -1 || endIndex == -1 { + return + } else { + out = out[startIndex+len(startTag) : endIndex] + } + // 获取解析结果 + var dep setupDep + if err := json.Unmarshal(out, &dep); err != nil { + logs.Warn(err) + } + root.Name = dep.Name + root.Version = model.NewVersion(dep.Version) + root.Licenses = append(root.Licenses, dep.License) + for _, pkg := range [][]string{dep.Packages, dep.InstallRequires, dep.Requires} { + for _, p := range pkg { + index := strings.IndexAny(p, "=<>") + sub := model.NewDepTree(root) + if index > -1 { + sub.Name = p[:index] + sub.Version = model.NewVersion(p[index:]) + } else { + sub.Name = p + } + } + } + }) + return +} diff --git a/cli/main.go b/cli/main.go index 6d36bf5..796942c 100644 --- a/cli/main.go +++ b/cli/main.go @@ -15,10 +15,12 @@ import ( "util/report" ) +var version string + func main() { args.Parse() - if len(args.Filepath) > 0 { - output(engine.NewEngine().ParseFile(args.Filepath)) + if len(args.Config.Path) > 0 { + output(engine.NewEngine().ParseFile(args.Config.Path)) } else { flag.PrintDefaults() } @@ -26,11 +28,12 @@ func main() { // output 输出结果 func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { + taskInfo.ToolVersion = version // 记录依赖 logs.Debug("\n" + depRoot.String()) // 输出结果 var reportFunc func(*model.DepTree, report.TaskInfo) []byte - switch path.Ext(args.Out) { + switch path.Ext(args.Config.Out) { case ".html": reportFunc = report.Html case ".json": @@ -38,8 +41,8 @@ func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { default: reportFunc = report.Json } - if args.Out != "" { - report.Save(reportFunc(depRoot, taskInfo), args.Out) + if args.Config.Out != "" { + report.Save(reportFunc(depRoot, taskInfo), args.Config.Out) } else { fmt.Println(string(reportFunc(depRoot, taskInfo))) } diff --git a/config.json b/config.json index 9381c40..63fc667 100644 --- a/config.json +++ b/config.json @@ -6,5 +6,6 @@ "out": "output.json", "cache": true, "vuln": false, - "progress": true + "progress": true, + "dedup": true } \ No newline at end of file diff --git a/util/args/args.go b/util/args/args.go index e9ae45f..d0f5406 100644 --- a/util/args/args.go +++ b/util/args/args.go @@ -6,52 +6,64 @@ package args import ( + "encoding/json" "flag" + "fmt" + "io/ioutil" + "path" "strings" + "util/temp" ) var ( - // 配置文件路径 - Config string - // 解析文件路径 - Filepath string - // 云服务地址 - Url string - IP string - // 云服务token - Token string - // 开启本地缓存 - Cache bool - // 输出文件 - Out string - // 仅展示有漏洞的组件 - OnlyVuln bool - // 本地漏洞库文件路径 - VulnDB string - // display progress bar - ProgressBar bool + ConfigPath string + Config = struct { + // detect option + Path string `json:"path"` + Out string `json:"out"` + Cache bool `json:"cache"` + Bar bool `json:"progress"` + OnlyVuln bool `json:"vuln"` + Dedup bool `json:"dedup"` + // remote vuldb + Url string `json:"url"` + Token string `json:"token"` + // local vuldb + VulnDB string `json:"db"` + }{} ) func init() { - // 设置参数信息 - flag.StringVar(&Config, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") - flag.StringVar(&Filepath, "path", "", "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") - flag.StringVar(&Url, "url", "", "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") - flag.StringVar(&IP, "ip", "", "(待废弃,删除)与url作用相同,兼容旧版本参数") - flag.StringVar(&Token, "token", "", "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") - flag.BoolVar(&Cache, "cache", false, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") - flag.BoolVar(&OnlyVuln, "vuln", false, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") - flag.StringVar(&Out, "out", "", "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") - flag.StringVar(&VulnDB, "db", "", "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") - flag.BoolVar(&ProgressBar, "progress", false, "(可选) 显示进度条") + flag.StringVar(&ConfigPath, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") + flag.StringVar(&Config.Path, "path", Config.Path, "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") + flag.StringVar(&Config.Url, "url", Config.Url, "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") + flag.StringVar(&Config.Token, "token", Config.Token, "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") + flag.BoolVar(&Config.Cache, "cache", Config.Cache, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") + flag.BoolVar(&Config.OnlyVuln, "vuln", Config.OnlyVuln, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") + flag.StringVar(&Config.Out, "out", Config.Out, "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") + flag.StringVar(&Config.VulnDB, "db", Config.VulnDB, "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") + flag.BoolVar(&Config.Bar, "progress", Config.Bar, "(可选) 显示进度条") + flag.BoolVar(&Config.Dedup, "dedup", Config.Dedup, "(可选) 相同组件去重") } func Parse() { flag.Parse() - // 兼容旧版本，待废弃 - if Url == "" && IP != "" { - Url = IP + if ConfigPath != "" { + if data, err := ioutil.ReadFile(ConfigPath); err != nil { + fmt.Printf("load config file error: %s\n", err) + } else { + if err = json.Unmarshal(data, &Config); err != nil { + fmt.Printf("parse config file error: %s\n", err) + } + } + } else { + // 默认读取目录下的config.json文件 + if data, err := ioutil.ReadFile(path.Join(temp.GetPwd(), "config.json")); err == nil { + // 不处理错误 + json.Unmarshal(data, &Config) + } } - loadConfigFile() - Url = strings.TrimSuffix(Url, "/") + // 再次调用Parse, 优先使用cli参数 + flag.Parse() + Config.Url = strings.TrimSuffix(Config.Url, "/") } diff --git a/util/args/config.go b/util/args/config.go deleted file mode 100644 index 4456d84..0000000 --- a/util/args/config.go +++ /dev/null @@ -1,63 +0,0 @@ -package args - -import ( - "encoding/json" - "os" - "util/logs" -) - -// loadConfigFile 加载配置文件 -func loadConfigFile() bool { - configFilePath := Config - if configFilePath == "" { - return false - } - if _, err := os.Stat(configFilePath); err != nil { - logs.Error(err) - return false - } - if data, err := os.ReadFile(configFilePath); err != nil { - logs.Error(err) - return false - } else { - config := struct { - Path string `json:"path"` - DB string `json:"db"` - Url string `json:"url"` - Token string `json:"token"` - Out string `json:"out"` - Cache *bool `json:"cache"` - OnlyVuln *bool `json:"vuln"` - ProgressBar *bool `json:"progress"` - }{} - if err = json.Unmarshal(data, &config); err != nil { - logs.Error(err) - return false - } - if Filepath == "" && config.Path != "" { - Filepath = config.Path - } - if VulnDB == "" && config.DB != "" { - VulnDB = config.DB - } - if Url == "" && config.Url != "" { - Url = config.Url - } - if Token == "" && config.Token != "" { - Token = config.Token - } - if Out == "" && config.Out != "" { - Out = config.Out - } - if !Cache && config.Cache != nil { - Cache = *config.Cache - } - if !OnlyVuln && config.OnlyVuln != nil { - OnlyVuln = *config.OnlyVuln - } - if !ProgressBar && config.ProgressBar != nil { - ProgressBar = *config.ProgressBar - } - return true - } -} diff --git a/util/bar/bar.go b/util/bar/bar.go index e7627b0..9085e4e 100644 --- a/util/bar/bar.go +++ b/util/bar/bar.go @@ -37,7 +37,7 @@ func newBar(text string) *Bar { // Add add progress func (b *Bar) Add(n int) { - if !args.ProgressBar { + if !args.Config.Bar { return } if b.id == -1 { diff --git a/util/cache/cache.go b/util/cache/cache.go index 8201134..7e44a4d 100644 --- a/util/cache/cache.go +++ b/util/cache/cache.go @@ -33,7 +33,7 @@ func init() { // save save cache file func save(filepath string, data []byte) { - if args.Cache { + if args.Config.Cache { if err := os.MkdirAll(path.Join(cacheDir, path.Dir(filepath)), os.ModeDir); err == nil { if f, err := os.Create(path.Join(cacheDir, filepath)); err == nil { defer f.Close() @@ -45,7 +45,7 @@ func save(filepath string, data []byte) { // load load cache file func load(filepath string) []byte { - if args.Cache { + if args.Config.Cache { if data, err := ioutil.ReadFile(path.Join(cacheDir, filepath)); err == nil { return data } else { diff --git a/util/client/client.go b/util/client/client.go index 046eb0a..c5a7660 100644 --- a/util/client/client.go +++ b/util/client/client.go @@ -20,6 +20,7 @@ import ( "regexp" "util/args" "util/logs" + "util/temp" "github.com/pkg/errors" ) @@ -60,35 +61,31 @@ type DetectRequst struct { func GetClientId() string { // 默认id id := "XXXXXXXXXXXXXXXX" - if pwd, err := os.Getwd(); err != nil { - logs.Error(err) - } else { - // 尝试读取.key文件 - idFile := path.Join(pwd, ".key") - if _, err := os.Stat(idFile); err != nil { - // 文件不存在则生成随机ID并保存 - if f, err := os.Create(idFile); err != nil { - logs.Error(err) - } else { - defer f.Close() - const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - idbyte := []byte(id) - for i := range idbyte { - idbyte[i] = chars[mrand.Intn(26)] - } - f.Write(idbyte) - id = string(idbyte) - } + // 尝试读取.key文件 + idFile := path.Join(temp.GetPwd(), ".key") + if _, err := os.Stat(idFile); err != nil { + // 文件不存在则生成随机ID并保存 + if f, err := os.Create(idFile); err != nil { + logs.Error(err) } else { - // 文件存在则读取ID - idbyte, err := os.ReadFile(idFile) - if err != nil { - logs.Error(err) + defer f.Close() + const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + idbyte := []byte(id) + for i := range idbyte { + idbyte[i] = chars[mrand.Intn(26)] } - if len(idbyte) == 16 { - if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { - id = string(idbyte) - } + f.Write(idbyte) + id = string(idbyte) + } + } else { + // 文件存在则读取ID + idbyte, err := os.ReadFile(idFile) + if err != nil { + logs.Error(err) + } + if len(idbyte) == 16 { + if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { + id = string(idbyte) } } } @@ -109,11 +106,11 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // aes加密 ciphertext, tag := encrypt(reqbody, key, nonce) // 构建请求 - url := args.Url + "/oss-saas/api-v1/open-sca-client/detect" + url := args.Config.Url + "/oss-saas/api-v1/open-sca-client/detect" // 添加参数 param := DetectRequst{} param.ClientId = GetClientId() - param.Token = args.Token + param.Token = args.Config.Token param.Tag = base64.StdEncoding.EncodeToString(tag) param.Nonce = base64.StdEncoding.EncodeToString(nonce) // base64编码 @@ -168,14 +165,14 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // getAesKey 获取aes-key func getAesKey() (key []byte, err error) { - u, err := url.Parse(args.Url + "/oss-saas/api-v1/open-sca-client/aes-key") + u, err := url.Parse(args.Config.Url + "/oss-saas/api-v1/open-sca-client/aes-key") if err != nil { return key, err } // 设置参数 param := url.Values{} param.Set("clientId", GetClientId()) - param.Set("ossToken", args.Token) + param.Set("ossToken", args.Config.Token) u.RawQuery = param.Encode() // 发送请求 rep, err := http.Get(u.String()) diff --git a/util/enum/language/language.go b/util/enum/language/language.go index a4d96eb..94c7175 100644 --- a/util/enum/language/language.go +++ b/util/enum/language/language.go @@ -21,7 +21,7 @@ const ( Golang Rust Erlang - Groovy + Python ) // String 语言类型 @@ -43,8 +43,8 @@ func (l Type) String() string { return "Rust" case Erlang: return "Erlang" - case Groovy: - return "Groovy" + case Python: + return "Python" default: return "None" } @@ -55,7 +55,7 @@ func (l Type) Vuln() string { switch l { case None: return "" - case Java, Groovy: + case Java: return "java" case JavaScript: return "js" @@ -69,6 +69,8 @@ func (l Type) Vuln() string { return "rust" case Erlang: return "" + case Python: + return "python" default: return "" } @@ -80,14 +82,14 @@ var ( func init() { lm := map[Type][]string{} - lm[Java] = []string{"java", "maven"} + lm[Java] = []string{"java", "maven", "groovy", "gradle"} lm[JavaScript] = []string{"js", "node", "nodejs", "javascript", "npm", "vue", "react"} lm[Php] = []string{"php", "composer"} lm[Ruby] = []string{"ruby"} lm[Golang] = []string{"golang", "go", "gomod"} lm[Rust] = []string{"rust", "cargo"} lm[Erlang] = []string{"erlang", "rebar"} - lm[Groovy] = []string{"groovy", "gradle"} + lm[Python] = []string{"python", "pip", "pipy"} for t, ls := range lm { for _, l := range ls { lanMap[l] = t diff --git a/util/filter/file.go b/util/filter/file.go index 331c1a9..81ffe09 100644 --- a/util/filter/file.go +++ b/util/filter/file.go @@ -82,5 +82,13 @@ var ( // groovy var ( - GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyGradle = filterFunc(strings.HasSuffix, ".gradle", ".gradle.kts") +) + +// python +var ( + PythonSetup = filterFunc(strings.HasSuffix, "setup.py") + PythonPipfile = filterFunc(strings.HasSuffix, "Pipfile") + PythonPipfileLock = filterFunc(strings.HasSuffix, "Pipfile.lock") ) diff --git a/util/model/dependency.go b/util/model/dependency.go index f6c5ba7..cfcfede 100644 --- a/util/model/dependency.go +++ b/util/model/dependency.go @@ -75,7 +75,8 @@ type DepTree struct { // 是否为直接依赖 Direct bool `json:"direct"` // 依赖路径 - Path string `json:"path,omitempty"` + Path string `json:"-"` + Paths []string `json:"paths,omitempty"` // 唯一的组件id，用来标识不同组件 ID int64 `json:"-"` // 父组件 @@ -97,6 +98,7 @@ func NewDepTree(parent *DepTree) *DepTree { Dependency: NewDependency(), Vulnerabilities: []*Vuln{}, Path: "", + Paths: nil, Parent: parent, Children: []*DepTree{}, licenseMap: map[string]struct{}{}, diff --git a/util/model/version.go b/util/model/version.go index 1808e86..01df4e2 100644 --- a/util/model/version.go +++ b/util/model/version.go @@ -40,7 +40,7 @@ func (ver *Version) weight() (weight int) { func NewVersion(verStr string) *Version { verStr = strings.TrimSpace(verStr) ver := &Version{Nums: []int{}, Org: verStr} - verStr = strings.TrimLeft(verStr, "vV^") + verStr = strings.TrimLeft(verStr, "vV^~=<>") // 获取后缀 index := strings.Index(verStr, "-") if index != -1 { diff --git a/util/report/format.go b/util/report/format.go index 641e296..4da0940 100644 --- a/util/report/format.go +++ b/util/report/format.go @@ -1,8 +1,9 @@ package report import ( + "fmt" "os" - "strings" + "util/args" "util/enum/language" "util/logs" "util/model" @@ -10,6 +11,7 @@ import ( // 任务检查信息 type TaskInfo struct { + ToolVersion string `json:"tool_version"` AppName string `json:"app_name"` Size int64 `json:"size"` StartTime string `json:"start_time"` @@ -22,18 +24,52 @@ type TaskInfo struct { // format 按照输出内容格式化(不可逆) func format(dep *model.DepTree) { q := []*model.DepTree{dep} + // 保留要导出的数据 for len(q) > 0 { - node := q[0] - q = append(q[1:], node.Children...) - if node.Language != language.None { - node.LanguageStr = node.Language.String() + n := q[0] + q = append(q[1:], n.Children...) + if n.Language != language.None { + n.LanguageStr = n.Language.String() } - if node.Version != nil { - node.VersionStr = node.Version.Org + if n.Version != nil { + n.VersionStr = n.Version.Org + } + if n.Path != "" { + n.Paths = []string{n.Path} + } + n.Language = language.None + n.Version = nil + } + // 去重 + if args.Config.Dedup { + q = []*model.DepTree{dep} + dm := map[string]*model.DepTree{} + for len(q) > 0 { + n := q[0] + q = append(q[1:], n.Children...) + // 去重 + k := fmt.Sprintf("%s:%s@%s#%s", n.Vendor, n.Name, n.VersionStr, n.LanguageStr) + if d, ok := dm[k]; !ok { + dm[k] = n + } else { + // 已存在相同组件 + d.Paths = append(d.Paths, n.Path) + // 从父组件中移除当前组件 + if n.Parent != nil { + for i, c := range n.Parent.Children { + if c.ID == n.ID { + n.Parent.Children = append(n.Parent.Children[:i], n.Parent.Children[i+1:]...) + break + } + } + } + // 将当前组件的子组件转移到已存在组件的子依赖中 + d.Children = append(d.Children, n.Children...) + for _, c := range n.Children { + c.Parent = d + } + } } - node.Path = node.Path[strings.Index(node.Path, "/")+1:] - node.Language = language.None - node.Version = nil } } diff --git a/util/report/html.go b/util/report/html.go index 3a7fbfb..527e079 100644 --- a/util/report/html.go +++ b/util/report/html.go @@ -8,7 +8,7 @@ import ( "util/model" ) -//go:embed index.html +//go:embed html_tpl var index []byte // Html 获取html格式报告数据 diff --git a/util/report/index.html b/util/report/html_tpl similarity index 30% rename from util/report/index.html rename to util/report/html_tpl index 76160e3..90a4ce3 100644 --- a/util/report/index.html +++ b/util/report/html_tpl @@ -1,2 +1,2 @@ -OpenSCA开源组件检测报告
\ No newline at end of file +OpenSCA开源组件检测报告
\ No newline at end of file diff --git a/util/report/json.go b/util/report/json.go index e92a317..addc702 100644 --- a/util/report/json.go +++ b/util/report/json.go @@ -13,11 +13,11 @@ func Json(dep *model.DepTree, taskInfo TaskInfo) []byte { taskInfo.ErrorString = taskInfo.Error.Error() } if data, err := json.Marshal(struct { - *model.DepTree TaskInfo TaskInfo `json:"task_info"` + *model.DepTree }{ - DepTree: dep, TaskInfo: taskInfo, + DepTree: dep, }); err != nil { logs.Error(err) } else { diff --git a/util/temp/temp.go b/util/temp/temp.go new file mode 100644 index 0000000..765a187 --- /dev/null +++ b/util/temp/temp.go @@ -0,0 +1,37 @@ +package temp + +import ( + "os" + "path" + "strconv" + "strings" + "time" + "util/logs" +) + +const tempdir = ".temp" + +func init() { + os.RemoveAll(path.Join(GetPwd(), tempdir)) +} + +// GetPwd 获取当前目录 +func GetPwd() string { + filepath, err := os.Executable() + if err != nil { + logs.Error(err) + return "" + } + return path.Dir(strings.ReplaceAll(filepath, `\`, `/`)) +} + +// DoInTempDir 在临时目录中执行 +func DoInTempDir(do func(tempdir string)) { + dir := path.Join(GetPwd(), tempdir, strconv.FormatInt(time.Now().UnixNano(), 10)) + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + logs.Warn(err) + } else { + defer os.RemoveAll(dir) + do(dir) + } +} diff --git a/util/vuln/local.go b/util/vuln/local.go index 66c99dd..90d62f8 100644 --- a/util/vuln/local.go +++ b/util/vuln/local.go @@ -29,9 +29,9 @@ var vulnDB map[string]map[string][]vulnInfo // loadVulnDB 加载本地漏洞 func loadVulnDB() { vulnDB = map[string]map[string][]vulnInfo{} - if args.VulnDB != "" { + if args.Config.VulnDB != "" { // 读取本地漏洞数据 - if data, err := ioutil.ReadFile(args.VulnDB); err != nil { + if data, err := ioutil.ReadFile(args.Config.VulnDB); err != nil { logs.Error(err) } else { // 解析本地漏洞 @@ -65,8 +65,8 @@ func GetLocalVulns(deps []model.Dependency) (vulns [][]*model.Vuln) { vulns[i] = []*model.Vuln{} if vs, ok := vulnDB[dep.Language.Vuln()][strings.ToLower(dep.Name)]; ok { for _, v := range vs { - switch dep.Language { - case language.Java: + switch dep.Language.Vuln() { + case language.Java.Vuln(): if !strings.EqualFold(v.Vendor, dep.Vendor) { continue } diff --git a/util/vuln/vuln.go b/util/vuln/vuln.go index 92c5d40..2a17832 100644 --- a/util/vuln/vuln.go +++ b/util/vuln/vuln.go @@ -29,14 +29,14 @@ func SearchVuln(root *model.DepTree) (err error) { for i, d := range deps { ds[i] = d.Dependency } - if args.VulnDB != "" { + if args.Config.VulnDB != "" { localVulns = GetLocalVulns(ds) } - if args.Url != "" && args.Token != "" { + if args.Config.Url != "" && args.Config.Token != "" { serverVulns, err = GetServerVuln(ds) - } else if args.VulnDB == "" && args.Url == "" && args.Token != "" { + } else if args.Config.VulnDB == "" && args.Config.Url == "" && args.Config.Token != "" { err = errors.New("url is null") - } else if args.VulnDB == "" && args.Url != "" && args.Token == "" { + } else if args.Config.VulnDB == "" && args.Config.Url != "" && args.Config.Token == "" { err = errors.New("token is null") } for i, dep := range deps { -- Gitee