diff --git a/.github/README.md b/.github/README.md index c8b1046b0922c24b0f5c2241b9c8e2be90c74fec..9548a15f6231ec2f3746386003564210a048d062 100644 --- a/.github/README.md +++ b/.github/README.md @@ -86,6 +86,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | Set the output file. The result defaults to json format. | `-out output.json` | | `db` | `string` | Set the local vulnerability database file. It helps when you prefer to use your own vulnerability database. The format of the vulnerability database is shown below. If the cloud and local vulnerability databases are both set, the result of detection will merge both. | `-db db.json` | | `progress` | `bool` | Show the progress bar. | `-progress` | +| `dedup` | `bool` | Same result deduplication | `-dedup` | ------ @@ -142,7 +143,7 @@ opensca-cli -db db.json -path ${project_path} OpenSCA is an open source project, we appreciate your help! -To contribute, please read our [Contributing Guideline](./docs/Contributing%20Guideline-en%20v1.0.md). +To contribute, please read our [Contributing Guideline](../docs/Contributing%20Guideline-en%20v1.0.md). diff --git a/README.md b/README.md index bc274800ec10cd166ac76c40f1fceb26beaac4f7..24a79fe1f01a287476973f2e1c2237e8c618becb 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | 将检测结果保存到指定文件，根据后缀生成不同格式的文件，默认为 `json` 格式 | `-out output.json` | | `db` | `string` | 指定本地漏洞库文件，希望使用自己漏洞库时可用，漏洞库文件为 `json` 格式，具体格式会在之后给出;若同时使用云端漏洞库与本地漏洞库，漏洞查询结果取并集 | `-db db.json` | | `progress` | `bool` | 显示进度条 | `-progress` | +| `dedup` | `bool` | 相同组件去重 | `-dedup` | --- diff --git a/analyzer/engine/archive.go b/analyzer/engine/archive.go index 11da4c35a9edc9d5c9a504021e94cc714e6aab9d..88d97a922a04bd8c78843763284ddcb08b8f8448 100644 --- a/analyzer/engine/archive.go +++ b/analyzer/engine/archive.go @@ -17,6 +17,7 @@ import ( "util/filter" "util/logs" "util/model" + "util/temp" "github.com/axgle/mahonia" "github.com/mholt/archiver" @@ -39,7 +40,6 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { filepath = strings.ReplaceAll(filepath, `\`, `/`) // 目录树根 root = model.NewDirTree() - root.Path = path.Base(filepath) var walker archiver.Walker if filter.Tar(filepath) { walker = archiver.NewTar() @@ -86,35 +86,27 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { // 支持解析的文件 root.AddFile(model.NewFileData(fileName, data)) } else if filter.AllPkg(fileName) { - // 支持检测的压缩包 - rootPath, _ := os.Executable() - rootPath = path.Dir(strings.ReplaceAll(rootPath, `\`, `/`)) - tempPath := path.Join(rootPath, ".temp_path") - // 创建临时文件夹 - os.Mkdir(tempPath, os.ModeDir) - targetPath := path.Join(tempPath, path.Base(fileName)) // 将压缩包解压到本地 - if out, err := os.Create(targetPath); err == nil { - _, err = out.Write(data) - out.Close() - if err != nil { - return errors.WithStack(err) - } - // 获取当前目录树 - dir := root.GetDir(fileName) - name := path.Base(fileName) - if _, ok := dir.SubDir[name]; !ok { - // 将压缩包的内容添加到当前目录树 - dir.DirList = append(dir.DirList, name) - dir.SubDir[name] = e.unArchiveFile(targetPath) - } - // 删除压缩包 - if err = os.Remove(targetPath); err != nil { + temp.DoInTempDir(func(tempdir string) { + targetPath := path.Join(tempdir, path.Base(fileName)) + if out, err := os.Create(targetPath); err == nil { + _, err = out.Write(data) + out.Close() + if err != nil { + logs.Error(err) + } + // 获取当前目录树 + dir := root.GetDir(fileName) + name := path.Base(fileName) + if _, ok := dir.SubDir[name]; !ok { + // 将压缩包的内容添加到当前目录树 + dir.DirList = append(dir.DirList, name) + dir.SubDir[name] = e.unArchiveFile(targetPath) + } + } else { logs.Error(err) } - } else { - logs.Error(err) - } + }) } } return nil diff --git a/analyzer/engine/engine.go b/analyzer/engine/engine.go index ed67841be19483501b74bd0d37b8b8ddc9e8b0fc..4a065050b00c277fee3bd925e2ba108eda86da59 100644 --- a/analyzer/engine/engine.go +++ b/analyzer/engine/engine.go @@ -24,6 +24,7 @@ import ( "analyzer/java" "analyzer/javascript" "analyzer/php" + "analyzer/python" "analyzer/ruby" "analyzer/rust" ) @@ -43,6 +44,9 @@ func NewEngine() Engine { rust.New(), golang.New(), erlang.New(), + // 暂不解析groovy文件 + // groovy.New(), + python.New(), }, } } @@ -97,7 +101,7 @@ func (e Engine) ParseFile(filepath string) (depRoot *model.DepTree, taskInfo rep // 获取漏洞 taskInfo.Error = vuln.SearchVuln(depRoot) // 是否仅保留漏洞组件 - if args.OnlyVuln { + if args.Config.OnlyVuln { root := model.NewDepTree(nil) q := model.NewQueue() q.Push(depRoot) diff --git a/analyzer/engine/parse.go b/analyzer/engine/parse.go index 5f61b5dd6c9b55da630eb65763b72c0421b35667..9049daa38b2357805036e1b5375fc307d02b6085 100644 --- a/analyzer/engine/parse.go +++ b/analyzer/engine/parse.go @@ -7,6 +7,7 @@ package engine import ( "path" + "strings" "util/filter" "util/model" ) @@ -36,7 +37,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for _, d := range analyzer.ParseFiles(files) { depRoot.Children = append(depRoot.Children, d) d.Parent = depRoot - if d.Name != "" && d.Version.Ok() { + if d.Name != "" && !strings.ContainsAny(d.Vendor+d.Name, "${}") && d.Version.Ok() { d.Path = path.Join(d.Path, d.Dependency.String()) } // 标识为直接依赖 @@ -73,7 +74,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for len(q) > 0 { n := q[0] q = append(q[1:], n.Children...) - if n.Name == "" || !n.Version.Ok() { + if n.Name == "" || strings.ContainsAny(n.Vendor+n.Name, "${}") || !n.Version.Ok() { n.Move(n.Parent) } } diff --git a/analyzer/java/analyzer.go b/analyzer/java/analyzer.go index 6087b26d5325354def9afb0277a29e81e0645a77..10c058e898c77a79fe1f4e4279b29f3eb9b91739 100644 --- a/analyzer/java/analyzer.go +++ b/analyzer/java/analyzer.go @@ -35,7 +35,7 @@ func (Analyzer) GetLanguage() language.Type { // CheckFile Check if it is a parsable file func (Analyzer) CheckFile(filename string) bool { - return filter.JavaPom(filename) + return filter.JavaPom(filename) || filter.GroovyGradle(filename) } // pomTree pom文件树 @@ -182,9 +182,15 @@ func (a Analyzer) ParseFiles(files []*model.FileInfo) (deps []*model.DepTree) { p.Path = f.Name poms = append(poms, p) } + if filter.GroovyGradle(f.Name) { + dep := model.NewDepTree(nil) + dep.Path = f.Name + parseGradle(dep, f) + deps = append(deps, dep) + } } // 构建jar树 - deps = buildJarTree(jarMap) + deps = append(deps, buildJarTree(jarMap)...) // 构建pom树 deps = append(deps, buildPomTree(poms).parsePomTree(jarMap)...) return diff --git a/analyzer/java/ext.go b/analyzer/java/ext.go index cc4478087f948fdfb3d5e3d955f0061f0b5a8a47..2afd04fa466e7933f3d6beaa2bfa156efd2f9960 100644 --- a/analyzer/java/ext.go +++ b/analyzer/java/ext.go @@ -18,17 +18,14 @@ import ( "util/enum/language" "util/logs" "util/model" + "util/temp" "github.com/pkg/errors" ) // MvnDepTree 调用mvn解析项目获取依赖树 func MvnDepTree(path string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(path) cmd := exec.Command("mvn", "dependency:tree", "--fail-never") out, _ := cmd.CombinedOutput() diff --git a/analyzer/java/gradle.go b/analyzer/java/gradle.go index 51f5e57171ae20b26ea729857910948f22b0ef67..673cae1892c1e98d804eebf5c4916f6cf751c085 100644 --- a/analyzer/java/gradle.go +++ b/analyzer/java/gradle.go @@ -6,9 +6,12 @@ import ( "encoding/json" "os" "os/exec" + "regexp" + "strings" "util/enum/language" "util/logs" "util/model" + "util/temp" ) //go:embed oss.gradle @@ -26,14 +29,10 @@ type gradleDep struct { // GradleDepTree 尝试获取 gradle 依赖树 func GradleDepTree(dirpath string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(dirpath) // 复制 oss.gradle - if err = os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { + if err := os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { logs.Warn(err) return } @@ -52,7 +51,7 @@ func GradleDepTree(dirpath string, root *model.DepTree) { data := out[startIndex+len(startTag) : endIndex] out = out[endIndex+1:] gdep := &gradleDep{MapDep: model.NewDepTree(root)} - err = json.Unmarshal(data, &gdep.Children) + err := json.Unmarshal(data, &gdep.Children) if err != nil { logs.Warn(err) } @@ -78,3 +77,35 @@ func GradleDepTree(dirpath string, root *model.DepTree) { } return } + +// parseGradle parse *.gradle or *.gradle.kts +func parseGradle(root *model.DepTree, file *model.FileInfo) { + regexs := []*regexp.Regexp{ + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?name: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?module: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`['"]([^\s:'"]+):([^\s:'"]+):([^\s:'"]+)['"]`), + } + for _, line := range strings.Split(string(file.Data), "\n") { + for _, re := range regexs { + match := re.FindStringSubmatch(line) + // 有捕获内容 + if len(match) == 4 && + // 不以注释开头 + !strings.HasPrefix(strings.TrimSpace(line), "/") && + // 不是测试组件 + !strings.Contains(strings.ToLower(line), "testimplementation") && + // 去掉非组件内容 + !strings.Contains(line, "//") { + ver := model.NewVersion(match[3]) + // 版本号正常 + if ver.Ok() { + dep := model.NewDepTree(root) + dep.Vendor = match[1] + dep.Name = match[2] + dep.Version = ver + break + } + } + } + } +} diff --git a/analyzer/java/pom.go b/analyzer/java/pom.go index ee4d686adfefed9d881020381a446d86213e5906..61ce5de069b5c8f31673603fe26ec84eebb87f1b 100644 --- a/analyzer/java/pom.go +++ b/analyzer/java/pom.go @@ -129,7 +129,7 @@ func (p *Pom) GetProperty(key string) string { return p.Version case "${project.groupId}", "${groupId}", "${pom.groupId}": return p.GroupId - case "${project.artifactId}": + case "${project.artifactId}", "${artifactId}", "${pom.artifactId}": return p.ArtifactId case "${project.parent.version}", "${parent.version}": return p.Parent.Version diff --git a/analyzer/python/analyzer.go b/analyzer/python/analyzer.go new file mode 100644 index 0000000000000000000000000000000000000000..047d3528504dd3fe62faee98ad789eb3dcf4a9b3 --- /dev/null +++ b/analyzer/python/analyzer.go @@ -0,0 +1,44 @@ +package python + +import ( + "util/enum/language" + "util/filter" + "util/model" +) + +type Analyzer struct { +} + +func New() Analyzer { + return Analyzer{} +} + +// GetLanguage get language of analyzer +func (Analyzer) GetLanguage() language.Type { + return language.Python +} + +// CheckFile check parsable file +func (Analyzer) CheckFile(filename string) bool { + return filter.PythonSetup(filename) || + filter.PythonPipfile(filename) || + filter.PythonPipfileLock(filename) +} + +// ParseFiles parse dependency from file +func (Analyzer) ParseFiles(files []*model.FileInfo) []*model.DepTree { + deps := []*model.DepTree{} + for _, f := range files { + dep := model.NewDepTree(nil) + dep.Path = f.Name + if filter.PythonSetup(f.Name) { + parseSetup(dep, f) + } else if filter.PythonPipfile(f.Name) { + parsePipfile(dep, f) + } else if filter.PythonPipfileLock(f.Name) { + parsePipfileLock(dep, f) + } + deps = append(deps, dep) + } + return deps +} diff --git a/analyzer/python/oss.py b/analyzer/python/oss.py new file mode 100644 index 0000000000000000000000000000000000000000..4179e6681c8db2fcc7c87363b758658ac73f98ed --- /dev/null +++ b/analyzer/python/oss.py @@ -0,0 +1,34 @@ +import re +import sys +import json + +def parse_setup_py(setup_py_path): + """解析setup.py文件""" + with open(setup_py_path, "r") as f: + pass_func = lambda **x: x + try: + import distutils + distutils.core.setup = pass_func + except Exception: + pass + try: + import setuptools + setuptools.setup = pass_func + except Exception: + pass + # 获取setup参数 + args = {} + code = re.sub('(?>oss_end'.format(json.dumps(info))) + +if __name__ == "__main__": + if len(sys.argv) > 1: + parse_setup_py(sys.argv[1]) \ No newline at end of file diff --git a/analyzer/python/pipfile.go b/analyzer/python/pipfile.go new file mode 100644 index 0000000000000000000000000000000000000000..4b720c9cf0fdbde0101f5c813f4b378c9ccd5d79 --- /dev/null +++ b/analyzer/python/pipfile.go @@ -0,0 +1,56 @@ +package python + +import ( + "encoding/json" + "util/logs" + "util/model" + + "github.com/BurntSushi/toml" +) + +// parsePipfile parse Pipfile file +func parsePipfile(root *model.DepTree, file *model.FileInfo) { + pip := struct { + DevPackages map[string]string `toml:"dev-packages"` + Packages map[string]string `toml:"packages"` + }{} + if err := toml.Unmarshal(file.Data, &pip); err != nil { + logs.Warn(err) + } + for name, version := range pip.Packages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } + for name, version := range pip.DevPackages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } +} + +// parsePipfileLock parse pipfile.lock file +func parsePipfileLock(root *model.DepTree, file *model.FileInfo) { + lock := struct { + Default map[string]struct { + Version string `json:"version"` + } `json:"default"` + }{} + err := json.Unmarshal(file.Data, &lock) + if err != nil { + logs.Warn(err) + } + names := []string{} + for n := range lock.Default { + names = append(names, n) + } + for _, n := range names { + v := lock.Default[n].Version + if v != "" { + dep := model.NewDepTree(root) + dep.Name = n + dep.Version = model.NewVersion(v) + } + } + return +} diff --git a/analyzer/python/setup.go b/analyzer/python/setup.go new file mode 100644 index 0000000000000000000000000000000000000000..5ce395056b901202229382a8902d2a69dbef5486 --- /dev/null +++ b/analyzer/python/setup.go @@ -0,0 +1,75 @@ +package python + +import ( + _ "embed" + "encoding/json" + "os" + "os/exec" + "path" + "strings" + "util/logs" + "util/model" + "util/temp" +) + +//go:embed oss.py +var ossPy []byte + +// oss.py 脚本输出的依赖结构 +type setupDep struct { + Name string `json:"name"` + Version string `json:"version"` + License string `json:"license"` + Packages []string `json:"packages"` + InstallRequires []string `json:"install_requires"` + Requires []string `json:"requires"` +} + +// parseSetup 解析 setup.py 文件 +func parseSetup(root *model.DepTree, file *model.FileInfo) { + temp.DoInTempDir(func(tempdir string) { + ossfile := path.Join(tempdir, "oss.py") + setupfile := path.Join(tempdir, "setup.py") + // 创建 oss.py + if err := os.WriteFile(ossfile, ossPy, 0444); err != nil { + logs.Warn(err) + return + } + // 创建 setup.py + if err := os.WriteFile(setupfile, file.Data, 0444); err != nil { + logs.Warn(err) + return + } + // 解析 setup.py + cmd := exec.Command("python", ossfile, setupfile) + out, _ := cmd.CombinedOutput() + startTag, endTag := `oss_start<<`, `>>oss_end` + startIndex, endIndex := strings.Index(string(out), startTag), strings.Index(string(out), endTag) + if startIndex == -1 || endIndex == -1 { + return + } else { + out = out[startIndex+len(startTag) : endIndex] + } + // 获取解析结果 + var dep setupDep + if err := json.Unmarshal(out, &dep); err != nil { + logs.Warn(err) + } + root.Name = dep.Name + root.Version = model.NewVersion(dep.Version) + root.Licenses = append(root.Licenses, dep.License) + for _, pkg := range [][]string{dep.Packages, dep.InstallRequires, dep.Requires} { + for _, p := range pkg { + index := strings.IndexAny(p, "=<>") + sub := model.NewDepTree(root) + if index > -1 { + sub.Name = p[:index] + sub.Version = model.NewVersion(p[index:]) + } else { + sub.Name = p + } + } + } + }) + return +} diff --git a/cli/main.go b/cli/main.go index 6d36bf58b2bbc20d11d2f836409b1b85f3fc822f..796942c5ccb90170707e4ab3e4936b2e84ec3c96 100644 --- a/cli/main.go +++ b/cli/main.go @@ -15,10 +15,12 @@ import ( "util/report" ) +var version string + func main() { args.Parse() - if len(args.Filepath) > 0 { - output(engine.NewEngine().ParseFile(args.Filepath)) + if len(args.Config.Path) > 0 { + output(engine.NewEngine().ParseFile(args.Config.Path)) } else { flag.PrintDefaults() } @@ -26,11 +28,12 @@ func main() { // output 输出结果 func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { + taskInfo.ToolVersion = version // 记录依赖 logs.Debug("\n" + depRoot.String()) // 输出结果 var reportFunc func(*model.DepTree, report.TaskInfo) []byte - switch path.Ext(args.Out) { + switch path.Ext(args.Config.Out) { case ".html": reportFunc = report.Html case ".json": @@ -38,8 +41,8 @@ func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { default: reportFunc = report.Json } - if args.Out != "" { - report.Save(reportFunc(depRoot, taskInfo), args.Out) + if args.Config.Out != "" { + report.Save(reportFunc(depRoot, taskInfo), args.Config.Out) } else { fmt.Println(string(reportFunc(depRoot, taskInfo))) } diff --git a/config.json b/config.json index 9381c40c04617691262c85f01656b15d2335eb42..63fc6672d1376cdf45d56a272e757c3b0114db3f 100644 --- a/config.json +++ b/config.json @@ -6,5 +6,6 @@ "out": "output.json", "cache": true, "vuln": false, - "progress": true + "progress": true, + "dedup": true } \ No newline at end of file diff --git a/util/args/args.go b/util/args/args.go index e9ae45ffaa214b9e8e97d5fa503a376517da51c6..d0f5406360fca279b02008ddea24c469e4be80ec 100644 --- a/util/args/args.go +++ b/util/args/args.go @@ -6,52 +6,64 @@ package args import ( + "encoding/json" "flag" + "fmt" + "io/ioutil" + "path" "strings" + "util/temp" ) var ( - // 配置文件路径 - Config string - // 解析文件路径 - Filepath string - // 云服务地址 - Url string - IP string - // 云服务token - Token string - // 开启本地缓存 - Cache bool - // 输出文件 - Out string - // 仅展示有漏洞的组件 - OnlyVuln bool - // 本地漏洞库文件路径 - VulnDB string - // display progress bar - ProgressBar bool + ConfigPath string + Config = struct { + // detect option + Path string `json:"path"` + Out string `json:"out"` + Cache bool `json:"cache"` + Bar bool `json:"progress"` + OnlyVuln bool `json:"vuln"` + Dedup bool `json:"dedup"` + // remote vuldb + Url string `json:"url"` + Token string `json:"token"` + // local vuldb + VulnDB string `json:"db"` + }{} ) func init() { - // 设置参数信息 - flag.StringVar(&Config, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") - flag.StringVar(&Filepath, "path", "", "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") - flag.StringVar(&Url, "url", "", "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") - flag.StringVar(&IP, "ip", "", "(待废弃,删除)与url作用相同,兼容旧版本参数") - flag.StringVar(&Token, "token", "", "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") - flag.BoolVar(&Cache, "cache", false, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") - flag.BoolVar(&OnlyVuln, "vuln", false, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") - flag.StringVar(&Out, "out", "", "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") - flag.StringVar(&VulnDB, "db", "", "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") - flag.BoolVar(&ProgressBar, "progress", false, "(可选) 显示进度条") + flag.StringVar(&ConfigPath, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") + flag.StringVar(&Config.Path, "path", Config.Path, "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") + flag.StringVar(&Config.Url, "url", Config.Url, "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") + flag.StringVar(&Config.Token, "token", Config.Token, "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") + flag.BoolVar(&Config.Cache, "cache", Config.Cache, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") + flag.BoolVar(&Config.OnlyVuln, "vuln", Config.OnlyVuln, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") + flag.StringVar(&Config.Out, "out", Config.Out, "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") + flag.StringVar(&Config.VulnDB, "db", Config.VulnDB, "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") + flag.BoolVar(&Config.Bar, "progress", Config.Bar, "(可选) 显示进度条") + flag.BoolVar(&Config.Dedup, "dedup", Config.Dedup, "(可选) 相同组件去重") } func Parse() { flag.Parse() - // 兼容旧版本，待废弃 - if Url == "" && IP != "" { - Url = IP + if ConfigPath != "" { + if data, err := ioutil.ReadFile(ConfigPath); err != nil { + fmt.Printf("load config file error: %s\n", err) + } else { + if err = json.Unmarshal(data, &Config); err != nil { + fmt.Printf("parse config file error: %s\n", err) + } + } + } else { + // 默认读取目录下的config.json文件 + if data, err := ioutil.ReadFile(path.Join(temp.GetPwd(), "config.json")); err == nil { + // 不处理错误 + json.Unmarshal(data, &Config) + } } - loadConfigFile() - Url = strings.TrimSuffix(Url, "/") + // 再次调用Parse, 优先使用cli参数 + flag.Parse() + Config.Url = strings.TrimSuffix(Config.Url, "/") } diff --git a/util/args/config.go b/util/args/config.go deleted file mode 100644 index 4456d84badc674cdf512b191b089654decadc36e..0000000000000000000000000000000000000000 --- a/util/args/config.go +++ /dev/null @@ -1,63 +0,0 @@ -package args - -import ( - "encoding/json" - "os" - "util/logs" -) - -// loadConfigFile 加载配置文件 -func loadConfigFile() bool { - configFilePath := Config - if configFilePath == "" { - return false - } - if _, err := os.Stat(configFilePath); err != nil { - logs.Error(err) - return false - } - if data, err := os.ReadFile(configFilePath); err != nil { - logs.Error(err) - return false - } else { - config := struct { - Path string `json:"path"` - DB string `json:"db"` - Url string `json:"url"` - Token string `json:"token"` - Out string `json:"out"` - Cache *bool `json:"cache"` - OnlyVuln *bool `json:"vuln"` - ProgressBar *bool `json:"progress"` - }{} - if err = json.Unmarshal(data, &config); err != nil { - logs.Error(err) - return false - } - if Filepath == "" && config.Path != "" { - Filepath = config.Path - } - if VulnDB == "" && config.DB != "" { - VulnDB = config.DB - } - if Url == "" && config.Url != "" { - Url = config.Url - } - if Token == "" && config.Token != "" { - Token = config.Token - } - if Out == "" && config.Out != "" { - Out = config.Out - } - if !Cache && config.Cache != nil { - Cache = *config.Cache - } - if !OnlyVuln && config.OnlyVuln != nil { - OnlyVuln = *config.OnlyVuln - } - if !ProgressBar && config.ProgressBar != nil { - ProgressBar = *config.ProgressBar - } - return true - } -} diff --git a/util/bar/bar.go b/util/bar/bar.go index e7627b015415529e89148893fa7a100cec0f35f3..9085e4ee2c48233c6dc4c0515febc683313e14af 100644 --- a/util/bar/bar.go +++ b/util/bar/bar.go @@ -37,7 +37,7 @@ func newBar(text string) *Bar { // Add add progress func (b *Bar) Add(n int) { - if !args.ProgressBar { + if !args.Config.Bar { return } if b.id == -1 { diff --git a/util/cache/cache.go b/util/cache/cache.go index 820113416acb5e6527d9bdb59ef6ab37614a4d07..7e44a4d078eeacf3a98a4ebb703e2bc208239f06 100644 --- a/util/cache/cache.go +++ b/util/cache/cache.go @@ -33,7 +33,7 @@ func init() { // save save cache file func save(filepath string, data []byte) { - if args.Cache { + if args.Config.Cache { if err := os.MkdirAll(path.Join(cacheDir, path.Dir(filepath)), os.ModeDir); err == nil { if f, err := os.Create(path.Join(cacheDir, filepath)); err == nil { defer f.Close() @@ -45,7 +45,7 @@ func save(filepath string, data []byte) { // load load cache file func load(filepath string) []byte { - if args.Cache { + if args.Config.Cache { if data, err := ioutil.ReadFile(path.Join(cacheDir, filepath)); err == nil { return data } else { diff --git a/util/client/client.go b/util/client/client.go index 046eb0a06c60d800609a357c8222a13c297f23bb..c5a766078cb05bb62a7022cbcd127ca54d72d3c3 100644 --- a/util/client/client.go +++ b/util/client/client.go @@ -20,6 +20,7 @@ import ( "regexp" "util/args" "util/logs" + "util/temp" "github.com/pkg/errors" ) @@ -60,35 +61,31 @@ type DetectRequst struct { func GetClientId() string { // 默认id id := "XXXXXXXXXXXXXXXX" - if pwd, err := os.Getwd(); err != nil { - logs.Error(err) - } else { - // 尝试读取.key文件 - idFile := path.Join(pwd, ".key") - if _, err := os.Stat(idFile); err != nil { - // 文件不存在则生成随机ID并保存 - if f, err := os.Create(idFile); err != nil { - logs.Error(err) - } else { - defer f.Close() - const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - idbyte := []byte(id) - for i := range idbyte { - idbyte[i] = chars[mrand.Intn(26)] - } - f.Write(idbyte) - id = string(idbyte) - } + // 尝试读取.key文件 + idFile := path.Join(temp.GetPwd(), ".key") + if _, err := os.Stat(idFile); err != nil { + // 文件不存在则生成随机ID并保存 + if f, err := os.Create(idFile); err != nil { + logs.Error(err) } else { - // 文件存在则读取ID - idbyte, err := os.ReadFile(idFile) - if err != nil { - logs.Error(err) + defer f.Close() + const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + idbyte := []byte(id) + for i := range idbyte { + idbyte[i] = chars[mrand.Intn(26)] } - if len(idbyte) == 16 { - if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { - id = string(idbyte) - } + f.Write(idbyte) + id = string(idbyte) + } + } else { + // 文件存在则读取ID + idbyte, err := os.ReadFile(idFile) + if err != nil { + logs.Error(err) + } + if len(idbyte) == 16 { + if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { + id = string(idbyte) } } } @@ -109,11 +106,11 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // aes加密 ciphertext, tag := encrypt(reqbody, key, nonce) // 构建请求 - url := args.Url + "/oss-saas/api-v1/open-sca-client/detect" + url := args.Config.Url + "/oss-saas/api-v1/open-sca-client/detect" // 添加参数 param := DetectRequst{} param.ClientId = GetClientId() - param.Token = args.Token + param.Token = args.Config.Token param.Tag = base64.StdEncoding.EncodeToString(tag) param.Nonce = base64.StdEncoding.EncodeToString(nonce) // base64编码 @@ -168,14 +165,14 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // getAesKey 获取aes-key func getAesKey() (key []byte, err error) { - u, err := url.Parse(args.Url + "/oss-saas/api-v1/open-sca-client/aes-key") + u, err := url.Parse(args.Config.Url + "/oss-saas/api-v1/open-sca-client/aes-key") if err != nil { return key, err } // 设置参数 param := url.Values{} param.Set("clientId", GetClientId()) - param.Set("ossToken", args.Token) + param.Set("ossToken", args.Config.Token) u.RawQuery = param.Encode() // 发送请求 rep, err := http.Get(u.String()) diff --git a/util/enum/language/language.go b/util/enum/language/language.go index a4d96eb6fd68f277a4e13d33f0709df1bd1154fe..94c717550601af436cc8c39c62d66c2a73f3dbe2 100644 --- a/util/enum/language/language.go +++ b/util/enum/language/language.go @@ -21,7 +21,7 @@ const ( Golang Rust Erlang - Groovy + Python ) // String 语言类型 @@ -43,8 +43,8 @@ func (l Type) String() string { return "Rust" case Erlang: return "Erlang" - case Groovy: - return "Groovy" + case Python: + return "Python" default: return "None" } @@ -55,7 +55,7 @@ func (l Type) Vuln() string { switch l { case None: return "" - case Java, Groovy: + case Java: return "java" case JavaScript: return "js" @@ -69,6 +69,8 @@ func (l Type) Vuln() string { return "rust" case Erlang: return "" + case Python: + return "python" default: return "" } @@ -80,14 +82,14 @@ var ( func init() { lm := map[Type][]string{} - lm[Java] = []string{"java", "maven"} + lm[Java] = []string{"java", "maven", "groovy", "gradle"} lm[JavaScript] = []string{"js", "node", "nodejs", "javascript", "npm", "vue", "react"} lm[Php] = []string{"php", "composer"} lm[Ruby] = []string{"ruby"} lm[Golang] = []string{"golang", "go", "gomod"} lm[Rust] = []string{"rust", "cargo"} lm[Erlang] = []string{"erlang", "rebar"} - lm[Groovy] = []string{"groovy", "gradle"} + lm[Python] = []string{"python", "pip", "pipy"} for t, ls := range lm { for _, l := range ls { lanMap[l] = t diff --git a/util/filter/file.go b/util/filter/file.go index 331c1a94b8830207a2e6ee7a2edfced2e29664eb..81ffe09f9b1cd99940ee01823340bf083a6e60be 100644 --- a/util/filter/file.go +++ b/util/filter/file.go @@ -82,5 +82,13 @@ var ( // groovy var ( - GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyGradle = filterFunc(strings.HasSuffix, ".gradle", ".gradle.kts") +) + +// python +var ( + PythonSetup = filterFunc(strings.HasSuffix, "setup.py") + PythonPipfile = filterFunc(strings.HasSuffix, "Pipfile") + PythonPipfileLock = filterFunc(strings.HasSuffix, "Pipfile.lock") ) diff --git a/util/model/dependency.go b/util/model/dependency.go index f6c5ba7223a25fc833694ba8854de008bcf25b08..cfcfede6aaacc038b56a5f80e237c556f764a3a2 100644 --- a/util/model/dependency.go +++ b/util/model/dependency.go @@ -75,7 +75,8 @@ type DepTree struct { // 是否为直接依赖 Direct bool `json:"direct"` // 依赖路径 - Path string `json:"path,omitempty"` + Path string `json:"-"` + Paths []string `json:"paths,omitempty"` // 唯一的组件id，用来标识不同组件 ID int64 `json:"-"` // 父组件 @@ -97,6 +98,7 @@ func NewDepTree(parent *DepTree) *DepTree { Dependency: NewDependency(), Vulnerabilities: []*Vuln{}, Path: "", + Paths: nil, Parent: parent, Children: []*DepTree{}, licenseMap: map[string]struct{}{}, diff --git a/util/model/version.go b/util/model/version.go index 1808e86f7341415bf40232378dade67b9910956b..01df4e2a38e2a9d702850b8d5040f8a380fb02bd 100644 --- a/util/model/version.go +++ b/util/model/version.go @@ -40,7 +40,7 @@ func (ver *Version) weight() (weight int) { func NewVersion(verStr string) *Version { verStr = strings.TrimSpace(verStr) ver := &Version{Nums: []int{}, Org: verStr} - verStr = strings.TrimLeft(verStr, "vV^") + verStr = strings.TrimLeft(verStr, "vV^~=<>") // 获取后缀 index := strings.Index(verStr, "-") if index != -1 { diff --git a/util/report/format.go b/util/report/format.go index 641e29627fdffe147617240ad202044309e3bd0b..4da0940e8058cdc48e7c79e614e68d918f4e4f5a 100644 --- a/util/report/format.go +++ b/util/report/format.go @@ -1,8 +1,9 @@ package report import ( + "fmt" "os" - "strings" + "util/args" "util/enum/language" "util/logs" "util/model" @@ -10,6 +11,7 @@ import ( // 任务检查信息 type TaskInfo struct { + ToolVersion string `json:"tool_version"` AppName string `json:"app_name"` Size int64 `json:"size"` StartTime string `json:"start_time"` @@ -22,18 +24,52 @@ type TaskInfo struct { // format 按照输出内容格式化(不可逆) func format(dep *model.DepTree) { q := []*model.DepTree{dep} + // 保留要导出的数据 for len(q) > 0 { - node := q[0] - q = append(q[1:], node.Children...) - if node.Language != language.None { - node.LanguageStr = node.Language.String() + n := q[0] + q = append(q[1:], n.Children...) + if n.Language != language.None { + n.LanguageStr = n.Language.String() } - if node.Version != nil { - node.VersionStr = node.Version.Org + if n.Version != nil { + n.VersionStr = n.Version.Org + } + if n.Path != "" { + n.Paths = []string{n.Path} + } + n.Language = language.None + n.Version = nil + } + // 去重 + if args.Config.Dedup { + q = []*model.DepTree{dep} + dm := map[string]*model.DepTree{} + for len(q) > 0 { + n := q[0] + q = append(q[1:], n.Children...) + // 去重 + k := fmt.Sprintf("%s:%s@%s#%s", n.Vendor, n.Name, n.VersionStr, n.LanguageStr) + if d, ok := dm[k]; !ok { + dm[k] = n + } else { + // 已存在相同组件 + d.Paths = append(d.Paths, n.Path) + // 从父组件中移除当前组件 + if n.Parent != nil { + for i, c := range n.Parent.Children { + if c.ID == n.ID { + n.Parent.Children = append(n.Parent.Children[:i], n.Parent.Children[i+1:]...) + break + } + } + } + // 将当前组件的子组件转移到已存在组件的子依赖中 + d.Children = append(d.Children, n.Children...) + for _, c := range n.Children { + c.Parent = d + } + } } - node.Path = node.Path[strings.Index(node.Path, "/")+1:] - node.Language = language.None - node.Version = nil } } diff --git a/util/report/html.go b/util/report/html.go index 3a7fbfb5aa500ab7d6f45ae0880ed7ec3b05e370..527e079adc272960ea1ed39902301fc9208d9d18 100644 --- a/util/report/html.go +++ b/util/report/html.go @@ -8,7 +8,7 @@ import ( "util/model" ) -//go:embed index.html +//go:embed html_tpl var index []byte // Html 获取html格式报告数据 diff --git a/util/report/index.html b/util/report/html_tpl similarity index 30% rename from util/report/index.html rename to util/report/html_tpl index 76160e38fb06631d2daff19d699eb09efd00bfcc..90a4ce35d3cfe8c292db6f479056be6349538d8f 100644 --- a/util/report/index.html +++ b/util/report/html_tpl @@ -1,2 +1,2 @@ -OpenSCA开源组件检测报告
\ No newline at end of file +OpenSCA开源组件检测报告
\ No newline at end of file diff --git a/util/report/json.go b/util/report/json.go index e92a317fc586376f8348b8ca60e3ecd7c2cda672..addc702219453384ae33406e82850aaf86bffafc 100644 --- a/util/report/json.go +++ b/util/report/json.go @@ -13,11 +13,11 @@ func Json(dep *model.DepTree, taskInfo TaskInfo) []byte { taskInfo.ErrorString = taskInfo.Error.Error() } if data, err := json.Marshal(struct { - *model.DepTree TaskInfo TaskInfo `json:"task_info"` + *model.DepTree }{ - DepTree: dep, TaskInfo: taskInfo, + DepTree: dep, }); err != nil { logs.Error(err) } else { diff --git a/util/temp/temp.go b/util/temp/temp.go new file mode 100644 index 0000000000000000000000000000000000000000..765a187422a228c3312c4beb51191df3a6e2f106 --- /dev/null +++ b/util/temp/temp.go @@ -0,0 +1,37 @@ +package temp + +import ( + "os" + "path" + "strconv" + "strings" + "time" + "util/logs" +) + +const tempdir = ".temp" + +func init() { + os.RemoveAll(path.Join(GetPwd(), tempdir)) +} + +// GetPwd 获取当前目录 +func GetPwd() string { + filepath, err := os.Executable() + if err != nil { + logs.Error(err) + return "" + } + return path.Dir(strings.ReplaceAll(filepath, `\`, `/`)) +} + +// DoInTempDir 在临时目录中执行 +func DoInTempDir(do func(tempdir string)) { + dir := path.Join(GetPwd(), tempdir, strconv.FormatInt(time.Now().UnixNano(), 10)) + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + logs.Warn(err) + } else { + defer os.RemoveAll(dir) + do(dir) + } +} diff --git a/util/vuln/local.go b/util/vuln/local.go index 66c99dd7313ee8fbc2784c691ccf822dc70e9bc0..90d62f8097f63d0c6675540d1448d0d514b78876 100644 --- a/util/vuln/local.go +++ b/util/vuln/local.go @@ -29,9 +29,9 @@ var vulnDB map[string]map[string][]vulnInfo // loadVulnDB 加载本地漏洞 func loadVulnDB() { vulnDB = map[string]map[string][]vulnInfo{} - if args.VulnDB != "" { + if args.Config.VulnDB != "" { // 读取本地漏洞数据 - if data, err := ioutil.ReadFile(args.VulnDB); err != nil { + if data, err := ioutil.ReadFile(args.Config.VulnDB); err != nil { logs.Error(err) } else { // 解析本地漏洞 @@ -65,8 +65,8 @@ func GetLocalVulns(deps []model.Dependency) (vulns [][]*model.Vuln) { vulns[i] = []*model.Vuln{} if vs, ok := vulnDB[dep.Language.Vuln()][strings.ToLower(dep.Name)]; ok { for _, v := range vs { - switch dep.Language { - case language.Java: + switch dep.Language.Vuln() { + case language.Java.Vuln(): if !strings.EqualFold(v.Vendor, dep.Vendor) { continue } diff --git a/util/vuln/vuln.go b/util/vuln/vuln.go index 92c5d40c0ee371bd7a130894a1b74e95485539f7..2a178327dc3e0a5620da7e9a97da2808b488377a 100644 --- a/util/vuln/vuln.go +++ b/util/vuln/vuln.go @@ -29,14 +29,14 @@ func SearchVuln(root *model.DepTree) (err error) { for i, d := range deps { ds[i] = d.Dependency } - if args.VulnDB != "" { + if args.Config.VulnDB != "" { localVulns = GetLocalVulns(ds) } - if args.Url != "" && args.Token != "" { + if args.Config.Url != "" && args.Config.Token != "" { serverVulns, err = GetServerVuln(ds) - } else if args.VulnDB == "" && args.Url == "" && args.Token != "" { + } else if args.Config.VulnDB == "" && args.Config.Url == "" && args.Config.Token != "" { err = errors.New("url is null") - } else if args.VulnDB == "" && args.Url != "" && args.Token == "" { + } else if args.Config.VulnDB == "" && args.Config.Url != "" && args.Config.Token == "" { err = errors.New("token is null") } for i, dep := range deps {