1 Star 0 Fork 0

青文杰 / colly

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
hackernews_comments.go 1.50 KB
一键复制 编辑 原始数据 按行查看 历史
package main
import (
"encoding/json"
"flag"
"log"
"os"
"strconv"
"strings"
"github.com/gocolly/colly"
)
type comment struct {
Author string `selector:"a.hnuser"`
URL string `selector:".age a[href]" attr:"href"`
Comment string `selector:".comment"`
Replies []*comment
depth int
}
func main() {
var itemID string
flag.StringVar(&itemID, "id", "", "hackernews post id")
flag.Parse()
if itemID == "" {
log.Println("Hackernews post id required")
os.Exit(1)
}
comments := make([]*comment, 0)
// Instantiate default collector
c := colly.NewCollector()
// Extract comment
c.OnHTML(".comment-tree tr.athing", func(e *colly.HTMLElement) {
width, err := strconv.Atoi(e.ChildAttr("td.ind img", "width"))
if err != nil {
return
}
// hackernews uses 40px spacers to indent comment replies,
// so we have to divide the width with it to get the depth
// of the comment
depth := width / 40
c := &comment{
Replies: make([]*comment, 0),
depth: depth,
}
e.Unmarshal(c)
c.Comment = strings.TrimSpace(c.Comment[:len(c.Comment)-5])
if depth == 0 {
comments = append(comments, c)
return
}
parent := comments[len(comments)-1]
// append comment to its parent
for i := 0; i < depth-1; i++ {
parent = parent.Replies[len(parent.Replies)-1]
}
parent.Replies = append(parent.Replies, c)
})
c.Visit("https://news.ycombinator.com/item?id=" + itemID)
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
// Dump json to the standard output
enc.Encode(comments)
}
1
https://gitee.com/qingwenjie/colly.git
git@gitee.com:qingwenjie/colly.git
qingwenjie
colly
colly
v1.1.0

搜索帮助