remoe fanqiangdang num limit

This commit is contained in:
zu1k
2020-08-21 09:05:17 +08:00
parent 6f70ee0b7b
commit f9d2dbbed9
2 changed files with 1 additions and 31 deletions

View File

@@ -24,10 +24,9 @@ sources:
num: 200
# 翻墙党论坛抓取
- type: web-fanqiangdang
- type: web-fanqiangdang-rss
options:
url: https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0
num: 200
# 某个网站抓取
- type: web-freessrxyz

View File

@@ -17,25 +17,11 @@ func init() {
type WebFanqiangdang struct {
c *colly.Collector
NumNeeded int
Url string
results proxy.ProxyList
}
func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
num, found := options["num"]
t := 200
switch num.(type) {
case int:
t = num.(int)
case float64:
t = int(num.(float64))
}
if !found || t <= 0 {
t = 200
}
urlInterface, found := options["url"]
if found {
url, err := AssertTypeStringNotNull(urlInterface)
@@ -44,7 +30,6 @@ func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
}
return &WebFanqiangdang{
c: colly.NewCollector(),
NumNeeded: t,
Url: url,
}, nil
}
@@ -64,7 +49,6 @@ func (w *WebFanqiangdang) Get() proxy.ProxyList {
w.c.OnHTML("th.new>a[href]", func(e *colly.HTMLElement) {
url := e.Attr("href")
if strings.HasPrefix(url, "https://fanqiangdang.com/thread") {
fmt.Println(url)
_ = e.Request.Visit(url)
}
})
@@ -94,19 +78,6 @@ type WebFanqiangdangRSS struct {
}
func NewWebFanqiangdangRSSGetter(options tool.Options) (getter Getter, err error) {
num, found := options["num"]
t := 200
switch num.(type) {
case int:
t = num.(int)
case float64:
t = int(num.(float64))
}
if !found || t <= 0 {
t = 200
}
urlInterface, found := options["url"]
if found {
url, err := AssertTypeStringNotNull(urlInterface)