82 lines
1.6 KiB
Go
82 lines
1.6 KiB
Go
package getter
|
|
|
|
import (
|
|
"fmt"
|
|
"sync"
|
|
|
|
"github.com/gocolly/colly"
|
|
"github.com/zu1k/proxypool/proxy"
|
|
"github.com/zu1k/proxypool/tool"
|
|
)
|
|
|
|
func init() {
|
|
Register("web-fanqiangdang", NewWebFanqiangdangGetter)
|
|
}
|
|
|
|
type WebFanqiangdang struct {
|
|
c *colly.Collector
|
|
NumNeeded int
|
|
Url string
|
|
results []string
|
|
}
|
|
|
|
func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
|
|
num, found := options["num"]
|
|
|
|
t := 200
|
|
switch num.(type) {
|
|
case int:
|
|
t = num.(int)
|
|
case float64:
|
|
t = int(num.(float64))
|
|
}
|
|
|
|
if !found || t <= 0 {
|
|
t = 200
|
|
}
|
|
urlInterface, found := options["url"]
|
|
if found {
|
|
url, err := AssertTypeStringNotNull(urlInterface)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &WebFanqiangdang{
|
|
c: colly.NewCollector(),
|
|
NumNeeded: t,
|
|
Url: url,
|
|
}, nil
|
|
}
|
|
return nil, ErrorUrlNotFound
|
|
}
|
|
|
|
func (w *WebFanqiangdang) Get() []proxy.Proxy {
|
|
w.results = make([]string, 0)
|
|
// 找到所有的文字消息
|
|
w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) {
|
|
w.results = append(w.results, GrepLinksFromString(e.Text)...)
|
|
})
|
|
|
|
// 从订阅中取出每一页，因为是订阅，所以都比较新
|
|
w.c.OnXML("//item//link", func(e *colly.XMLElement) {
|
|
if len(w.results) < w.NumNeeded {
|
|
_ = e.Request.Visit(e.Text)
|
|
}
|
|
})
|
|
|
|
w.results = make([]string, 0)
|
|
err := w.c.Visit(w.Url)
|
|
if err != nil {
|
|
_ = fmt.Errorf("%s", err.Error())
|
|
}
|
|
|
|
return StringArray2ProxyArray(w.results)
|
|
}
|
|
|
|
func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
|
|
defer wg.Done()
|
|
nodes := w.Get()
|
|
for _, node := range nodes {
|
|
pc <- node
|
|
}
|
|
}
|