欢迎关注tg频道：@peekfun
抓取程序已开源：https://github.com/zu1k/proxypool {{ .version }}
diff --git a/go.mod b/go.mod index e7bf3da..41c7b87 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/gocolly/colly v1.2.0 github.com/golang/protobuf v1.4.2 // indirect github.com/heroku/x v0.0.25 + github.com/ivpusic/grpool v1.0.0 github.com/jasonlvhit/gocron v0.0.1 github.com/json-iterator/go v1.1.10 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect diff --git a/go.sum b/go.sum index 42dbb52..f4bf22a 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/heroku/x v0.0.25/go.mod h1:qE/I0jp6rIeTBBosrPYV4ygRX3OMhqmC/A6x8ewodJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ivpusic/grpool v1.0.0 h1:+FCiCo3GhfsvzfXuJWnpJUNb/VaqyYVgG8C+qvh07Rc= +github.com/ivpusic/grpool v1.0.0/go.mod h1:WPmiAI5ExAn06vg+0JzyPzXMQutJmpb7TrBtyLJkOHQ= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= diff --git a/internal/app/getter.go b/internal/app/getter.go index 068ff12..3ee198a 100644 --- a/internal/app/getter.go +++ b/internal/app/getter.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" + "github.com/zu1k/proxypool/internal/cache" + "github.com/ghodss/yaml" "github.com/zu1k/proxypool/config" @@ -48,4 +50,5 @@ func initGetters(sourceFiles []string) { } } fmt.Println("Getter count:", len(Getters)) + cache.GettersCount = len(Getters) } diff --git a/internal/app/task.go b/internal/app/task.go index bb5cc4f..0e441a9 100644 --- a/internal/app/task.go +++ b/internal/app/task.go @@ -33,20 +33,28 @@ func CrawlGo() { proxies = proxies.Deduplication() log.Println("CrawlGo node count:", len(proxies)) proxies = provider.Clash{Proxies: proxies}.CleanProxies() - proxies.NameAddCounrty().Sort().NameAddIndex() + log.Println("CrawlGo cleaned node count:", len(proxies)) + proxies.NameAddCounrty().Sort().NameAddIndex().NameAddTG() + log.Println("Proxy rename DONE!") + cache.SetProxies("allproxies", proxies) - cache.GettersCount = len(Getters) cache.AllProxiesCount = proxies.Len() + log.Println("AllProxiesCount:", cache.AllProxiesCount) cache.SSProxiesCount = proxies.TypeLen("ss") + log.Println("SSProxiesCount:", cache.SSProxiesCount) cache.SSRProxiesCount = proxies.TypeLen("ssr") + log.Println("SSRProxiesCount:", cache.SSRProxiesCount) cache.VmessProxiesCount = proxies.TypeLen("vmess") + log.Println("VmessProxiesCount:", cache.VmessProxiesCount) cache.TrojanProxiesCount = proxies.TypeLen("trojan") + log.Println("TrojanProxiesCount:", cache.TrojanProxiesCount) cache.LastCrawlTime = time.Now().In(location).Format("2006-01-02 15:04:05") // 可用性检测 - proxies = proxy.CleanBadProxies(proxies) - log.Println("CrawlGo clash useable node count:", len(proxies)) - proxies.NameAddCounrty().Sort().NameAddIndex() + log.Println("Now check proxy health...") + proxies = proxy.CleanBadProxiesWithGrpool(proxies) + log.Println("CrawlGo clash usefull node count:", len(proxies)) + proxies.NameReIndex() cache.SetProxies("proxies", proxies) cache.UsefullProxiesCount = proxies.Len() diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 8e33dda..8bb1a42 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -1,7 +1,6 @@ package cache import ( - "log" "time" "github.com/patrickmn/go-cache" @@ -13,10 +12,8 @@ var c = cache.New(cache.NoExpiration, 10*time.Minute) func GetProxies(key string) proxy.ProxyList { result, found := c.Get(key) if found { - log.Println("found cache for:", key, "length:", len(result.(proxy.ProxyList))) return result.(proxy.ProxyList) } - log.Println("cache not found:", key) return nil } diff --git a/pkg/getter/tgchannel.go b/pkg/getter/tgchannel.go index 9a5b520..ebe9e13 100644 --- a/pkg/getter/tgchannel.go +++ b/pkg/getter/tgchannel.go @@ -40,7 +40,7 @@ func NewTGChannelGetter(options tool.Options) (getter Getter, err error) { return nil, err } return &TGChannelGetter{ - c: colly.NewCollector(), + c: tool.GetColly(), NumNeeded: t, Url: "https://t.me/s/" + url, }, nil diff --git a/pkg/getter/web_fanqiangdang.go b/pkg/getter/web_fanqiangdang.go index 126ab5e..22901e5 100644 --- a/pkg/getter/web_fanqiangdang.go +++ b/pkg/getter/web_fanqiangdang.go @@ -16,9 +16,9 @@ func init() { } type WebFanqiangdang struct { - c *colly.Collector - Url string - results proxy.ProxyList + c *colly.Collector + Url string + results proxy.ProxyList } func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) { @@ -29,8 +29,8 @@ func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) { return nil, err } return &WebFanqiangdang{ - c: colly.NewCollector(), - Url: url, + c: colly.NewCollector(), + Url: url, }, nil } return nil, ErrorUrlNotFound @@ -70,11 +70,10 @@ func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) { } } - type WebFanqiangdangRSS struct { - c *colly.Collector - Url string - results []string + c *colly.Collector + Url string + results []string } func NewWebFanqiangdangRSSGetter(options tool.Options) (getter Getter, err error) { @@ -85,8 +84,8 @@ func NewWebFanqiangdangRSSGetter(options tool.Options) (getter Getter, err error return nil, err } return &WebFanqiangdangRSS{ - c: colly.NewCollector(), - Url: url, + c: tool.GetColly(), + Url: url, }, nil } return nil, ErrorUrlNotFound diff --git a/pkg/proxy/base.go b/pkg/proxy/base.go index 4d50040..82ba827 100644 --- a/pkg/proxy/base.go +++ b/pkg/proxy/base.go @@ -19,6 +19,10 @@ func (b *Base) SetName(name string) { b.Name = name } +func (b *Base) SetIP(ip string) { + b.Server = ip +} + func (b *Base) BaseInfo() *Base { return b } @@ -34,6 +38,7 @@ type Proxy interface { ToSurge() string Identifier() string SetName(name string) + SetIP(ip string) TypeName() string BaseInfo() *Base Clone() Proxy diff --git a/pkg/proxy/check.go b/pkg/proxy/check.go index e9c545d..7d26da7 100644 --- a/pkg/proxy/check.go +++ b/pkg/proxy/check.go @@ -4,8 +4,11 @@ import ( "context" "encoding/json" "fmt" + "sync" "time" + "github.com/ivpusic/grpool" + "github.com/Dreamacro/clash/adapters/outbound" ) @@ -35,21 +38,75 @@ func testDelay(p Proxy) (delay uint16, err error) { return delay, err } +func CleanBadProxiesWithGrpool(proxies []Proxy) (cproxies []Proxy) { + pool := grpool.NewPool(500, 200) + + c := make(chan checkResult) + defer close(c) + + pool.WaitCount(len(proxies)) + go func() { + for _, p := range proxies { + pp := p + pool.JobQueue <- func() { + defer pool.JobDone() + delay, err := testDelay(pp) + if err == nil { + c <- checkResult{ + name: pp.Identifier(), + delay: delay, + } + } + } + } + }() + done := make(chan struct{}) + defer close(done) + + go func() { + pool.WaitAll() + pool.Release() + done <- struct{}{} + }() + + okMap := make(map[string]struct{}) + for { + select { + case r := <-c: + if r.delay > 0 { + okMap[r.name] = struct{}{} + } + case <-done: + cproxies = make(ProxyList, 0, 500) + for _, p := range proxies { + if _, ok := okMap[p.Identifier()]; ok { + cproxies = append(cproxies, p.Clone()) + } + } + return + } + } +} + func CleanBadProxies(proxies []Proxy) (cproxies []Proxy) { c := make(chan checkResult, 40) - defer close(c) + wg := &sync.WaitGroup{} + wg.Add(len(proxies)) for _, p := range proxies { - go testProxyDelayToChan(p, c) + go testProxyDelayToChan(p, c, wg) } + go func() { + wg.Wait() + close(c) + }() + okMap := make(map[string]struct{}) - size := len(proxies) - for i := 0; i < size; i++ { - r := <-c + for r := range c { if r.delay > 0 { okMap[r.name] = struct{}{} } } - cproxies = make([]Proxy, 0) + cproxies = make(ProxyList, 0, 500) for _, p := range proxies { if _, ok := okMap[p.Identifier()]; ok { cproxies = append(cproxies, p.Clone()) @@ -63,17 +120,13 @@ type checkResult struct { delay uint16 } -func testProxyDelayToChan(p Proxy, c chan checkResult) { +func testProxyDelayToChan(p Proxy, c chan checkResult, wg *sync.WaitGroup) { + defer wg.Done() delay, err := testDelay(p) - if err != nil { + if err == nil { c <- checkResult{ name: p.Identifier(), - delay: 0, + delay: delay, } - return - } - c <- checkResult{ - name: p.Identifier(), - delay: delay, } } diff --git a/pkg/proxy/geoip.go b/pkg/proxy/geoip.go index 62b0f75..e47005c 100644 --- a/pkg/proxy/geoip.go +++ b/pkg/proxy/geoip.go @@ -37,15 +37,15 @@ func NewGeoIP(filePath string) (geoip GeoIP) { } // find ip info -func (g GeoIP) Find(ipORdomain string) (country string, err error) { +func (g GeoIP) Find(ipORdomain string) (ip, country string, err error) { ips, err := net.LookupIP(ipORdomain) if err != nil { - return "", err + return "", "", err } ipData := net.ParseIP(ips[0].String()) record, err := g.db.City(ipData) if err != nil { - return "", err + return "", "", err } - return record.Country.IsoCode, nil + return ips[0].String(), record.Country.IsoCode, nil } diff --git a/pkg/proxy/proxies.go b/pkg/proxy/proxies.go index 92f21fd..4b091ae 100644 --- a/pkg/proxy/proxies.go +++ b/pkg/proxy/proxies.go @@ -3,6 +3,7 @@ package proxy import ( "fmt" "sort" + "sync" ) type ProxyList []Proxy @@ -63,13 +64,21 @@ func (ps ProxyList) Sort() ProxyList { func (ps ProxyList) NameAddCounrty() ProxyList { num := len(ps) + wg := &sync.WaitGroup{} + wg.Add(num) for i := 0; i < num; i++ { - country, err := geoIp.Find(ps[i].BaseInfo().Server) - if err != nil || country == "" { - country = "Earth" - } - ps[i].SetName(fmt.Sprintf("%s", country)) + ii := i + go func() { + defer wg.Done() + _, country, err := geoIp.Find(ps[ii].BaseInfo().Server) + if err != nil || country == "" { + country = "ZZ" + } + ps[ii].SetName(fmt.Sprintf("%s", country)) + //ps[ii].SetIP(ip) + }() } + wg.Wait() return ps } @@ -81,6 +90,24 @@ func (ps ProxyList) NameAddIndex() ProxyList { return ps } +func (ps ProxyList) NameReIndex() ProxyList { + num := len(ps) + for i := 0; i < num; i++ { + originName := ps[i].BaseInfo().Name + country := string([]rune(originName)[:2]) + ps[i].SetName(fmt.Sprintf("%s_%d", country, i+1)) + } + return ps +} + +func (ps ProxyList) NameAddTG() ProxyList { + num := len(ps) + for i := 0; i < num; i++ { + ps[i].SetName(fmt.Sprintf("%s %s", ps[i].BaseInfo().Name, "@peekfun")) + } + return ps +} + func Deduplication(src ProxyList) ProxyList { result := make(ProxyList, 0, len(src)) temp := map[string]struct{}{} diff --git a/pkg/proxy/shadowsocksr.go b/pkg/proxy/shadowsocksr.go index f1caa83..489efe5 100644 --- a/pkg/proxy/shadowsocksr.go +++ b/pkg/proxy/shadowsocksr.go @@ -34,7 +34,7 @@ type ShadowsocksR struct { } func (ssr ShadowsocksR) Identifier() string { - return net.JoinHostPort(ssr.Server, strconv.Itoa(ssr.Port)) + ssr.ProtocolParam + return net.JoinHostPort(ssr.Server, strconv.Itoa(ssr.Port)) + ssr.Password + ssr.ProtocolParam } func (ssr ShadowsocksR) String() string { diff --git a/pkg/tool/colly.go b/pkg/tool/colly.go new file mode 100644 index 0000000..adc3c53 --- /dev/null +++ b/pkg/tool/colly.go @@ -0,0 +1,27 @@ +package tool + +import ( + "net" + "net/http" + "time" + + "github.com/gocolly/colly" +) + +func GetColly() *colly.Collector { + c := colly.NewCollector( + colly.UserAgent(UserAgent), + ) + c.WithTransport(&http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, // 超时时间 + KeepAlive: 10 * time.Second, // keepAlive 超时时间 + }).DialContext, + MaxIdleConns: 100, // 最大空闲连接数 + IdleConnTimeout: 20 * time.Second, // 空闲连接超时 + TLSHandshakeTimeout: 10 * time.Second, // TLS 握手超时 + ExpectContinueTimeout: 10 * time.Second, + }) + return c +} diff --git a/pkg/tool/httpclient.go b/pkg/tool/httpclient.go index cff518a..93312de 100644 --- a/pkg/tool/httpclient.go +++ b/pkg/tool/httpclient.go @@ -6,6 +6,8 @@ import ( "time" ) +const UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + type HttpClient struct { *http.Client } @@ -28,7 +30,7 @@ func (c *HttpClient) Get(url string) (resp *http.Response, err error) { return nil, err } req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36") + req.Header.Set("User-Agent", UserAgent) return c.Do(req) } @@ -38,6 +40,6 @@ func (c *HttpClient) Post(url string, body io.Reader) (resp *http.Response, err return nil, err } req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36") + req.Header.Set("User-Agent", UserAgent) return c.Do(req) }