WarlockFish/proxypool
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

opti name

Browse Source
This commit is contained in:
zu1k
2020-08-17 11:08:47 +08:00
parent 12da2a73d6
commit 4fd88553ba
16 changed files with 107 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/cache/cache.go vendored
View File

@@ -10,17 +10,17 @@ import (
var c = cache.New(cache.NoExpiration, 10*time.Minute)
func GetProxies() []proxy.Proxy {
func GetProxies() proxy.ProxyList {
result, found := c.Get("proxies")
if found {
log.Println(len(result.([]proxy.Proxy)))
return result.([]proxy.Proxy)
log.Println(len(result.(proxy.ProxyList)))
return result.(proxy.ProxyList)
}
log.Println("Cache not found")
return nil
}
func SetProxies(proxies []proxy.Proxy) {
func SetProxies(proxies proxy.ProxyList) {
c.Set("proxies", proxies, cache.NoExpiration)
}

12
app/global.go
View File

@@ -1,12 +0,0 @@
package app
import "github.com/zu1k/proxypool/proxy"
var (
GeoIp proxy.GeoIP
ProjectName = "proxypool"
)
func InitGeoIpDB() {
GeoIp = proxy.NewGeoIP("assets/GeoLite2-City.mmdb")
}

41
app/task.go
View File

@@ -4,42 +4,18 @@ import (
"fmt"
"io/ioutil"
"log"
"math/rand"
"strconv"
"sync"
"github.com/zu1k/proxypool/config"
"gopkg.in/yaml.v2"
"github.com/zu1k/proxypool/app/cache"
"github.com/zu1k/proxypool/config"
"github.com/zu1k/proxypool/provider"
"github.com/zu1k/proxypool/proxy"
"github.com/zu1k/proxypool/tool"
"gopkg.in/yaml.v2"
)
var NeedFetchNewConfigFile = false
func Crawl() {
if NeedFetchNewConfigFile {
FetchNewConfigFileThenInit()
}
proxies := make([]proxy.Proxy, 0)
for _, g := range Getters {
proxies = append(proxies, g.Get()...)
}
proxies = append(proxies, cache.GetProxies()...)
proxies = proxy.Deduplication(proxies)
num := len(proxies)
for i := 0; i < num; i++ {
proxies[i].SetName(strconv.Itoa(rand.Int()))
}
log.Println("Crawl node count:", num)
cache.SetProxies(proxies)
cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
cache.SetString("surgeproxies", provider.Surge{Proxies: proxies}.Provide())
}
func CrawlGo() {
if NeedFetchNewConfigFile {
FetchNewConfigFileThenInit()
@@ -60,19 +36,14 @@ func CrawlGo() {
proxies = append(proxies, node)
}
}
proxies = proxy.Deduplication(proxies)
proxies = proxies.Deduplication()
log.Println("CrawlGo node count:", len(proxies))
proxies = proxy.CleanProxies(provider.Clash{Proxies: proxies}.CleanProxies())
log.Println("CrawlGo clash useable node count:", len(proxies))
num := len(proxies)
for i := 0; i < num; i++ {
country, err := GeoIp.Find(proxies[i].BaseInfo().Server)
if err != nil || country == "" {
country = "Earth"
}
proxies[i].SetName(fmt.Sprintf("%s_%d_%s", ProjectName, i+1, country))
}
// 排序和重命名
proxies.NameAddCounrty().Sort().NameAddIndex()
cache.SetProxies(proxies)
cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
cache.SetString("surgeproxies", provider.Surge{Proxies: proxies}.Provide())

8
getter/base.go
View File

@@ -10,7 +10,7 @@ import (
)
type Getter interface {
Get() []proxy.Proxy
Get() proxy.ProxyList
Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup)
}
@@ -46,8 +46,8 @@ func String2Proxy(link string) proxy.Proxy {
return data
}
func StringArray2ProxyArray(origin []string) []proxy.Proxy {
results := make([]proxy.Proxy, 0)
func StringArray2ProxyArray(origin []string) proxy.ProxyList {
results := make(proxy.ProxyList, 0)
for _, link := range origin {
results = append(results, String2Proxy(link))
}
@@ -61,7 +61,7 @@ func GrepLinksFromString(text string) []string {
return results
}
func FuzzParseProxyFromString(text string) []proxy.Proxy {
func FuzzParseProxyFromString(text string) proxy.ProxyList {
return StringArray2ProxyArray(GrepLinksFromString(text))
}

2
getter/subscribe.go
View File

@@ -17,7 +17,7 @@ type Subscribe struct {
Url string
}
func (s *Subscribe) Get() []proxy.Proxy {
func (s *Subscribe) Get() proxy.ProxyList {
resp, err := tool.GetHttpClient().Get(s.Url)
if err != nil {
return nil

2
getter/tgchannel.go
View File

@@ -48,7 +48,7 @@ func NewTGChannelGetter(options tool.Options) (getter Getter, err error) {
return nil, ErrorUrlNotFound
}
func (g *TGChannelGetter) Get() []proxy.Proxy {
func (g *TGChannelGetter) Get() proxy.ProxyList {
g.results = make([]string, 0)
// 找到所有的文字消息
g.c.OnHTML("div.tgme_widget_message_text", func(e *colly.HTMLElement) {

2
getter/web_fanqiangdang.go
View File

@@ -49,7 +49,7 @@ func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
return nil, ErrorUrlNotFound
}
func (w *WebFanqiangdang) Get() []proxy.Proxy {
func (w *WebFanqiangdang) Get() proxy.ProxyList {
w.results = make([]string, 0)
// 找到所有的文字消息
w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) {

4
getter/web_free_ssr_xyz.go
View File

@@ -25,7 +25,7 @@ func NewWebFreessrxyzGetter(options tool.Options) (getter Getter, err error) {
return &WebFreessrXyz{}, nil
}
func (w *WebFreessrXyz) Get() []proxy.Proxy {
func (w *WebFreessrXyz) Get() proxy.ProxyList {
results := freessrxyzFetch(freessrxyzSsrLink)
results = append(results, freessrxyzFetch(freessrxyzV2rayLink)...)
return results
@@ -39,7 +39,7 @@ func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
}
}
func freessrxyzFetch(link string) []proxy.Proxy {
func freessrxyzFetch(link string) proxy.ProxyList {
resp, err := tool.GetHttpClient().Get(link)
if err != nil {
return nil

2
getter/web_fuzz.go
View File

@@ -16,7 +16,7 @@ type WebFuzz struct {
Url string
}
func (w *WebFuzz) Get() []proxy.Proxy {
func (w *WebFuzz) Get() proxy.ProxyList {
resp, err := tool.GetHttpClient().Get(w.Url)
if err != nil {
return nil

2
getter/web_lucn_org.go
View File

@@ -23,7 +23,7 @@ func NewWebLucnorg(options tool.Options) (getter Getter, err error) {
return &WebLucnOrg{}, nil
}
func (w *WebLucnOrg) Get() []proxy.Proxy {
func (w *WebLucnOrg) Get() proxy.ProxyList {
resp, err := tool.GetHttpClient().Post(lucnorgSsrLink, nil)
if err != nil {
return nil

3
main.go
View File

@@ -9,6 +9,7 @@ import (
_ "github.com/mkevac/debugcharts"
"github.com/zu1k/proxypool/api"
"github.com/zu1k/proxypool/app"
"github.com/zu1k/proxypool/proxy"
)
var (
@@ -33,7 +34,7 @@ func main() {
fmt.Println(err)
}
}
app.InitGeoIpDB()
proxy.InitGeoIpDB()
go app.Cron()
fmt.Println("Do the first crawl...")

8
provider/clash.go
View File

@@ -7,12 +7,12 @@ import (
)
type Clash struct {
Proxies []proxy.Proxy `yaml:"proxies"`
Types string `yaml:"type"`
Proxies proxy.ProxyList `yaml:"proxies"`
Types string `yaml:"type"`
}
func (c Clash) CleanProxies() (proxies []proxy.Proxy) {
proxies = make([]proxy.Proxy, 0)
func (c Clash) CleanProxies() (proxies proxy.ProxyList) {
proxies = make(proxy.ProxyList, 0)
for _, p := range c.Proxies {
if checkClashSupport(p) {
proxies = append(proxies, p)

2
provider/surge.go
View File

@@ -7,7 +7,7 @@ import (
)
type Surge struct {
Proxies []proxy.Proxy `yaml:"proxies"`
Proxies proxy.ProxyList `yaml:"proxies"`
}
func (s Surge) Provide() string {

14
proxy/base.go
View File

@@ -32,17 +32,3 @@ type Proxy interface {
TypeName() string
BaseInfo() *Base
}
func Deduplication(src []Proxy) []Proxy {
result := make([]Proxy, 0, len(src))
temp := map[string]struct{}{}
for _, item := range src {
if item != nil {
if _, ok := temp[item.Identifier()]; !ok {
temp[item.Identifier()] = struct{}{}
result = append(result, item)
}
}
}
return result
}

6
proxy/geoip.go
View File

@@ -8,6 +8,12 @@ import (
"github.com/oschwald/geoip2-golang"
)
var geoIp GeoIP
func InitGeoIpDB() {
geoIp = NewGeoIP("assets/GeoLite2-City.mmdb")
}
// GeoIP2
type GeoIP struct {
db *geoip2.Reader

73
proxy/proxies.go Normal file
View File

@@ -0,0 +1,73 @@
package proxy
import (
"fmt"
"sort"
)
type ProxyList []Proxy
func (ps ProxyList) Len() int {
return len(ps)
}
func (ps ProxyList) Less(i, j int) bool {
return ps[i].BaseInfo().Name < ps[j].BaseInfo().Name
}
func (ps ProxyList) Swap(i, j int) {
ps[i], ps[j] = ps[j], ps[i]
}
func (ps ProxyList) Deduplication() ProxyList {
result := make(ProxyList, 0, len(ps))
temp := map[string]struct{}{}
for _, item := range ps {
if item != nil {
if _, ok := temp[item.Identifier()]; !ok {
temp[item.Identifier()] = struct{}{}
result = append(result, item)
}
}
}
return result
}
func (ps ProxyList) Sort() ProxyList {
sort.Sort(ps)
return ps
}
func (ps ProxyList) NameAddCounrty() ProxyList {
num := len(ps)
for i := 0; i < num; i++ {
country, err := geoIp.Find(ps[i].BaseInfo().Server)
if err != nil || country == "" {
country = "Earth"
}
ps[i].SetName(fmt.Sprintf("%s", country))
}
return ps
}
func (ps ProxyList) NameAddIndex() ProxyList {
num := len(ps)
for i := 0; i < num; i++ {
ps[i].SetName(fmt.Sprintf("%s_%d", ps[i].BaseInfo().Name, i+1))
}
return ps
}
func Deduplication(src ProxyList) ProxyList {
result := make(ProxyList, 0, len(src))
temp := map[string]struct{}{}
for _, item := range src {
if item != nil {
if _, ok := temp[item.Identifier()]; !ok {
temp[item.Identifier()] = struct{}{}
result = append(result, item)
}
}
}
return result
}