From 4fd88553bad141859160aa673ee32e6301606c7b Mon Sep 17 00:00:00 2001 From: zu1k Date: Mon, 17 Aug 2020 11:08:47 +0800 Subject: [PATCH] opti name --- app/cache/cache.go | 8 ++--- app/global.go | 12 ------- app/task.go | 41 ++++----------------- getter/base.go | 8 ++--- getter/subscribe.go | 2 +- getter/tgchannel.go | 2 +- getter/web_fanqiangdang.go | 2 +- getter/web_free_ssr_xyz.go | 4 +-- getter/web_fuzz.go | 2 +- getter/web_lucn_org.go | 2 +- main.go | 3 +- provider/clash.go | 8 ++--- provider/surge.go | 2 +- proxy/base.go | 14 -------- proxy/geoip.go | 6 ++++ proxy/proxies.go | 73 ++++++++++++++++++++++++++++++++++++++ 16 files changed, 107 insertions(+), 82 deletions(-) delete mode 100644 app/global.go create mode 100644 proxy/proxies.go diff --git a/app/cache/cache.go b/app/cache/cache.go index f008986..27beba2 100644 --- a/app/cache/cache.go +++ b/app/cache/cache.go @@ -10,17 +10,17 @@ import ( var c = cache.New(cache.NoExpiration, 10*time.Minute) -func GetProxies() []proxy.Proxy { +func GetProxies() proxy.ProxyList { result, found := c.Get("proxies") if found { - log.Println(len(result.([]proxy.Proxy))) - return result.([]proxy.Proxy) + log.Println(len(result.(proxy.ProxyList))) + return result.(proxy.ProxyList) } log.Println("Cache not found") return nil } -func SetProxies(proxies []proxy.Proxy) { +func SetProxies(proxies proxy.ProxyList) { c.Set("proxies", proxies, cache.NoExpiration) } diff --git a/app/global.go b/app/global.go deleted file mode 100644 index d0e52b6..0000000 --- a/app/global.go +++ /dev/null @@ -1,12 +0,0 @@ -package app - -import "github.com/zu1k/proxypool/proxy" - -var ( - GeoIp proxy.GeoIP - ProjectName = "proxypool" -) - -func InitGeoIpDB() { - GeoIp = proxy.NewGeoIP("assets/GeoLite2-City.mmdb") -} diff --git a/app/task.go b/app/task.go index 37051de..b63ae53 100644 --- a/app/task.go +++ b/app/task.go @@ -4,42 +4,18 @@ import ( "fmt" "io/ioutil" "log" - "math/rand" - "strconv" "sync" - "github.com/zu1k/proxypool/config" - "gopkg.in/yaml.v2" - "github.com/zu1k/proxypool/app/cache" + "github.com/zu1k/proxypool/config" "github.com/zu1k/proxypool/provider" "github.com/zu1k/proxypool/proxy" "github.com/zu1k/proxypool/tool" + "gopkg.in/yaml.v2" ) var NeedFetchNewConfigFile = false -func Crawl() { - if NeedFetchNewConfigFile { - FetchNewConfigFileThenInit() - } - proxies := make([]proxy.Proxy, 0) - for _, g := range Getters { - proxies = append(proxies, g.Get()...) - } - proxies = append(proxies, cache.GetProxies()...) - proxies = proxy.Deduplication(proxies) - - num := len(proxies) - for i := 0; i < num; i++ { - proxies[i].SetName(strconv.Itoa(rand.Int())) - } - log.Println("Crawl node count:", num) - cache.SetProxies(proxies) - cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide()) - cache.SetString("surgeproxies", provider.Surge{Proxies: proxies}.Provide()) -} - func CrawlGo() { if NeedFetchNewConfigFile { FetchNewConfigFileThenInit() @@ -60,19 +36,14 @@ func CrawlGo() { proxies = append(proxies, node) } } - proxies = proxy.Deduplication(proxies) + proxies = proxies.Deduplication() log.Println("CrawlGo node count:", len(proxies)) proxies = proxy.CleanProxies(provider.Clash{Proxies: proxies}.CleanProxies()) log.Println("CrawlGo clash useable node count:", len(proxies)) - num := len(proxies) - for i := 0; i < num; i++ { - country, err := GeoIp.Find(proxies[i].BaseInfo().Server) - if err != nil || country == "" { - country = "Earth" - } - proxies[i].SetName(fmt.Sprintf("%s_%d_%s", ProjectName, i+1, country)) - } + // 排序和重命名 + proxies.NameAddCounrty().Sort().NameAddIndex() + cache.SetProxies(proxies) cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide()) cache.SetString("surgeproxies", provider.Surge{Proxies: proxies}.Provide()) diff --git a/getter/base.go b/getter/base.go index bdf73f7..be38f03 100644 --- a/getter/base.go +++ b/getter/base.go @@ -10,7 +10,7 @@ import ( ) type Getter interface { - Get() []proxy.Proxy + Get() proxy.ProxyList Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) } @@ -46,8 +46,8 @@ func String2Proxy(link string) proxy.Proxy { return data } -func StringArray2ProxyArray(origin []string) []proxy.Proxy { - results := make([]proxy.Proxy, 0) +func StringArray2ProxyArray(origin []string) proxy.ProxyList { + results := make(proxy.ProxyList, 0) for _, link := range origin { results = append(results, String2Proxy(link)) } @@ -61,7 +61,7 @@ func GrepLinksFromString(text string) []string { return results } -func FuzzParseProxyFromString(text string) []proxy.Proxy { +func FuzzParseProxyFromString(text string) proxy.ProxyList { return StringArray2ProxyArray(GrepLinksFromString(text)) } diff --git a/getter/subscribe.go b/getter/subscribe.go index e299549..b9bfe8a 100644 --- a/getter/subscribe.go +++ b/getter/subscribe.go @@ -17,7 +17,7 @@ type Subscribe struct { Url string } -func (s *Subscribe) Get() []proxy.Proxy { +func (s *Subscribe) Get() proxy.ProxyList { resp, err := tool.GetHttpClient().Get(s.Url) if err != nil { return nil diff --git a/getter/tgchannel.go b/getter/tgchannel.go index ba89cc7..38b7eb0 100644 --- a/getter/tgchannel.go +++ b/getter/tgchannel.go @@ -48,7 +48,7 @@ func NewTGChannelGetter(options tool.Options) (getter Getter, err error) { return nil, ErrorUrlNotFound } -func (g *TGChannelGetter) Get() []proxy.Proxy { +func (g *TGChannelGetter) Get() proxy.ProxyList { g.results = make([]string, 0) // 找到所有的文字消息 g.c.OnHTML("div.tgme_widget_message_text", func(e *colly.HTMLElement) { diff --git a/getter/web_fanqiangdang.go b/getter/web_fanqiangdang.go index f87d2dd..a07fa56 100644 --- a/getter/web_fanqiangdang.go +++ b/getter/web_fanqiangdang.go @@ -49,7 +49,7 @@ func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) { return nil, ErrorUrlNotFound } -func (w *WebFanqiangdang) Get() []proxy.Proxy { +func (w *WebFanqiangdang) Get() proxy.ProxyList { w.results = make([]string, 0) // 找到所有的文字消息 w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) { diff --git a/getter/web_free_ssr_xyz.go b/getter/web_free_ssr_xyz.go index a5be559..ada5262 100644 --- a/getter/web_free_ssr_xyz.go +++ b/getter/web_free_ssr_xyz.go @@ -25,7 +25,7 @@ func NewWebFreessrxyzGetter(options tool.Options) (getter Getter, err error) { return &WebFreessrXyz{}, nil } -func (w *WebFreessrXyz) Get() []proxy.Proxy { +func (w *WebFreessrXyz) Get() proxy.ProxyList { results := freessrxyzFetch(freessrxyzSsrLink) results = append(results, freessrxyzFetch(freessrxyzV2rayLink)...) return results @@ -39,7 +39,7 @@ func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) { } } -func freessrxyzFetch(link string) []proxy.Proxy { +func freessrxyzFetch(link string) proxy.ProxyList { resp, err := tool.GetHttpClient().Get(link) if err != nil { return nil diff --git a/getter/web_fuzz.go b/getter/web_fuzz.go index 2e84333..33a834c 100644 --- a/getter/web_fuzz.go +++ b/getter/web_fuzz.go @@ -16,7 +16,7 @@ type WebFuzz struct { Url string } -func (w *WebFuzz) Get() []proxy.Proxy { +func (w *WebFuzz) Get() proxy.ProxyList { resp, err := tool.GetHttpClient().Get(w.Url) if err != nil { return nil diff --git a/getter/web_lucn_org.go b/getter/web_lucn_org.go index d1a5191..6ed6f62 100644 --- a/getter/web_lucn_org.go +++ b/getter/web_lucn_org.go @@ -23,7 +23,7 @@ func NewWebLucnorg(options tool.Options) (getter Getter, err error) { return &WebLucnOrg{}, nil } -func (w *WebLucnOrg) Get() []proxy.Proxy { +func (w *WebLucnOrg) Get() proxy.ProxyList { resp, err := tool.GetHttpClient().Post(lucnorgSsrLink, nil) if err != nil { return nil diff --git a/main.go b/main.go index 2ffb45e..1da69d3 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( _ "github.com/mkevac/debugcharts" "github.com/zu1k/proxypool/api" "github.com/zu1k/proxypool/app" + "github.com/zu1k/proxypool/proxy" ) var ( @@ -33,7 +34,7 @@ func main() { fmt.Println(err) } } - app.InitGeoIpDB() + proxy.InitGeoIpDB() go app.Cron() fmt.Println("Do the first crawl...") diff --git a/provider/clash.go b/provider/clash.go index 0c5b87d..ecced52 100644 --- a/provider/clash.go +++ b/provider/clash.go @@ -7,12 +7,12 @@ import ( ) type Clash struct { - Proxies []proxy.Proxy `yaml:"proxies"` - Types string `yaml:"type"` + Proxies proxy.ProxyList `yaml:"proxies"` + Types string `yaml:"type"` } -func (c Clash) CleanProxies() (proxies []proxy.Proxy) { - proxies = make([]proxy.Proxy, 0) +func (c Clash) CleanProxies() (proxies proxy.ProxyList) { + proxies = make(proxy.ProxyList, 0) for _, p := range c.Proxies { if checkClashSupport(p) { proxies = append(proxies, p) diff --git a/provider/surge.go b/provider/surge.go index ff06ca2..a3ac18d 100644 --- a/provider/surge.go +++ b/provider/surge.go @@ -7,7 +7,7 @@ import ( ) type Surge struct { - Proxies []proxy.Proxy `yaml:"proxies"` + Proxies proxy.ProxyList `yaml:"proxies"` } func (s Surge) Provide() string { diff --git a/proxy/base.go b/proxy/base.go index b5c85fd..069713d 100644 --- a/proxy/base.go +++ b/proxy/base.go @@ -32,17 +32,3 @@ type Proxy interface { TypeName() string BaseInfo() *Base } - -func Deduplication(src []Proxy) []Proxy { - result := make([]Proxy, 0, len(src)) - temp := map[string]struct{}{} - for _, item := range src { - if item != nil { - if _, ok := temp[item.Identifier()]; !ok { - temp[item.Identifier()] = struct{}{} - result = append(result, item) - } - } - } - return result -} diff --git a/proxy/geoip.go b/proxy/geoip.go index 29175c6..62b0f75 100644 --- a/proxy/geoip.go +++ b/proxy/geoip.go @@ -8,6 +8,12 @@ import ( "github.com/oschwald/geoip2-golang" ) +var geoIp GeoIP + +func InitGeoIpDB() { + geoIp = NewGeoIP("assets/GeoLite2-City.mmdb") +} + // GeoIP2 type GeoIP struct { db *geoip2.Reader diff --git a/proxy/proxies.go b/proxy/proxies.go new file mode 100644 index 0000000..c19fa73 --- /dev/null +++ b/proxy/proxies.go @@ -0,0 +1,73 @@ +package proxy + +import ( + "fmt" + "sort" +) + +type ProxyList []Proxy + +func (ps ProxyList) Len() int { + return len(ps) +} + +func (ps ProxyList) Less(i, j int) bool { + return ps[i].BaseInfo().Name < ps[j].BaseInfo().Name +} + +func (ps ProxyList) Swap(i, j int) { + ps[i], ps[j] = ps[j], ps[i] +} + +func (ps ProxyList) Deduplication() ProxyList { + result := make(ProxyList, 0, len(ps)) + temp := map[string]struct{}{} + for _, item := range ps { + if item != nil { + if _, ok := temp[item.Identifier()]; !ok { + temp[item.Identifier()] = struct{}{} + result = append(result, item) + } + } + } + return result +} + +func (ps ProxyList) Sort() ProxyList { + sort.Sort(ps) + return ps +} + +func (ps ProxyList) NameAddCounrty() ProxyList { + num := len(ps) + for i := 0; i < num; i++ { + country, err := geoIp.Find(ps[i].BaseInfo().Server) + if err != nil || country == "" { + country = "Earth" + } + ps[i].SetName(fmt.Sprintf("%s", country)) + } + return ps +} + +func (ps ProxyList) NameAddIndex() ProxyList { + num := len(ps) + for i := 0; i < num; i++ { + ps[i].SetName(fmt.Sprintf("%s_%d", ps[i].BaseInfo().Name, i+1)) + } + return ps +} + +func Deduplication(src ProxyList) ProxyList { + result := make(ProxyList, 0, len(src)) + temp := map[string]struct{}{} + for _, item := range src { + if item != nil { + if _, ok := temp[item.Identifier()]; !ok { + temp[item.Identifier()] = struct{}{} + result = append(result, item) + } + } + } + return result +}