add gen bindata shell

Add country flag emoji before country code in each proxy name field, some typo fix

.github/workflows/docker.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2.3.1
+        uses: actions/checkout@v2.3.2
 
       - name: Build and push Docker images
         uses: docker/build-push-action@v1.1.0

.github/workflows/go.yml

@@ -12,7 +12,7 @@ jobs:
           go-version: 1.14.x
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2.3.1
+        uses: actions/checkout@v2.3.2
 
       - name: Cache go module
         uses: actions/cache@v2
@@ -26,6 +26,12 @@ jobs:
         run: |
           go test ./...
 
+      - name: gen go-bindata
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          go get -u github.com/go-bindata/go-bindata/...
+          go-bindata -o internal/bindata/geoip/geoip.go -pkg bingeoip  assets/GeoLite2-City.mmdb assets/flags.json
+
       - name: Build
         if: startsWith(github.ref, 'refs/tags/')
         env:

.gitignore

@@ -21,4 +21,4 @@ vendor
 # macOS file
 .DS_Store
 
-*.exe
+assets/html/

Dockerfile

@@ -9,7 +9,7 @@ RUN go mod download && \
 
 FROM alpine:latest
 
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates tzdata
 WORKDIR /proxypool-src
 COPY ./assets /proxypool-src/assets
 COPY --from=builder /proxypool /proxypool-src/

Makefile

@@ -20,11 +20,8 @@ PLATFORM_LIST = \
 	freebsd-386 \
 	freebsd-amd64
 
-WINDOWS_ARCH_LIST = \
-	windows-386 \
-	windows-amd64
 
-all: linux-amd64 darwin-amd64 windows-amd64 # Most used
+all: linux-amd64 darwin-amd64
 
 docker:
 	$(GOBUILD) -o $(BINDIR)/$(NAME)-$@
@@ -74,24 +71,14 @@ freebsd-386:
 freebsd-amd64:
 	GOARCH=amd64 GOOS=freebsd $(GOBUILD) -o $(BINDIR)/$(NAME)-$@
 
-windows-386:
-	GOARCH=386 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe
-
-windows-amd64:
-	GOARCH=amd64 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe
-
 gz_releases=$(addsuffix .gz, $(PLATFORM_LIST))
-zip_releases=$(addsuffix .zip, $(WINDOWS_ARCH_LIST))
 
 $(gz_releases): %.gz : %
 	chmod +x $(BINDIR)/$(NAME)-$(basename $@)
 	gzip -f -S -$(VERSION).gz $(BINDIR)/$(NAME)-$(basename $@)
 
-$(zip_releases): %.zip : %
-	zip -m -j $(BINDIR)/$(NAME)-$(basename $@)-$(VERSION).zip $(BINDIR)/$(NAME)-$(basename $@).exe
+all-arch: $(PLATFORM_LIST)
 
-all-arch: $(PLATFORM_LIST) $(WINDOWS_ARCH_LIST)
-
-releases: $(gz_releases) $(zip_releases)
+releases: $(gz_releases)
 clean:
 	rm $(BINDIR)/*

README.md

@@ -32,7 +32,7 @@
 
 点击按钮进入部署页面，填写基本信息然后运行
 
-其中 `DOMAIN` 需要填写为你需要绑定的域名，`CONFIG_FILE` 需要填写你的配置文件路径，配置文件模板见 config/source.yaml 文件
+其中 `DOMAIN` 需要填写为你需要绑定的域名，`CONFIG_FILE` 需要填写你的配置文件路径，配置文件模板见 config/config.yaml 文件
 
 `CF` 开头的选项暂不需要填写，不影响程序运行
 
@@ -63,16 +63,16 @@ docker pull docker.pkg.github.com/zu1k/proxypool/proxypool:latest
 
 ### 修改配置文件
 
-首先修改 source.yaml 中的必要配置信息，cf开头的选项不需要填写
+首先修改 config.yaml 中的必要配置信息，cf开头的选项不需要填写
 
-配置文件中定义了抓取源，需要定期手动维护更新
+source.yaml 文件中定义了抓取源，需要定期手动维护更新
 
 ### 启动程序
 
 使用 `-c` 参数指定配置文件路径，支持http链接
 
 ```shell
-proxypool -c source.yaml
+proxypool -c config.yaml
 ```
 
 ## 截图

api/router.go

@@ -1,28 +1,36 @@
 package api
 
 import (
+	"html/template"
 	"net/http"
 	"os"
 
-	"github.com/zu1k/proxypool/config"
-
 	"github.com/gin-gonic/gin"
 	_ "github.com/heroku/x/hmetrics/onload"
+	"github.com/zu1k/proxypool/config"
+	binhtml "github.com/zu1k/proxypool/internal/bindata/html"
 	"github.com/zu1k/proxypool/internal/cache"
 	"github.com/zu1k/proxypool/pkg/provider"
 )
 
-const version = "v0.3.0"
+const version = "v0.3.4"
 
 var router *gin.Engine
 
 func setupRouter() {
-	router = gin.Default()
-	router.LoadHTMLGlob("assets/html/*")
+	gin.SetMode(gin.ReleaseMode)
+	router = gin.New()
+	router.Use(gin.Recovery())
+	temp, err := loadTemplate()
+	if err != nil {
+		panic(err)
+	}
+	router.SetHTMLTemplate(temp)
 
 	router.GET("/", func(c *gin.Context) {
-		c.HTML(http.StatusOK, "index.html", gin.H{
+		c.HTML(http.StatusOK, "assets/html/index.html", gin.H{
 			"domain":               config.Config.Domain,
+			"getters_count":        cache.GettersCount,
 			"all_proxies_count":    cache.AllProxiesCount,
 			"ss_proxies_count":     cache.SSProxiesCount,
 			"ssr_proxies_count":    cache.SSRProxiesCount,
@@ -35,25 +43,25 @@ func setupRouter() {
 	})
 
 	router.GET("/clash", func(c *gin.Context) {
-		c.HTML(http.StatusOK, "clash.html", gin.H{
+		c.HTML(http.StatusOK, "assets/html/clash.html", gin.H{
 			"domain": config.Config.Domain,
 		})
 	})
 
 	router.GET("/surge", func(c *gin.Context) {
-		c.HTML(http.StatusOK, "surge.html", gin.H{
+		c.HTML(http.StatusOK, "assets/html/surge.html", gin.H{
 			"domain": config.Config.Domain,
 		})
 	})
 
 	router.GET("/clash/config", func(c *gin.Context) {
-		c.HTML(http.StatusOK, "clash-config.yaml", gin.H{
+		c.HTML(http.StatusOK, "assets/html/clash-config.yaml", gin.H{
 			"domain": config.Config.Domain,
 		})
 	})
 
 	router.GET("/surge/config", func(c *gin.Context) {
-		c.HTML(http.StatusOK, "surge.conf", gin.H{
+		c.HTML(http.StatusOK, "assets/html/surge.conf", gin.H{
 			"domain": config.Config.Domain,
 		})
 	})
@@ -61,8 +69,9 @@ func setupRouter() {
 	router.GET("/clash/proxies", func(c *gin.Context) {
 		proxyTypes := c.DefaultQuery("type", "")
 		proxyCountry := c.DefaultQuery("c", "")
+		proxyNotCountry := c.DefaultQuery("nc", "")
 		text := ""
-		if proxyTypes == "" && proxyCountry == "" {
+		if proxyTypes == "" && proxyCountry == "" && proxyNotCountry == "" {
 			text = cache.GetString("clashproxies")
 			if text == "" {
 				proxies := cache.GetProxies("proxies")
@@ -72,11 +81,21 @@ func setupRouter() {
 			}
 		} else if proxyTypes == "all" {
 			proxies := cache.GetProxies("allproxies")
-			clash := provider.Clash{Proxies: proxies, Types: proxyTypes, Country: proxyCountry}
+			clash := provider.Clash{
+				Proxies:    proxies,
+				Types:      proxyTypes,
+				Country:    proxyCountry,
+				NotCountry: proxyNotCountry,
+			}
 			text = clash.Provide()
 		} else {
 			proxies := cache.GetProxies("proxies")
-			clash := provider.Clash{Proxies: proxies, Types: proxyTypes, Country: proxyCountry}
+			clash := provider.Clash{
+				Proxies:    proxies,
+				Types:      proxyTypes,
+				Country:    proxyCountry,
+				NotCountry: proxyNotCountry,
+			}
 			text = clash.Provide()
 		}
 		c.String(200, text)
@@ -101,3 +120,15 @@ func Run() {
 	}
 	router.Run(":" + port)
 }
+
+func loadTemplate() (t *template.Template, err error) {
+	t = template.New("")
+	for _, fileName := range binhtml.AssetNames() {
+		data := binhtml.MustAsset(fileName)
+		t, err = t.New(fileName).Parse(string(data))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return t, nil
+}

assets/html/clash.html

@@ -1,129 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="HandheldFriendly" content="True">
-    <title>免费Clash节点</title>
-    <style type='text/css'>
-        body {
-            background-color: white;
-            color: #333333;
-            font-family: Arial, sans-serif;
-            margin: 0;
-            padding: 36px;
-            line-height: 1;
-            font-size: 14px;
-        }
-
-        .section {
-            margin-bottom: 36px;
-        }
-
-        .section.friendly {
-            color: #222222;
-        }
-
-        .section.friendly h1 {
-            font-size: 26px;
-            background-color: #dad8e4;
-            padding: 18px 22px 15px 22px;
-            margin: 0;
-            overflow: hidden;
-        }
-
-        .section.friendly h1 strong {
-            display: inline-block;
-            float: left;
-        }
-
-        .section.friendly h1 small {
-            display: inline-block;
-            float: right;
-            text-align: right;
-            font-size: 18px;
-            padding-top: 4px;
-            color: #333333;
-        }
-
-        .section.friendly .article {
-            border: 4px solid #dad8e4;
-            padding: 24px 18px 18px 18px;
-        }
-
-        .section.friendly .article h3 {
-            font-size: 20px;
-            margin: 0 0 18px 0;
-        }
-
-        .section.friendly .article a {
-            color: #6b6ceb;
-        }
-
-        .section.friendly .article a:visited {
-            color: #1d1d3b;
-        }
-
-        .section.friendly .article p {
-            font-size: 14px;
-        }
-
-        .section.friendly .article ul {
-            list-style-type: square;
-        }
-
-        .section.original {
-            background-color: #eeeeee;
-            color: #444444;
-        }
-
-        .section.original h2 {
-            background-color: #dddddd;
-            margin: 0;
-            padding: 18px 22px 18px 22px;
-            font-size: 20px;
-        }
-
-        .section.original pre {
-            margin: 0;
-            padding: 18px 22px 18px 22px;
-            overflow: auto;
-            font-family: monaco, monospaced;
-        }
-
-        .section.original pre code {
-            display: block;
-            font-size: 11px;
-            width: 100%;
-        }
-    </style>
-</head>
-<body>
-<div class='container'>
-    <div class='section friendly'>
-        <h1><strong>免费Clash节点</strong></h1>
-        <div class='article'>
-            <p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess、trojan节点信息，聚合去重后提供clash配置，每15分钟更新</p>
-            <br>
-            <p>Clash配置文件：https://{{ .domain }}/clash/config <a href="clash://install-config?url=https://{{ .domain }}/clash/config">一键导入</a></p>
-            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://{{ .domain }}/clash/proxies">https://{{ .domain }}/clash/proxies</a></p>
-            <br>
-			<p>筛选代理类型(此种方式你只能自己维护配置文件)：https://{{ .domain }}/clash/proxies?type=ss,ssr,vmess</p>
-			<p>筛选国家(此种方式你只能自己维护配置文件)：https://{{ .domain }}/clash/proxies?type=ss,ssr,vmess&c=HK,TW,US</p>
-            <p>所有节点的Provider(不是都可以用)：<a href="https://{{ .domain }}/clash/proxies?type=all">https://{{ .domain }}/clash/proxies?type=all</a></p>
-            <br>
-            <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a></p>
-        </div>
-    </div>
-</div>
-<!-- Global site tag (gtag.js) - Google Analytics -->
-<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
-<script>
-    window.dataLayer = window.dataLayer || [];
-    function gtag(){dataLayer.push(arguments);}
-    gtag('js', new Date());
-
-    gtag('config', 'G-DC6CR61FHE');
-</script>
-</body>
-</html>

assets/html/index.html

@@ -1,147 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="HandheldFriendly" content="True">
-    <title>免费节点</title>
-    <meta property="og:site_name" content="免费节点">
-    <meta property="og:description" content="免费ss,免费ssr,免费v2ray,免费vmess,免费vless,免费shadowsocks,免费shadowsocksr,免费shadowsocksrr,免费shadowrocket,免费quan,免费quanx,免费surge,免费小火箭,白嫖ss,白嫖ssr,白嫖v2ray,白嫖vmess,白嫖vless,白嫖shadowsocks,白嫖shadowsocksr,白嫖shadowsocksrr,白嫖shadowrocket,白嫖quan,白嫖quanx,白嫖surge,白嫖小火箭,高速ss,高速ssr,高速v2ray,高速vmess,高速vless,高速shadowsocks,高速shadowsocksr,高速shadowsocksrr,高速shadowrocket,高速quan,高速quanx,高速surge,高速小火箭,公益ss,公益ssr,公益v2ray,公益vmess,公益vless,公益shadowsocks,公益shadowsocksr,公益shadowsocksrr,公益shadowrocket,公益quan,公益quanx,公益surge,公益小火箭,windowsss,windowsssr,windowsv2ray,windowsvmess,windowsvless,windowsshadowsocks,windowsshadowsocksr,windowsshadowsocksrr,windowsshadowrocket,windowsquan,windowsquanx,windowssurge,windows小火箭,androidss,androidssr,androidv2ray,androidvmess,androidvless,androidshadowsocks,androidshadowsocksr,androidshadowsocksrr,androidshadowrocket,androidquan,androidquanx,androidsurge,android小火箭,iosss,iosssr,iosv2ray,iosvmess,iosvless,iosshadowsocks,iosshadowsocksr,iosshadowsocksrr,iosshadowrocket,iosquan,iosquanx,iossurge,ios小火箭,macss,macssr,macv2ray,macvmess,macvless,macshadowsocks,macshadowsocksr,macshadowsocksrr,macshadowrocket,macquan,macquanx,macsurge,mac小火箭,苹果ss,苹果ssr,苹果v2ray,苹果vmess,苹果vless,苹果shadowsocks,苹果shadowsocksr,苹果shadowsocksrr,苹果shadowrocket,苹果quan,苹果quanx,苹果surge,苹果小火箭,安卓ss,安卓ssr,安卓v2ray,安卓vmess,安卓vless,安卓shadowsocks,安卓shadowsocksr,安卓shadowsocksrr,安卓shadowrocket,安卓quan,安卓quanx,安卓surge,安卓小火箭">
-    <meta name="keywords" content="免费ss,免费ssr,免费v2ray,免费vmess,免费vless,免费shadowsocks,免费shadowsocksr,免费shadowsocksrr,免费shadowrocket,免费quan,免费quanx,免费surge,免费小火箭,白嫖ss,白嫖ssr,白嫖v2ray,白嫖vmess,白嫖vless,白嫖shadowsocks,白嫖shadowsocksr,白嫖shadowsocksrr,白嫖shadowrocket,白嫖quan,白嫖quanx,白嫖surge,白嫖小火箭,高速ss,高速ssr,高速v2ray,高速vmess,高速vless,高速shadowsocks,高速shadowsocksr,高速shadowsocksrr,高速shadowrocket,高速quan,高速quanx,高速surge,高速小火箭,公益ss,公益ssr,公益v2ray,公益vmess,公益vless,公益shadowsocks,公益shadowsocksr,公益shadowsocksrr,公益shadowrocket,公益quan,公益quanx,公益surge,公益小火箭,windowsss,windowsssr,windowsv2ray,windowsvmess,windowsvless,windowsshadowsocks,windowsshadowsocksr,windowsshadowsocksrr,windowsshadowrocket,windowsquan,windowsquanx,windowssurge,windows小火箭,androidss,androidssr,androidv2ray,androidvmess,androidvless,androidshadowsocks,androidshadowsocksr,androidshadowsocksrr,androidshadowrocket,androidquan,androidquanx,androidsurge,android小火箭,iosss,iosssr,iosv2ray,iosvmess,iosvless,iosshadowsocks,iosshadowsocksr,iosshadowsocksrr,iosshadowrocket,iosquan,iosquanx,iossurge,ios小火箭,macss,macssr,macv2ray,macvmess,macvless,macshadowsocks,macshadowsocksr,macshadowsocksrr,macshadowrocket,macquan,macquanx,macsurge,mac小火箭,苹果ss,苹果ssr,苹果v2ray,苹果vmess,苹果vless,苹果shadowsocks,苹果shadowsocksr,苹果shadowsocksrr,苹果shadowrocket,苹果quan,苹果quanx,苹果surge,苹果小火箭,安卓ss,安卓ssr,安卓v2ray,安卓vmess,安卓vless,安卓shadowsocks,安卓shadowsocksr,安卓shadowsocksrr,安卓shadowrocket,安卓quan,安卓quanx,安卓surge,安卓小火箭">
-    <style type='text/css'>
-        body {
-            background-color: white;
-            color: #333333;
-            font-family: Arial, sans-serif;
-            margin: 0;
-            padding: 36px;
-            line-height: 1;
-            font-size: 14px;
-        }
-
-        .section {
-            margin-bottom: 36px;
-        }
-
-        .section.friendly {
-            color: #222222;
-        }
-
-        .section.friendly h1 {
-            font-size: 26px;
-            background-color: #dad8e4;
-            padding: 18px 22px 15px 22px;
-            margin: 0;
-            overflow: hidden;
-        }
-
-        .section.friendly h1 strong {
-            display: inline-block;
-            float: left;
-        }
-
-        .section.friendly h1 small {
-            display: inline-block;
-            float: right;
-            text-align: right;
-            font-size: 18px;
-            padding-top: 4px;
-            color: #333333;
-        }
-
-        .section.friendly .article {
-            border: 4px solid #dad8e4;
-            padding: 24px 18px 18px 18px;
-        }
-
-        .section.friendly .article h3 {
-            font-size: 20px;
-            margin: 0 0 18px 0;
-        }
-
-        .section.friendly .article a {
-            color: #6b6ceb;
-        }
-
-        .section.friendly .article a:visited {
-            color: #1d1d3b;
-        }
-
-        .section.friendly .article p {
-            font-size: 14px;
-        }
-
-        .section.friendly .article ul {
-            list-style-type: square;
-        }
-
-        .section.original {
-            background-color: #eeeeee;
-            color: #444444;
-        }
-
-        .section.original h2 {
-            background-color: #dddddd;
-            margin: 0;
-            padding: 18px 22px 18px 22px;
-            font-size: 20px;
-        }
-
-        .section.original pre {
-            margin: 0;
-            padding: 18px 22px 18px 22px;
-            overflow: auto;
-            font-family: monaco, monospaced;
-        }
-
-        .section.original pre code {
-            display: block;
-            font-size: 11px;
-            width: 100%;
-        }
-    </style>
-</head>
-<body>
-<div class='container'>
-    <div class='section friendly'>
-        <h1><strong>免费节点</strong></h1>
-        <div class='article'>
-            <p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess、trojan节点信息，聚合去重后提供节点列表，每15分钟更新</p>
-            <p>汇总节点数量：{{ .all_proxies_count }}</p>
-            <p>ss节点数量：{{ .ss_proxies_count }}</p>
-            <p>ssr节点数量：{{ .ssr_proxies_count }}</p>
-            <p>vmess节点数量：{{ .vmess_proxies_count }}</p>
-            <p>trojan节点数量：{{ .trojan_proxies_count }}</p>
-            <p>可用节点数量：{{ .useful_proxies_count }}</p>
-            <p>最后更新时间：{{ .last_crawl_time }}</p>
-            <br>
-            <h4><a href="/clash">Clash</a></h4>
-            <p>Clash配置文件：https://{{ .domain }}/clash/config <a
-                        href="clash://install-config?url=https://{{ .domain }}/clash/config">一键导入</a></p>
-            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://{{ .domain }}/clash/proxies">https://{{ .domain }}/clash/proxies</a>
-            </p>
-            <p>筛选代理类型：https://{{ .domain }}/clash/proxies?type=ss,ssr,vmess</p>
-            <p>筛选国家：https://{{ .domain }}/clash/proxies?c=HK,TW,US</p>
-			<br>
-			<h4><a href="/surge">Surge</a></h4>
-            <p>Surge配置文件：https://{{ .domain }}/surge/config <a
-                        href="surge3:///install-config?url=https://{{ .domain }}/surge/config">一键导入</a></p>
-            <p>Surge proxy list：<a href="https://{{ .domain }}/surge/proxies">https://{{ .domain }}/surge/proxies</a>
-            </p>
-            <br>
-            {{- /* 所有使用本代码提供服务的禁止删除该行*/}}
-            <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a> {{ .version }}</p>
-        </div>
-    </div>
-</div>
-<!-- Global site tag (gtag.js) - Google Analytics -->
-<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
-<script>
-    window.dataLayer = window.dataLayer || [];
-    function gtag(){dataLayer.push(arguments);}
-    gtag('js', new Date());
-
-    gtag('config', 'G-DC6CR61FHE');
-</script>
-</body>
-</html>

assets/html/surge.conf

@@ -1,45 +0,0 @@
-[Proxy]
-Direct = direct
-
-[Proxy Group]
-Proxy = select, 延迟最低, 失败切换, 手动选择
-延迟最低 = url-test, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
-失败切换 = fallback, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
-手动选择 = select, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
-Apple = select, Direct, Proxy
-Adblock = select, Direct, REJECT, REJECT-TINYGIF
-
-[Rule]
-# RULESET
-RULE-SET,SYSTEM,Direct
-# 
-# Unbreak 后续规则修正
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Unbreak.list,Adblock
-# Advertising 广告
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Guard/Advertising.list,Adblock
-# Privacy 隐私
-# RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Guard/Privacy.list,Adblock
-# Hijacking 运营商劫持或恶意网站
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Guard/Hijacking.list,Adblock
-# 
-# Apple
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Extra/Apple/Apple.list,Apple
-# 
-# 代理
-# Streaming 国际流媒体服务
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/StreamingMedia/Streaming.list,Proxy
-# StreamingSE 中国流媒体服务（面向海外版本）
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/StreamingMedia/StreamingSE.list,Proxy
-# Global 全球加速
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Global.list,Proxy
-# 
-RULE-SET,https://github.com/Hackl0us/SS-Rule-Snippet/raw/master/Rulesets/App/social/Telegram.list,Proxy
-RULE-SET,https://github.com/Hackl0us/SS-Rule-Snippet/raw/master/Rulesets/App/social/WhatsApp.list,Proxy
-RULE-SET,https://github.com/Hackl0us/SS-Rule-Snippet/raw/master/Rulesets/App/social/LINE.list,Proxy
-# 
-# Direct
-RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Extra/ChinaIP.list,Direct
-# 
-RULE-SET,LAN,Direct
-# GEOIP,CN,Direct
-FINAL,Proxy,dns-failed

assets/html/surge.html

@@ -1,95 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8">
-  <meta http-equiv="X-UA-Compatible" content="IE=edge">
-  <meta name="HandheldFriendly" content="True">
-	<title>免费Surge节点</title>
-	<style type='text/css'>
-	  body {
-		background-color: white;
-		color: #333333;
-		  font-family: Arial, sans-serif;
-		margin: 0;
-		padding: 36px;
-		line-height: 1;
-		font-size: 14px; }
-
-	  .section {
-		margin-bottom: 36px; }
-		.section.friendly {
-		  color: #222222; }
-		  .section.friendly h1 {
-			font-size: 26px;
-			background-color: #dad8e4;
-			padding: 18px 22px 15px 22px;
-			margin: 0;
-			overflow: hidden; }
-			.section.friendly h1 strong {
-			  display: inline-block;
-			  float: left; }
-			.section.friendly h1 small {
-			  display: inline-block;
-			  float: right;
-			  text-align: right;
-			  font-size: 18px;
-			  padding-top: 4px;
-			  color: #333333; }
-		  .section.friendly .article {
-			border: 4px solid #dad8e4;
-			padding: 24px 18px 18px 18px; }
-			.section.friendly .article h3 {
-			  font-size: 20px;
-			  margin: 0 0 18px 0; }
-			.section.friendly .article a {
-			  color: #6b6ceb; }
-			  .section.friendly .article a:visited {
-				color: #1d1d3b; }
-			.section.friendly .article p {
-			  font-size: 14px; }
-			.section.friendly .article ul {
-			  list-style-type: square; }
-	  .section.original {
-		background-color: #eeeeee;
-		color: #444444; }
-		.section.original h2 {
-		  background-color: #dddddd;
-		  margin: 0;
-		  padding: 18px 22px 18px 22px;
-		  font-size: 20px; }
-		.section.original pre {
-		  margin: 0;
-		  padding: 18px 22px 18px 22px;
-		  overflow: auto;
-		  font-family: monaco, monospaced; }
-		  .section.original pre code {
-			display: block;
-			font-size: 11px;
-			width: 100%; }
-	</style>
-</head>
-<body>
-<div class='container'>
-  <div class='section friendly'>
-	<h1><strong>免费Surge节点</strong></h1>
-	<div class='article'>
-	  <p>自动抓取tg频道、订阅地址、公开互联网上的ss、vmess节点信息，聚合去重后提供Surge节点列表，每15分钟更新</p>
-		<br>
-	  <p>Surge配置文件：https://{{ .domain }}/surge/config <a href="surge3:///install-config?url=https://{{ .domain }}/surge/config">一键导入</a></p>
-	  <p>Surge proxy list：<a href="https://{{ .domain }}/surge/proxies">https://{{ .domain }}/surge/proxies</a></p>
-		<br>
-	  <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a></p>
-	</div>
-  </div>
-</div>
-<!-- Global site tag (gtag.js) - Google Analytics -->
-<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
-<script>
-	window.dataLayer = window.dataLayer || [];
-	function gtag(){dataLayer.push(arguments);}
-	gtag('js', new Date());
-
-	gtag('config', 'G-DC6CR61FHE');
-</script>
-</body>
-</html>

config/config.yaml

@@ -1,6 +1,13 @@
+# your domain
 domain: example.com
+
+# cloudflare api
 cf_email: ""
 cf_key: ""
+
+# source definition file
 source-files:
-  - ./source.yaml
+  # use local file
+  - source.yaml
+  # use web file
   - https://example.com/source.yaml

config/source.yaml

@@ -20,9 +20,9 @@
     num: 200
 
 # 翻墙党论坛抓取
-- type: web-fanqiangdang-rss
+- type: web-fanqiangdang
   options:
-    url: https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0
+    url: https://fanqiangdang.com/forum-48-1.html
 
 # 某个网站抓取
 - type: web-freessrxyz

docs/genbindata.sh

@@ -0,0 +1,2 @@
+go-bindata -o internal/bindata/html/html.go -pkg binhtml  assets/html/
+go-bindata -o internal/bindata/geoip/geoip.go -pkg bingeoip  assets/GeoLite2-City.mmdb assets/flags.json

go.mod

@@ -9,14 +9,16 @@ require (
 	github.com/antchfx/htmlquery v1.2.3 // indirect
 	github.com/antchfx/xmlquery v1.2.4 // indirect
 	github.com/antchfx/xpath v1.1.8 // indirect
-	github.com/cloudflare/cloudflare-go v0.13.0
+	github.com/cloudflare/cloudflare-go v0.13.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.6.3
+	github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect
 	github.com/go-playground/validator/v10 v10.3.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gocolly/colly v1.2.0
 	github.com/golang/protobuf v1.4.2 // indirect
 	github.com/heroku/x v0.0.25
+	github.com/ivpusic/grpool v1.0.0
 	github.com/jasonlvhit/gocron v0.0.1
 	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
@@ -28,7 +30,6 @@ require (
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/temoto/robotstxt v1.1.1 // indirect
 	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect
-	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
 	golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect
 	golang.org/x/text v0.3.3 // indirect
 	google.golang.org/appengine v1.6.6 // indirect

go.sum

@@ -28,6 +28,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.13.0 h1:Mx2gNz/r4H0eHETENq5ZPtc/kOX/uzEfT2UFuA+/XAU=
 github.com/cloudflare/cloudflare-go v0.13.0/go.mod h1:6rZ6s/XWxP8WIWMATgDM7aUop0Z0ZOAfcm/4rEd0lOY=
+github.com/cloudflare/cloudflare-go v0.13.1 h1:8i8E5CubfAlD/2zWM7VGKoIPS2/x/lO/nRnqDl0j5Dk=
+github.com/cloudflare/cloudflare-go v0.13.1/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -45,6 +47,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/go-bindata/go-bindata v1.0.0 h1:DZ34txDXWn1DyWa+vQf7V9ANc2ILTtrEjtlsdJRF26M=
+github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE=
+github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
@@ -120,6 +125,8 @@ github.com/heroku/x v0.0.25/go.mod h1:qE/I0jp6rIeTBBosrPYV4ygRX3OMhqmC/A6x8ewodJ
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/ivpusic/grpool v1.0.0 h1:+FCiCo3GhfsvzfXuJWnpJUNb/VaqyYVgG8C+qvh07Rc=
+github.com/ivpusic/grpool v1.0.0/go.mod h1:WPmiAI5ExAn06vg+0JzyPzXMQutJmpb7TrBtyLJkOHQ=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -264,6 +271,8 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

internal/app/getter.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/zu1k/proxypool/internal/cache"
+
 	"github.com/ghodss/yaml"
 
 	"github.com/zu1k/proxypool/config"
@@ -48,4 +50,5 @@ func initGetters(sourceFiles []string) {
 		}
 	}
 	fmt.Println("Getter count:", len(Getters))
+	cache.GettersCount = len(Getters)
 }

internal/app/task.go

@@ -33,19 +33,28 @@ func CrawlGo() {
 	proxies = proxies.Deduplication()
 	log.Println("CrawlGo node count:", len(proxies))
 	proxies = provider.Clash{Proxies: proxies}.CleanProxies()
-	proxies.NameAddCounrty().Sort().NameAddIndex()
+	log.Println("CrawlGo cleaned node count:", len(proxies))
+	proxies.NameAddCounrty().Sort().NameAddIndex().NameAddTG()
+	log.Println("Proxy rename DONE!")
+
 	cache.SetProxies("allproxies", proxies)
 	cache.AllProxiesCount = proxies.Len()
+	log.Println("AllProxiesCount:", cache.AllProxiesCount)
 	cache.SSProxiesCount = proxies.TypeLen("ss")
+	log.Println("SSProxiesCount:", cache.SSProxiesCount)
 	cache.SSRProxiesCount = proxies.TypeLen("ssr")
+	log.Println("SSRProxiesCount:", cache.SSRProxiesCount)
 	cache.VmessProxiesCount = proxies.TypeLen("vmess")
+	log.Println("VmessProxiesCount:", cache.VmessProxiesCount)
 	cache.TrojanProxiesCount = proxies.TypeLen("trojan")
+	log.Println("TrojanProxiesCount:", cache.TrojanProxiesCount)
 	cache.LastCrawlTime = time.Now().In(location).Format("2006-01-02 15:04:05")
 
 	// 可用性检测
-	proxies = proxy.CleanBadProxies(proxies)
-	log.Println("CrawlGo clash useable node count:", len(proxies))
-	proxies.NameAddCounrty().Sort().NameAddIndex()
+	log.Println("Now proceed proxy health check...")
+	proxies = proxy.CleanBadProxiesWithGrpool(proxies)
+	log.Println("CrawlGo clash usable node count:", len(proxies))
+	proxies.NameReIndex()
 	cache.SetProxies("proxies", proxies)
 	cache.UsefullProxiesCount = proxies.Len()
 

internal/bindata/geoip/geoip.go

@@ -0,0 +1,213 @@
+// Code generated for package bingeoip by go-bindata DO NOT EDIT. (@generated)
+// sources:
+// assets/GeoLite2-City.mmdb
+// assets/flags.json
+package bingeoip
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// bindataRead reads the given file from disk. It returns an error on failure.
+func bindataRead(path, name string) ([]byte, error) {
+	buf, err := ioutil.ReadFile(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err)
+	}
+	return buf, err
+}
+
+type asset struct {
+	bytes []byte
+	info  os.FileInfo
+}
+
+// assetsGeolite2CityMmdb reads file data from disk. It returns an error on failure.
+func assetsGeolite2CityMmdb() (*asset, error) {
+	path := "assets/GeoLite2-City.mmdb"
+	name := "assets/GeoLite2-City.mmdb"
+	bytes, err := bindataRead(path, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
+	}
+
+	a := &asset{bytes: bytes, info: fi}
+	return a, err
+}
+
+// assetsFlagsJson reads file data from disk. It returns an error on failure.
+func assetsFlagsJson() (*asset, error) {
+	path := "assets/flags.json"
+	name := "assets/flags.json"
+	bytes, err := bindataRead(path, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
+	}
+
+	a := &asset{bytes: bytes, info: fi}
+	return a, err
+}
+
+// Asset loads and returns the asset for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func Asset(name string) ([]byte, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
+		}
+		return a.bytes, nil
+	}
+	return nil, fmt.Errorf("Asset %s not found", name)
+}
+
+// MustAsset is like Asset but panics when Asset would return an error.
+// It simplifies safe initialization of global variables.
+func MustAsset(name string) []byte {
+	a, err := Asset(name)
+	if err != nil {
+		panic("asset: Asset(" + name + "): " + err.Error())
+	}
+
+	return a
+}
+
+// AssetInfo loads and returns the asset info for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func AssetInfo(name string) (os.FileInfo, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
+		}
+		return a.info, nil
+	}
+	return nil, fmt.Errorf("AssetInfo %s not found", name)
+}
+
+// AssetNames returns the names of the assets.
+func AssetNames() []string {
+	names := make([]string, 0, len(_bindata))
+	for name := range _bindata {
+		names = append(names, name)
+	}
+	return names
+}
+
+// _bindata is a table, holding each asset generator, mapped to its name.
+var _bindata = map[string]func() (*asset, error){
+	"assets/GeoLite2-City.mmdb": assetsGeolite2CityMmdb,
+	"assets/flags.json":         assetsFlagsJson,
+}
+
+// AssetDir returns the file names below a certain
+// directory embedded in the file by go-bindata.
+// For example if you run go-bindata on data/... and data contains the
+// following hierarchy:
+//     data/
+//       foo.txt
+//       img/
+//         a.png
+//         b.png
+// then AssetDir("data") would return []string{"foo.txt", "img"}
+// AssetDir("data/img") would return []string{"a.png", "b.png"}
+// AssetDir("foo.txt") and AssetDir("notexist") would return an error
+// AssetDir("") will return []string{"data"}.
+func AssetDir(name string) ([]string, error) {
+	node := _bintree
+	if len(name) != 0 {
+		cannonicalName := strings.Replace(name, "\\", "/", -1)
+		pathList := strings.Split(cannonicalName, "/")
+		for _, p := range pathList {
+			node = node.Children[p]
+			if node == nil {
+				return nil, fmt.Errorf("Asset %s not found", name)
+			}
+		}
+	}
+	if node.Func != nil {
+		return nil, fmt.Errorf("Asset %s not found", name)
+	}
+	rv := make([]string, 0, len(node.Children))
+	for childName := range node.Children {
+		rv = append(rv, childName)
+	}
+	return rv, nil
+}
+
+type bintree struct {
+	Func     func() (*asset, error)
+	Children map[string]*bintree
+}
+
+var _bintree = &bintree{nil, map[string]*bintree{
+	"assets": &bintree{nil, map[string]*bintree{
+		"GeoLite2-City.mmdb": &bintree{assetsGeolite2CityMmdb, map[string]*bintree{}},
+		"flags.json":         &bintree{assetsFlagsJson, map[string]*bintree{}},
+	}},
+}}
+
+// RestoreAsset restores an asset under the given directory
+func RestoreAsset(dir, name string) error {
+	data, err := Asset(name)
+	if err != nil {
+		return err
+	}
+	info, err := AssetInfo(name)
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
+	if err != nil {
+		return err
+	}
+	err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// RestoreAssets restores an asset under the given directory recursively
+func RestoreAssets(dir, name string) error {
+	children, err := AssetDir(name)
+	// File
+	if err != nil {
+		return RestoreAsset(dir, name)
+	}
+	// Dir
+	for _, child := range children {
+		err = RestoreAssets(dir, filepath.Join(name, child))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func _filePath(dir, name string) string {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
+}

internal/cache/cache.go

@@ -1,7 +1,6 @@
 package cache
 
 import (
-	"log"
 	"time"
 
 	"github.com/patrickmn/go-cache"
@@ -13,10 +12,8 @@ var c = cache.New(cache.NoExpiration, 10*time.Minute)
 func GetProxies(key string) proxy.ProxyList {
 	result, found := c.Get(key)
 	if found {
-		log.Println("found cache for:", key, "length:", len(result.(proxy.ProxyList)))
 		return result.(proxy.ProxyList)
 	}
-	log.Println("cache not found:", key)
 	return nil
 }
 

internal/cache/vars.go

@@ -1,6 +1,8 @@
 package cache
 
 var (
+	GettersCount = 0
+
 	AllProxiesCount    = 0
 	SSRProxiesCount    = 0
 	SSProxiesCount     = 0

pkg/getter/subscribe.go

@@ -2,6 +2,7 @@ package getter
 
 import (
 	"io/ioutil"
+	"log"
 	"strings"
 	"sync"
 
@@ -41,6 +42,7 @@ func (s *Subscribe) Get() proxy.ProxyList {
 func (s *Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := s.Get()
+	log.Printf("STATISTIC: Subscribe\tcount=%d\turl=%s\n", len(nodes), s.Url)
 	for _, node := range nodes {
 		pc <- node
 	}

pkg/getter/tgchannel.go

@@ -2,6 +2,7 @@ package getter
 
 import (
 	"fmt"
+	"log"
 	"sync"
 
 	"github.com/gocolly/colly"
@@ -40,7 +41,7 @@ func NewTGChannelGetter(options tool.Options) (getter Getter, err error) {
 			return nil, err
 		}
 		return &TGChannelGetter{
-			c:         colly.NewCollector(),
+			c:         tool.GetColly(),
 			NumNeeded: t,
 			Url:       "https://t.me/s/" + url,
 		}, nil
@@ -79,6 +80,7 @@ func (g *TGChannelGetter) Get() proxy.ProxyList {
 func (g *TGChannelGetter) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := g.Get()
+	log.Printf("STATISTIC: TGChannel\tcount=%d\turl=%s\n", len(nodes), g.Url)
 	for _, node := range nodes {
 		pc <- node
 	}

pkg/getter/web_fanqiangdang.go

@@ -2,6 +2,7 @@ package getter
 
 import (
 	"fmt"
+	"log"
 	"strings"
 	"sync"
 
@@ -11,14 +12,13 @@ import (
 )
 
 func init() {
-	Register("web-fanqiangdang-rss", NewWebFanqiangdangRSSGetter)
 	Register("web-fanqiangdang", NewWebFanqiangdangGetter)
 }
 
 type WebFanqiangdang struct {
-	c         *colly.Collector
-	Url       string
-	results   proxy.ProxyList
+	c       *colly.Collector
+	Url     string
+	results proxy.ProxyList
 }
 
 func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
@@ -29,8 +29,8 @@ func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
 			return nil, err
 		}
 		return &WebFanqiangdang{
-			c:         colly.NewCollector(),
-			Url:       url,
+			c:   colly.NewCollector(),
+			Url: url,
 		}, nil
 	}
 	return nil, ErrorUrlNotFound
@@ -65,55 +65,7 @@ func (w *WebFanqiangdang) Get() proxy.ProxyList {
 func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
-	for _, node := range nodes {
-		pc <- node
-	}
-}
-
-
-type WebFanqiangdangRSS struct {
-	c         *colly.Collector
-	Url       string
-	results   []string
-}
-
-func NewWebFanqiangdangRSSGetter(options tool.Options) (getter Getter, err error) {
-	urlInterface, found := options["url"]
-	if found {
-		url, err := AssertTypeStringNotNull(urlInterface)
-		if err != nil {
-			return nil, err
-		}
-		return &WebFanqiangdangRSS{
-			c:         colly.NewCollector(),
-			Url:       url,
-		}, nil
-	}
-	return nil, ErrorUrlNotFound
-}
-
-func (w *WebFanqiangdangRSS) Get() proxy.ProxyList {
-	w.results = make([]string, 0)
-	w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) {
-		w.results = append(w.results, GrepLinksFromString(e.Text)...)
-	})
-
-	w.c.OnXML("//item//link", func(e *colly.XMLElement) {
-		_ = e.Request.Visit(e.Text)
-	})
-
-	w.results = make([]string, 0)
-	err := w.c.Visit(w.Url)
-	if err != nil {
-		_ = fmt.Errorf("%s", err.Error())
-	}
-
-	return StringArray2ProxyArray(w.results)
-}
-
-func (w *WebFanqiangdangRSS) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
-	defer wg.Done()
-	nodes := w.Get()
+	log.Printf("STATISTIC: Fanqiangdang\tcount=%d\turl=%s\n", len(nodes), w.Url)
 	for _, node := range nodes {
 		pc <- node
 	}

pkg/getter/web_free_ssr_xyz.go

@@ -3,6 +3,7 @@ package getter
 import (
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"sync"
 
 	"github.com/zu1k/proxypool/pkg/proxy"
@@ -34,6 +35,7 @@ func (w *WebFreessrXyz) Get() proxy.ProxyList {
 func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
+	log.Printf("STATISTIC: FreeSSRxyz\tcount=%d\turl=%s\n", len(nodes), "api.free-ssr.xyz")
 	for _, node := range nodes {
 		pc <- node
 	}

pkg/getter/web_fuzz.go

@@ -2,6 +2,7 @@ package getter
 
 import (
 	"io/ioutil"
+	"log"
 	"sync"
 
 	"github.com/zu1k/proxypool/pkg/proxy"
@@ -33,6 +34,7 @@ func (w *WebFuzz) Get() proxy.ProxyList {
 func (w *WebFuzz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
+	log.Printf("STATISTIC: WebFuzz\tcount=%d\turl=%s\n", len(nodes), w.Url)
 	for _, node := range nodes {
 		pc <- node
 	}

pkg/getter/web_fuzz_sub.go

@@ -2,6 +2,7 @@ package getter
 
 import (
 	"io/ioutil"
+	"log"
 	"regexp"
 	"sync"
 
@@ -39,6 +40,7 @@ func (w *WebFuzzSub) Get() proxy.ProxyList {
 func (w *WebFuzzSub) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
+	log.Printf("STATISTIC: WebFuzzSub\tcount=%d\turl=%s\n", len(nodes), w.Url)
 	for _, node := range nodes {
 		pc <- node
 	}

pkg/provider/clash.go

@@ -7,9 +7,10 @@ import (
 )
 
 type Clash struct {
-	Proxies proxy.ProxyList `yaml:"proxies"`
-	Types   string          `yaml:"type"`
-	Country string          `yaml:"country"`
+	Proxies    proxy.ProxyList `yaml:"proxies"`
+	Types      string          `yaml:"type"`
+	Country    string          `yaml:"country"`
+	NotCountry string          `yaml:"not_country"`
 }
 
 func (c Clash) CleanProxies() (proxies proxy.ProxyList) {
@@ -26,44 +27,63 @@ func (c Clash) Provide() string {
 	var resultBuilder strings.Builder
 	resultBuilder.WriteString("proxies:\n")
 
-	noNeedFilterType := false
-	noNeedFilterCountry := false
+	needFilterType := true
+	needFilterCountry := true
+	needFilterNotCountry := true
 	if c.Types == "" || c.Types == "all" {
-		noNeedFilterType = true
+		needFilterType = false
 	}
 	if c.Country == "" || c.Country == "all" {
-		noNeedFilterCountry = true
+		needFilterCountry = false
+	}
+	if c.NotCountry == "" {
+		needFilterNotCountry = false
 	}
 	types := strings.Split(c.Types, ",")
 	countries := strings.Split(c.Country, ",")
+	notCountries := strings.Split(c.NotCountry, ",")
 
 	for _, p := range c.Proxies {
 		if !checkClashSupport(p) {
 			continue
 		}
 
-		typeOk := false
-		countryOk := false
-		if !noNeedFilterType {
+		if needFilterType {
+			typeOk := false
 			for _, t := range types {
 				if p.TypeName() == t {
 					typeOk = true
 					break
 				}
 			}
+			if !typeOk {
+				goto exclude
+			}
 		}
-		if !noNeedFilterCountry {
-			for _, c := range countries {
-				if strings.HasPrefix(p.BaseInfo().Name, c) {
-					countryOk = true
-					break
+
+		if needFilterNotCountry {
+			for _, c := range notCountries {
+				if strings.Contains(p.BaseInfo().Name, c) {
+					goto exclude
 				}
 			}
 		}
 
-		if (noNeedFilterType || typeOk) && (noNeedFilterCountry || countryOk) {
-			resultBuilder.WriteString(p.ToClash() + "\n")
+		if needFilterCountry {
+			countryOk := false
+			for _, c := range countries {
+				if strings.Contains(p.BaseInfo().Name, c) {
+					countryOk = true
+					break
+				}
+			}
+			if !countryOk {
+				goto exclude
+			}
 		}
+
+		resultBuilder.WriteString(p.ToClash() + "\n")
+	exclude:
 	}
 
 	return resultBuilder.String()

pkg/proxy/base.go

@@ -19,6 +19,10 @@ func (b *Base) SetName(name string) {
 	b.Name = name
 }
 
+func (b *Base) SetIP(ip string) {
+	b.Server = ip
+}
+
 func (b *Base) BaseInfo() *Base {
 	return b
 }
@@ -34,6 +38,7 @@ type Proxy interface {
 	ToSurge() string
 	Identifier() string
 	SetName(name string)
+	SetIP(ip string)
 	TypeName() string
 	BaseInfo() *Base
 	Clone() Proxy

pkg/proxy/check.go

@@ -4,8 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync"
 	"time"
 
+	"github.com/ivpusic/grpool"
+
 	"github.com/Dreamacro/clash/adapters/outbound"
 )
 
@@ -35,21 +38,75 @@ func testDelay(p Proxy) (delay uint16, err error) {
 	return delay, err
 }
 
+func CleanBadProxiesWithGrpool(proxies []Proxy) (cproxies []Proxy) {
+	pool := grpool.NewPool(500, 200)
+
+	c := make(chan checkResult)
+	defer close(c)
+
+	pool.WaitCount(len(proxies))
+	go func() {
+		for _, p := range proxies {
+			pp := p
+			pool.JobQueue <- func() {
+				defer pool.JobDone()
+				delay, err := testDelay(pp)
+				if err == nil {
+					c <- checkResult{
+						name:  pp.Identifier(),
+						delay: delay,
+					}
+				}
+			}
+		}
+	}()
+	done := make(chan struct{})
+	defer close(done)
+
+	go func() {
+		pool.WaitAll()
+		pool.Release()
+		done <- struct{}{}
+	}()
+
+	okMap := make(map[string]struct{})
+	for {
+		select {
+		case r := <-c:
+			if r.delay > 0 {
+				okMap[r.name] = struct{}{}
+			}
+		case <-done:
+			cproxies = make(ProxyList, 0, 500)
+			for _, p := range proxies {
+				if _, ok := okMap[p.Identifier()]; ok {
+					cproxies = append(cproxies, p.Clone())
+				}
+			}
+			return
+		}
+	}
+}
+
 func CleanBadProxies(proxies []Proxy) (cproxies []Proxy) {
 	c := make(chan checkResult, 40)
-	defer close(c)
+	wg := &sync.WaitGroup{}
+	wg.Add(len(proxies))
 	for _, p := range proxies {
-		go testProxyDelayToChan(p, c)
+		go testProxyDelayToChan(p, c, wg)
 	}
+	go func() {
+		wg.Wait()
+		close(c)
+	}()
+
 	okMap := make(map[string]struct{})
-	size := len(proxies)
-	for i := 0; i < size; i++ {
-		r := <-c
+	for r := range c {
 		if r.delay > 0 {
 			okMap[r.name] = struct{}{}
 		}
 	}
-	cproxies = make([]Proxy, 0)
+	cproxies = make(ProxyList, 0, 500)
 	for _, p := range proxies {
 		if _, ok := okMap[p.Identifier()]; ok {
 			cproxies = append(cproxies, p.Clone())
@@ -63,17 +120,13 @@ type checkResult struct {
 	delay uint16
 }
 
-func testProxyDelayToChan(p Proxy, c chan checkResult) {
+func testProxyDelayToChan(p Proxy, c chan checkResult, wg *sync.WaitGroup) {
+	defer wg.Done()
 	delay, err := testDelay(p)
-	if err != nil {
+	if err == nil {
 		c <- checkResult{
 			name:  p.Identifier(),
-			delay: 0,
+			delay: delay,
 		}
-		return
-	}
-	c <- checkResult{
-		name:  p.Identifier(),
-		delay: delay,
 	}
 }

pkg/proxy/geoip.go

@@ -1,51 +1,105 @@
 package proxy
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
 	"log"
 	"net"
 	"os"
 
 	"github.com/oschwald/geoip2-golang"
+	bingeoip "github.com/zu1k/proxypool/internal/bindata/geoip"
 )
 
 var geoIp GeoIP
 
 func InitGeoIpDB() {
-	geoIp = NewGeoIP("assets/GeoLite2-City.mmdb")
+	err := bingeoip.RestoreAsset("", "assets/GeoLite2-City.mmdb")
+	if err != nil {
+		panic(err)
+	}
+	err = bingeoip.RestoreAsset("", "assets/flags.json")
+	if err != nil {
+		panic(err)
+	}
+	geoIp = NewGeoIP("assets/GeoLite2-City.mmdb", "assets/flags.json")
 }
 
 // GeoIP2
 type GeoIP struct {
-	db *geoip2.Reader
+	db       *geoip2.Reader
+	emojiMap map[string]string
+}
+
+type CountryEmoji struct {
+	Code  string `json:"code"`
+	Emoji string `json:"emoji"`
 }
 
 // new geoip from db file
-func NewGeoIP(filePath string) (geoip GeoIP) {
+func NewGeoIP(geodb, flags string) (geoip GeoIP) {
 	// 判断文件是否存在
-	_, err := os.Stat(filePath)
+	_, err := os.Stat(geodb)
 	if err != nil && os.IsNotExist(err) {
-		log.Println("文件不存在，请自行下载 Geoip2 City库，并保存在", filePath)
+		log.Println("文件不存在，请自行下载 Geoip2 City库，并保存在", geodb)
 		os.Exit(1)
 	} else {
-		db, err := geoip2.Open(filePath)
+		db, err := geoip2.Open(geodb)
 		if err != nil {
 			log.Fatal(err)
 		}
-		geoip = GeoIP{db: db}
+		geoip.db = db
+	}
+
+	_, err = os.Stat(flags)
+	if err != nil && os.IsNotExist(err) {
+		log.Println("flags 文件不存在，请自行下载 flags.json，并保存在", flags)
+		os.Exit(1)
+	} else {
+		data, err := ioutil.ReadFile(flags)
+		if err != nil {
+			log.Fatal(err)
+			return
+		}
+		var countryEmojiList = make([]CountryEmoji, 0)
+		err = json.Unmarshal(data, &countryEmojiList)
+		if err != nil {
+			log.Fatalln(err.Error())
+			return
+		}
+
+		emojiMap := make(map[string]string)
+		for _, i := range countryEmojiList {
+			emojiMap[i.Code] = i.Emoji
+		}
+		geoip.emojiMap = emojiMap
 	}
 	return
 }
 
 // find ip info
-func (g GeoIP) Find(ipORdomain string) (country string, err error) {
+func (g GeoIP) Find(ipORdomain string) (ip, country string, err error) {
 	ips, err := net.LookupIP(ipORdomain)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
-	ipData := net.ParseIP(ips[0].String())
-	record, err := g.db.City(ipData)
+	ip = ips[0].String()
+
+	var record *geoip2.City
+	record, err = g.db.City(ips[0])
 	if err != nil {
-		return "", err
+		return
 	}
-	return record.Country.IsoCode, nil
+	countryIsoCode := record.Country.IsoCode
+	if countryIsoCode == "" {
+		country = fmt.Sprintf("🏁 ZZ")
+	}
+	emoji, found := g.emojiMap[countryIsoCode]
+	if found {
+		country = fmt.Sprintf("%v %v", emoji, countryIsoCode)
+	} else {
+		country = fmt.Sprintf("🏁 ZZ")
+	}
+	return
 }

pkg/proxy/proxies.go

@@ -3,6 +3,8 @@ package proxy
 import (
 	"fmt"
 	"sort"
+	"strings"
+	"sync"
 )
 
 type ProxyList []Proxy
@@ -63,20 +65,46 @@ func (ps ProxyList) Sort() ProxyList {
 
 func (ps ProxyList) NameAddCounrty() ProxyList {
 	num := len(ps)
+	wg := &sync.WaitGroup{}
+	wg.Add(num)
 	for i := 0; i < num; i++ {
-		country, err := geoIp.Find(ps[i].BaseInfo().Server)
-		if err != nil || country == "" {
-			country = "Earth"
-		}
-		ps[i].SetName(fmt.Sprintf("%s", country))
+		ii := i
+		go func() {
+			defer wg.Done()
+			_, country, err := geoIp.Find(ps[ii].BaseInfo().Server)
+			if err != nil {
+				country = "🏁 ZZ"
+			}
+			ps[ii].SetName(fmt.Sprintf("%s", country))
+			//ps[ii].SetIP(ip)
+		}()
 	}
+	wg.Wait()
 	return ps
 }
 
 func (ps ProxyList) NameAddIndex() ProxyList {
 	num := len(ps)
 	for i := 0; i < num; i++ {
-		ps[i].SetName(fmt.Sprintf("%s_%d", ps[i].BaseInfo().Name, i+1))
+		ps[i].SetName(fmt.Sprintf("%s_%+02v", ps[i].BaseInfo().Name, i+1))
+	}
+	return ps
+}
+
+func (ps ProxyList) NameReIndex() ProxyList {
+	num := len(ps)
+	for i := 0; i < num; i++ {
+		originName := ps[i].BaseInfo().Name
+		country := strings.SplitN(originName, "_", 2)[0]
+		ps[i].SetName(fmt.Sprintf("%s_%+02v", country, i+1))
+	}
+	return ps
+}
+
+func (ps ProxyList) NameAddTG() ProxyList {
+	num := len(ps)
+	for i := 0; i < num; i++ {
+		ps[i].SetName(fmt.Sprintf("%s %s", ps[i].BaseInfo().Name, "@peekfun"))
 	}
 	return ps
 }

pkg/proxy/shadowsocksr.go

@@ -34,7 +34,7 @@ type ShadowsocksR struct {
 }
 
 func (ssr ShadowsocksR) Identifier() string {
-	return net.JoinHostPort(ssr.Server, strconv.Itoa(ssr.Port)) + ssr.ProtocolParam
+	return net.JoinHostPort(ssr.Server, strconv.Itoa(ssr.Port)) + ssr.Password + ssr.ProtocolParam
 }
 
 func (ssr ShadowsocksR) String() string {

pkg/tool/colly.go

@@ -0,0 +1,27 @@
+package tool
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/gocolly/colly"
+)
+
+func GetColly() *colly.Collector {
+	c := colly.NewCollector(
+		colly.UserAgent(UserAgent),
+	)
+	c.WithTransport(&http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   10 * time.Second, // 超时时间
+			KeepAlive: 10 * time.Second, // keepAlive 超时时间
+		}).DialContext,
+		MaxIdleConns:          100,              // 最大空闲连接数
+		IdleConnTimeout:       20 * time.Second, // 空闲连接超时
+		TLSHandshakeTimeout:   10 * time.Second, // TLS 握手超时
+		ExpectContinueTimeout: 10 * time.Second,
+	})
+	return c
+}

pkg/tool/httpclient.go

@@ -6,6 +6,8 @@ import (
 	"time"
 )
 
+const UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36"
+
 type HttpClient struct {
 	*http.Client
 }
@@ -28,7 +30,7 @@ func (c *HttpClient) Get(url string) (resp *http.Response, err error) {
 		return nil, err
 	}
 	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
+	req.Header.Set("User-Agent", UserAgent)
 	return c.Do(req)
 }
 
@@ -38,6 +40,6 @@ func (c *HttpClient) Post(url string, body io.Reader) (resp *http.Response, err
 		return nil, err
 	}
 	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
+	req.Header.Set("User-Agent", UserAgent)
 	return c.Do(req)
 }