add proxies count show

README.md

@@ -34,10 +34,13 @@
 
 ### 使用Heroku
 
-首先 Fork 仓库到你的用户名下，然后修改 assets 文件夹下的html文件，将里面的域名修改为你自己的，最后点击按钮进行部署
+点击按钮进入部署页面，填写基本信息然后运行
 
 [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy)
 
+> 因为爬虫程序需要持续运行，所以至少选择 $7/月 的配置
+> 免费配置长时间无人访问会被heroku强制停止
+
 ### 从源码编译
 
 需要安装Golang
@@ -58,30 +61,30 @@ docker pull docker.pkg.github.com/zu1k/proxypool/proxypool:latest
 
 ## 使用
 
-### 共享抓取配置文件
+### 修改配置文件
 
-每一次抓取前会自动从github下载最新的抓取配置文件
+首先修改 source.yaml 中的必要配置信息，其中域名修改为你自己的域名，cf开头的配置信息可以留空
 
-直接运行 `proxypool` 程序即可
-
-### 自定义抓取配置文件
+### 启动程序
 
 ```shell
 proxypool -c source.yaml
 ```
 
-### 帮助更新配置文件
-
-可以在 https://github.com/zu1k/proxypool/issues/3 进行留言
-
 ### 用户使用
 
 目前公开版本： https://proxy.tgbot.co
 
-直接在clash添加配置文件即可使用： https://proxy.tgbot.co/clash/config
+访问页面，按照相关指导进行使用
 
 ## 截图
 
 ![Speedtest](docs/speedtest.png)
 
 ![Fast](docs/fast.png)
+
+## 声明
+
+本项目遵循 GNU General Public License v3.0 开源，在此基础上，所有使用本项目提供服务者都必须在网站保留指向本项目的链接
+
+禁止使用本项目进行营利和做其他违法事情，产生的一切后果本项目概不负责

api/router.go

@@ -1,24 +1,60 @@
 package api
 
 import (
+	"fmt"
+	"net/http"
 	"os"
 
+	"github.com/zu1k/proxypool/config"
+
 	"github.com/gin-gonic/gin"
 	_ "github.com/heroku/x/hmetrics/onload"
-	"github.com/zu1k/proxypool/app/cache"
-	"github.com/zu1k/proxypool/provider"
+	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/pkg/provider"
 )
 
 var router *gin.Engine
+var domain = "proxy.tgbot.co"
 
 func setupRouter() {
-	router = gin.Default()
+	domain = config.SourceConfig.Domain
+	fmt.Println("Domain:", domain)
+
+	router = gin.Default()
+	router.LoadHTMLGlob("assets/html/*")
+
+	router.GET("/", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "index.html", gin.H{
+			"domain":               domain,
+			"all_proxies_count":    cache.AllProxiesCount,
+			"useful_proxies_count": cache.UsefullProxiesCount,
+		})
+	})
+
+	router.GET("/clash", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "clash.html", gin.H{
+			"domain": domain,
+		})
+	})
+
+	router.GET("/surge", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "surge.html", gin.H{
+			"domain": domain,
+		})
+	})
+
+	router.GET("/clash/config", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "clash-config.yaml", gin.H{
+			"domain": domain,
+		})
+	})
+
+	router.GET("/surge/config", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "surge.conf", gin.H{
+			"domain": domain,
+		})
+	})
 
-	router.StaticFile("/", "assets/index.html")
-	router.StaticFile("/clash", "assets/clash.html")
-	router.StaticFile("/surge", "assets/surge.html")
-	router.StaticFile("/clash/config", "assets/clash-config.yaml")
-	router.StaticFile("/surge/config", "assets/surge.conf")
 	router.GET("/clash/proxies", func(c *gin.Context) {
 		proxyTypes := c.DefaultQuery("type", "")
 		text := ""

app.json

@@ -1,7 +1,20 @@
 {
   "name": "proxypool",
   "description": "自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess节点信息，聚合去重测试可用性后提供节点列表",
+  "website": "https://proxy.tgbot.co/",
   "repository": "https://github.com/zu1k/proxypool",
+  "success_url": "/",
   "logo": "https://raw.githubusercontent.com/zu1k/proxypool/master/assets/proxy.jpg",
-  "keywords": ["golang", "ss", "ssr", "vmess", "shadowsocks", "shadowsocksr"]
+  "keywords": ["golang", "ss", "ssr", "vmess", "shadowsocks", "shadowsocksr"],
+  "env": {
+    "DOMAIN": {
+      "description": "Domain to use."
+    },
+    "CF_API_EMAIL": {
+      "description": "Cloudflare Email."
+    },
+    "CF_API_KEY": {
+      "description": "Cloudflare API key."
+    }
+  }
 }

app/cron.go

@@ -1,10 +0,0 @@
-package app
-
-import (
-	"github.com/jasonlvhit/gocron"
-)
-
-func Cron() {
-	_ = gocron.Every(15).Minutes().Do(CrawlGo)
-	<-gocron.Start()
-}

assets/html/clash-config.yaml

@@ -54,7 +54,7 @@ proxy-groups:
 proxy-providers:
   provider:
     type: http
-    url: "https://proxy.tgbot.co/clash/proxies"
+    url: "https://{{ .domain }}/clash/proxies"
     interval: 3600
     path: ./provider.yaml
     health-check:

assets/html/clash.html

@@ -105,15 +105,24 @@
         <div class='article'>
             <p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess节点信息，聚合去重后提供clash配置，每15分钟更新</p>
             <br>
-            <p>Clash配置文件：<a href="https://proxy.tgbot.co/clash/config">https://proxy.tgbot.co/clash/config</a> <a href="clash://install-config?url=https://proxy.tgbot.co/clash/config">一键导入</a></p>
-            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://proxy.tgbot.co/clash/proxies">https://proxy.tgbot.co/clash/proxies</a></p>
+            <p>Clash配置文件：<a href="https://{{ .domain }}/clash/config">https://{{ .domain }}/clash/config</a> <a href="clash://install-config?url=https://{{ .domain }}/clash/config">一键导入</a></p>
+            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://{{ .domain }}/clash/proxies">https://{{ .domain }}/clash/proxies</a></p>
             <br>
-			<p>筛选代理类型(此种方式你只能自己维护配置文件)：https://proxy.tgbot.co/clash/proxies?type=ss,ssr,vmess</p>
-            <p>所有节点的Provider(不是都可以用)：<a href="https://proxy.tgbot.co/clash/proxies?type=all">https://proxy.tgbot.co/clash/proxies?type=all</a></p>
+			<p>筛选代理类型(此种方式你只能自己维护配置文件)：https://{{ .domain }}/clash/proxies?type=ss,ssr,vmess</p>
+            <p>所有节点的Provider(不是都可以用)：<a href="https://{{ .domain }}/clash/proxies?type=all">https://{{ .domain }}/clash/proxies?type=all</a></p>
             <br>
             <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a></p>
         </div>
     </div>
 </div>
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
+<script>
+    window.dataLayer = window.dataLayer || [];
+    function gtag(){dataLayer.push(arguments);}
+    gtag('js', new Date());
+
+    gtag('config', 'G-DC6CR61FHE');
+</script>
 </body>
 </html>

assets/html/index.html

@@ -107,24 +107,35 @@
         <h1><strong>免费节点</strong></h1>
         <div class='article'>
             <p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess节点信息，聚合去重后提供节点列表，每15分钟更新</p>
+            <p>汇总节点数量：{{ .all_proxies_count }}</p>
+            <p>可用节点数量：{{ .useful_proxies_count }}</p>
             <br>
             <h5><a href="/clash">Clash</a></h5>
-            <p>Clash配置文件：<a href="https://proxy.tgbot.co/clash/config">https://proxy.tgbot.co/clash/config</a> <a
-                        href="clash://install-config?url=https://proxy.tgbot.co/clash/config">一键导入</a></p>
-            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://proxy.tgbot.co/clash/proxies">https://proxy.tgbot.co/clash/proxies</a>
+            <p>Clash配置文件：<a href="https://{{ .domain }}/clash/config">https://{{ .domain }}/clash/config</a> <a
+                        href="clash://install-config?url=https://{{ .domain }}/clash/config">一键导入</a></p>
+            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://{{ .domain }}/clash/proxies">https://{{ .domain }}/clash/proxies</a>
             </p>
-			<p>所有节点的Provider(不是都可以用)：<a href="https://proxy.tgbot.co/clash/proxies?type=all">https://proxy.tgbot.co/clash/proxies?type=all</a>
+			<p>所有节点的Provider(不是都可以用)：<a href="https://{{ .domain }}/clash/proxies?type=all">https://{{ .domain }}/clash/proxies?type=all</a>
 			</p>
 			<br>
 			<h5><a href="/surge">Surge</a></h5>
-            <p>Surge配置文件：<a href="https://proxy.tgbot.co/surge/config">https://proxy.tgbot.co/surge/config</a> <a
-                        href="surge3:///install-config?url=https://proxy.tgbot.co/surge/config">一键导入</a></p>
-            <p>Surge proxy list：<a href="https://proxy.tgbot.co/surge/proxies">https://proxy.tgbot.co/surge/proxies</a>
+            <p>Surge配置文件：<a href="https://{{ .domain }}/surge/config">https://{{ .domain }}/surge/config</a> <a
+                        href="surge3:///install-config?url=https://{{ .domain }}/surge/config">一键导入</a></p>
+            <p>Surge proxy list：<a href="https://{{ .domain }}/surge/proxies">https://{{ .domain }}/surge/proxies</a>
             </p>
             <br>
             <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a></p>
         </div>
     </div>
 </div>
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
+<script>
+    window.dataLayer = window.dataLayer || [];
+    function gtag(){dataLayer.push(arguments);}
+    gtag('js', new Date());
+
+    gtag('config', 'G-DC6CR61FHE');
+</script>
 </body>
 </html>

assets/html/surge.conf

@@ -3,9 +3,9 @@ Direct = direct
 
 [Proxy Group]
 Proxy = select, 延迟最低, 失败切换, 手动选择
-延迟最低 = url-test, policy-path=https://proxy.tgbot.co/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
-失败切换 = fallback, policy-path=https://proxy.tgbot.co/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
-手动选择 = select, policy-path=https://proxy.tgbot.co/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
+延迟最低 = url-test, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
+失败切换 = fallback, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
+手动选择 = select, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
 Apple = select, Direct, Proxy
 Adblock = select, Direct, REJECT, REJECT-TINYGIF
 

assets/html/surge.html

@@ -75,12 +75,21 @@
 	<div class='article'>
 	  <p>自动抓取tg频道、订阅地址、公开互联网上的ss、vmess节点信息，聚合去重后提供Surge节点列表，每15分钟更新</p>
 		<br>
-	  <p>Surge配置文件：<a href="https://proxy.tgbot.co/surge/config">https://proxy.tgbot.co/surge/config</a> <a href="surge3:///install-config?url=https://proxy.tgbot.co/surge/config">一键导入</a></p>
-	  <p>Surge proxy list：<a href="https://proxy.tgbot.co/surge/proxies">https://proxy.tgbot.co/surge/proxies</a></p>
+	  <p>Surge配置文件：<a href="https://{{ .domain }}/surge/config">https://{{ .domain }}/surge/config</a> <a href="surge3:///install-config?url=https://{{ .domain }}/surge/config">一键导入</a></p>
+	  <p>Surge proxy list：<a href="https://{{ .domain }}/surge/proxies">https://{{ .domain }}/surge/proxies</a></p>
 		<br>
 	  <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a></p>
 	</div>
   </div>
 </div>
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
+<script>
+	window.dataLayer = window.dataLayer || [];
+	function gtag(){dataLayer.push(arguments);}
+	gtag('js', new Date());
+
+	gtag('config', 'G-DC6CR61FHE');
+</script>
 </body>
 </html>

config/config.go

@@ -6,7 +6,7 @@ import (
 	"os"
 
 	"github.com/ghodss/yaml"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 type Source struct {
@@ -15,6 +15,9 @@ type Source struct {
 }
 
 type Config struct {
+	Domain  string   `json:"domain" yaml:"domain"`
+	CFEmail string   `json:"cf_email" yaml:"cf_email"`
+	CFKey   string   `json:"cf_key" yaml:"cf_key"`
 	Sources []Source `json:"sources" yaml:"sources"`
 }
 
@@ -30,6 +33,15 @@ func Parse(path string) (*Config, error) {
 	if err != nil {
 		return nil, err
 	}
+	if domain := os.Getenv("DOMAIN"); domain != "" {
+		SourceConfig.Domain = domain
+	}
+	if cfEmail := os.Getenv("CF_API_EMAIL"); cfEmail != "" {
+		SourceConfig.CFEmail = cfEmail
+	}
+	if cfKey := os.Getenv("CF_API_KEY"); cfKey != "" {
+		SourceConfig.CFKey = cfKey
+	}
 	return &SourceConfig, nil
 }
 

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/antchfx/htmlquery v1.2.3 // indirect
 	github.com/antchfx/xmlquery v1.2.4 // indirect
 	github.com/antchfx/xpath v1.1.8 // indirect
+	github.com/cloudflare/cloudflare-go v0.13.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-playground/validator/v10 v10.3.0 // indirect

go.sum

@@ -28,7 +28,10 @@ github.com/aws/aws-sdk-go v1.13.10/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41Zo
 github.com/axiomhq/hyperloglog v0.0.0-20180317131949-fe9507de0228/go.mod h1:IOXAcuKIFq/mDyuQ4wyJuJ79XLMsmLM+5RdQ+vWrL7o=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/cloudflare-go v0.13.0 h1:Mx2gNz/r4H0eHETENq5ZPtc/kOX/uzEfT2UFuA+/XAU=
+github.com/cloudflare/cloudflare-go v0.13.0/go.mod h1:6rZ6s/XWxP8WIWMATgDM7aUop0Z0ZOAfcm/4rEd0lOY=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -163,6 +166,7 @@ github.com/lstoll/grpce v1.7.0/go.mod h1:XiCWl3R+avNCT7KsTjv3qCblgsSqd0SC4ymySrH
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615 h1:/mD+ABZyXD39BzJI2XyRJlqdZG11gXFo0SSynL+OFeU=
@@ -176,6 +180,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -190,6 +195,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.2-0.20190227000051-27936f6d90f9/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -198,6 +205,7 @@ github.com/rcrowley/go-metrics v0.0.0-20160613154715-cfa5a85e9f0a/go.mod h1:bCqn
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
 github.com/rollbar/rollbar-go v1.2.0/go.mod h1:czC86b8U4xdUH7W2C6gomi2jutLm8qK0OtrF5WMvpcc=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
@@ -207,6 +215,7 @@ github.com/shirou/gopsutil v2.20.7+incompatible h1:Ymv4OD12d6zm+2yONe39VSmp2XooJ
 github.com/shirou/gopsutil v2.20.7+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -231,6 +240,8 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/unrolled/secure v1.0.1/go.mod h1:R6rugAuzh4TQpbFAq69oqZggyBQxFRFQIewtz5z7Jsc=
 github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
+github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -277,6 +288,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -317,6 +329,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

internal/app/config_test.go

@@ -7,11 +7,11 @@ import (
 	"testing"
 
 	"github.com/zu1k/proxypool/config"
-	"github.com/zu1k/proxypool/getter"
+	"github.com/zu1k/proxypool/pkg/getter"
 )
 
 func TestConfigFile(t *testing.T) {
-	c, err := config.Parse("../source.yaml")
+	c, err := config.Parse("../../source.yaml")
 	if err != nil {
 		t.Error(err)
 		return

internal/app/getter.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	"github.com/zu1k/proxypool/config"
-	"github.com/zu1k/proxypool/getter"
+	"github.com/zu1k/proxypool/pkg/getter"
 )
 
 var Getters = make([]getter.Getter, 0)

internal/app/task.go

@@ -6,11 +6,11 @@ import (
 	"log"
 	"sync"
 
-	"github.com/zu1k/proxypool/app/cache"
 	"github.com/zu1k/proxypool/config"
-	"github.com/zu1k/proxypool/provider"
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/pkg/provider"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 	"gopkg.in/yaml.v2"
 )
 
@@ -42,12 +42,14 @@ func CrawlGo() {
 	proxies = provider.Clash{Proxies: proxies}.CleanProxies()
 	proxies.NameAddCounrty().Sort().NameAddIndex()
 	cache.SetProxies("allproxies", proxies)
+	cache.AllProxiesCount = proxies.Len()
 
 	// 可用性检测
 	proxies = proxy.CleanBadProxies(proxies)
 	log.Println("CrawlGo clash useable node count:", len(proxies))
 	proxies.NameAddCounrty().Sort().NameAddIndex()
 	cache.SetProxies("proxies", proxies)
+	cache.UsefullProxiesCount = proxies.Len()
 
 	cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
 	cache.SetString("surgeproxies", provider.Surge{Proxies: proxies}.Provide())

internal/cache/cache.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/patrickmn/go-cache"
-	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
 var c = cache.New(cache.NoExpiration, 10*time.Minute)

internal/cache/vars.go

@@ -0,0 +1,6 @@
+package cache
+
+var (
+	AllProxiesCount     = 0
+	UsefullProxiesCount = 0
+)

internal/cloudflare/cache.go

@@ -0,0 +1,30 @@
+package cloudflare
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/cloudflare/cloudflare-go"
+	"github.com/zu1k/proxypool/config"
+)
+
+func test() {
+	api, err := cloudflare.New(config.SourceConfig.CFKey, config.SourceConfig.CFKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Fetch the zone ID
+	id, err := api.ZoneIDByName(config.SourceConfig.Domain)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Fetch zone details
+	zone, err := api.ZoneDetails(id)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Print zone details
+	fmt.Println(zone)
+}

internal/cron/cron.go

@@ -0,0 +1,11 @@
+package cron
+
+import (
+	"github.com/jasonlvhit/gocron"
+	"github.com/zu1k/proxypool/internal/app"
+)
+
+func Cron() {
+	_ = gocron.Every(15).Minutes().Do(app.CrawlGo)
+	<-gocron.Start()
+}

main.go

@@ -6,10 +6,12 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 
+	"github.com/zu1k/proxypool/internal/cron"
+
 	_ "github.com/mkevac/debugcharts"
 	"github.com/zu1k/proxypool/api"
-	"github.com/zu1k/proxypool/app"
-	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/internal/app"
+	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
 var (
@@ -28,6 +30,7 @@ func main() {
 
 	if configFilePath == "" {
 		app.NeedFetchNewConfigFile = true
+		app.FetchNewConfigFileThenInit()
 	} else {
 		err := app.InitConfigAndGetters(configFilePath)
 		if err != nil {
@@ -36,7 +39,7 @@ func main() {
 	}
 	proxy.InitGeoIpDB()
 
-	go app.Cron()
+	go cron.Cron()
 	fmt.Println("Do the first crawl...")
 	go app.CrawlGo()
 	api.Run()

pkg/getter/base.go

@@ -5,8 +5,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 type Getter interface {

pkg/getter/subscribe.go

@@ -5,8 +5,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 func init() {

pkg/getter/tgchannel.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/gocolly/colly"
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 func init() {

pkg/getter/web_fanqiangdang.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/gocolly/colly"
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 func init() {

pkg/getter/web_free_ssr_xyz.go

@@ -5,8 +5,8 @@ import (
 	"io/ioutil"
 	"sync"
 
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 func init() {

pkg/getter/web_fuzz.go

@@ -4,8 +4,8 @@ import (
 	"io/ioutil"
 	"sync"
 
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 func init() {

pkg/getter/web_fuzz_sub.go

@@ -5,8 +5,8 @@ import (
 	"regexp"
 	"sync"
 
-	"github.com/zu1k/proxypool/proxy"
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 func init() {

pkg/provider/clash.go

@@ -3,7 +3,7 @@ package provider
 import (
 	"strings"
 
-	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
 type Clash struct {

pkg/provider/surge.go

@@ -3,7 +3,7 @@ package provider
 import (
 	"strings"
 
-	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
 type Surge struct {

pkg/proxy/shadowsocks.go

@@ -11,7 +11,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 var (

pkg/proxy/shadowsocksr.go

@@ -10,7 +10,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 var (

pkg/proxy/vmess.go

@@ -11,7 +11,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/tool"
 )
 
 var (

source.yaml

@@ -1,3 +1,6 @@
+domain: proxy.tgbot.co
+cf_email: ""
+cf_key: ""
 sources:
   - type: webfuzzsub
     options:
@@ -229,7 +232,7 @@ sources:
       url: https://raw.githubusercontent.com/pojiezhiyuanjun/freev2/master/all.txt
   - type: webfuzz
     options:
-      url: https://raw.githubusercontent.com/zu1k/ssrtool-crawler/master/data/ssr.txt
+      url: https://ssrtoolcrawler.tgbot.co/data/ssr.txt
   - type: webfuzz
     options:
       url: https://raw.githubusercontent.com/freefq/free/master/README.md
@@ -294,6 +297,10 @@ sources:
     options:
       channel: freeshadowsock
       num: 200
+  - type: tgchannel
+    options:
+      channel: mianfeifenxiang
+      num: 200
 
   # 翻墙党，后面的东西都无需修改
   - type: web-fanqiangdang