fix source.uaml

fix bug
fix name
2020-08-23 17:57:42 +08:00 · 2020-08-23 17:32:55 +08:00 · 2020-08-23 17:21:21 +08:00 · 2020-08-23 17:14:04 +08:00 · 2020-08-23 00:14:00 +08:00 · 2020-08-23 00:06:27 +08:00
64 changed files with 2319 additions and 496 deletions
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
 liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
-custom: ['https://blog.lgf.im/donate/']# Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -43,7 +43,7 @@ jobs:
          draft: true
          prerelease: true

-      - uses: actions/upload-artifact@v2.1.3
+      - uses: actions/upload-artifact@v2.1.4
        if: startsWith(github.ref, 'refs/tags/')
        with:
          name: build
--- a/6
+++ b/6
@@ -10,5 +10,7 @@ RUN go mod download && \
 FROM alpine:latest

 RUN apk add --no-cache ca-certificates
-COPY --from=builder /proxypool /
-ENTRYPOINT ["/proxypool"]
+WORKDIR /proxypool-src
+COPY ./assets /proxypool-src/assets
+COPY --from=builder /proxypool /proxypool-src/
+ENTRYPOINT ["/proxypool-src/proxypool"]
--- a/README.md
+++ b/README.md
@@ -0,0 +1,88 @@
+<h1 align="center">
+  <br>proxypool<br>
+</h1>
+
+<h5 align="center">自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess、trojan节点信息，聚合去重测试可用性后提供节点列表</h5>
+
+<p align="center">
+  <a href="https://github.com/zu1k/proxypool/actions">
+    <img src="https://img.shields.io/github/workflow/status/zu1k/proxypool/Go?style=flat-square" alt="Github Actions">
+  </a>
+  <a href="https://goreportcard.com/report/github.com/zu1k/proxypool">
+    <img src="https://goreportcard.com/badge/github.com/zu1k/proxypool?style=flat-square">
+  </a>
+  <a href="https://github.com/zu1k/proxypool/releases">
+    <img src="https://img.shields.io/github/release/zu1k/proxypool/all.svg?style=flat-square">
+  </a>
+</p>
+
+## 支持
+
+- 支持ss、ssr、vmess、trojan多种类型
+- Telegram频道抓取
+- 订阅地址抓取解析
+- 公开互联网页面模糊抓取
+- 定时抓取自动更新
+- 通过配置文件设置抓取源
+- 自动检测节点可用性
+
+## 安装
+
+### 使用Heroku
+
+点击按钮进入部署页面，填写基本信息然后运行
+
+其中 `DOMAIN` 需要填写为你需要绑定的域名，`CONFIG_FILE` 需要填写你的配置文件路径，配置文件模板见 config/source.yaml 文件
+
+`CF` 开头的选项暂不需要填写，不影响程序运行
+
+[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy)
+
+> 因为爬虫程序需要持续运行，所以至少选择 $7/月 的配置
+> 免费配置长时间无人访问会被heroku强制停止
+
+### 从源码编译
+
+需要安装Golang
+
+```sh
+$ go get -u -v github.com/zu1k/proxypool
+```
+
+### 下载预编译程序
+
+从这里下载预编译好的程序 [release](https://github.com/zu1k/proxypool/releases)
+
+### 使用docker
+
+```sh
+docker pull docker.pkg.github.com/zu1k/proxypool/proxypool:latest
+```
+
+## 使用
+
+### 修改配置文件
+
+首先修改 source.yaml 中的必要配置信息，cf开头的选项不需要填写
+
+配置文件中定义了抓取源，需要定期手动维护更新
+
+### 启动程序
+
+使用 `-c` 参数指定配置文件路径，支持http链接
+
+```shell
+proxypool -c source.yaml
+```
+
+## 截图
+
+![Speedtest](docs/speedtest.png)
+
+![Fast](docs/fast.png)
+
+## 声明
+
+本项目遵循 GNU General Public License v3.0 开源，在此基础上，所有使用本项目提供服务者都必须在网站首页保留指向本项目的链接
+
+禁止使用本项目进行营利和做其他违法事情，产生的一切后果本项目概不负责
--- a/api/router.go
+++ b/api/router.go
@@ -1,28 +1,93 @@
 package api

 import (
+	"net/http"
 	"os"

-	"github.com/zu1k/proxypool/app/cache"
+	"github.com/zu1k/proxypool/config"

 	"github.com/gin-gonic/gin"
 	_ "github.com/heroku/x/hmetrics/onload"
-	"github.com/zu1k/proxypool/provider"
+	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/pkg/provider"
 )

+const version = "v0.3.0"
+
 var router *gin.Engine

 func setupRouter() {
 	router = gin.Default()
+	router.LoadHTMLGlob("assets/html/*")
+
+	router.GET("/", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "index.html", gin.H{
+			"domain":               config.Config.Domain,
+			"all_proxies_count":    cache.AllProxiesCount,
+			"ss_proxies_count":     cache.SSProxiesCount,
+			"ssr_proxies_count":    cache.SSRProxiesCount,
+			"vmess_proxies_count":  cache.VmessProxiesCount,
+			"trojan_proxies_count": cache.TrojanProxiesCount,
+			"useful_proxies_count": cache.UsefullProxiesCount,
+			"last_crawl_time":      cache.LastCrawlTime,
+			"version":              version,
+		})
+	})
+
+	router.GET("/clash", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "clash.html", gin.H{
+			"domain": config.Config.Domain,
+		})
+	})
+
+	router.GET("/surge", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "surge.html", gin.H{
+			"domain": config.Config.Domain,
+		})
+	})
+
+	router.GET("/clash/config", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "clash-config.yaml", gin.H{
+			"domain": config.Config.Domain,
+		})
+	})
+
+	router.GET("/surge/config", func(c *gin.Context) {
+		c.HTML(http.StatusOK, "surge.conf", gin.H{
+			"domain": config.Config.Domain,
+		})
+	})

-	router.StaticFile("/clash/config", "example/clash-config.yaml")
 	router.GET("/clash/proxies", func(c *gin.Context) {
-		text := cache.GetString("clashproxies")
-		if text == "" {
-			proxies := cache.GetProxies()
-			clash := provider.Clash{Proxies: proxies}
+		proxyTypes := c.DefaultQuery("type", "")
+		proxyCountry := c.DefaultQuery("c", "")
+		text := ""
+		if proxyTypes == "" && proxyCountry == "" {
+			text = cache.GetString("clashproxies")
+			if text == "" {
+				proxies := cache.GetProxies("proxies")
+				clash := provider.Clash{Proxies: proxies}
+				text = clash.Provide()
+				cache.SetString("clashproxies", text)
+			}
+		} else if proxyTypes == "all" {
+			proxies := cache.GetProxies("allproxies")
+			clash := provider.Clash{Proxies: proxies, Types: proxyTypes, Country: proxyCountry}
 			text = clash.Provide()
-			cache.SetString("clashproxies", text)
+		} else {
+			proxies := cache.GetProxies("proxies")
+			clash := provider.Clash{Proxies: proxies, Types: proxyTypes, Country: proxyCountry}
+			text = clash.Provide()
+		}
+		c.String(200, text)
+	})
+	router.GET("/surge/proxies", func(c *gin.Context) {
+		text := cache.GetString("surgeproxies")
+		if text == "" {
+			proxies := cache.GetProxies("proxies")
+			surge := provider.Surge{Proxies: proxies}
+			text = surge.Provide()
+			cache.SetString("surgeproxies", text)
 		}
 		c.String(200, text)
 	})
--- a/app.json
+++ b/app.json
@@ -0,0 +1,25 @@
+{
+  "name": "proxypool",
+  "description": "自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess节点信息，聚合去重测试可用性后提供节点列表",
+  "website": "https://proxy.tgbot.co/",
+  "repository": "https://github.com/zu1k/proxypool",
+  "success_url": "/",
+  "logo": "https://raw.githubusercontent.com/zu1k/proxypool/heroku/assets/proxy.jpg",
+  "keywords": ["golang", "ss", "ssr", "vmess", "shadowsocks", "shadowsocksr", "trojan"],
+  "env": {
+    "CONFIG_FILE": {
+      "description": "Path to config file, could be a url."
+    },
+    "DOMAIN": {
+      "description": "Domain to use."
+    },
+    "CF_API_EMAIL": {
+      "description": "Cloudflare Email.",
+      "required": false
+    },
+    "CF_API_KEY": {
+      "description": "Cloudflare API key.",
+      "required": false
+    }
+  }
+}
--- a/app/cache/cache.go
+++ b/app/cache/cache.go
@@ -1,37 +0,0 @@
-package cache
-
-import (
-	"log"
-	"time"
-
-	"github.com/patrickmn/go-cache"
-	"github.com/zu1k/proxypool/proxy"
-)
-
-var c = cache.New(cache.NoExpiration, 10*time.Minute)
-
-func GetProxies() []proxy.Proxy {
-	result, found := c.Get("proxies")
-	if found {
-		log.Println(len(result.([]proxy.Proxy)))
-		return result.([]proxy.Proxy)
-	}
-	log.Println("Cache not found")
-	return nil
-}
-
-func SetProxies(proxies []proxy.Proxy) {
-	c.Set("proxies", proxies, cache.NoExpiration)
-}
-
-func SetString(key, value string) {
-	c.Set(key, value, cache.NoExpiration)
-}
-
-func GetString(key string) string {
-	result, found := c.Get(key)
-	if found {
-		return result.(string)
-	}
-	return ""
-}
--- a/app/cron.go
+++ b/app/cron.go
@@ -1,11 +0,0 @@
-package app
-
-import (
-	"github.com/jasonlvhit/gocron"
-)
-
-func Cron() {
-	gocron.Every(10).Minutes().Do(CrawlTGChannel)
-
-	<-gocron.Start()
-}
--- a/app/task.go
+++ b/app/task.go
@@ -1,48 +0,0 @@
-package app
-
-import (
-	"math/rand"
-	"strconv"
-
-	"github.com/zu1k/proxypool/provider"
-
-	"github.com/zu1k/proxypool/app/cache"
-
-	"github.com/zu1k/proxypool/getter"
-	"github.com/zu1k/proxypool/proxy"
-)
-
-func CrawlTGChannel() {
-	proxies := make([]proxy.Proxy, 0)
-
-	// tg上各种节点分享频道
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrList", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/SSRSUB", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/FreeSSRNode", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrlists", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrshares", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/V2List", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrtool", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/vmessr", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/FreeSSR666", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/fanqiang666", 200).Get()...)
-
-	// 各种网站上公开的
-	proxies = append(proxies, getter.WebFreessrXyz{}.Get()...)
-	proxies = append(proxies, getter.WebLucnOrg{}.Get()...)
-
-	// 从web页面模糊获取
-	proxies = append(proxies, getter.NewWebFuzz("https://zfjvpn.gitbook.io/").Get()...)
-	proxies = append(proxies, getter.NewWebFuzz("https://www.freefq.com/d/file/free-ssr/20200811/1f3e9d0d0064f662457062712dcf1b66.txt").Get()...)
-	proxies = append(proxies, getter.NewWebFuzz("https://merlinblog.xyz/wiki/freess.html").Get()...)
-
-	proxies = append(proxies, cache.GetProxies()...)
-	proxies = proxy.Deduplication(proxies)
-
-	num := len(proxies)
-	for i := 0; i < num; i++ {
-		proxies[i].SetName("@tgbotlist_" + strconv.Itoa(rand.Int()))
-	}
-	cache.SetProxies(proxies)
-	cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
-}
--- a/assets/GeoLite2-City.mmdb
+++ b/assets/GeoLite2-City.mmdb
--- a/assets/html/clash-config.yaml
+++ b/assets/html/clash-config.yaml
@@ -1,3 +1,6 @@
+# 所有节点均从公开互联网上抓取，使用clash自动检查可用性即可
+# 抓取程序已开源： https://github.com/zu1k/proxypool
+
 # port of HTTP
 port: 7890

@@ -51,7 +54,7 @@ proxy-groups:
 proxy-providers:
  provider:
    type: http
-    url: "https://proxy.tgbot.co/clash/proxies"
+    url: "https://{{ .domain }}/clash/proxies"
    interval: 3600
    path: ./provider.yaml
    health-check:
--- a/assets/html/clash.html
+++ b/assets/html/clash.html
@@ -0,0 +1,129 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="HandheldFriendly" content="True">
+    <title>免费Clash节点</title>
+    <style type='text/css'>
+        body {
+            background-color: white;
+            color: #333333;
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 36px;
+            line-height: 1;
+            font-size: 14px;
+        }
+
+        .section {
+            margin-bottom: 36px;
+        }
+
+        .section.friendly {
+            color: #222222;
+        }
+
+        .section.friendly h1 {
+            font-size: 26px;
+            background-color: #dad8e4;
+            padding: 18px 22px 15px 22px;
+            margin: 0;
+            overflow: hidden;
+        }
+
+        .section.friendly h1 strong {
+            display: inline-block;
+            float: left;
+        }
+
+        .section.friendly h1 small {
+            display: inline-block;
+            float: right;
+            text-align: right;
+            font-size: 18px;
+            padding-top: 4px;
+            color: #333333;
+        }
+
+        .section.friendly .article {
+            border: 4px solid #dad8e4;
+            padding: 24px 18px 18px 18px;
+        }
+
+        .section.friendly .article h3 {
+            font-size: 20px;
+            margin: 0 0 18px 0;
+        }
+
+        .section.friendly .article a {
+            color: #6b6ceb;
+        }
+
+        .section.friendly .article a:visited {
+            color: #1d1d3b;
+        }
+
+        .section.friendly .article p {
+            font-size: 14px;
+        }
+
+        .section.friendly .article ul {
+            list-style-type: square;
+        }
+
+        .section.original {
+            background-color: #eeeeee;
+            color: #444444;
+        }
+
+        .section.original h2 {
+            background-color: #dddddd;
+            margin: 0;
+            padding: 18px 22px 18px 22px;
+            font-size: 20px;
+        }
+
+        .section.original pre {
+            margin: 0;
+            padding: 18px 22px 18px 22px;
+            overflow: auto;
+            font-family: monaco, monospaced;
+        }
+
+        .section.original pre code {
+            display: block;
+            font-size: 11px;
+            width: 100%;
+        }
+    </style>
+</head>
+<body>
+<div class='container'>
+    <div class='section friendly'>
+        <h1><strong>免费Clash节点</strong></h1>
+        <div class='article'>
+            <p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess、trojan节点信息，聚合去重后提供clash配置，每15分钟更新</p>
+            <br>
+            <p>Clash配置文件：https://{{ .domain }}/clash/config <a href="clash://install-config?url=https://{{ .domain }}/clash/config">一键导入</a></p>
+            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://{{ .domain }}/clash/proxies">https://{{ .domain }}/clash/proxies</a></p>
+            <br>
+			<p>筛选代理类型(此种方式你只能自己维护配置文件)：https://{{ .domain }}/clash/proxies?type=ss,ssr,vmess</p>
+			<p>筛选国家(此种方式你只能自己维护配置文件)：https://{{ .domain }}/clash/proxies?type=ss,ssr,vmess&c=HK,TW,US</p>
+            <p>所有节点的Provider(不是都可以用)：<a href="https://{{ .domain }}/clash/proxies?type=all">https://{{ .domain }}/clash/proxies?type=all</a></p>
+            <br>
+            <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a></p>
+        </div>
+    </div>
+</div>
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
+<script>
+    window.dataLayer = window.dataLayer || [];
+    function gtag(){dataLayer.push(arguments);}
+    gtag('js', new Date());
+
+    gtag('config', 'G-DC6CR61FHE');
+</script>
+</body>
+</html>
--- a/assets/html/index.html
+++ b/assets/html/index.html
@@ -0,0 +1,147 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="HandheldFriendly" content="True">
+    <title>免费节点</title>
+    <meta property="og:site_name" content="免费节点">
+    <meta property="og:description" content="免费ss,免费ssr,免费v2ray,免费vmess,免费vless,免费shadowsocks,免费shadowsocksr,免费shadowsocksrr,免费shadowrocket,免费quan,免费quanx,免费surge,免费小火箭,白嫖ss,白嫖ssr,白嫖v2ray,白嫖vmess,白嫖vless,白嫖shadowsocks,白嫖shadowsocksr,白嫖shadowsocksrr,白嫖shadowrocket,白嫖quan,白嫖quanx,白嫖surge,白嫖小火箭,高速ss,高速ssr,高速v2ray,高速vmess,高速vless,高速shadowsocks,高速shadowsocksr,高速shadowsocksrr,高速shadowrocket,高速quan,高速quanx,高速surge,高速小火箭,公益ss,公益ssr,公益v2ray,公益vmess,公益vless,公益shadowsocks,公益shadowsocksr,公益shadowsocksrr,公益shadowrocket,公益quan,公益quanx,公益surge,公益小火箭,windowsss,windowsssr,windowsv2ray,windowsvmess,windowsvless,windowsshadowsocks,windowsshadowsocksr,windowsshadowsocksrr,windowsshadowrocket,windowsquan,windowsquanx,windowssurge,windows小火箭,androidss,androidssr,androidv2ray,androidvmess,androidvless,androidshadowsocks,androidshadowsocksr,androidshadowsocksrr,androidshadowrocket,androidquan,androidquanx,androidsurge,android小火箭,iosss,iosssr,iosv2ray,iosvmess,iosvless,iosshadowsocks,iosshadowsocksr,iosshadowsocksrr,iosshadowrocket,iosquan,iosquanx,iossurge,ios小火箭,macss,macssr,macv2ray,macvmess,macvless,macshadowsocks,macshadowsocksr,macshadowsocksrr,macshadowrocket,macquan,macquanx,macsurge,mac小火箭,苹果ss,苹果ssr,苹果v2ray,苹果vmess,苹果vless,苹果shadowsocks,苹果shadowsocksr,苹果shadowsocksrr,苹果shadowrocket,苹果quan,苹果quanx,苹果surge,苹果小火箭,安卓ss,安卓ssr,安卓v2ray,安卓vmess,安卓vless,安卓shadowsocks,安卓shadowsocksr,安卓shadowsocksrr,安卓shadowrocket,安卓quan,安卓quanx,安卓surge,安卓小火箭">
+    <meta name="keywords" content="免费ss,免费ssr,免费v2ray,免费vmess,免费vless,免费shadowsocks,免费shadowsocksr,免费shadowsocksrr,免费shadowrocket,免费quan,免费quanx,免费surge,免费小火箭,白嫖ss,白嫖ssr,白嫖v2ray,白嫖vmess,白嫖vless,白嫖shadowsocks,白嫖shadowsocksr,白嫖shadowsocksrr,白嫖shadowrocket,白嫖quan,白嫖quanx,白嫖surge,白嫖小火箭,高速ss,高速ssr,高速v2ray,高速vmess,高速vless,高速shadowsocks,高速shadowsocksr,高速shadowsocksrr,高速shadowrocket,高速quan,高速quanx,高速surge,高速小火箭,公益ss,公益ssr,公益v2ray,公益vmess,公益vless,公益shadowsocks,公益shadowsocksr,公益shadowsocksrr,公益shadowrocket,公益quan,公益quanx,公益surge,公益小火箭,windowsss,windowsssr,windowsv2ray,windowsvmess,windowsvless,windowsshadowsocks,windowsshadowsocksr,windowsshadowsocksrr,windowsshadowrocket,windowsquan,windowsquanx,windowssurge,windows小火箭,androidss,androidssr,androidv2ray,androidvmess,androidvless,androidshadowsocks,androidshadowsocksr,androidshadowsocksrr,androidshadowrocket,androidquan,androidquanx,androidsurge,android小火箭,iosss,iosssr,iosv2ray,iosvmess,iosvless,iosshadowsocks,iosshadowsocksr,iosshadowsocksrr,iosshadowrocket,iosquan,iosquanx,iossurge,ios小火箭,macss,macssr,macv2ray,macvmess,macvless,macshadowsocks,macshadowsocksr,macshadowsocksrr,macshadowrocket,macquan,macquanx,macsurge,mac小火箭,苹果ss,苹果ssr,苹果v2ray,苹果vmess,苹果vless,苹果shadowsocks,苹果shadowsocksr,苹果shadowsocksrr,苹果shadowrocket,苹果quan,苹果quanx,苹果surge,苹果小火箭,安卓ss,安卓ssr,安卓v2ray,安卓vmess,安卓vless,安卓shadowsocks,安卓shadowsocksr,安卓shadowsocksrr,安卓shadowrocket,安卓quan,安卓quanx,安卓surge,安卓小火箭">
+    <style type='text/css'>
+        body {
+            background-color: white;
+            color: #333333;
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 36px;
+            line-height: 1;
+            font-size: 14px;
+        }
+
+        .section {
+            margin-bottom: 36px;
+        }
+
+        .section.friendly {
+            color: #222222;
+        }
+
+        .section.friendly h1 {
+            font-size: 26px;
+            background-color: #dad8e4;
+            padding: 18px 22px 15px 22px;
+            margin: 0;
+            overflow: hidden;
+        }
+
+        .section.friendly h1 strong {
+            display: inline-block;
+            float: left;
+        }
+
+        .section.friendly h1 small {
+            display: inline-block;
+            float: right;
+            text-align: right;
+            font-size: 18px;
+            padding-top: 4px;
+            color: #333333;
+        }
+
+        .section.friendly .article {
+            border: 4px solid #dad8e4;
+            padding: 24px 18px 18px 18px;
+        }
+
+        .section.friendly .article h3 {
+            font-size: 20px;
+            margin: 0 0 18px 0;
+        }
+
+        .section.friendly .article a {
+            color: #6b6ceb;
+        }
+
+        .section.friendly .article a:visited {
+            color: #1d1d3b;
+        }
+
+        .section.friendly .article p {
+            font-size: 14px;
+        }
+
+        .section.friendly .article ul {
+            list-style-type: square;
+        }
+
+        .section.original {
+            background-color: #eeeeee;
+            color: #444444;
+        }
+
+        .section.original h2 {
+            background-color: #dddddd;
+            margin: 0;
+            padding: 18px 22px 18px 22px;
+            font-size: 20px;
+        }
+
+        .section.original pre {
+            margin: 0;
+            padding: 18px 22px 18px 22px;
+            overflow: auto;
+            font-family: monaco, monospaced;
+        }
+
+        .section.original pre code {
+            display: block;
+            font-size: 11px;
+            width: 100%;
+        }
+    </style>
+</head>
+<body>
+<div class='container'>
+    <div class='section friendly'>
+        <h1><strong>免费节点</strong></h1>
+        <div class='article'>
+            <p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess、trojan节点信息，聚合去重后提供节点列表，每15分钟更新</p>
+            <p>汇总节点数量：{{ .all_proxies_count }}</p>
+            <p>ss节点数量：{{ .ss_proxies_count }}</p>
+            <p>ssr节点数量：{{ .ssr_proxies_count }}</p>
+            <p>vmess节点数量：{{ .vmess_proxies_count }}</p>
+            <p>trojan节点数量：{{ .trojan_proxies_count }}</p>
+            <p>可用节点数量：{{ .useful_proxies_count }}</p>
+            <p>最后更新时间：{{ .last_crawl_time }}</p>
+            <br>
+            <h4><a href="/clash">Clash</a></h4>
+            <p>Clash配置文件：https://{{ .domain }}/clash/config <a
+                        href="clash://install-config?url=https://{{ .domain }}/clash/config">一键导入</a></p>
+            <p>Clash proxy-provider(Shadowrocket添加订阅方式可用)：<a href="https://{{ .domain }}/clash/proxies">https://{{ .domain }}/clash/proxies</a>
+            </p>
+            <p>筛选代理类型：https://{{ .domain }}/clash/proxies?type=ss,ssr,vmess</p>
+            <p>筛选国家：https://{{ .domain }}/clash/proxies?c=HK,TW,US</p>
+			<br>
+			<h4><a href="/surge">Surge</a></h4>
+            <p>Surge配置文件：https://{{ .domain }}/surge/config <a
+                        href="surge3:///install-config?url=https://{{ .domain }}/surge/config">一键导入</a></p>
+            <p>Surge proxy list：<a href="https://{{ .domain }}/surge/proxies">https://{{ .domain }}/surge/proxies</a>
+            </p>
+            <br>
+            {{- /* 所有使用本代码提供服务的禁止删除该行*/}}
+            <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a> {{ .version }}</p>
+        </div>
+    </div>
+</div>
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
+<script>
+    window.dataLayer = window.dataLayer || [];
+    function gtag(){dataLayer.push(arguments);}
+    gtag('js', new Date());
+
+    gtag('config', 'G-DC6CR61FHE');
+</script>
+</body>
+</html>
--- a/assets/html/surge.conf
+++ b/assets/html/surge.conf
@@ -0,0 +1,45 @@
+[Proxy]
+Direct = direct
+
+[Proxy Group]
+Proxy = select, 延迟最低, 失败切换, 手动选择
+延迟最低 = url-test, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
+失败切换 = fallback, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
+手动选择 = select, policy-path=https://{{ .domain }}/surge/proxies, url=http://www.qualcomm.cn/generate_204, update-interval=3600, interval = 600s, tolerance = 100ms, timeout = 5s, evaluate-before-use = true
+Apple = select, Direct, Proxy
+Adblock = select, Direct, REJECT, REJECT-TINYGIF
+
+[Rule]
+# RULESET
+RULE-SET,SYSTEM,Direct
+# 
+# Unbreak 后续规则修正
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Unbreak.list,Adblock
+# Advertising 广告
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Guard/Advertising.list,Adblock
+# Privacy 隐私
+# RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Guard/Privacy.list,Adblock
+# Hijacking 运营商劫持或恶意网站
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Guard/Hijacking.list,Adblock
+# 
+# Apple
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Extra/Apple/Apple.list,Apple
+# 
+# 代理
+# Streaming 国际流媒体服务
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/StreamingMedia/Streaming.list,Proxy
+# StreamingSE 中国流媒体服务（面向海外版本）
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/StreamingMedia/StreamingSE.list,Proxy
+# Global 全球加速
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Global.list,Proxy
+# 
+RULE-SET,https://github.com/Hackl0us/SS-Rule-Snippet/raw/master/Rulesets/App/social/Telegram.list,Proxy
+RULE-SET,https://github.com/Hackl0us/SS-Rule-Snippet/raw/master/Rulesets/App/social/WhatsApp.list,Proxy
+RULE-SET,https://github.com/Hackl0us/SS-Rule-Snippet/raw/master/Rulesets/App/social/LINE.list,Proxy
+# 
+# Direct
+RULE-SET,https://github.com/DivineEngine/Profiles/raw/master/Surge/Ruleset/Extra/ChinaIP.list,Direct
+# 
+RULE-SET,LAN,Direct
+# GEOIP,CN,Direct
+FINAL,Proxy,dns-failed
--- a/assets/html/surge.html
+++ b/assets/html/surge.html
@@ -0,0 +1,95 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="HandheldFriendly" content="True">
+	<title>免费Surge节点</title>
+	<style type='text/css'>
+	  body {
+		background-color: white;
+		color: #333333;
+		  font-family: Arial, sans-serif;
+		margin: 0;
+		padding: 36px;
+		line-height: 1;
+		font-size: 14px; }
+
+	  .section {
+		margin-bottom: 36px; }
+		.section.friendly {
+		  color: #222222; }
+		  .section.friendly h1 {
+			font-size: 26px;
+			background-color: #dad8e4;
+			padding: 18px 22px 15px 22px;
+			margin: 0;
+			overflow: hidden; }
+			.section.friendly h1 strong {
+			  display: inline-block;
+			  float: left; }
+			.section.friendly h1 small {
+			  display: inline-block;
+			  float: right;
+			  text-align: right;
+			  font-size: 18px;
+			  padding-top: 4px;
+			  color: #333333; }
+		  .section.friendly .article {
+			border: 4px solid #dad8e4;
+			padding: 24px 18px 18px 18px; }
+			.section.friendly .article h3 {
+			  font-size: 20px;
+			  margin: 0 0 18px 0; }
+			.section.friendly .article a {
+			  color: #6b6ceb; }
+			  .section.friendly .article a:visited {
+				color: #1d1d3b; }
+			.section.friendly .article p {
+			  font-size: 14px; }
+			.section.friendly .article ul {
+			  list-style-type: square; }
+	  .section.original {
+		background-color: #eeeeee;
+		color: #444444; }
+		.section.original h2 {
+		  background-color: #dddddd;
+		  margin: 0;
+		  padding: 18px 22px 18px 22px;
+		  font-size: 20px; }
+		.section.original pre {
+		  margin: 0;
+		  padding: 18px 22px 18px 22px;
+		  overflow: auto;
+		  font-family: monaco, monospaced; }
+		  .section.original pre code {
+			display: block;
+			font-size: 11px;
+			width: 100%; }
+	</style>
+</head>
+<body>
+<div class='container'>
+  <div class='section friendly'>
+	<h1><strong>免费Surge节点</strong></h1>
+	<div class='article'>
+	  <p>自动抓取tg频道、订阅地址、公开互联网上的ss、vmess节点信息，聚合去重后提供Surge节点列表，每15分钟更新</p>
+		<br>
+	  <p>Surge配置文件：https://{{ .domain }}/surge/config <a href="surge3:///install-config?url=https://{{ .domain }}/surge/config">一键导入</a></p>
+	  <p>Surge proxy list：<a href="https://{{ .domain }}/surge/proxies">https://{{ .domain }}/surge/proxies</a></p>
+		<br>
+	  <p>抓取程序已开源：<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a></p>
+	</div>
+  </div>
+</div>
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-DC6CR61FHE"></script>
+<script>
+	window.dataLayer = window.dataLayer || [];
+	function gtag(){dataLayer.push(arguments);}
+	gtag('js', new Date());
+
+	gtag('config', 'G-DC6CR61FHE');
+</script>
+</body>
+</html>
--- a/assets/proxy.jpg
+++ b/assets/proxy.jpg
--- a/config/config.go
+++ b/config/config.go
@@ -0,0 +1,71 @@
+package config
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+var configFilePath = "config.yaml"
+
+type ConfigOptions struct {
+	Domain      string   `json:"domain" yaml:"domain"`
+	CFEmail     string   `json:"cf_email" yaml:"cf_email"`
+	CFKey       string   `json:"cf_key" yaml:"cf_key"`
+	SourceFiles []string `json:"source-files" yaml:"source-files"`
+}
+
+// Config 配置
+var Config ConfigOptions
+
+// Parse 解析配置文件，支持本地文件系统和网络链接
+func Parse(path string) error {
+	if path == "" {
+		path = configFilePath
+	} else {
+		configFilePath = path
+	}
+	fileData, err := ReadFile(path)
+	if err != nil {
+		return err
+	}
+	Config = ConfigOptions{}
+	err = yaml.Unmarshal(fileData, &Config)
+	if err != nil {
+		return err
+	}
+
+	// 部分配置环境变量优先
+	if domain := os.Getenv("DOMAIN"); domain != "" {
+		Config.Domain = domain
+	}
+	if cfEmail := os.Getenv("CF_API_EMAIL"); cfEmail != "" {
+		Config.CFEmail = cfEmail
+	}
+	if cfKey := os.Getenv("CF_API_KEY"); cfKey != "" {
+		Config.CFKey = cfKey
+	}
+
+	return nil
+}
+
+// 从本地文件或者http链接读取配置文件内容
+func ReadFile(path string) ([]byte, error) {
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		resp, err := tool.GetHttpClient().Get(path)
+		if err != nil {
+			return nil, errors.New("config file http get fail")
+		}
+		defer resp.Body.Close()
+		return ioutil.ReadAll(resp.Body)
+	} else {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			return nil, err
+		}
+		return ioutil.ReadFile(path)
+	}
+}
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -0,0 +1,6 @@
+domain: example.com
+cf_email: ""
+cf_key: ""
+source-files:
+  - ./source.yaml
+  - https://example.com/source.yaml
--- a/config/source.go
+++ b/config/source.go
@@ -0,0 +1,8 @@
+package config
+
+import "github.com/zu1k/proxypool/pkg/tool"
+
+type Source struct {
+	Type    string       `json:"type" yaml:"type"`
+	Options tool.Options `json:"options" yaml:"options"`
+}
--- a/config/source.yaml
+++ b/config/source.yaml
@@ -0,0 +1,29 @@
+# 模糊抓取订阅链接
+- type: webfuzzsub
+  options:
+    url: https://raw.githubusercontent.com/du5/free/master/sub.list
+
+# 订阅链接
+- type: subscribe
+  options:
+    url: https://raw.githubusercontent.com/ssrsub/ssr/master/v2ray
+
+# 网页模糊抓取
+- type: webfuzz
+  options:
+    url: https://merlinblog.xyz/wiki/freess.html
+
+# tg频道抓取
+- type: tgchannel
+  options:
+    channel: ssrList
+    num: 200
+
+# 翻墙党论坛抓取
+- type: web-fanqiangdang-rss
+  options:
+    url: https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0
+
+# 某个网站抓取
+- type: web-freessrxyz
+  options:
--- a/docs/fast.png
+++ b/docs/fast.png
--- a/docs/speedtest.png
+++ b/docs/speedtest.png
--- a/getter/base.go
+++ b/getter/base.go
@@ -1,43 +0,0 @@
-package getter
-
-import (
-	"strings"
-
-	"github.com/zu1k/proxypool/proxy"
-)
-
-type Getter interface {
-	Get() []proxy.Proxy
-}
-
-func String2Proxy(link string) proxy.Proxy {
-	var err error
-	var data proxy.Proxy
-	if strings.HasPrefix(link, "ssr://") {
-		data, err = proxy.ParseSSRLink(link)
-	} else if strings.HasPrefix(link, "vmess://") {
-		data, err = proxy.ParseVmessLink(link)
-	}
-	if err != nil {
-		return nil
-	}
-	return data
-}
-
-func StringArray2ProxyArray(origin []string) []proxy.Proxy {
-	var err error
-	results := make([]proxy.Proxy, 0)
-	for _, link := range origin {
-		var data proxy.Proxy
-		if strings.HasPrefix(link, "ssr://") {
-			data, err = proxy.ParseSSRLink(link)
-		} else if strings.HasPrefix(link, "vmess://") {
-			data, err = proxy.ParseVmessLink(link)
-		}
-		if err != nil {
-			continue
-		}
-		results = append(results, data)
-	}
-	return results
-}
--- a/getter/subscribe.go
+++ b/getter/subscribe.go
@@ -1,13 +0,0 @@
-package getter
-
-import "github.com/zu1k/proxypool/proxy"
-
-type Subscribe struct {
-	NumNeeded int
-	Results   []string
-	Url       string
-}
-
-func (s Subscribe) Get() []proxy.Proxy {
-	return nil
-}
--- a/getter/tgchannel.go
+++ b/getter/tgchannel.go
@@ -1,50 +0,0 @@
-package getter
-
-import (
-	"fmt"
-
-	"github.com/gocolly/colly"
-	"github.com/zu1k/proxypool/proxy"
-)
-
-type TGChannelGetter struct {
-	c         *colly.Collector
-	NumNeeded int
-	Results   []string
-	Url       string
-}
-
-func NewTGChannelGetter(url string, numNeeded int) *TGChannelGetter {
-	if numNeeded <= 0 {
-		numNeeded = 200
-	}
-	return &TGChannelGetter{
-		c:         colly.NewCollector(),
-		NumNeeded: numNeeded,
-		Results:   make([]string, 0),
-		Url:       url,
-	}
-}
-
-func (g TGChannelGetter) Get() []proxy.Proxy {
-	// 找到所有的文字消息
-	g.c.OnHTML("div.tgme_widget_message_text", func(e *colly.HTMLElement) {
-		g.Results = append(g.Results, proxy.GrepSSRLinkFromString(e.Text)...)
-		g.Results = append(g.Results, proxy.GrepVmessLinkFromString(e.Text)...)
-	})
-
-	// 找到之前消息页面的链接，加入访问队列
-	g.c.OnHTML("link[rel=prev]", func(e *colly.HTMLElement) {
-		if len(g.Results) < g.NumNeeded {
-			_ = e.Request.Visit(e.Attr("href"))
-		}
-	})
-
-	g.Results = make([]string, 0)
-	err := g.c.Visit(g.Url)
-	if err != nil {
-		_ = fmt.Errorf("%s", err.Error())
-	}
-
-	return StringArray2ProxyArray(g.Results)
-}
--- a/getter/web_fuzz.go
+++ b/getter/web_fuzz.go
@@ -1,34 +0,0 @@
-package getter
-
-import (
-	"io/ioutil"
-	"net/http"
-
-	"github.com/zu1k/proxypool/proxy"
-)
-
-type WebFuzz struct {
-	Url string
-}
-
-func (w WebFuzz) Get() []proxy.Proxy {
-	resp, err := http.Get(w.Url)
-	if err != nil {
-		return nil
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil
-	}
-	text := string(body)
-
-	results := proxy.GrepSSRLinkFromString(text)
-	results = append(results, proxy.GrepVmessLinkFromString(text)...)
-
-	return StringArray2ProxyArray(results)
-}
-
-func NewWebFuzz(url string) *WebFuzz {
-	return &WebFuzz{Url: url}
-}
--- a/getter/web_lucn_org.go
+++ b/getter/web_lucn_org.go
@@ -1,70 +0,0 @@
-package getter
-
-import (
-	"encoding/base64"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/zu1k/proxypool/tool"
-
-	"github.com/zu1k/proxypool/proxy"
-)
-
-const lucnorgSsrLink = "https://lncn.org/api/ssrList"
-
-type WebLucnOrg struct {
-}
-
-func (w WebLucnOrg) Get() []proxy.Proxy {
-	resp, err := http.Post(lucnorgSsrLink, "", nil)
-	if err != nil {
-		return nil
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil
-	}
-
-	response := struct {
-		Code string `json:"code"`
-		Ssrs string `json:"ssrs"`
-	}{}
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return nil
-	}
-
-	dec := decryptAesForLucn(response.Code, response.Ssrs)
-	if dec == nil {
-		return nil
-	}
-
-	type node struct {
-		Url string `json:"url"`
-	}
-	ssrs := make([]node, 0)
-	err = json.Unmarshal(dec, &ssrs)
-	if err != nil {
-		return nil
-	}
-
-	result := make([]string, 0)
-	for _, node := range ssrs {
-		result = append(result, node.Url)
-	}
-	return StringArray2ProxyArray(result)
-}
-
-func decryptAesForLucn(code string, c string) []byte {
-	if code == "" {
-		code = "abclnv561cqqfg30"
-	}
-	cipher, err := base64.StdEncoding.DecodeString(c)
-	if err != nil {
-		return nil
-	}
-	result := tool.AesEcbDecryptWithPKCS7Unpadding(cipher, []byte(code))
-	return result
-}
--- a/getter/web_test.go
+++ b/getter/web_test.go
@@ -1,18 +0,0 @@
-package getter
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestWebLucnOrg_Get(t *testing.T) {
-	fmt.Println(WebLucnOrg{}.Get())
-}
-
-func TestWebFreessrXyz_Get(t *testing.T) {
-	fmt.Println(WebFreessrXyz{}.Get())
-}
-
-func TestWebFuzz_Get(t *testing.T) {
-	fmt.Println(NewWebFuzz("https://merlinblog.xyz/wiki/freess.html").Get())
-}
--- a/go.mod
+++ b/go.mod
@@ -3,11 +3,14 @@ module github.com/zu1k/proxypool
 go 1.14

 require (
+	github.com/Dreamacro/clash v1.0.1-0.20200812125056-8b7c731fd629
 	github.com/PuerkitoBio/goquery v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.2.0 // indirect
 	github.com/antchfx/htmlquery v1.2.3 // indirect
 	github.com/antchfx/xmlquery v1.2.4 // indirect
 	github.com/antchfx/xpath v1.1.8 // indirect
+	github.com/cloudflare/cloudflare-go v0.13.0
+	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-playground/validator/v10 v10.3.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
@@ -19,12 +22,16 @@ require (
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/oschwald/geoip2-golang v1.4.0
+	github.com/oschwald/maxminddb-golang v1.7.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/temoto/robotstxt v1.1.1 // indirect
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
-	golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed // indirect
+	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect
+	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
+	golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect
+	golang.org/x/text v0.3.3 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -3,6 +3,10 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 contrib.go.opencensus.io/exporter/ocagent v0.6.0/go.mod h1:zmKjrJcdo0aYcVS7bmEeSEBLPA9YJp5bjrofdU3pIXs=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Dreamacro/clash v1.0.1-0.20200812125056-8b7c731fd629 h1:876g402+S9ROmgHU7wGdSssWz/TNiJJsDsphd2qsbRc=
+github.com/Dreamacro/clash v1.0.1-0.20200812125056-8b7c731fd629/go.mod h1:LMbDBvj4kuiE5TGhZ8JxhEdRpBtZRjQllo2r7RfUw1c=
+github.com/Dreamacro/go-shadowsocks2 v0.1.6-0.20200722122336-8e5c7db4f96a h1:JhQFrFOkCpRB8qsN6PrzHFzjy/8iQpFFk5cbOiplh6s=
+github.com/Dreamacro/go-shadowsocks2 v0.1.6-0.20200722122336-8e5c7db4f96a/go.mod h1:LSXCjyHesPY3pLjhwff1mQX72ItcBT/N2xNC685cYeU=
 github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
@@ -22,21 +26,28 @@ github.com/aws/aws-sdk-go v1.13.10/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41Zo
 github.com/axiomhq/hyperloglog v0.0.0-20180317131949-fe9507de0228/go.mod h1:IOXAcuKIFq/mDyuQ4wyJuJ79XLMsmLM+5RdQ+vWrL7o=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/cloudflare-go v0.13.0 h1:Mx2gNz/r4H0eHETENq5ZPtc/kOX/uzEfT2UFuA+/XAU=
+github.com/cloudflare/cloudflare-go v0.13.0/go.mod h1:6rZ6s/XWxP8WIWMATgDM7aUop0Z0ZOAfcm/4rEd0lOY=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
 github.com/go-ini/ini v1.33.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@@ -47,7 +58,6 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o=
 github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
@@ -57,6 +67,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
 github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
+github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
+github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -95,6 +107,8 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s=
 github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.6/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -110,7 +124,6 @@ github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/joeshaw/envdecode v0.0.0-20180129163420-d5f34bca07f3/go.mod h1:Q+alOFAXgW5SrcfMPt/G4B2oN+qEcQRJjkn/f4mKL04=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -139,23 +152,31 @@ github.com/lstoll/grpce v1.7.0/go.mod h1:XiCWl3R+avNCT7KsTjv3qCblgsSqd0SC4ymySrH
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug=
+github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
+github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w=
+github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc=
+github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.2-0.20190227000051-27936f6d90f9/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -164,10 +185,12 @@ github.com/rcrowley/go-metrics v0.0.0-20160613154715-cfa5a85e9f0a/go.mod h1:bCqn
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
 github.com/rollbar/rollbar-go v1.2.0/go.mod h1:czC86b8U4xdUH7W2C6gomi2jutLm8qK0OtrF5WMvpcc=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -180,8 +203,9 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
 github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
@@ -190,6 +214,8 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/unrolled/secure v1.0.1/go.mod h1:R6rugAuzh4TQpbFAq69oqZggyBQxFRFQIewtz5z7Jsc=
 github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
+github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -199,6 +225,12 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -207,6 +239,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -223,11 +256,14 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -237,6 +273,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -248,17 +285,21 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed h1:WBkVNH1zd9jg/dK4HCM4lNANnmd12EHC9z+LmcCG4ns=
-golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed h1:J22ig1FUekjjkmZUM7pTKixYm8DvrYsvrBZdunYeIuQ=
+golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -271,6 +312,8 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20190502212712-4a2eb0188cbc/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
@@ -310,8 +353,10 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/caio/go-tdigest.v2 v2.3.0/go.mod h1:HPfh/CLN8UWDMOC76lqxVeKa5E24ypoVuTj4BLMb9cU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/eapache/channels.v1 v1.1.0/go.mod h1:BHIBujSvu9yMTrTYbTCjDD43gUhtmaOtTWDe7sTv1js=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
@@ -322,6 +367,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
--- a/internal/app/getter.go
+++ b/internal/app/getter.go
@@ -0,0 +1,51 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ghodss/yaml"
+
+	"github.com/zu1k/proxypool/config"
+	"github.com/zu1k/proxypool/pkg/getter"
+)
+
+var Getters = make([]getter.Getter, 0)
+
+func InitConfigAndGetters(path string) (err error) {
+	err = config.Parse(path)
+	if err != nil {
+		return
+	}
+	if s := config.Config.SourceFiles; len(s) == 0 {
+		return errors.New("no sources")
+	} else {
+		initGetters(s)
+	}
+	return
+}
+
+func initGetters(sourceFiles []string) {
+	Getters = make([]getter.Getter, 0)
+	for _, path := range sourceFiles {
+		data, err := config.ReadFile(path)
+		if err != nil {
+			fmt.Errorf("Init SourceFile Error: %s\n", err.Error())
+			continue
+		}
+		sourceList := make([]config.Source, 0)
+		err = yaml.Unmarshal(data, &sourceList)
+		if err != nil {
+			fmt.Errorf("Init SourceFile Error: %s\n", err.Error())
+			continue
+		}
+		for _, source := range sourceList {
+			g, err := getter.NewGetter(source.Type, source.Options)
+			if err == nil && g != nil {
+				Getters = append(Getters, g)
+				fmt.Println("init getter:", source.Type, source.Options)
+			}
+		}
+	}
+	fmt.Println("Getter count:", len(Getters))
+}
--- a/internal/app/task.go
+++ b/internal/app/task.go
@@ -0,0 +1,54 @@
+package app
+
+import (
+	"log"
+	"sync"
+	"time"
+
+	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/pkg/provider"
+	"github.com/zu1k/proxypool/pkg/proxy"
+)
+
+var location, _ = time.LoadLocation("PRC")
+
+func CrawlGo() {
+	wg := &sync.WaitGroup{}
+	var pc = make(chan proxy.Proxy)
+	for _, g := range Getters {
+		wg.Add(1)
+		go g.Get2Chan(pc, wg)
+	}
+	proxies := cache.GetProxies("proxies")
+	go func() {
+		wg.Wait()
+		close(pc)
+	}()
+	for node := range pc {
+		if node != nil {
+			proxies = append(proxies, node)
+		}
+	}
+	// 节点去重
+	proxies = proxies.Deduplication()
+	log.Println("CrawlGo node count:", len(proxies))
+	proxies = provider.Clash{Proxies: proxies}.CleanProxies()
+	proxies.NameAddCounrty().Sort().NameAddIndex()
+	cache.SetProxies("allproxies", proxies)
+	cache.AllProxiesCount = proxies.Len()
+	cache.SSProxiesCount = proxies.TypeLen("ss")
+	cache.SSRProxiesCount = proxies.TypeLen("ssr")
+	cache.VmessProxiesCount = proxies.TypeLen("vmess")
+	cache.TrojanProxiesCount = proxies.TypeLen("trojan")
+	cache.LastCrawlTime = time.Now().In(location).Format("2006-01-02 15:04:05")
+
+	// 可用性检测
+	proxies = proxy.CleanBadProxies(proxies)
+	log.Println("CrawlGo clash useable node count:", len(proxies))
+	proxies.NameAddCounrty().Sort().NameAddIndex()
+	cache.SetProxies("proxies", proxies)
+	cache.UsefullProxiesCount = proxies.Len()
+
+	cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
+	cache.SetString("surgeproxies", provider.Surge{Proxies: proxies}.Provide())
+}
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -0,0 +1,37 @@
+package cache
+
+import (
+	"log"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+	"github.com/zu1k/proxypool/pkg/proxy"
+)
+
+var c = cache.New(cache.NoExpiration, 10*time.Minute)
+
+func GetProxies(key string) proxy.ProxyList {
+	result, found := c.Get(key)
+	if found {
+		log.Println("found cache for:", key, "length:", len(result.(proxy.ProxyList)))
+		return result.(proxy.ProxyList)
+	}
+	log.Println("cache not found:", key)
+	return nil
+}
+
+func SetProxies(key string, proxies proxy.ProxyList) {
+	c.Set(key, proxies, cache.NoExpiration)
+}
+
+func SetString(key, value string) {
+	c.Set(key, value, cache.NoExpiration)
+}
+
+func GetString(key string) string {
+	result, found := c.Get(key)
+	if found {
+		return result.(string)
+	}
+	return ""
+}
--- a/internal/cache/vars.go
+++ b/internal/cache/vars.go
@@ -0,0 +1,13 @@
+package cache
+
+var (
+	AllProxiesCount    = 0
+	SSRProxiesCount    = 0
+	SSProxiesCount     = 0
+	VmessProxiesCount  = 0
+	TrojanProxiesCount = 0
+
+	UsefullProxiesCount = 0
+
+	LastCrawlTime = "程序刚升级完成，正在爬取中..."
+)
--- a/internal/cloudflare/cache.go
+++ b/internal/cloudflare/cache.go
@@ -0,0 +1,30 @@
+package cloudflare
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/cloudflare/cloudflare-go"
+	"github.com/zu1k/proxypool/config"
+)
+
+func test() {
+	api, err := cloudflare.New(config.Config.CFKey, config.Config.CFKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Fetch the zone ID
+	id, err := api.ZoneIDByName(config.Config.Domain)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Fetch zone details
+	zone, err := api.ZoneDetails(id)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Print zone details
+	fmt.Println(zone)
+}
--- a/internal/cron/cron.go
+++ b/internal/cron/cron.go
@@ -0,0 +1,16 @@
+package cron
+
+import (
+	"github.com/jasonlvhit/gocron"
+	"github.com/zu1k/proxypool/internal/app"
+)
+
+func Cron() {
+	_ = gocron.Every(15).Minutes().Do(crawlTask)
+	<-gocron.Start()
+}
+
+func crawlTask() {
+	_ = app.InitConfigAndGetters("")
+	app.CrawlGo()
+}
--- a/main.go
+++ b/main.go
@@ -1,14 +1,36 @@
 package main

 import (
+	"flag"
 	"fmt"
+	"os"

 	"github.com/zu1k/proxypool/api"
-	"github.com/zu1k/proxypool/app"
+	"github.com/zu1k/proxypool/internal/app"
+	"github.com/zu1k/proxypool/internal/cron"
+	"github.com/zu1k/proxypool/pkg/proxy"
 )

+var configFilePath = ""
+
 func main() {
+	flag.StringVar(&configFilePath, "c", "", "path to config file: config.yaml")
+	flag.Parse()
+
+	if configFilePath == "" {
+		configFilePath = os.Getenv("CONFIG_FILE")
+	}
+	if configFilePath == "" {
+		configFilePath = "config.yaml"
+	}
+	err := app.InitConfigAndGetters(configFilePath)
+	if err != nil {
+		panic(err)
+	}
+
+	proxy.InitGeoIpDB()
 	fmt.Println("Do the first crawl...")
-	app.CrawlTGChannel()
+	go app.CrawlGo()
+	go cron.Cron()
 	api.Run()
 }
--- a/pkg/getter/base.go
+++ b/pkg/getter/base.go
@@ -0,0 +1,88 @@
+package getter
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+type Getter interface {
+	Get() proxy.ProxyList
+	Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup)
+}
+
+type creator func(options tool.Options) (getter Getter, err error)
+
+var creatorMap = make(map[string]creator)
+
+func Register(sourceType string, c creator) {
+	creatorMap[sourceType] = c
+}
+
+func NewGetter(sourceType string, options tool.Options) (getter Getter, err error) {
+	c, ok := creatorMap[sourceType]
+	if ok {
+		return c(options)
+	}
+	return nil, ErrorCreaterNotSupported
+}
+
+func String2Proxy(link string) proxy.Proxy {
+	var err error
+	var data proxy.Proxy
+	if strings.HasPrefix(link, "ssr://") {
+		data, err = proxy.ParseSSRLink(link)
+	} else if strings.HasPrefix(link, "vmess://") {
+		data, err = proxy.ParseVmessLink(link)
+	} else if strings.HasPrefix(link, "ss://") {
+		data, err = proxy.ParseSSLink(link)
+	} else if strings.HasPrefix(link, "trojan://") {
+		data, err = proxy.ParseTrojanLink(link)
+	}
+	if err != nil {
+		return nil
+	}
+	return data
+}
+
+func StringArray2ProxyArray(origin []string) proxy.ProxyList {
+	results := make(proxy.ProxyList, 0)
+	for _, link := range origin {
+		results = append(results, String2Proxy(link))
+	}
+	return results
+}
+
+func GrepLinksFromString(text string) []string {
+	results := proxy.GrepSSRLinkFromString(text)
+	results = append(results, proxy.GrepVmessLinkFromString(text)...)
+	results = append(results, proxy.GrepSSLinkFromString(text)...)
+	results = append(results, proxy.GrepTrojanLinkFromString(text)...)
+	return results
+}
+
+func FuzzParseProxyFromString(text string) proxy.ProxyList {
+	return StringArray2ProxyArray(GrepLinksFromString(text))
+}
+
+var (
+	ErrorUrlNotFound         = errors.New("url should be specified")
+	ErrorCreaterNotSupported = errors.New("type not supported")
+)
+
+func AssertTypeStringNotNull(i interface{}) (str string, err error) {
+	switch i.(type) {
+	case string:
+		str = i.(string)
+		if str == "" {
+			return "", errors.New("string is null")
+		}
+		return str, nil
+	default:
+		return "", errors.New("type is not string")
+	}
+	return "", nil
+}
--- a/pkg/getter/subscribe.go
+++ b/pkg/getter/subscribe.go
@@ -0,0 +1,61 @@
+package getter
+
+import (
+	"io/ioutil"
+	"strings"
+	"sync"
+
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+func init() {
+	Register("subscribe", NewSubscribe)
+}
+
+type Subscribe struct {
+	Url string
+}
+
+func (s *Subscribe) Get() proxy.ProxyList {
+	resp, err := tool.GetHttpClient().Get(s.Url)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil
+	}
+
+	nodesString, err := tool.Base64DecodeString(string(body))
+	if err != nil {
+		return nil
+	}
+	nodesString = strings.ReplaceAll(nodesString, "\t", "")
+
+	nodes := strings.Split(nodesString, "\n")
+	return StringArray2ProxyArray(nodes)
+}
+
+func (s *Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := s.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
+func NewSubscribe(options tool.Options) (getter Getter, err error) {
+	urlInterface, found := options["url"]
+	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
+		return &Subscribe{
+			Url: url,
+		}, nil
+	}
+	return nil, ErrorUrlNotFound
+}
--- a/pkg/getter/tgchannel.go
+++ b/pkg/getter/tgchannel.go
@@ -0,0 +1,85 @@
+package getter
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/gocolly/colly"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+func init() {
+	Register("tgchannel", NewTGChannelGetter)
+}
+
+type TGChannelGetter struct {
+	c         *colly.Collector
+	NumNeeded int
+	results   []string
+	Url       string
+}
+
+func NewTGChannelGetter(options tool.Options) (getter Getter, err error) {
+	num, found := options["num"]
+	t := 200
+	switch num.(type) {
+	case int:
+		t = num.(int)
+	case float64:
+		t = int(num.(float64))
+	}
+
+	if !found || t <= 0 {
+		t = 200
+	}
+	urlInterface, found := options["channel"]
+	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
+		return &TGChannelGetter{
+			c:         colly.NewCollector(),
+			NumNeeded: t,
+			Url:       "https://t.me/s/" + url,
+		}, nil
+	}
+	return nil, ErrorUrlNotFound
+}
+
+func (g *TGChannelGetter) Get() proxy.ProxyList {
+	result := make(proxy.ProxyList, 0)
+	g.results = make([]string, 0)
+	// 找到所有的文字消息
+	g.c.OnHTML("div.tgme_widget_message_text", func(e *colly.HTMLElement) {
+		g.results = append(g.results, GrepLinksFromString(e.Text)...)
+		// 抓取到http链接，有可能是订阅链接或其他链接，无论如何试一下
+		subUrls := urlRe.FindAllString(e.Text, -1)
+		for _, url := range subUrls {
+			result = append(result, (&Subscribe{Url: url}).Get()...)
+		}
+	})
+
+	// 找到之前消息页面的链接，加入访问队列
+	g.c.OnHTML("link[rel=prev]", func(e *colly.HTMLElement) {
+		if len(g.results) < g.NumNeeded {
+			_ = e.Request.Visit(e.Attr("href"))
+		}
+	})
+
+	g.results = make([]string, 0)
+	err := g.c.Visit(g.Url)
+	if err != nil {
+		_ = fmt.Errorf("%s", err.Error())
+	}
+	return append(result, StringArray2ProxyArray(g.results)...)
+}
+
+func (g *TGChannelGetter) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := g.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
--- a/pkg/getter/web_fanqiangdang.go
+++ b/pkg/getter/web_fanqiangdang.go
@@ -0,0 +1,120 @@
+package getter
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/gocolly/colly"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+func init() {
+	Register("web-fanqiangdang-rss", NewWebFanqiangdangRSSGetter)
+	Register("web-fanqiangdang", NewWebFanqiangdangGetter)
+}
+
+type WebFanqiangdang struct {
+	c         *colly.Collector
+	Url       string
+	results   proxy.ProxyList
+}
+
+func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
+	urlInterface, found := options["url"]
+	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
+		return &WebFanqiangdang{
+			c:         colly.NewCollector(),
+			Url:       url,
+		}, nil
+	}
+	return nil, ErrorUrlNotFound
+}
+
+func (w *WebFanqiangdang) Get() proxy.ProxyList {
+	w.results = make(proxy.ProxyList, 0)
+	w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) {
+		w.results = append(w.results, FuzzParseProxyFromString(e.Text)...)
+		subUrls := urlRe.FindAllString(e.Text, -1)
+		for _, url := range subUrls {
+			w.results = append(w.results, (&Subscribe{Url: url}).Get()...)
+		}
+	})
+
+	w.c.OnHTML("th.new>a[href]", func(e *colly.HTMLElement) {
+		url := e.Attr("href")
+		if strings.HasPrefix(url, "https://fanqiangdang.com/thread") {
+			_ = e.Request.Visit(url)
+		}
+	})
+
+	w.results = make(proxy.ProxyList, 0)
+	err := w.c.Visit(w.Url)
+	if err != nil {
+		_ = fmt.Errorf("%s", err.Error())
+	}
+
+	return w.results
+}
+
+func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
+
+type WebFanqiangdangRSS struct {
+	c         *colly.Collector
+	Url       string
+	results   []string
+}
+
+func NewWebFanqiangdangRSSGetter(options tool.Options) (getter Getter, err error) {
+	urlInterface, found := options["url"]
+	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
+		return &WebFanqiangdangRSS{
+			c:         colly.NewCollector(),
+			Url:       url,
+		}, nil
+	}
+	return nil, ErrorUrlNotFound
+}
+
+func (w *WebFanqiangdangRSS) Get() proxy.ProxyList {
+	w.results = make([]string, 0)
+	w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) {
+		w.results = append(w.results, GrepLinksFromString(e.Text)...)
+	})
+
+	w.c.OnXML("//item//link", func(e *colly.XMLElement) {
+		_ = e.Request.Visit(e.Text)
+	})
+
+	w.results = make([]string, 0)
+	err := w.c.Visit(w.Url)
+	if err != nil {
+		_ = fmt.Errorf("%s", err.Error())
+	}
+
+	return StringArray2ProxyArray(w.results)
+}
+
+func (w *WebFanqiangdangRSS) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
--- a/pkg/getter/web_free_ssr_xyz.go
+++ b/pkg/getter/web_free_ssr_xyz.go
@@ -3,11 +3,16 @@ package getter
 import (
 	"encoding/json"
 	"io/ioutil"
-	"net/http"
+	"sync"

-	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
 )

+func init() {
+	Register("web-freessrxyz", NewWebFreessrxyzGetter)
+}
+
 const (
 	freessrxyzSsrLink   = "https://api.free-ssr.xyz/ssr"
 	freessrxyzV2rayLink = "https://api.free-ssr.xyz/v2ray"
@@ -16,14 +21,26 @@ const (
 type WebFreessrXyz struct {
 }

-func (w WebFreessrXyz) Get() []proxy.Proxy {
+func NewWebFreessrxyzGetter(options tool.Options) (getter Getter, err error) {
+	return &WebFreessrXyz{}, nil
+}
+
+func (w *WebFreessrXyz) Get() proxy.ProxyList {
 	results := freessrxyzFetch(freessrxyzSsrLink)
 	results = append(results, freessrxyzFetch(freessrxyzV2rayLink)...)
 	return results
 }

-func freessrxyzFetch(link string) []proxy.Proxy {
-	resp, err := http.Get(link)
+func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
+func freessrxyzFetch(link string) proxy.ProxyList {
+	resp, err := tool.GetHttpClient().Get(link)
 	if err != nil {
 		return nil
 	}
--- a/pkg/getter/web_fuzz.go
+++ b/pkg/getter/web_fuzz.go
@@ -0,0 +1,51 @@
+package getter
+
+import (
+	"io/ioutil"
+	"sync"
+
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+func init() {
+	Register("webfuzz", NewWebFuzzGetter)
+}
+
+type WebFuzz struct {
+	Url string
+}
+
+func (w *WebFuzz) Get() proxy.ProxyList {
+	resp, err := tool.GetHttpClient().Get(w.Url)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil
+	}
+
+	return FuzzParseProxyFromString(string(body))
+}
+
+func (w *WebFuzz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
+func NewWebFuzzGetter(options tool.Options) (getter Getter, err error) {
+	urlInterface, found := options["url"]
+	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
+		return &WebFuzz{Url: url}, nil
+	}
+	return nil, ErrorUrlNotFound
+}
--- a/pkg/getter/web_fuzz_sub.go
+++ b/pkg/getter/web_fuzz_sub.go
@@ -0,0 +1,89 @@
+package getter
+
+import (
+	"io/ioutil"
+	"regexp"
+	"sync"
+
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+func init() {
+	Register("webfuzzsub", NewWebFuzzSubGetter)
+}
+
+type WebFuzzSub struct {
+	Url string
+}
+
+func (w *WebFuzzSub) Get() proxy.ProxyList {
+	resp, err := tool.GetHttpClient().Get(w.Url)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil
+	}
+	text := string(body)
+	subUrls := urlRe.FindAllString(text, -1)
+	result := make(proxy.ProxyList, 0)
+	for _, url := range subUrls {
+		result = append(result, (&Subscribe{Url: url}).Get()...)
+	}
+	return result
+}
+
+func (w *WebFuzzSub) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
+func NewWebFuzzSubGetter(options tool.Options) (getter Getter, err error) {
+	urlInterface, found := options["url"]
+	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
+		return &WebFuzzSub{Url: url}, nil
+	}
+	return nil, ErrorUrlNotFound
+}
+
+var urlRe = regexp.MustCompile(urlPattern)
+
+const (
+	// 匹配 IP4
+	ip4Pattern = `((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)`
+
+	// 匹配 IP6，参考以下网页内容：
+	// http://blog.csdn.net/jiangfeng08/article/details/7642018
+	ip6Pattern = `(([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|` +
+		`(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|` +
+		`(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|` +
+		`(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
+		`(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
+		`(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
+		`(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
+		`(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))`
+
+	// 同时匹配 IP4 和 IP6
+	ipPattern = "(" + ip4Pattern + ")|(" + ip6Pattern + ")"
+
+	// 匹配域名
+	domainPattern = `[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}(\.[a-zA-Z0-9][a-zA-Z0-9_-]{0,62})*(\.[a-zA-Z][a-zA-Z0-9]{0,10}){1}`
+
+	// 匹配 URL
+	urlPattern = `((https|http)?://)?` + // 协议
+		`(([0-9a-zA-Z]+:)?[0-9a-zA-Z_-]+@)?` + // pwd:user@
+		"(" + ipPattern + "|(" + domainPattern + "))" + // IP 或域名
+		`(:\d{1,5})?` + // 端口
+		`(/+[a-zA-Z0-9][a-zA-Z0-9_.-]*)*/*` + // path
+		`(\?([a-zA-Z0-9_-]+(=.*&?)*)*)*` // query
+)
--- a/pkg/provider/base.go
+++ b/pkg/provider/base.go
@@ -0,0 +1,14 @@
+package provider
+
+type Provider interface {
+	Provide() string
+}
+
+func checkInList(list []string, item string) bool {
+	for _, i := range list {
+		if item == i {
+			return true
+		}
+	}
+	return false
+}
--- a/pkg/provider/clash.go
+++ b/pkg/provider/clash.go
@@ -3,19 +3,65 @@ package provider
 import (
 	"strings"

-	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/pkg/proxy"
 )

 type Clash struct {
-	Proxies []proxy.Proxy `yaml:"proxies"`
+	Proxies proxy.ProxyList `yaml:"proxies"`
+	Types   string          `yaml:"type"`
+	Country string          `yaml:"country"`
+}
+
+func (c Clash) CleanProxies() (proxies proxy.ProxyList) {
+	proxies = make(proxy.ProxyList, 0)
+	for _, p := range c.Proxies {
+		if checkClashSupport(p) {
+			proxies = append(proxies, p)
+		}
+	}
+	return
 }

 func (c Clash) Provide() string {
 	var resultBuilder strings.Builder
-
 	resultBuilder.WriteString("proxies:\n")
+
+	noNeedFilterType := false
+	noNeedFilterCountry := false
+	if c.Types == "" || c.Types == "all" {
+		noNeedFilterType = true
+	}
+	if c.Country == "" || c.Country == "all" {
+		noNeedFilterCountry = true
+	}
+	types := strings.Split(c.Types, ",")
+	countries := strings.Split(c.Country, ",")
+
 	for _, p := range c.Proxies {
-		if checkClashSupport(p) {
+		if !checkClashSupport(p) {
+			continue
+		}
+
+		typeOk := false
+		countryOk := false
+		if !noNeedFilterType {
+			for _, t := range types {
+				if p.TypeName() == t {
+					typeOk = true
+					break
+				}
+			}
+		}
+		if !noNeedFilterCountry {
+			for _, c := range countries {
+				if strings.HasPrefix(p.BaseInfo().Name, c) {
+					countryOk = true
+					break
+				}
+			}
+		}
+
+		if (noNeedFilterType || typeOk) && (noNeedFilterCountry || countryOk) {
 			resultBuilder.WriteString(p.ToClash() + "\n")
 		}
 	}
@@ -24,28 +70,26 @@ func (c Clash) Provide() string {
 }

 func checkClashSupport(p proxy.Proxy) bool {
-	switch p.(type) {
-	case *proxy.ShadowsocksR:
+	switch p.TypeName() {
+	case "ssr":
 		ssr := p.(*proxy.ShadowsocksR)
 		if checkInList(ssrCipherList, ssr.Cipher) && checkInList(ssrProtocolList, ssr.Protocol) && checkInList(ssrObfsList, ssr.Obfs) {
 			return true
 		}
-	case *proxy.Vmess:
+	case "vmess":
 		vmess := p.(*proxy.Vmess)
 		if checkInList(vmessCipherList, vmess.Cipher) {
 			return true
 		}
-	default:
-		return false
-	}
-	return false
-}
-
-func checkInList(list []string, item string) bool {
-	for _, i := range list {
-		if item == i {
+	case "ss":
+		ss := p.(*proxy.Shadowsocks)
+		if checkInList(ssCipherList, ss.Cipher) {
 			return true
 		}
+	case "trojan":
+		return true
+	default:
+		return false
 	}
 	return false
 }
@@ -102,3 +146,20 @@ var vmessCipherList = []string{
 	"chacha20-poly1305",
 	"none",
 }
+
+var ssCipherList = []string{
+	"aes-128-gcm",
+	"aes-192-gcm",
+	"aes-256-gcm",
+	"aes-128-cfb",
+	"aes-192-cfb",
+	"aes-256-cfb",
+	"aes-128-ctr",
+	"aes-192-ctr",
+	"aes-256-ctr",
+	"rc4-md5",
+	"chacha20-ietf",
+	"xchacha20",
+	"chacha20-ietf-poly1305",
+	"xchacha20-ietf-poly1305",
+}
--- a/pkg/provider/surge.go
+++ b/pkg/provider/surge.go
@@ -0,0 +1,40 @@
+package provider
+
+import (
+	"strings"
+
+	"github.com/zu1k/proxypool/pkg/proxy"
+)
+
+type Surge struct {
+	Proxies proxy.ProxyList `yaml:"proxies"`
+}
+
+func (s Surge) Provide() string {
+	var resultBuilder strings.Builder
+
+	for _, p := range s.Proxies {
+		if checkSurgeSupport(p) {
+			resultBuilder.WriteString(p.ToSurge() + "\n")
+		}
+	}
+
+	return resultBuilder.String()
+}
+
+func checkSurgeSupport(p proxy.Proxy) bool {
+	switch p.(type) {
+	case *proxy.ShadowsocksR:
+		return false
+	case *proxy.Vmess:
+		return true
+	case *proxy.Shadowsocks:
+		ss := p.(*proxy.Shadowsocks)
+		if checkInList(ssCipherList, ss.Cipher) {
+			return true
+		}
+	default:
+		return false
+	}
+	return false
+}
--- a/pkg/proxy/base.go
+++ b/pkg/proxy/base.go
@@ -8,21 +8,33 @@ type Base struct {
 	UDP    bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
 }

+func (b *Base) TypeName() string {
+	if b.Type == "" {
+		return "unknown"
+	}
+	return b.Type
+}
+
+func (b *Base) SetName(name string) {
+	b.Name = name
+}
+
+func (b *Base) BaseInfo() *Base {
+	return b
+}
+
+func (b *Base) Clone() Base {
+	c := *b
+	return c
+}
+
 type Proxy interface {
 	String() string
 	ToClash() string
+	ToSurge() string
 	Identifier() string
 	SetName(name string)
-}
-
-func Deduplication(src []Proxy) []Proxy {
-	result := make([]Proxy, 0, len(src))
-	temp := map[string]struct{}{}
-	for _, item := range src {
-		if _, ok := temp[item.Identifier()]; !ok {
-			temp[item.Identifier()] = struct{}{}
-			result = append(result, item)
-		}
-	}
-	return result
+	TypeName() string
+	BaseInfo() *Base
+	Clone() Proxy
 }
--- a/pkg/proxy/check.go
+++ b/pkg/proxy/check.go
@@ -0,0 +1,79 @@
+package proxy
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/Dreamacro/clash/adapters/outbound"
+)
+
+const defaultURLTestTimeout = time.Second * 5
+
+func testDelay(p Proxy) (delay uint16, err error) {
+	pmap := make(map[string]interface{})
+	err = json.Unmarshal([]byte(p.String()), &pmap)
+	if err != nil {
+		return
+	}
+
+	pmap["port"] = int(pmap["port"].(float64))
+	if p.TypeName() == "vmess" {
+		pmap["alterId"] = int(pmap["alterId"].(float64))
+	}
+
+	clashProxy, err := outbound.ParseProxy(pmap)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultURLTestTimeout)
+	delay, err = clashProxy.URLTest(ctx, "http://www.gstatic.com/generate_204")
+	cancel()
+	return delay, err
+}
+
+func CleanBadProxies(proxies []Proxy) (cproxies []Proxy) {
+	c := make(chan checkResult, 40)
+	defer close(c)
+	for _, p := range proxies {
+		go testProxyDelayToChan(p, c)
+	}
+	okMap := make(map[string]struct{})
+	size := len(proxies)
+	for i := 0; i < size; i++ {
+		r := <-c
+		if r.delay > 0 {
+			okMap[r.name] = struct{}{}
+		}
+	}
+	cproxies = make([]Proxy, 0)
+	for _, p := range proxies {
+		if _, ok := okMap[p.Identifier()]; ok {
+			cproxies = append(cproxies, p.Clone())
+		}
+	}
+	return
+}
+
+type checkResult struct {
+	name  string
+	delay uint16
+}
+
+func testProxyDelayToChan(p Proxy, c chan checkResult) {
+	delay, err := testDelay(p)
+	if err != nil {
+		c <- checkResult{
+			name:  p.Identifier(),
+			delay: 0,
+		}
+		return
+	}
+	c <- checkResult{
+		name:  p.Identifier(),
+		delay: delay,
+	}
+}
--- a/pkg/proxy/geoip.go
+++ b/pkg/proxy/geoip.go
@@ -0,0 +1,51 @@
+package proxy
+
+import (
+	"log"
+	"net"
+	"os"
+
+	"github.com/oschwald/geoip2-golang"
+)
+
+var geoIp GeoIP
+
+func InitGeoIpDB() {
+	geoIp = NewGeoIP("assets/GeoLite2-City.mmdb")
+}
+
+// GeoIP2
+type GeoIP struct {
+	db *geoip2.Reader
+}
+
+// new geoip from db file
+func NewGeoIP(filePath string) (geoip GeoIP) {
+	// 判断文件是否存在
+	_, err := os.Stat(filePath)
+	if err != nil && os.IsNotExist(err) {
+		log.Println("文件不存在，请自行下载 Geoip2 City库，并保存在", filePath)
+		os.Exit(1)
+	} else {
+		db, err := geoip2.Open(filePath)
+		if err != nil {
+			log.Fatal(err)
+		}
+		geoip = GeoIP{db: db}
+	}
+	return
+}
+
+// find ip info
+func (g GeoIP) Find(ipORdomain string) (country string, err error) {
+	ips, err := net.LookupIP(ipORdomain)
+	if err != nil {
+		return "", err
+	}
+	ipData := net.ParseIP(ips[0].String())
+	record, err := g.db.City(ipData)
+	if err != nil {
+		return "", err
+	}
+	return record.Country.IsoCode, nil
+}
--- a/pkg/proxy/proxies.go
+++ b/pkg/proxy/proxies.go
@@ -0,0 +1,106 @@
+package proxy
+
+import (
+	"fmt"
+	"sort"
+)
+
+type ProxyList []Proxy
+
+func (ps ProxyList) Len() int {
+	return len(ps)
+}
+
+func (ps ProxyList) TypeLen(t string) int {
+	l := 0
+	for _, p := range ps {
+		if p.TypeName() == t {
+			l++
+		}
+	}
+	return l
+}
+
+var sortType = make(map[string]int)
+
+func init() {
+	sortType["ss"] = 1
+	sortType["ssr"] = 2
+	sortType["vmess"] = 3
+	sortType["trojan"] = 4
+}
+
+func (ps ProxyList) Less(i, j int) bool {
+	if ps[i].BaseInfo().Name == ps[j].BaseInfo().Name {
+		return sortType[ps[i].BaseInfo().Type] < sortType[ps[j].BaseInfo().Type]
+	} else {
+		return ps[i].BaseInfo().Name < ps[j].BaseInfo().Name
+	}
+}
+
+func (ps ProxyList) Swap(i, j int) {
+	ps[i], ps[j] = ps[j], ps[i]
+}
+
+func (ps ProxyList) Deduplication() ProxyList {
+	result := make(ProxyList, 0, len(ps))
+	temp := map[string]struct{}{}
+	for _, item := range ps {
+		if item != nil {
+			if _, ok := temp[item.Identifier()]; !ok {
+				temp[item.Identifier()] = struct{}{}
+				result = append(result, item)
+			}
+		}
+	}
+	return result
+}
+
+func (ps ProxyList) Sort() ProxyList {
+	sort.Sort(ps)
+	return ps
+}
+
+func (ps ProxyList) NameAddCounrty() ProxyList {
+	num := len(ps)
+	for i := 0; i < num; i++ {
+		country, err := geoIp.Find(ps[i].BaseInfo().Server)
+		if err != nil || country == "" {
+			country = "Earth"
+		}
+		ps[i].SetName(fmt.Sprintf("%s", country))
+	}
+	return ps
+}
+
+func (ps ProxyList) NameAddIndex() ProxyList {
+	num := len(ps)
+	for i := 0; i < num; i++ {
+		ps[i].SetName(fmt.Sprintf("%s_%d", ps[i].BaseInfo().Name, i+1))
+	}
+	return ps
+}
+
+func Deduplication(src ProxyList) ProxyList {
+	result := make(ProxyList, 0, len(src))
+	temp := map[string]struct{}{}
+	for _, item := range src {
+		if item != nil {
+			if _, ok := temp[item.Identifier()]; !ok {
+				temp[item.Identifier()] = struct{}{}
+				result = append(result, item)
+			}
+		}
+	}
+	return result
+}
+
+func (ps ProxyList) Clone() ProxyList {
+	result := make(ProxyList, 0, len(ps))
+	for _, pp := range ps {
+		if pp != nil {
+			result = append(result, pp.Clone())
+		}
+	}
+	return result
+}
--- a/pkg/proxy/shadowsocks.go
+++ b/pkg/proxy/shadowsocks.go
@@ -0,0 +1,155 @@
+package proxy
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+var (
+	ErrorNotSSLink = errors.New("not a correct ss link")
+)
+
+type Shadowsocks struct {
+	Base
+	Password   string                 `yaml:"password" json:"password"`
+	Cipher     string                 `yaml:"cipher" json:"cipher"`
+	Plugin     string                 `yaml:"plugin,omitempty" json:"plugin,omitempty"`
+	PluginOpts map[string]interface{} `yaml:"plugin-opts,omitempty" json:"plugin-opts,omitempty"`
+}
+
+func (ss Shadowsocks) Identifier() string {
+	return net.JoinHostPort(ss.Server, strconv.Itoa(ss.Port)) + ss.Password
+}
+
+func (ss Shadowsocks) String() string {
+	data, err := json.Marshal(ss)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+func (ss Shadowsocks) ToClash() string {
+	data, err := json.Marshal(ss)
+	if err != nil {
+		return ""
+	}
+	return "- " + string(data)
+}
+
+func (ss Shadowsocks) ToSurge() string {
+	// node1 = ss, server, port, encrypt-method=, password=, obfs=, obfs-host=, udp-relay=false
+	if ss.Plugin == "obfs" {
+		text := fmt.Sprintf("%s = ss, %s, %d, encrypt-method=%s, password=%s, obfs=%s, udp-relay=false",
+			ss.Name, ss.Server, ss.Port, ss.Cipher, ss.Password, ss.PluginOpts["mode"])
+		if ss.PluginOpts["host"].(string) != "" {
+			text += ", obfs-host=" + ss.PluginOpts["host"].(string)
+		}
+		return text
+	} else {
+		return fmt.Sprintf("%s = ss, %s, %d, encrypt-method=%s, password=%s, udp-relay=false",
+			ss.Name, ss.Server, ss.Port, ss.Cipher, ss.Password)
+	}
+}
+
+func (ss Shadowsocks) Clone() Proxy {
+	return &ss
+}
+
+func ParseSSLink(link string) (*Shadowsocks, error) {
+	if !strings.HasPrefix(link, "ss://") {
+		return nil, ErrorNotSSRLink
+	}
+
+	uri, err := url.Parse(link)
+	if err != nil {
+		return nil, ErrorNotSSLink
+	}
+
+	cipher := ""
+	password := ""
+	if uri.User.String() == "" {
+		// base64的情况
+		infos, err := tool.Base64DecodeString(uri.Hostname())
+		if err != nil {
+			return nil, err
+		}
+		uri, err = url.Parse("ss://" + infos)
+		if err != nil {
+			return nil, err
+		}
+		cipher = uri.User.Username()
+		password, _ = uri.User.Password()
+	} else {
+		cipherInfoString, err := tool.Base64DecodeString(uri.User.Username())
+		if err != nil {
+			return nil, ErrorPasswordParseFail
+		}
+		cipherInfo := strings.SplitN(cipherInfoString, ":", 2)
+		if len(cipherInfo) < 2 {
+			return nil, ErrorPasswordParseFail
+		}
+		cipher = strings.ToLower(cipherInfo[0])
+		password = cipherInfo[1]
+	}
+	server := uri.Hostname()
+	port, _ := strconv.Atoi(uri.Port())
+
+	moreInfos := uri.Query()
+	pluginString := moreInfos.Get("plugin")
+	plugin := ""
+	pluginOpts := make(map[string]interface{})
+	if strings.Contains(pluginString, ";") {
+		pluginInfos, err := url.ParseQuery(pluginString)
+		if err == nil {
+			if strings.Contains(pluginString, "obfs") {
+				plugin = "obfs"
+				pluginOpts["mode"] = pluginInfos.Get("obfs")
+				pluginOpts["host"] = pluginInfos.Get("obfs-host")
+			} else if strings.Contains(pluginString, "v2ray") {
+				plugin = "v2ray-plugin"
+				pluginOpts["mode"] = pluginInfos.Get("mode")
+				pluginOpts["host"] = pluginInfos.Get("host")
+				pluginOpts["tls"] = strings.Contains(pluginString, "tls")
+			}
+		}
+	}
+	if port == 0 || cipher == "" {
+		return nil, ErrorNotSSLink
+	}
+
+	return &Shadowsocks{
+		Base: Base{
+			Name:   strconv.Itoa(rand.Int()),
+			Server: server,
+			Port:   port,
+			Type:   "ss",
+		},
+		Password:   password,
+		Cipher:     cipher,
+		Plugin:     plugin,
+		PluginOpts: pluginOpts,
+	}, nil
+}
+
+var (
+	ssPlainRe = regexp.MustCompile("ss://([A-Za-z0-9+/_&?=@:%.-])+")
+)
+
+func GrepSSLinkFromString(text string) []string {
+	results := make([]string, 0)
+	texts := strings.Split(text, "ss://")
+	for _, text := range texts {
+		results = append(results, ssPlainRe.FindAllString("ss://"+text, -1)...)
+	}
+	return results
+}
--- a/pkg/proxy/shadowsocksr.go
+++ b/pkg/proxy/shadowsocksr.go
@@ -10,7 +10,7 @@ import (
 	"strconv"
 	"strings"

-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/tool"
 )

 var (
@@ -53,8 +53,12 @@ func (ssr ShadowsocksR) ToClash() string {
 	return "- " + string(data)
 }

-func (ssr *ShadowsocksR) SetName(name string) {
-	ssr.Name = name
+func (ssr ShadowsocksR) ToSurge() string {
+	return ""
+}
+
+func (ssr ShadowsocksR) Clone() Proxy {
+	return &ssr
 }

 func ParseSSRLink(link string) (*ShadowsocksR, error) {
--- a/pkg/proxy/trojan.go
+++ b/pkg/proxy/trojan.go
@@ -0,0 +1,147 @@
+package proxy
+
+import (
+	"encoding/json"
+	"errors"
+	"math/rand"
+	"net"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	ErrorNotTrojanink = errors.New("not a correct trojan link")
+)
+
+type Trojan struct {
+	Base
+	Password       string   `yaml:"password" json:"password"`
+	ALPN           []string `yaml:"alpn,omitempty" json:"alpn,omitempty"`
+	SNI            string   `yaml:"sni,omitempty" json:"sni,omitempty"`
+	SkipCertVerify bool     `yaml:"skip-cert-verify,omitempty" json:"skip-cert-verify,omitempty"`
+	UDP            bool     `yaml:"udp,omitempty" json:"udp,omitempty"`
+}
+
+/**
+  - name: "trojan"
+    type: trojan
+    server: server
+    port: 443
+    password: yourpsk
+    # udp: true
+    # sni: example.com # aka server name
+    # alpn:
+    #   - h2
+    #   - http/1.1
+    # skip-cert-verify: true
+*/
+
+func (t Trojan) Identifier() string {
+	return net.JoinHostPort(t.Server, strconv.Itoa(t.Port)) + t.Password
+}
+
+func (t Trojan) String() string {
+	data, err := json.Marshal(t)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+func (t Trojan) ToClash() string {
+	data, err := json.Marshal(t)
+	if err != nil {
+		return ""
+	}
+	return "- " + string(data)
+}
+
+func (t Trojan) ToSurge() string {
+	return ""
+}
+
+func (t Trojan) Clone() Proxy {
+	return &t
+}
+
+func ParseTrojanLink(link string) (*Trojan, error) {
+	if !strings.HasPrefix(link, "trojan://") && !strings.HasPrefix(link, "trojan-go://") {
+		return nil, ErrorNotTrojanink
+	}
+
+	/**
+	trojan-go://
+	    $(trojan-password)
+	    @
+	    trojan-host
+	    :
+	    port
+	/?
+	    sni=$(tls-sni.com)&
+	    type=$(original|ws|h2|h2+ws)&
+	        host=$(websocket-host.com)&
+	        path=$(/websocket/path)&
+	    encryption=$(ss;aes-256-gcm;ss-password)&
+	    plugin=$(...)
+	#$(descriptive-text)
+	*/
+
+	uri, err := url.Parse(link)
+	if err != nil {
+		return nil, ErrorNotSSLink
+	}
+
+	password := uri.User.Username()
+	password, _ = url.QueryUnescape(password)
+
+	server := uri.Hostname()
+	port, _ := strconv.Atoi(uri.Port())
+
+	moreInfos := uri.Query()
+	sni := moreInfos.Get("sni")
+	sni, _ = url.QueryUnescape(sni)
+	transformType := moreInfos.Get("type")
+	transformType, _ = url.QueryUnescape(transformType)
+	host := moreInfos.Get("host")
+	host, _ = url.QueryUnescape(host)
+	path := moreInfos.Get("path")
+	path, _ = url.QueryUnescape(path)
+
+	alpn := make([]string, 0)
+	if transformType == "h2" {
+		alpn = append(alpn, "h2")
+	}
+
+	if port == 0 {
+		return nil, ErrorNotTrojanink
+	}
+
+	return &Trojan{
+		Base: Base{
+			Name:   strconv.Itoa(rand.Int()),
+			Server: server,
+			Port:   port,
+			Type:   "trojan",
+		},
+		Password:       password,
+		ALPN:           alpn,
+		UDP:            true,
+		SNI:            host,
+		SkipCertVerify: true,
+	}, nil
+}
+
+var (
+	trojanPlainRe = regexp.MustCompile("trojan(-go)?://([A-Za-z0-9+/_&?=@:%.-])+")
+)
+
+func GrepTrojanLinkFromString(text string) []string {
+	results := make([]string, 0)
+	texts := strings.Split(text, "trojan://")
+	for _, text := range texts {
+		results = append(results, trojanPlainRe.FindAllString("trojan://"+text, -1)...)
+	}
+	return results
+}
--- a/pkg/proxy/vmess.go
+++ b/pkg/proxy/vmess.go
@@ -11,7 +11,7 @@ import (
 	"strconv"
 	"strings"

-	"github.com/zu1k/proxypool/tool"
+	"github.com/zu1k/proxypool/pkg/tool"
 )

 var (
@@ -40,7 +40,7 @@ type HTTPOptions struct {
 }

 func (v Vmess) Identifier() string {
-	return net.JoinHostPort(v.Server, strconv.Itoa(v.Port)) + v.Cipher
+	return net.JoinHostPort(v.Server, strconv.Itoa(v.Port)) + v.Cipher + v.UUID
 }

 func (v Vmess) String() string {
@@ -59,8 +59,31 @@ func (v Vmess) ToClash() string {
 	return "- " + string(data)
 }

-func (v *Vmess) SetName(name string) {
-	v.Name = name
+func (v Vmess) ToSurge() string {
+	// node2 = vmess, server, port, username=, ws=true, ws-path=, ws-headers=
+	if v.Network == "ws" {
+		wsHeasers := ""
+		for k, v := range v.WSHeaders {
+			if wsHeasers == "" {
+				wsHeasers = k + ":" + v
+			} else {
+				wsHeasers += "|" + k + ":" + v
+			}
+		}
+		text := fmt.Sprintf("%s = vmess, %s, %d, username=%s, ws=true, tls=%t, ws-path=%s",
+			v.Name, v.Server, v.Port, v.UUID, v.TLS, v.WSPath)
+		if wsHeasers != "" {
+			text += ", ws-headers=" + wsHeasers
+		}
+		return text
+	} else {
+		return fmt.Sprintf("%s = vmess, %s, %d, username=%s, tls=%t",
+			v.Name, v.Server, v.Port, v.UUID, v.TLS)
+	}
+}
+
+func (v Vmess) Clone() Proxy {
+	return &v
 }

 type vmessLinkJson struct {
@@ -103,7 +126,6 @@ func ParseVmessLink(link string) (*Vmess, error) {
 		if err != nil {
 			return nil, ErrorVmessPayloadParseFail
 		}
-		fmt.Println(baseInfo)
 		baseInfoPath := strings.Split(baseInfo, ":")
 		if len(baseInfoPath) < 3 {
 			return nil, ErrorPathNotComplete
@@ -123,7 +145,6 @@ func ParseVmessLink(link string) (*Vmess, error) {
 		}

 		moreInfo, _ := url.ParseQuery(infoPayloads[1])
-		fmt.Println(moreInfo)
 		remarks := moreInfo.Get("remarks")
 		obfs := moreInfo.Get("obfs")
 		network := "tcp"
@@ -132,6 +153,9 @@ func ParseVmessLink(link string) (*Vmess, error) {
 		}
 		//obfsParam := moreInfo.Get("obfsParam")
 		path := moreInfo.Get("path")
+		if path == "" {
+			path = "/"
+		}
 		tls := moreInfo.Get("tls") == "1"

 		wsHeaders := make(map[string]string)
@@ -179,8 +203,13 @@ func ParseVmessLink(link string) (*Vmess, error) {
 		tls := vmessJson.Tls == "tls"

 		wsHeaders := make(map[string]string)
-		wsHeaders["HOST"] = vmessJson.Host
+		if vmessJson.Host != "" {
+			wsHeaders["HOST"] = vmessJson.Host
+		}

+		if vmessJson.Path == "" {
+			vmessJson.Path = "/"
+		}
 		return &Vmess{
 			Base: Base{
 				Name:   vmessJson.Ps + "_" + strconv.Itoa(rand.Int()),
@@ -204,7 +233,7 @@ func ParseVmessLink(link string) (*Vmess, error) {
 }

 var (
-	vmessPlainRe = regexp.MustCompile("vmess://([A-Za-z0-9+/_-])+")
+	vmessPlainRe = regexp.MustCompile("vmess://([A-Za-z0-9+/_?&=-])+")
 )

 func GrepVmessLinkFromString(text string) []string {
--- a/pkg/tool/base64.go
+++ b/pkg/tool/base64.go
@@ -8,15 +8,8 @@ func Base64DecodeString(src string) (dst string, err error) {
 	if src == "" {
 		return "", nil
 	}
-
-	origin := src
-	//if i := len(src) % 4; i != 0 {
-	//	for k := 0; k < i; k++ {
-	//		src += string(base64.StdPadding)
-	//	}
-	//}
 	var dstbytes []byte
-	dstbytes, err = base64.RawURLEncoding.DecodeString(origin)
+	dstbytes, err = base64.RawURLEncoding.DecodeString(src)

 	if err != nil {
 		dstbytes, err = base64.RawStdEncoding.DecodeString(src)
--- a/pkg/tool/httpclient.go
+++ b/pkg/tool/httpclient.go
@@ -0,0 +1,43 @@
+package tool
+
+import (
+	"io"
+	"net/http"
+	"time"
+)
+
+type HttpClient struct {
+	*http.Client
+}
+
+var httpClient *HttpClient
+
+func init() {
+	httpClient = &HttpClient{http.DefaultClient}
+	httpClient.Timeout = time.Second * 10
+}
+
+func GetHttpClient() *HttpClient {
+	c := *httpClient
+	return &c
+}
+
+func (c *HttpClient) Get(url string) (resp *http.Response, err error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
+	return c.Do(req)
+}
+
+func (c *HttpClient) Post(url string, body io.Reader) (resp *http.Response, err error) {
+	req, err := http.NewRequest(http.MethodPost, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
+	return c.Do(req)
+}
--- a/pkg/tool/option.go
+++ b/pkg/tool/option.go
@@ -0,0 +1,3 @@
+package tool
+
+type Options map[string]interface{}
--- a/pkg/tool/unicode.go
+++ b/pkg/tool/unicode.go
--- a/provider/base.go
+++ b/provider/base.go
@@ -1,5 +0,0 @@
-package provider
-
-type Provider interface {
-	Provide() string
-}
--- a/proxy/vmess_test.go
+++ b/proxy/vmess_test.go
@@ -1,13 +0,0 @@
-package proxy
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestParseVmessLink(t *testing.T) {
-	//link := "vmess://YXV0bzo5MGQ0YTM3NC1kYWU4LTExZWEtODY3Zi01NjAwMDJlY2MzNDlAZml2ZWRlbWFuZHMubWw6NDQz?remarks=%E6%97%A5%E6%9C%AC5%EF%BC%9A%E7%94%B5%E6%8A%A5%E9%A2%91%E9%81%93%EF%BC%9A"
-
-	link := "vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjI5LeS7mOi0ueaOqOiNkDp0LmNuL0VHSkl5cmwiLA0KICAiYWRkIjogImJ1cmdlcmtpbmdnb29kLm1sIiwNCiAgInBvcnQiOiAiNDQzIiwNCiAgImlkIjogIjRiNWJhMzkwLWRhZjQtMTFlYS04ODEwLTU2MDAwMmVjYzk2NyIsDQogICJhaWQiOiAiNDYiLA0KICAibmV0IjogIndzIiwNCiAgInR5cGUiOiAibm9uZSIsDQogICJob3N0IjogImJ1cmdlcmtpbmdnb29kLm1sIiwNCiAgInBhdGgiOiAiL0M2bGt2UzNLLyIsDQogICJ0bHMiOiAidGxzIg0KfQ=="
-	fmt.Println(ParseVmessLink(link))
-}
--- a/tool/aes.go
+++ b/tool/aes.go
@@ -1,43 +0,0 @@
-package tool
-
-import (
-	"bytes"
-	"crypto/aes"
-)
-
-func PKCS7Padding(ciphertext []byte, blockSize int) []byte {
-	padding := blockSize - len(ciphertext)%blockSize
-	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
-	return append(ciphertext, padtext...)
-}
-
-func PKCS7UnPadding(origData []byte) []byte {
-	length := len(origData)
-	unpadding := int(origData[length-1])
-	return origData[:(length - unpadding)]
-}
-
-func AesEcbDecryptWithPKCS7Unpadding(data, key []byte) []byte {
-	block, _ := aes.NewCipher(key)
-	decrypted := make([]byte, len(data))
-	size := block.BlockSize()
-
-	for bs, be := 0, size; bs < len(data); bs, be = bs+size, be+size {
-		block.Decrypt(decrypted[bs:be], data[bs:be])
-	}
-
-	return PKCS7UnPadding(decrypted)
-}
-
-func AesEcbEncryptWithPKCS7Padding(data, key []byte) []byte {
-	block, _ := aes.NewCipher(key)
-	data = PKCS7Padding(data, block.BlockSize())
-	decrypted := make([]byte, len(data))
-	size := block.BlockSize()
-
-	for bs, be := 0, size; bs < len(data); bs, be = bs+size, be+size {
-		block.Encrypt(decrypted[bs:be], data[bs:be])
-	}
-
-	return decrypted
-}
--- a/tool/base64_test.go
+++ b/tool/base64_test.go
@@ -1,12 +0,0 @@
-package tool
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestBase64DecodeString(t *testing.T) {
-	str := "OTEuMjA2LjkyLjM4OjI5MzE6b3JpZ2luOnJjNDpwbGFpbjpiRzVqYmk1dmNtY2dNak5zLz9vYmZzcGFyYW09JnJlbWFya3M9NTctNzVhS1o1WVdhWm1GdWNXbGhibWRrWVc1bkxtTnZiUSY9NUwtRTZMLWM1TGljUkEmZ3JvdXA9NTctNzVhS1o1WVdhWm1GdWNXbGhibWRrWVc1bkxtTnZiUSY9VEc1amJpNXZjbWM"
-	fmt.Println(Base64DecodeString(str))
-	fmt.Println(string([]byte(str)[232]))
-}
--- a/tool/httpclient.go
+++ b/tool/httpclient.go
@@ -1,18 +0,0 @@
-package tool
-
-import (
-	"net/http"
-	"time"
-)
-
-var httpClient = http.DefaultClient
-
-func init() {
-	httpClient.Timeout = time.Second * 10
-	http.DefaultClient.Timeout = time.Second * 10
-}
-
-func GetHttpClient() *http.Client {
-	c := *httpClient
-	return &c
-}
Author	SHA1	Message	Date
zu1k	526487be04	fix source.uaml	2020-08-23 17:57:42 +08:00
zu1k	8ca93891ff	fix bug	2020-08-23 17:32:55 +08:00
zu1k	32c07bcd60	fix name	2020-08-23 17:21:21 +08:00
zu1k	42035022c6	splite source from config	2020-08-23 17:14:04 +08:00
zu1k	2e9a2b4281	add version show	2020-08-23 00:14:00 +08:00
zu1k	4ee01fbc6d	go mod tidy	2020-08-23 00:06:27 +08:00
zu1k	dfa76a0255	mod	2020-08-23 00:04:31 +08:00
zu1k	03604fa900	update html	2020-08-22 16:26:24 +08:00
zu1k	7e1dd60837	add country filter	2020-08-22 16:25:24 +08:00
zu1k	a52e63cf65	fix domain race	2020-08-22 15:49:35 +08:00
zu1k	71b9af384c	fix domain race	2020-08-22 15:47:56 +08:00
zu1k	095733101b	add config error show	2020-08-22 15:43:12 +08:00
zu1k	579a5dc750	Merge branch 'heroku' of github.com:zu1k/proxypool into heroku	2020-08-22 15:33:57 +08:00
zu1k	03a9795ae2	show config file	2020-08-22 15:30:19 +08:00
zu1k	1936ef319e	Update app.json	2020-08-22 09:14:54 +08:00
zu1k	20b6066c6a	update github	2020-08-21 16:15:03 +08:00
zu1k	53813e71b2	update readme	2020-08-21 16:14:07 +08:00
zu1k	1ea57c65f5	update readme	2020-08-21 16:12:54 +08:00
zu1k	fe6c8e603b	add crawl time show	2020-08-21 13:25:38 +08:00
zu1k	45275bc2d3	fix fetch http config file not parse env	2020-08-21 13:01:00 +08:00
zu1k	e6d0f19c0a	proxy sort order: name \ type	2020-08-21 11:00:32 +08:00
zu1k	2c4a6a833d	update html	2020-08-21 10:21:11 +08:00
zu1k	9f6ae0cb9b	add type len	2020-08-21 09:53:08 +08:00
zu1k	6262f510e9	mod html	2020-08-21 09:48:05 +08:00
zu1k	eea6a692e6	fix trojan provider	2020-08-21 09:38:10 +08:00
zu1k	f9d2dbbed9	remoe fanqiangdang num limit	2020-08-21 09:05:17 +08:00
zu1k	6f70ee0b7b	update web fanqiangdang	2020-08-21 09:00:26 +08:00
zu1k	bb6600cdd6	fix trojan	2020-08-21 08:31:45 +08:00
zu1k	5a2b0a4188	comment on source.yaml	2020-08-21 08:16:24 +08:00
zu1k	5c2b14f836	update readme	2020-08-20 22:44:15 +08:00
zu1k	345c6f89b6	add trojan link grep	2020-08-20 22:42:40 +08:00
zu1k	097c2d2241	add trojan support	2020-08-20 22:30:59 +08:00
zu1k	023de15f0e	config file not found	2020-08-20 20:32:03 +08:00
zu1k	70dea95241	remove config test	2020-08-20 19:25:30 +08:00
zu1k	1a713ed8e4	use local config source default	2020-08-20 19:21:51 +08:00
zu1k	d92cd20d88	add config file path param	2020-08-20 19:06:25 +08:00
zu1k	17c051cee3	add proxies count show	2020-08-20 18:32:41 +08:00
zu1k	07a91fd647	fix for github action	2020-08-20 18:25:23 +08:00
zu1k	c6c5a4c707	can modify domain	2020-08-20 18:23:32 +08:00
zu1k	008183b40e	change source url	2020-08-20 15:58:31 +08:00
zu1k	21551f7a67	update readme	2020-08-20 15:44:01 +08:00
zu1k	35c6d1348b	add google analytics	2020-08-20 09:39:17 +08:00
zu1k	1e10be9153	update readme	2020-08-20 09:29:27 +08:00
zu1k	8d988f0f5b	add more tgchannel	2020-08-19 22:54:10 +08:00
zu1k	a7a8a9ff8b	vmess identifier add uuid	2020-08-19 21:20:08 +08:00
zu1k	a9fa94c8c1	opti	2020-08-19 21:12:49 +08:00
zu1k	4cfb143f88	crowl sub in tgchannel‘	2020-08-19 21:06:19 +08:00
zu1k	3fbdb91e7d	update index.html	2020-08-19 09:28:24 +08:00
zu1k	6b462f6310	update readme	2020-08-19 08:34:52 +08:00
zu1k	32c1c7c127	update readme	2020-08-19 08:10:11 +08:00
zu1k	be46f914de	add heroku app.json	2020-08-19 08:05:19 +08:00
zu1k	89338d7152	remove nouse getter	2020-08-19 06:50:45 +08:00
zu1k	1fee5e87ff	rename func	2020-08-18 19:07:52 +08:00
zu1k	2ba481e388	clean for all	2020-08-18 19:06:47 +08:00
zu1k	daea47552d	opti	2020-08-18 18:37:38 +08:00
zu1k	59ea871190	fix proxy list clone	2020-08-18 18:31:45 +08:00
zu1k	ef44ef7436	use deep copy	2020-08-18 16:42:30 +08:00
zu1k	f1ec6c6bf8	fix port long time no bind	2020-08-18 16:24:20 +08:00
zu1k	e91c024e7b	change type all	2020-08-18 16:18:08 +08:00
zu1k	32dadde123	add more tgchannel	2020-08-18 11:14:38 +08:00
zu1k	573f8cb312	add more webfuzz	2020-08-18 10:53:39 +08:00
zu1k	67457a5a9e	add more subs	2020-08-18 10:49:56 +08:00
zu1k	b61dd2d8ec	add heroku support	2020-08-18 10:08:38 +08:00
zu1k	c0ee23cb98	add heroku metrics	2020-08-18 09:51:24 +08:00
zu1k	f3c3397844	add ssrtool	2020-08-18 09:00:26 +08:00
zu1k	8b2b109ca9	add more source	2020-08-17 20:56:07 +08:00
zu1k	b2b7c94b68	add more sources	2020-08-17 17:31:26 +08:00
zu1k	119873e704	add webfuzzsub getter	2020-08-17 11:54:38 +08:00
zu1k	4fd88553ba	opti name	2020-08-17 11:08:47 +08:00
zu1k	12da2a73d6	remove duplicate code	2020-08-17 09:17:39 +08:00
zu1k	a29d02c88e	close chan when noneed	2020-08-16 22:32:09 +08:00
zu1k	7d0cac882c	add debug charts	2020-08-16 21:15:03 +08:00
zu1k	36d199a200	fix country name is none	2020-08-16 12:45:51 +08:00
zu1k	7b74ed4419	fix proxy checker goroutine leak	2020-08-16 12:41:31 +08:00
zu1k	892ba586ed	fix for go test	2020-08-16 12:16:54 +08:00
zu1k	151184389a	Merge branch 'master' of github.com:zu1k/proxypool	2020-08-16 11:35:41 +08:00
zu1k	23a337e845	add geo parse	2020-08-16 11:35:28 +08:00
zu1k	b1b82b5d4e	Update source.yaml	2020-08-16 10:16:35 +08:00
zu1k	e13892a984	update readme	2020-08-16 06:57:48 +08:00
zu1k	95fa5a72c6	fix index out of range	2020-08-16 06:42:56 +08:00
zu1k	996c1d7cf3	check health for all proxies	2020-08-16 06:41:16 +08:00
zu1k	30a9cae5aa	fix	2020-08-15 20:10:41 +08:00
zu1k	8ca9893650	add source config test	2020-08-15 20:06:58 +08:00
zu1k	d7aa135666	add more subs	2020-08-15 19:29:36 +08:00
zu1k	e4b97202cd	Merge branch 'master' of github.com:zu1k/proxypool	2020-08-15 19:26:28 +08:00
zu1k	b0b72c19c9	add debug tool	2020-08-15 19:26:14 +08:00
zu1k	ac203e945a	Merge pull request #7 from hulb/fix fix type assert and Dockerfile	2020-08-15 17:08:33 +08:00
zu1k	222800134f	surge check	2020-08-15 16:42:07 +08:00
zu1k	960b469d9b	remove debug code	2020-08-15 15:17:55 +08:00
zu1k	6d947f4fbb	add base health check	2020-08-15 15:14:52 +08:00
hulb	ff3f5ae893	fix Dockerfile	2020-08-15 11:43:14 +08:00
hulb	84fc16357e	type assert fix	2020-08-15 10:50:50 +08:00
zu1k	cfec56c9b5	update html	2020-08-14 13:18:32 +08:00
zu1k	3ca9e86a29	add type filter	2020-08-14 13:13:11 +08:00
zu1k	a32b2a79bf	update html	2020-08-14 12:54:51 +08:00
zu1k	00cc03d6dc	update html	2020-08-14 12:03:39 +08:00
zu1k	8cb970c010	update html	2020-08-14 11:19:59 +08:00
zu1k	bc8002e656	add return	2020-08-14 11:08:06 +08:00
zu1k	ce1e4938d2	try to fix obfs-host none	2020-08-14 11:07:30 +08:00
zu1k	d9790fb7c6	fix ws-path	2020-08-14 11:02:26 +08:00
zu1k	01341f2894	ws-path not none	2020-08-14 10:58:32 +08:00
zu1k	36f7fb11b0	fix ss cipher auto	2020-08-14 10:51:58 +08:00
zu1k	a1b46e9fa4	fix vmess wsheader none	2020-08-14 10:46:38 +08:00
zu1k	3c68aafd18	update readme’	2020-08-14 10:40:09 +08:00
zu1k	5ec8cfcc43	vmess ws-header ignore none	2020-08-14 10:34:43 +08:00
zu1k	272a1c1277	ignore parse ss	2020-08-14 10:32:55 +08:00
zu1k	23bdc67444	checkSurgesupport	2020-08-14 10:30:10 +08:00
zu1k	8f5d76a471	mod assets	2020-08-14 10:27:40 +08:00
zu1k	5222a11d20	try surge list	2020-08-14 10:24:15 +08:00
zu1k	5329a7b545	fix’	2020-08-14 08:18:56 +08:00
zu1k	180c5b7943	??	2020-08-14 08:16:57 +08:00
zu1k	0d30186ea4	fix config file parse err	2020-08-14 08:14:21 +08:00
zu1k	956910881e	Merge branch 'master' of github.com:zu1k/proxypool	2020-08-14 08:11:14 +08:00
zu1k	d885b747fa	add ss another link support	2020-08-14 08:11:07 +08:00
zu1k	14b8577186	add tg sslist	2020-08-14 07:43:03 +08:00
zu1k	3653819e28	add github link	2020-08-13 21:16:25 +08:00
zu1k	055e1f3db0	update readme	2020-08-13 08:57:42 +08:00
zu1k	15b4ce563c	update readme	2020-08-13 08:54:02 +08:00
zu1k	abb65edfb2	add readme	2020-08-13 08:50:09 +08:00
zu1k	cc61798fdf	add tgchannel	2020-08-13 08:33:32 +08:00
zu1k	1a6ad785e6	add tips	2020-08-13 08:27:30 +08:00
zu1k	7db05c31d0	fix source config	2020-08-13 08:22:35 +08:00
zu1k	8a2181f822	auto fetch new config & add subs	2020-08-13 08:18:52 +08:00
zu1k	f0cc329db2	add subs	2020-08-13 08:07:23 +08:00
zu1k	ab9c13d852	use custom http client	2020-08-12 20:49:07 +08:00
zu1k	88469325bd	add subs	2020-08-12 20:24:05 +08:00
zu1k	b16e1d6b89	add sub	2020-08-12 20:13:16 +08:00
zu1k	d12a581575	order imports	2020-08-12 20:08:49 +08:00
zu1k	8f510c21fe	fix send on close channel	2020-08-12 19:57:57 +08:00
zu1k	21f55ac81d	use config file	2020-08-12 19:52:57 +08:00
zu1k	c87b5872e2	show count	2020-08-12 17:58:51 +08:00
zu1k	304de88e96	Merge branch 'master' of github.com:zu1k/proxypool	2020-08-12 17:54:36 +08:00
zu1k	db47de2747	add coroutine	2020-08-12 17:54:22 +08:00
zu1k	e6c935f13f	add shadowsocks and subscribe support	2020-08-12 17:09:11 +08:00
zu1k	a96d21f4c7	fix vmess re	2020-08-12 16:07:25 +08:00
zu1k	c7f1dc8830	Merge pull request #2 from zu1k/dependabot/github_actions/actions/upload-artifact-v2.1.4 Bump actions/upload-artifact from v2.1.3 to v2.1.4	2020-08-12 14:09:25 +08:00
zu1k	b35e73cbd4	add clash html	2020-08-12 14:01:12 +08:00
zu1k	1d890b0257	add clash	2020-08-12 13:59:26 +08:00
dependabot[bot]	4ba0f7337a	Bump actions/upload-artifact from v2.1.3 to v2.1.4 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from v2.1.3 to v2.1.4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v2.1.3...58740802ef971a2d71eff71e63d48ab68d1f5507) Signed-off-by: dependabot[bot] <support@github.com>	2020-08-12 05:51:44 +00:00
zu1k	3127fe6e2c	cron	2020-08-12 13:50:29 +08:00