order imports

fix send on close channel
use config file
2020-08-12 20:08:49 +08:00 · 2020-08-12 19:57:57 +08:00 · 2020-08-12 19:52:57 +08:00 · 2020-08-12 17:58:51 +08:00 · 2020-08-12 17:54:36 +08:00 · 2020-08-12 17:54:22 +08:00
28 changed files with 849 additions and 287 deletions
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -43,7 +43,7 @@ jobs:
          draft: true
          prerelease: true

-      - uses: actions/upload-artifact@v2.1.3
+      - uses: actions/upload-artifact@v2.1.4
        if: startsWith(github.ref, 'refs/tags/')
        with:
          name: build
--- a/api/router.go
+++ b/api/router.go
@@ -3,10 +3,8 @@ package api
 import (
 	"os"

-	"github.com/zu1k/proxypool/app/cache"
-
 	"github.com/gin-gonic/gin"
-	_ "github.com/heroku/x/hmetrics/onload"
+	"github.com/zu1k/proxypool/app/cache"
 	"github.com/zu1k/proxypool/provider"
 )

@@ -15,6 +13,7 @@ var router *gin.Engine
 func setupRouter() {
 	router = gin.Default()

+	router.StaticFile("/clash", "example/clash.html")
 	router.StaticFile("/clash/config", "example/clash-config.yaml")
 	router.GET("/clash/proxies", func(c *gin.Context) {
 		text := cache.GetString("clashproxies")
--- a/app/cron.go
+++ b/app/cron.go
@@ -5,7 +5,6 @@ import (
 )

 func Cron() {
-	gocron.Every(10).Minutes().Do(CrawlTGChannel)
-
+	_ = gocron.Every(10).Minutes().Do(CrawlGo)
 	<-gocron.Start()
 }
--- a/app/getter.go
+++ b/app/getter.go
@@ -0,0 +1,21 @@
+package app
+
+import (
+	"fmt"
+
+	"github.com/zu1k/proxypool/config"
+	"github.com/zu1k/proxypool/getter"
+)
+
+var Getters = make([]getter.Getter, 0)
+
+func InitGetters(sources []config.Source) {
+	for _, source := range sources {
+		g := getter.NewGetter(source.Type, source.Options)
+		if g != nil {
+			Getters = append(Getters, g)
+			fmt.Println("init getter:", source.Type, source.Options)
+		}
+	}
+	fmt.Println("Getter count:", len(Getters))
+}
--- a/app/task.go
+++ b/app/task.go
@@ -1,48 +1,57 @@
 package app

 import (
+	"log"
 	"math/rand"
 	"strconv"
-
-	"github.com/zu1k/proxypool/provider"
+	"sync"

 	"github.com/zu1k/proxypool/app/cache"
-
-	"github.com/zu1k/proxypool/getter"
+	"github.com/zu1k/proxypool/provider"
 	"github.com/zu1k/proxypool/proxy"
 )

-func CrawlTGChannel() {
+func Crawl() {
 	proxies := make([]proxy.Proxy, 0)
-
-	// tg上各种节点分享频道
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrList", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/SSRSUB", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/FreeSSRNode", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrlists", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrshares", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/V2List", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrtool", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/vmessr", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/FreeSSR666", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/fanqiang666", 200).Get()...)
-
-	// 各种网站上公开的
-	proxies = append(proxies, getter.WebFreessrXyz{}.Get()...)
-	proxies = append(proxies, getter.WebLucnOrg{}.Get()...)
-
-	// 从web页面模糊获取
-	proxies = append(proxies, getter.NewWebFuzz("https://zfjvpn.gitbook.io/").Get()...)
-	proxies = append(proxies, getter.NewWebFuzz("https://www.freefq.com/d/file/free-ssr/20200811/1f3e9d0d0064f662457062712dcf1b66.txt").Get()...)
-	proxies = append(proxies, getter.NewWebFuzz("https://merlinblog.xyz/wiki/freess.html").Get()...)
-
+	for _, g := range Getters {
+		proxies = append(proxies, g.Get()...)
+	}
 	proxies = append(proxies, cache.GetProxies()...)
 	proxies = proxy.Deduplication(proxies)

 	num := len(proxies)
 	for i := 0; i < num; i++ {
-		proxies[i].SetName("@tgbotlist_" + strconv.Itoa(rand.Int()))
+		proxies[i].SetName(strconv.Itoa(rand.Int()))
 	}
+	log.Println("Crawl node count:", num)
+	cache.SetProxies(proxies)
+	cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
+}
+
+func CrawlGo() {
+	wg := &sync.WaitGroup{}
+	var pc = make(chan proxy.Proxy)
+	for _, g := range Getters {
+		wg.Add(1)
+		go g.Get2Chan(pc, wg)
+	}
+	proxies := cache.GetProxies()
+	go func() {
+		wg.Wait()
+		close(pc)
+	}()
+	for node := range pc {
+		if node != nil {
+			proxies = append(proxies, node)
+		}
+	}
+	proxies = proxy.Deduplication(proxies)
+
+	num := len(proxies)
+	for i := 0; i < num; i++ {
+		proxies[i].SetName(strconv.Itoa(rand.Int()))
+	}
+	log.Println("CrawlGo node count:", num)
 	cache.SetProxies(proxies)
 	cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
 }
--- a/config/config.go
+++ b/config/config.go
@@ -0,0 +1,50 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/ghodss/yaml"
+	"github.com/zu1k/proxypool/tool"
+)
+
+type Source struct {
+	Type    string       `json:"type" yaml:"type"`
+	Options tool.Options `json:"options" yaml:"options"`
+}
+
+type Config struct {
+	Sources []Source `json:"sources" yaml:"sources"`
+}
+
+var SourceConfig = Config{}
+
+func Parse(path string) (*Config, error) {
+	fileData, err := readFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(fileData, &SourceConfig)
+	if err != nil {
+		return nil, err
+	}
+	return &SourceConfig, nil
+}
+
+func readFile(path string) ([]byte, error) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil, err
+	}
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(data) == 0 {
+		return nil, fmt.Errorf("Configuration file %s is empty", path)
+	}
+
+	return data, err
+}
--- a/example/clash.html
+++ b/example/clash.html
@@ -0,0 +1,83 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="HandheldFriendly" content="True">
+	<title>免费Clash节点</title>
+	<style type='text/css'>
+	  body {
+		background-color: white;
+		color: #333333;
+		  font-family: Arial, sans-serif;
+		margin: 0;
+		padding: 36px;
+		line-height: 1;
+		font-size: 14px; }
+
+	  .section {
+		margin-bottom: 36px; }
+		.section.friendly {
+		  color: #222222; }
+		  .section.friendly h1 {
+			font-size: 26px;
+			background-color: #dad8e4;
+			padding: 18px 22px 15px 22px;
+			margin: 0;
+			overflow: hidden; }
+			.section.friendly h1 strong {
+			  display: inline-block;
+			  float: left; }
+			.section.friendly h1 small {
+			  display: inline-block;
+			  float: right;
+			  text-align: right;
+			  font-size: 18px;
+			  padding-top: 4px;
+			  color: #333333; }
+		  .section.friendly .article {
+			border: 4px solid #dad8e4;
+			padding: 24px 18px 18px 18px; }
+			.section.friendly .article h3 {
+			  font-size: 20px;
+			  margin: 0 0 18px 0; }
+			.section.friendly .article a {
+			  color: #6b6ceb; }
+			  .section.friendly .article a:visited {
+				color: #1d1d3b; }
+			.section.friendly .article p {
+			  font-size: 14px; }
+			.section.friendly .article ul {
+			  list-style-type: square; }
+	  .section.original {
+		background-color: #eeeeee;
+		color: #444444; }
+		.section.original h2 {
+		  background-color: #dddddd;
+		  margin: 0;
+		  padding: 18px 22px 18px 22px;
+		  font-size: 20px; }
+		.section.original pre {
+		  margin: 0;
+		  padding: 18px 22px 18px 22px;
+		  overflow: auto;
+		  font-family: monaco, monospaced; }
+		  .section.original pre code {
+			display: block;
+			font-size: 11px;
+			width: 100%; }
+	</style>
+</head>
+<body>
+<div class='container'>
+  <div class='section friendly'>
+	<h1><strong>免费Clash节点</strong></h1>
+	<div class='article'>
+	  <p>所有节点均从公开互联网上爬取，每10分钟自动更新</p>
+	  <p>Clash配置文件：<a href="https://proxy.tgbot.co/clash/config">https://proxy.tgbot.co/clash/config</a></p>
+	  <p>Clash proxy-provider：<a href="https://proxy.tgbot.co/clash/proxies">https://proxy.tgbot.co/clash/proxies</a></p>
+	</div>
+  </div>
+</div>
+</body>
+</html>
--- a/example/source.yaml
+++ b/example/source.yaml
@@ -0,0 +1,24 @@
+sources:
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/v2ray
+
+  - type: webfuzz
+    options:
+      url: https://merlinblog.xyz/wiki/freess.html
+
+  - type: tgchannel
+    options:
+      channel: ssrList
+      num: 200
+
+  - type: web-fanqiangdang
+    options:
+      url: https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0
+      num: 200
+
+  - type: web-freessrxyz
+    options:
+
+  - type: web-lucnorg
+    options:
--- a/getter/base.go
+++ b/getter/base.go
@@ -2,12 +2,31 @@ package getter

 import (
 	"strings"
+	"sync"

 	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
 )

 type Getter interface {
 	Get() []proxy.Proxy
+	Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup)
+}
+
+type creator func(options tool.Options) Getter
+
+var creatorMap = make(map[string]creator)
+
+func Register(sourceType string, c creator) {
+	creatorMap[sourceType] = c
+}
+
+func NewGetter(sourceType string, options tool.Options) Getter {
+	c, ok := creatorMap[sourceType]
+	if ok {
+		return c(options)
+	}
+	return nil
 }

 func String2Proxy(link string) proxy.Proxy {
@@ -17,6 +36,8 @@ func String2Proxy(link string) proxy.Proxy {
 		data, err = proxy.ParseSSRLink(link)
 	} else if strings.HasPrefix(link, "vmess://") {
 		data, err = proxy.ParseVmessLink(link)
+	} else if strings.HasPrefix(link, "ss://") {
+		data, err = proxy.ParseSSLink(link)
 	}
 	if err != nil {
 		return nil
@@ -25,19 +46,20 @@ func String2Proxy(link string) proxy.Proxy {
 }

 func StringArray2ProxyArray(origin []string) []proxy.Proxy {
-	var err error
 	results := make([]proxy.Proxy, 0)
 	for _, link := range origin {
-		var data proxy.Proxy
-		if strings.HasPrefix(link, "ssr://") {
-			data, err = proxy.ParseSSRLink(link)
-		} else if strings.HasPrefix(link, "vmess://") {
-			data, err = proxy.ParseVmessLink(link)
-		}
-		if err != nil {
-			continue
-		}
-		results = append(results, data)
+		results = append(results, String2Proxy(link))
 	}
 	return results
 }
+
+func GrepLinksFromString(text string) []string {
+	results := proxy.GrepSSRLinkFromString(text)
+	results = append(results, proxy.GrepVmessLinkFromString(text)...)
+	results = append(results, proxy.GrepSSLinkFromString(text)...)
+	return results
+}
+
+func FuzzParseProxyFromString(text string) []proxy.Proxy {
+	return StringArray2ProxyArray(GrepLinksFromString(text))
+}
--- a/getter/subscribe.go
+++ b/getter/subscribe.go
@@ -1,13 +1,58 @@
 package getter

-import "github.com/zu1k/proxypool/proxy"
+import (
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
+)
+
+func init() {
+	Register("subscribe", NewSubscribe)
+}

 type Subscribe struct {
-	NumNeeded int
-	Results   []string
-	Url       string
+	Url string
 }

-func (s Subscribe) Get() []proxy.Proxy {
+func (s *Subscribe) Get() []proxy.Proxy {
+	resp, err := http.Get(s.Url)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil
+	}
+
+	nodesString, err := tool.Base64DecodeString(string(body))
+	if err != nil {
+		return nil
+	}
+	nodesString = strings.ReplaceAll(nodesString, "\t", "")
+
+	nodes := strings.Split(nodesString, "\n")
+	return StringArray2ProxyArray(nodes)
+}
+
+func (s *Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := s.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
+func NewSubscribe(options tool.Options) Getter {
+	url, found := options["url"]
+	if found {
+		return &Subscribe{
+			Url: url.(string),
+		}
+	}
 	return nil
 }
--- a/getter/tgchannel.go
+++ b/getter/tgchannel.go
@@ -2,49 +2,67 @@ package getter

 import (
 	"fmt"
+	"sync"

 	"github.com/gocolly/colly"
 	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
 )

+func init() {
+	Register("tgchannel", NewTGChannelGetter)
+}
+
 type TGChannelGetter struct {
 	c         *colly.Collector
 	NumNeeded int
-	Results   []string
+	results   []string
 	Url       string
 }

-func NewTGChannelGetter(url string, numNeeded int) *TGChannelGetter {
-	if numNeeded <= 0 {
-		numNeeded = 200
+func NewTGChannelGetter(options tool.Options) Getter {
+	num, found := options["num"]
+	if !found || int(num.(float64)) <= 0 {
+		num = 200
 	}
-	return &TGChannelGetter{
-		c:         colly.NewCollector(),
-		NumNeeded: numNeeded,
-		Results:   make([]string, 0),
-		Url:       url,
+	url, found := options["channel"]
+	if found {
+		return &TGChannelGetter{
+			c:         colly.NewCollector(),
+			NumNeeded: int(num.(float64)),
+			Url:       "https://t.me/s/" + url.(string),
+		}
 	}
+	return nil
 }

-func (g TGChannelGetter) Get() []proxy.Proxy {
+func (g *TGChannelGetter) Get() []proxy.Proxy {
+	g.results = make([]string, 0)
 	// 找到所有的文字消息
 	g.c.OnHTML("div.tgme_widget_message_text", func(e *colly.HTMLElement) {
-		g.Results = append(g.Results, proxy.GrepSSRLinkFromString(e.Text)...)
-		g.Results = append(g.Results, proxy.GrepVmessLinkFromString(e.Text)...)
+		g.results = append(g.results, GrepLinksFromString(e.Text)...)
 	})

 	// 找到之前消息页面的链接，加入访问队列
 	g.c.OnHTML("link[rel=prev]", func(e *colly.HTMLElement) {
-		if len(g.Results) < g.NumNeeded {
+		if len(g.results) < g.NumNeeded {
 			_ = e.Request.Visit(e.Attr("href"))
 		}
 	})

-	g.Results = make([]string, 0)
+	g.results = make([]string, 0)
 	err := g.c.Visit(g.Url)
 	if err != nil {
 		_ = fmt.Errorf("%s", err.Error())
 	}

-	return StringArray2ProxyArray(g.Results)
+	return StringArray2ProxyArray(g.results)
+}
+
+func (g *TGChannelGetter) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := g.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
 }
--- a/getter/web_fanqiangdang.go
+++ b/getter/web_fanqiangdang.go
@@ -0,0 +1,68 @@
+package getter
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/gocolly/colly"
+	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
+)
+
+func init() {
+	Register("web-fanqiangdang", NewWebFanqiangdangGetter)
+}
+
+type WebFanqiangdang struct {
+	c         *colly.Collector
+	NumNeeded int
+	Url       string
+	results   []string
+}
+
+func NewWebFanqiangdangGetter(options tool.Options) Getter {
+	num, found := options["num"]
+	if !found || int(num.(float64)) <= 0 {
+		num = 200
+	}
+	url, found := options["url"]
+	if found {
+		return &WebFanqiangdang{
+			c:         colly.NewCollector(),
+			NumNeeded: int(num.(float64)),
+			Url:       url.(string),
+		}
+	}
+	return nil
+}
+
+func (w *WebFanqiangdang) Get() []proxy.Proxy {
+	w.results = make([]string, 0)
+	// 找到所有的文字消息
+	w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) {
+		w.results = append(w.results, GrepLinksFromString(e.Text)...)
+	})
+
+	// 从订阅中取出每一页，因为是订阅，所以都比较新
+	w.c.OnXML("//item//link", func(e *colly.XMLElement) {
+		if len(w.results) < w.NumNeeded {
+			_ = e.Request.Visit(e.Text)
+		}
+	})
+
+	w.results = make([]string, 0)
+	err := w.c.Visit(w.Url)
+	if err != nil {
+		_ = fmt.Errorf("%s", err.Error())
+	}
+
+	return StringArray2ProxyArray(w.results)
+}
+
+func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
--- a/getter/web_free_ssr_xyz.go
+++ b/getter/web_free_ssr_xyz.go
@@ -4,10 +4,16 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
+	"sync"

 	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
 )

+func init() {
+	Register("web-freessrxyz", NewWebFreessrxyzGetter)
+}
+
 const (
 	freessrxyzSsrLink   = "https://api.free-ssr.xyz/ssr"
 	freessrxyzV2rayLink = "https://api.free-ssr.xyz/v2ray"
@@ -16,12 +22,24 @@ const (
 type WebFreessrXyz struct {
 }

-func (w WebFreessrXyz) Get() []proxy.Proxy {
+func NewWebFreessrxyzGetter(options tool.Options) Getter {
+	return &WebFreessrXyz{}
+}
+
+func (w *WebFreessrXyz) Get() []proxy.Proxy {
 	results := freessrxyzFetch(freessrxyzSsrLink)
 	results = append(results, freessrxyzFetch(freessrxyzV2rayLink)...)
 	return results
 }

+func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
 func freessrxyzFetch(link string) []proxy.Proxy {
 	resp, err := http.Get(link)
 	if err != nil {
--- a/getter/web_fuzz.go
+++ b/getter/web_fuzz.go
@@ -3,15 +3,21 @@ package getter
 import (
 	"io/ioutil"
 	"net/http"
+	"sync"

 	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
 )

+func init() {
+	Register("webfuzz", NewWebFuzzGetter)
+}
+
 type WebFuzz struct {
 	Url string
 }

-func (w WebFuzz) Get() []proxy.Proxy {
+func (w *WebFuzz) Get() []proxy.Proxy {
 	resp, err := http.Get(w.Url)
 	if err != nil {
 		return nil
@@ -21,14 +27,21 @@ func (w WebFuzz) Get() []proxy.Proxy {
 	if err != nil {
 		return nil
 	}
-	text := string(body)
-
-	results := proxy.GrepSSRLinkFromString(text)
-	results = append(results, proxy.GrepVmessLinkFromString(text)...)
-
-	return StringArray2ProxyArray(results)
+	return FuzzParseProxyFromString(string(body))
 }

-func NewWebFuzz(url string) *WebFuzz {
-	return &WebFuzz{Url: url}
+func (w *WebFuzz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
+func NewWebFuzzGetter(options tool.Options) Getter {
+	url, found := options["url"]
+	if found {
+		return &WebFuzz{Url: url.(string)}
+	}
+	return nil
 }
--- a/getter/web_lucn_org.go
+++ b/getter/web_lucn_org.go
@@ -5,18 +5,26 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
-
-	"github.com/zu1k/proxypool/tool"
+	"sync"

 	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
 )

+func init() {
+	Register("web-lucnorg", NewWebLucnorg)
+}
+
 const lucnorgSsrLink = "https://lncn.org/api/ssrList"

 type WebLucnOrg struct {
 }

-func (w WebLucnOrg) Get() []proxy.Proxy {
+func NewWebLucnorg(options tool.Options) Getter {
+	return &WebLucnOrg{}
+}
+
+func (w *WebLucnOrg) Get() []proxy.Proxy {
 	resp, err := http.Post(lucnorgSsrLink, "", nil)
 	if err != nil {
 		return nil
@@ -57,6 +65,14 @@ func (w WebLucnOrg) Get() []proxy.Proxy {
 	return StringArray2ProxyArray(result)
 }

+func (w *WebLucnOrg) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+	defer wg.Done()
+	nodes := w.Get()
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
 func decryptAesForLucn(code string, c string) []byte {
 	if code == "" {
 		code = "abclnv561cqqfg30"
--- a/getter/web_test.go
+++ b/getter/web_test.go
@@ -1,18 +0,0 @@
-package getter
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestWebLucnOrg_Get(t *testing.T) {
-	fmt.Println(WebLucnOrg{}.Get())
-}
-
-func TestWebFreessrXyz_Get(t *testing.T) {
-	fmt.Println(WebFreessrXyz{}.Get())
-}
-
-func TestWebFuzz_Get(t *testing.T) {
-	fmt.Println(NewWebFuzz("https://merlinblog.xyz/wiki/freess.html").Get())
-}
--- a/go.mod
+++ b/go.mod
@@ -8,23 +8,26 @@ require (
 	github.com/antchfx/htmlquery v1.2.3 // indirect
 	github.com/antchfx/xmlquery v1.2.4 // indirect
 	github.com/antchfx/xpath v1.1.8 // indirect
+	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-playground/validator/v10 v10.3.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gocolly/colly v1.2.0
 	github.com/golang/protobuf v1.4.2 // indirect
-	github.com/heroku/x v0.0.25
 	github.com/jasonlvhit/gocron v0.0.1
 	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/stretchr/testify v1.5.1 // indirect
 	github.com/temoto/robotstxt v1.1.1 // indirect
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
+	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
 	golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -1,12 +1,7 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-contrib.go.opencensus.io/exporter/ocagent v0.6.0/go.mod h1:zmKjrJcdo0aYcVS7bmEeSEBLPA9YJp5bjrofdU3pIXs=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
-github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
 github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
@@ -17,30 +12,20 @@ github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7
 github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
 github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk=
 github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
-github.com/armon/go-proxyproto v0.0.0-20190211145416-68259f75880e/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU=
-github.com/aws/aws-sdk-go v1.13.10/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
-github.com/axiomhq/hyperloglog v0.0.0-20180317131949-fe9507de0228/go.mod h1:IOXAcuKIFq/mDyuQ4wyJuJ79XLMsmLM+5RdQ+vWrL7o=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
-github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-ini/ini v1.33.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
@@ -52,22 +37,14 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o=
 github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
 github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
-github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -80,8 +57,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/gomodule/redigo v1.8.1/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -89,54 +64,22 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gops v0.3.8-0.20200229223415-3a98d6d24562/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s=
-github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.6/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/heroku/rollrus v0.2.0/go.mod h1:B3MwEcr9nmf4xj0Sr5l9eSht7wLKMa1C+9ajgAU79ek=
-github.com/heroku/x v0.0.25 h1:6/2rENs6rfUds9jOxy9oO59b0fiOx/FeTCGOyLMXcZw=
-github.com/heroku/x v0.0.25/go.mod h1:qE/I0jp6rIeTBBosrPYV4ygRX3OMhqmC/A6x8ewodJQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/joeshaw/envdecode v0.0.0-20180129163420-d5f34bca07f3/go.mod h1:Q+alOFAXgW5SrcfMPt/G4B2oN+qEcQRJjkn/f4mKL04=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
-github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/leesper/go_rng v0.0.0-20171009123644-5344a9259b21/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/lstoll/grpce v1.7.0/go.mod h1:XiCWl3R+avNCT7KsTjv3qCblgsSqd0SC4ymySrH226g=
-github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
@@ -147,37 +90,17 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.2-0.20190227000051-27936f6d90f9/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
-github.com/rcrowley/go-metrics v0.0.0-20160613154715-cfa5a85e9f0a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
-github.com/rollbar/rollbar-go v1.2.0/go.mod h1:czC86b8U4xdUH7W2C6gomi2jutLm8qK0OtrF5WMvpcc=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
-github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soveran/redisurl v0.0.0-20180322091936-eb325bc7a4b8/go.mod h1:FVJ8jbHu7QrNFs3bZEsv/L5JjearIAY9N0oXh2wk+6Y=
-github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
@@ -188,114 +111,57 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/unrolled/secure v1.0.1/go.mod h1:R6rugAuzh4TQpbFAq69oqZggyBQxFRFQIewtz5z7Jsc=
-github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
-github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed h1:WBkVNH1zd9jg/dK4HCM4lNANnmd12EHC9z+LmcCG4ns=
 golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.0.0-20190502212712-4a2eb0188cbc/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
-gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -308,23 +174,15 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-gopkg.in/caio/go-tdigest.v2 v2.3.0/go.mod h1:HPfh/CLN8UWDMOC76lqxVeKa5E24ypoVuTj4BLMb9cU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
--- a/main.go
+++ b/main.go
@@ -1,14 +1,31 @@
 package main

 import (
+	"flag"
 	"fmt"
+	"os"

 	"github.com/zu1k/proxypool/api"
 	"github.com/zu1k/proxypool/app"
+	"github.com/zu1k/proxypool/config"
 )

 func main() {
+	filePath := flag.String("c", "source.yaml", "path to config file: source.yaml")
+	flag.Parse()
+	c, err := config.Parse(*filePath)
+	if err != nil {
+		fmt.Println("Error: ", err.Error())
+		os.Exit(1)
+	}
+	if c == nil {
+		fmt.Println("Error: no sources")
+		os.Exit(2)
+	}
+
+	app.InitGetters(c.Sources)
+	go app.Cron()
 	fmt.Println("Do the first crawl...")
-	app.CrawlTGChannel()
+	app.CrawlGo()
 	api.Run()
 }
--- a/provider/clash.go
+++ b/provider/clash.go
@@ -35,6 +35,11 @@ func checkClashSupport(p proxy.Proxy) bool {
 		if checkInList(vmessCipherList, vmess.Cipher) {
 			return true
 		}
+	case *proxy.Shadowsocks:
+		ss := p.(*proxy.Shadowsocks)
+		if checkInList(ssCipherList, ss.Cipher) {
+			return true
+		}
 	default:
 		return false
 	}
@@ -102,3 +107,20 @@ var vmessCipherList = []string{
 	"chacha20-poly1305",
 	"none",
 }
+
+var ssCipherList = []string{
+	"aes-128-gcm",
+	"aes-192-gcm",
+	"aes-256-gcm",
+	"aes-128-cfb",
+	"aes-192-cfb",
+	"aes-256-cfb",
+	"aes-128-ctr",
+	"aes-192-ctr",
+	"aes-256-ctr",
+	"rc4-md5",
+	"chacha20-ietf",
+	"xchacha20",
+	"chacha20-ietf-poly1305",
+	"xchacha20-ietf-poly1305",
+}
--- a/proxy/base.go
+++ b/proxy/base.go
@@ -19,9 +19,11 @@ func Deduplication(src []Proxy) []Proxy {
 	result := make([]Proxy, 0, len(src))
 	temp := map[string]struct{}{}
 	for _, item := range src {
-		if _, ok := temp[item.Identifier()]; !ok {
-			temp[item.Identifier()] = struct{}{}
-			result = append(result, item)
+		if item != nil {
+			if _, ok := temp[item.Identifier()]; !ok {
+				temp[item.Identifier()] = struct{}{}
+				result = append(result, item)
+			}
 		}
 	}
 	return result
--- a/proxy/shadowsocks.go
+++ b/proxy/shadowsocks.go
@@ -0,0 +1,121 @@
+package proxy
+
+import (
+	"encoding/json"
+	"errors"
+	"math/rand"
+	"net"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/zu1k/proxypool/tool"
+)
+
+var (
+	ErrorNotSSLink = errors.New("not a correct ss link")
+)
+
+type Shadowsocks struct {
+	Base
+	Password   string                 `yaml:"password" json:"password"`
+	Cipher     string                 `yaml:"cipher" json:"cipher"`
+	Plugin     string                 `yaml:"plugin,omitempty" json:"plugin,omitempty"`
+	PluginOpts map[string]interface{} `yaml:"plugin-opts,omitempty" json:"plugin-opts,omitempty"`
+}
+
+func (ss Shadowsocks) Identifier() string {
+	return net.JoinHostPort(ss.Server, strconv.Itoa(ss.Port)) + ss.Password
+}
+
+func (ss Shadowsocks) String() string {
+	data, err := json.Marshal(ss)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+func (ss Shadowsocks) ToClash() string {
+	data, err := json.Marshal(ss)
+	if err != nil {
+		return ""
+	}
+	return "- " + string(data)
+}
+
+func (ss *Shadowsocks) SetName(name string) {
+	ss.Name = name
+}
+
+func ParseSSLink(link string) (*Shadowsocks, error) {
+	if !strings.HasPrefix(link, "ss://") {
+		return nil, ErrorNotSSRLink
+	}
+
+	uri, err := url.Parse(link)
+	if err != nil {
+		return nil, ErrorNotSSLink
+	}
+
+	server := uri.Hostname()
+	port, _ := strconv.Atoi(uri.Port())
+
+	cipherInfoString, err := tool.Base64DecodeString(uri.User.Username())
+	if err != nil {
+		return nil, ErrorPasswordParseFail
+	}
+	cipherInfo := strings.SplitN(cipherInfoString, ":", 2)
+	if len(cipherInfo) < 2 {
+		return nil, ErrorPasswordParseFail
+	}
+	cipher := strings.ToLower(cipherInfo[0])
+	password := cipherInfo[1]
+
+	moreInfos := uri.Query()
+	pluginString := moreInfos.Get("plugin")
+	plugin := ""
+	pluginOpts := make(map[string]interface{})
+	if strings.Contains(pluginString, ";") {
+		pluginInfos, err := url.ParseQuery(pluginString)
+		if err == nil {
+			if strings.Contains(pluginString, "obfs") {
+				plugin = "obfs"
+				pluginOpts["mode"] = pluginInfos.Get("obfs")
+				pluginOpts["host"] = pluginInfos.Get("obfs-host")
+			} else if strings.Contains(pluginString, "v2ray") {
+				plugin = "v2ray-plugin"
+				pluginOpts["mode"] = pluginInfos.Get("mode")
+				pluginOpts["host"] = pluginInfos.Get("host")
+				pluginOpts["tls"] = strings.Contains(pluginString, "tls")
+			}
+		}
+	}
+
+	return &Shadowsocks{
+		Base: Base{
+			Name:   strconv.Itoa(rand.Int()),
+			Server: server,
+			Port:   port,
+			Type:   "ss",
+		},
+		Password:   password,
+		Cipher:     cipher,
+		Plugin:     plugin,
+		PluginOpts: pluginOpts,
+	}, nil
+}
+
+var (
+	ssPlainRe = regexp.MustCompile("ss://([A-Za-z0-9+/_&?=@:%.-])+")
+)
+
+func GrepSSLinkFromString(text string) []string {
+	results := make([]string, 0)
+	texts := strings.Split(text, "ss://")
+	for _, text := range texts {
+		results = append(results, ssPlainRe.FindAllString("ss://"+text, -1)...)
+	}
+	return results
+}
--- a/proxy/vmess.go
+++ b/proxy/vmess.go
@@ -3,7 +3,6 @@ package proxy
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"math/rand"
 	"net"
 	"net/url"
@@ -103,7 +102,6 @@ func ParseVmessLink(link string) (*Vmess, error) {
 		if err != nil {
 			return nil, ErrorVmessPayloadParseFail
 		}
-		fmt.Println(baseInfo)
 		baseInfoPath := strings.Split(baseInfo, ":")
 		if len(baseInfoPath) < 3 {
 			return nil, ErrorPathNotComplete
@@ -123,7 +121,6 @@ func ParseVmessLink(link string) (*Vmess, error) {
 		}

 		moreInfo, _ := url.ParseQuery(infoPayloads[1])
-		fmt.Println(moreInfo)
 		remarks := moreInfo.Get("remarks")
 		obfs := moreInfo.Get("obfs")
 		network := "tcp"
@@ -204,7 +201,7 @@ func ParseVmessLink(link string) (*Vmess, error) {
 }

 var (
-	vmessPlainRe = regexp.MustCompile("vmess://([A-Za-z0-9+/_-])+")
+	vmessPlainRe = regexp.MustCompile("vmess://([A-Za-z0-9+/_?&=-])+")
 )

 func GrepVmessLinkFromString(text string) []string {
--- a/proxy/vmess_test.go
+++ b/proxy/vmess_test.go
@@ -1,13 +0,0 @@
-package proxy
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestParseVmessLink(t *testing.T) {
-	//link := "vmess://YXV0bzo5MGQ0YTM3NC1kYWU4LTExZWEtODY3Zi01NjAwMDJlY2MzNDlAZml2ZWRlbWFuZHMubWw6NDQz?remarks=%E6%97%A5%E6%9C%AC5%EF%BC%9A%E7%94%B5%E6%8A%A5%E9%A2%91%E9%81%93%EF%BC%9A"
-
-	link := "vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjI5LeS7mOi0ueaOqOiNkDp0LmNuL0VHSkl5cmwiLA0KICAiYWRkIjogImJ1cmdlcmtpbmdnb29kLm1sIiwNCiAgInBvcnQiOiAiNDQzIiwNCiAgImlkIjogIjRiNWJhMzkwLWRhZjQtMTFlYS04ODEwLTU2MDAwMmVjYzk2NyIsDQogICJhaWQiOiAiNDYiLA0KICAibmV0IjogIndzIiwNCiAgInR5cGUiOiAibm9uZSIsDQogICJob3N0IjogImJ1cmdlcmtpbmdnb29kLm1sIiwNCiAgInBhdGgiOiAiL0M2bGt2UzNLLyIsDQogICJ0bHMiOiAidGxzIg0KfQ=="
-	fmt.Println(ParseVmessLink(link))
-}
--- a/source.yaml
+++ b/source.yaml
@@ -0,0 +1,204 @@
+sources:
+  - type: subscribe
+    options:
+      url: https://www.suncloud.fun/link/Su683BQEyUdxtyuY?mu=1&extend=1
+  - type: subscribe
+    options:
+      url: https://sub88.xyz/link/egy4oyqok9aavpcy?sub=3
+  - type: subscribe
+    options:
+      url: https://qiaomenzhuanfx.netlify.com/
+  - type: subscribe
+    options:
+      url: https://sub88.xyz/link/lkY8Jt5Q3SHlffra?mu=2
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/hotsymbol/vpnsetting/master/v2rayopen
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/satrom/V2SSR/master/V2RAY/Sub.txt
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/v2ray
+  - type: subscribe
+    options:
+      url: http://qe83xk711.sabkt.gdipper.com/freev2ray.txt
+  - type: subscribe
+    options:
+      url: https://iwxf.netlify.app/
+  - type: subscribe
+    options:
+      url: https://rss.getfree.win/link/w9ted9eZq1zbC3gx?sub=1
+  - type: subscribe
+    options:
+      url: https://youlianboshi.netlify.com/
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/ss-sub
+  - type: subscribe
+    options:
+      url: https://bujidao302.com/link/IJ6wKEt96Otboilb?mu=2
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/satrom/V2SSR/master/SSR/Day.txt
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ntkernel/lantern/master/vmess_base64.txt
+  - type: subscribe
+    options:
+      url: https://lei-su.me//link/Dw3lmyXXJJTnvsBA?sub=3&extend=1
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/cdp2020/v2ray/master/README.md
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/ssrsub
+  - type: subscribe
+    options:
+      url: https://www.liesauer.net/yogurt/subscribe?ACCESS_TOKEN=DAYxR3mMaZAsaqUb
+  - type: subscribe
+    options:
+      url: https://rss.cnrss.xyz/link/Rcr8h8fvZIWE001U?mu=2
+  - type: subscribe
+    options:
+      url: https://12o.ooo/link/JLisT80gOhRkiFnX?sub=1
+  - type: subscribe
+    options:
+      url: https://sub1.m87sub.xyz/link/pZDgnxhgL6Poh1GQ?sub=3
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/satrom/V2SSR/master/SSR/Sub.txt
+  - type: subscribe
+    options:
+      url: https://sub88.xyz/link/lkY8Jt5Q3SHlffra?mu=2
+  - type: subscribe
+    options:
+      url: https://s.sublank.xyz/subscribe/66056/0yzuMVawjpr/ssr/
+  - type: subscribe
+    options:
+      url: https://sub1.m87sub.xyz/link/pZDgnxhgL6Poh1GQ?sub=1
+  - type: subscribe
+    options:
+      url: http://qe83xk711.sabkt.gdipper.com/freess.txt
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/pojiezhiyuanjun/freev2/master/20200808.txt
+  - type: subscribe
+    options:
+      url: https://sub1.m87sub.xyz/link/2488O6IM4n4nksMG?sub=3
+  - type: subscribe
+    options:
+      url: http://103.72.166.89/v2ray.php
+  - type: subscribe
+    options:
+      url: https://fly.lo-lita.ru/api/v1/client/subscribe?token=94be7fa3d65791197dd10c6a0b757571
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/RaymondHarris971/ssrsub/master/9a075bdee5.txt
+  - type: subscribe
+    options:
+      url: https://bit.ly/2D5fWhX
+  - type: subscribe
+    options:
+      url: https://www.suncloud.fun/link/ja3OIcMTrky6VhN6?mu=1&extend=1
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/voken100g/AutoSSR/master/recent
+  - type: subscribe
+    options:
+      url: https://jiang.netlify.com/
+
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/v2ray
+
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/ssrsub
+
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/ss-sub
+
+  - type: webfuzz
+    options:
+      url: https://zfjvpn.gitbook.io/
+
+  - type: webfuzz
+    options:
+      url: https://www.freefq.com/d/file/free-ssr/20200811/1f3e9d0d0064f662457062712dcf1b66.txt
+
+  - type: webfuzz
+    options:
+      url: https://merlinblog.xyz/wiki/freess.html
+
+  - type: tgchannel
+    options:
+      channel: ssrList
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: SSRSUB
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: FreeSSRNode
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: ssrlists
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: ssrshares
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: V2List
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: ssrtool
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: vmessr
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: FreeSSR666
+      num: 200
+
+  - type: tgchannel
+    options:
+      channel: fanqiang666
+      num: 200
+
+  - type: web-fanqiangdang
+    options:
+      url: https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0
+      num: 200
+
+  - type: web-fanqiangdang
+    options:
+      url: https://fanqiangdang.com/forum.php?mod=rss&fid=2&auth=0
+      num: 200
+
+  - type: web-fanqiangdang
+    options:
+      url: https://fanqiangdang.com/forum.php?mod=rss&fid=36&auth=0
+      num: 200
+
+  - type: web-freessrxyz
+    options:
+
+  - type: web-lucnorg
+    options:
--- a/tool/base64.go
+++ b/tool/base64.go
@@ -8,15 +8,8 @@ func Base64DecodeString(src string) (dst string, err error) {
 	if src == "" {
 		return "", nil
 	}
-
-	origin := src
-	//if i := len(src) % 4; i != 0 {
-	//	for k := 0; k < i; k++ {
-	//		src += string(base64.StdPadding)
-	//	}
-	//}
 	var dstbytes []byte
-	dstbytes, err = base64.RawURLEncoding.DecodeString(origin)
+	dstbytes, err = base64.RawURLEncoding.DecodeString(src)

 	if err != nil {
 		dstbytes, err = base64.RawStdEncoding.DecodeString(src)
--- a/tool/base64_test.go
+++ b/tool/base64_test.go
@@ -1,12 +0,0 @@
-package tool
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestBase64DecodeString(t *testing.T) {
-	str := "OTEuMjA2LjkyLjM4OjI5MzE6b3JpZ2luOnJjNDpwbGFpbjpiRzVqYmk1dmNtY2dNak5zLz9vYmZzcGFyYW09JnJlbWFya3M9NTctNzVhS1o1WVdhWm1GdWNXbGhibWRrWVc1bkxtTnZiUSY9NUwtRTZMLWM1TGljUkEmZ3JvdXA9NTctNzVhS1o1WVdhWm1GdWNXbGhibWRrWVc1bkxtTnZiUSY9VEc1amJpNXZjbWM"
-	fmt.Println(Base64DecodeString(str))
-	fmt.Println(string([]byte(str)[232]))
-}
--- a/tool/option.go
+++ b/tool/option.go
@@ -0,0 +1,3 @@
+package tool
+
+type Options map[string]interface{}
Author	SHA1	Message	Date
zu1k	d12a581575	order imports	2020-08-12 20:08:49 +08:00
zu1k	8f510c21fe	fix send on close channel	2020-08-12 19:57:57 +08:00
zu1k	21f55ac81d	use config file	2020-08-12 19:52:57 +08:00
zu1k	c87b5872e2	show count	2020-08-12 17:58:51 +08:00
zu1k	304de88e96	Merge branch 'master' of github.com:zu1k/proxypool	2020-08-12 17:54:36 +08:00
zu1k	db47de2747	add coroutine	2020-08-12 17:54:22 +08:00
zu1k	e6c935f13f	add shadowsocks and subscribe support	2020-08-12 17:09:11 +08:00
zu1k	a96d21f4c7	fix vmess re	2020-08-12 16:07:25 +08:00
zu1k	c7f1dc8830	Merge pull request #2 from zu1k/dependabot/github_actions/actions/upload-artifact-v2.1.4 Bump actions/upload-artifact from v2.1.3 to v2.1.4	2020-08-12 14:09:25 +08:00
zu1k	b35e73cbd4	add clash html	2020-08-12 14:01:12 +08:00
zu1k	1d890b0257	add clash	2020-08-12 13:59:26 +08:00
dependabot[bot]	4ba0f7337a	Bump actions/upload-artifact from v2.1.3 to v2.1.4 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from v2.1.3 to v2.1.4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v2.1.3...58740802ef971a2d71eff71e63d48ab68d1f5507) Signed-off-by: dependabot[bot] <support@github.com>	2020-08-12 05:51:44 +00:00
zu1k	3127fe6e2c	cron	2020-08-12 13:50:29 +08:00