fetch proxies from db every task & bump version

use miner patch size
manual gc every crawl
2020-09-06 09:17:50 +08:00 · 2020-09-05 22:30:05 +08:00 · 2020-09-05 22:16:06 +08:00 · 2020-09-05 21:08:58 +08:00 · 2020-09-05 17:40:07 +08:00 · 2020-09-05 12:23:53 +08:00
26 changed files with 723 additions and 114 deletions
--- a/api/router.go
+++ b/api/router.go
@@ -4,6 +4,7 @@ import (
 	"html/template"
 	"net/http"
 	"os"
+	"strconv"

 	"github.com/gin-gonic/gin"
 	_ "github.com/heroku/x/hmetrics/onload"
@@ -13,7 +14,7 @@ import (
 	"github.com/zu1k/proxypool/pkg/provider"
 )

-const version = "v0.3.4"
+const version = "v0.3.7"

 var router *gin.Engine

@@ -75,41 +76,124 @@ func setupRouter() {
 			text = cache.GetString("clashproxies")
 			if text == "" {
 				proxies := cache.GetProxies("proxies")
-				clash := provider.Clash{Proxies: proxies}
+				clash := provider.Clash{
+					provider.Base{
+						Proxies: &proxies,
+					},
+				}
 				text = clash.Provide()
 				cache.SetString("clashproxies", text)
 			}
 		} else if proxyTypes == "all" {
 			proxies := cache.GetProxies("allproxies")
 			clash := provider.Clash{
-				Proxies:    proxies,
-				Types:      proxyTypes,
-				Country:    proxyCountry,
-				NotCountry: proxyNotCountry,
+				provider.Base{
+					Proxies:    &proxies,
+					Types:      proxyTypes,
+					Country:    proxyCountry,
+					NotCountry: proxyNotCountry,
+				},
 			}
 			text = clash.Provide()
 		} else {
 			proxies := cache.GetProxies("proxies")
 			clash := provider.Clash{
-				Proxies:    proxies,
-				Types:      proxyTypes,
-				Country:    proxyCountry,
-				NotCountry: proxyNotCountry,
+				provider.Base{
+					Proxies:    &proxies,
+					Types:      proxyTypes,
+					Country:    proxyCountry,
+					NotCountry: proxyNotCountry,
+				},
 			}
 			text = clash.Provide()
 		}
 		c.String(200, text)
 	})
 	router.GET("/surge/proxies", func(c *gin.Context) {
-		text := cache.GetString("surgeproxies")
-		if text == "" {
-			proxies := cache.GetProxies("proxies")
-			surge := provider.Surge{Proxies: proxies}
+		proxyTypes := c.DefaultQuery("type", "")
+		proxyCountry := c.DefaultQuery("c", "")
+		proxyNotCountry := c.DefaultQuery("nc", "")
+		text := ""
+		if proxyTypes == "" && proxyCountry == "" && proxyNotCountry == "" {
+			text = cache.GetString("surgeproxies")
+			if text == "" {
+				proxies := cache.GetProxies("proxies")
+				surge := provider.Surge{
+					provider.Base{
+						Proxies: &proxies,
+					},
+				}
+				text = surge.Provide()
+				cache.SetString("surgeproxies", text)
+			}
+		} else if proxyTypes == "all" {
+			proxies := cache.GetProxies("allproxies")
+			surge := provider.Surge{
+				provider.Base{
+					Proxies:    &proxies,
+					Types:      proxyTypes,
+					Country:    proxyCountry,
+					NotCountry: proxyNotCountry,
+				},
+			}
+			text = surge.Provide()
+		} else {
+			proxies := cache.GetProxies("proxies")
+			surge := provider.Surge{
+				provider.Base{
+					Proxies:    &proxies,
+					Types:      proxyTypes,
+					Country:    proxyCountry,
+					NotCountry: proxyNotCountry,
+				},
+			}
 			text = surge.Provide()
-			cache.SetString("surgeproxies", text)
 		}
 		c.String(200, text)
 	})
+
+	router.GET("/ss/sub", func(c *gin.Context) {
+		proxies := cache.GetProxies("proxies")
+		ssSub := provider.SSSub{
+			provider.Base{
+				Proxies: &proxies,
+				Types:   "ss",
+			},
+		}
+		c.String(200, ssSub.Provide())
+	})
+	router.GET("/ssr/sub", func(c *gin.Context) {
+		proxies := cache.GetProxies("proxies")
+		ssrSub := provider.SSRSub{
+			provider.Base{
+				Proxies: &proxies,
+				Types:   "ssr",
+			},
+		}
+		c.String(200, ssrSub.Provide())
+	})
+	router.GET("/vmess/sub", func(c *gin.Context) {
+		proxies := cache.GetProxies("proxies")
+		vmessSub := provider.VmessSub{
+			provider.Base{
+				Proxies: &proxies,
+				Types:   "vmess",
+			},
+		}
+		c.String(200, vmessSub.Provide())
+	})
+	router.GET("/link/:id", func(c *gin.Context) {
+		idx := c.Param("id")
+		proxies := cache.GetProxies("allproxies")
+		id, err := strconv.Atoi(idx)
+		if err != nil {
+			c.String(500, err.Error())
+		}
+		if id >= proxies.Len() {
+			c.String(500, "id too big")
+		}
+		c.String(200, proxies[id].Link())
+	})
 }

 func Run() {
@@ -122,6 +206,7 @@ func Run() {
 }

 func loadTemplate() (t *template.Template, err error) {
+	_ = binhtml.RestoreAssets("", "assets/html")
 	t = template.New("")
 	for _, fileName := range binhtml.AssetNames() {
 		data := binhtml.MustAsset(fileName)
--- a/config/config.go
+++ b/config/config.go
@@ -14,6 +14,7 @@ var configFilePath = "config.yaml"

 type ConfigOptions struct {
 	Domain      string   `json:"domain" yaml:"domain"`
+	DatabaseUrl string   `json:"database_url" yaml:"database_url"`
 	CFEmail     string   `json:"cf_email" yaml:"cf_email"`
 	CFKey       string   `json:"cf_key" yaml:"cf_key"`
 	SourceFiles []string `json:"source-files" yaml:"source-files"`
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -1,6 +1,9 @@
 # your domain
 domain: example.com

+# database url
+database_url: ""
+
 # cloudflare api
 cf_email: ""
 cf_key: ""
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,5 @@
+// +heroku goVersion go1.14
+
 module github.com/zu1k/proxypool

 go 1.14
@@ -9,16 +11,16 @@ require (
 	github.com/antchfx/htmlquery v1.2.3 // indirect
 	github.com/antchfx/xmlquery v1.2.4 // indirect
 	github.com/antchfx/xpath v1.1.8 // indirect
-	github.com/cloudflare/cloudflare-go v0.13.1
+	github.com/cloudflare/cloudflare-go v0.13.2
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.6.3
-	github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect
 	github.com/go-playground/validator/v10 v10.3.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gocolly/colly v1.2.0
 	github.com/golang/protobuf v1.4.2 // indirect
 	github.com/heroku/x v0.0.25
 	github.com/ivpusic/grpool v1.0.0
+	github.com/jackc/pgproto3/v2 v2.0.4 // indirect
 	github.com/jasonlvhit/gocron v0.0.1
 	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
@@ -29,10 +31,12 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/temoto/robotstxt v1.1.1 // indirect
-	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
 	golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect
-	golang.org/x/text v0.3.3 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gorm.io/driver/postgres v1.0.0
+	gorm.io/gorm v1.20.0
 )
--- a/go.sum
+++ b/go.sum
@@ -26,12 +26,15 @@ github.com/aws/aws-sdk-go v1.13.10/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41Zo
 github.com/axiomhq/hyperloglog v0.0.0-20180317131949-fe9507de0228/go.mod h1:IOXAcuKIFq/mDyuQ4wyJuJ79XLMsmLM+5RdQ+vWrL7o=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudflare/cloudflare-go v0.13.0 h1:Mx2gNz/r4H0eHETENq5ZPtc/kOX/uzEfT2UFuA+/XAU=
-github.com/cloudflare/cloudflare-go v0.13.0/go.mod h1:6rZ6s/XWxP8WIWMATgDM7aUop0Z0ZOAfcm/4rEd0lOY=
-github.com/cloudflare/cloudflare-go v0.13.1 h1:8i8E5CubfAlD/2zWM7VGKoIPS2/x/lO/nRnqDl0j5Dk=
-github.com/cloudflare/cloudflare-go v0.13.1/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM=
+github.com/cloudflare/cloudflare-go v0.13.2 h1:bhMGoNhAg21DuqJjU9jQepRRft6vYfo6pejT3NN4V6A=
+github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM=
+github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
+github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -47,9 +50,6 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/go-bindata/go-bindata v1.0.0 h1:DZ34txDXWn1DyWa+vQf7V9ANc2ILTtrEjtlsdJRF26M=
-github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE=
-github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
@@ -72,6 +72,7 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
 github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -109,6 +110,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gops v0.3.8-0.20200229223415-3a98d6d24562/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -127,8 +129,64 @@ github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:q
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ivpusic/grpool v1.0.0 h1:+FCiCo3GhfsvzfXuJWnpJUNb/VaqyYVgG8C+qvh07Rc=
 github.com/ivpusic/grpool v1.0.0/go.mod h1:WPmiAI5ExAn06vg+0JzyPzXMQutJmpb7TrBtyLJkOHQ=
+github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
+github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
+github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
+github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
+github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
+github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
+github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
+github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.6.4 h1:S7T6cx5o2OqmxdHaXLH1ZeD1SbI8jBznyYE9Ec0RCQ8=
+github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
+github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
+github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
+github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.4 h1:RHkX5ZUD9bl/kn0f9dYUWs1N7Nwvo1wwUYvKiR26Zco=
+github.com/jackc/pgproto3/v2 v2.0.4/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
+github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
+github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
+github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
+github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
+github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
+github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY=
+github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
+github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
+github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
+github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
+github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
+github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
+github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
+github.com/jackc/pgx/v4 v4.8.1 h1:SUbCLP2pXvf/Sr/25KsuI4aTxiFYIvpfk4l6aTSdyCw=
+github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
+github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/joeshaw/envdecode v0.0.0-20180129163420-d5f34bca07f3/go.mod h1:Q+alOFAXgW5SrcfMPt/G4B2oN+qEcQRJjkn/f4mKL04=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -150,13 +208,26 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leesper/go_rng v0.0.0-20171009123644-5344a9259b21/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
+github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lstoll/grpce v1.7.0/go.mod h1:XiCWl3R+avNCT7KsTjv3qCblgsSqd0SC4ymySrH226g=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -190,14 +261,23 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
 github.com/rcrowley/go-metrics v0.0.0-20160613154715-cfa5a85e9f0a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
 github.com/rollbar/rollbar-go v1.2.0/go.mod h1:czC86b8U4xdUH7W2C6gomi2jutLm8qK0OtrF5WMvpcc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
+github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY=
+github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -207,6 +287,7 @@ github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -222,22 +303,32 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY
 github.com/unrolled/secure v1.0.1/go.mod h1:R6rugAuzh4TQpbFAq69oqZggyBQxFRFQIewtz5z7Jsc=
 github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
-golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -246,6 +337,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -262,15 +355,13 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
-golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -285,18 +376,24 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed h1:J22ig1FUekjjkmZUM7pTKixYm8DvrYsvrBZdunYeIuQ=
@@ -319,12 +416,21 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20190502212712-4a2eb0188cbc/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -366,7 +472,9 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/eapache/channels.v1 v1.1.0/go.mod h1:BHIBujSvu9yMTrTYbTCjDD43gUhtmaOtTWDe7sTv1js=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
@@ -378,9 +486,15 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/postgres v1.0.0 h1:Yh4jyFQ0a7F+JPU0Gtiam/eKmpT/XFc1FKxotGqc6FM=
+gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to=
+gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v1.20.0 h1:qfIlyaZvrF7kMWY3jBdEBXkXJ2M5MFYMTppjILxS3fQ=
+gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
--- a/internal/app/task.go
+++ b/internal/app/task.go
@@ -6,6 +6,7 @@ import (
 	"time"

 	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/provider"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
@@ -19,7 +20,8 @@ func CrawlGo() {
 		wg.Add(1)
 		go g.Get2Chan(pc, wg)
 	}
-	proxies := cache.GetProxies("proxies")
+	proxies := cache.GetProxies("allproxies")
+	proxies = append(proxies, database.GetAllProxies()...)
 	go func() {
 		wg.Wait()
 		close(pc)
@@ -32,11 +34,18 @@ func CrawlGo() {
 	// 节点去重
 	proxies = proxies.Deduplication()
 	log.Println("CrawlGo node count:", len(proxies))
-	proxies = provider.Clash{Proxies: proxies}.CleanProxies()
+	proxies = provider.Clash{
+		provider.Base{
+			Proxies: &proxies,
+		},
+	}.CleanProxies()
 	log.Println("CrawlGo cleaned node count:", len(proxies))
 	proxies.NameAddCounrty().Sort().NameAddIndex().NameAddTG()
 	log.Println("Proxy rename DONE!")

+	// 全节点存储到数据库
+	database.SaveProxyList(proxies)
+
 	cache.SetProxies("allproxies", proxies)
 	cache.AllProxiesCount = proxies.Len()
 	log.Println("AllProxiesCount:", cache.AllProxiesCount)
@@ -58,6 +67,14 @@ func CrawlGo() {
 	cache.SetProxies("proxies", proxies)
 	cache.UsefullProxiesCount = proxies.Len()

-	cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
-	cache.SetString("surgeproxies", provider.Surge{Proxies: proxies}.Provide())
+	cache.SetString("clashproxies", provider.Clash{
+		provider.Base{
+			Proxies: &proxies,
+		},
+	}.Provide())
+	cache.SetString("surgeproxies", provider.Surge{
+		provider.Base{
+			Proxies: &proxies,
+		},
+	}.Provide())
 }
--- a/internal/cron/cron.go
+++ b/internal/cron/cron.go
@@ -1,6 +1,8 @@
 package cron

 import (
+	"runtime"
+
 	"github.com/jasonlvhit/gocron"
 	"github.com/zu1k/proxypool/internal/app"
 )
@@ -13,4 +15,6 @@ func Cron() {
 func crawlTask() {
 	_ = app.InitConfigAndGetters("")
 	app.CrawlGo()
+	app.Getters = nil
+	runtime.GC()
 }
--- a/internal/database/db.go
+++ b/internal/database/db.go
@@ -0,0 +1,31 @@
+package database
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/zu1k/proxypool/config"
+
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var DB *gorm.DB
+
+func connect() (err error) {
+	dsn := "user=proxypool password=proxypool dbname=proxypool port=5432 sslmode=disable TimeZone=Asia/Shanghai"
+	if url := config.Config.DatabaseUrl; url != "" {
+		dsn = url
+	}
+	if url := os.Getenv("DATABASE_URL"); url != "" {
+		dsn = url
+	}
+	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err == nil {
+		fmt.Println("DB connect success: ", DB.Name())
+	}
+	return
+}
--- a/internal/database/db_test.go
+++ b/internal/database/db_test.go
@@ -0,0 +1,9 @@
+package database
+
+import "testing"
+
+func TestConnect(t *testing.T) {
+	t.SkipNow()
+	//connect()
+	InitTables()
+}
--- a/internal/database/proxy.go
+++ b/internal/database/proxy.go
@@ -0,0 +1,68 @@
+package database
+
+import (
+	"github.com/zu1k/proxypool/pkg/getter"
+	"github.com/zu1k/proxypool/pkg/proxy"
+	"gorm.io/gorm"
+)
+
+type Proxy struct {
+	gorm.Model
+	proxy.Base
+	Link       string
+	Identifier string `gorm:"unique"`
+}
+
+func InitTables() {
+	if DB == nil {
+		err := connect()
+		if err != nil {
+			return
+		}
+	}
+	err := DB.AutoMigrate(&Proxy{})
+	if err != nil {
+		panic(err)
+	}
+}
+
+const roundSize = 100
+
+func SaveProxyList(pl proxy.ProxyList) {
+	if DB == nil {
+		return
+	}
+
+	size := pl.Len()
+	round := (size + roundSize - 1) / roundSize
+
+	for r := 0; r < round; r++ {
+		proxies := make([]Proxy, 0, roundSize)
+		for i, j := r*roundSize, (r+1)*roundSize-1; i < j && i < size; i++ {
+			p := pl[i]
+			proxies = append(proxies, Proxy{
+				Base:       *p.BaseInfo(),
+				Link:       p.Link(),
+				Identifier: p.Identifier(),
+			})
+		}
+		DB.Create(&proxies)
+	}
+}
+
+func GetAllProxies() (proxies proxy.ProxyList) {
+	proxies = make(proxy.ProxyList, 0)
+	if DB == nil {
+		return
+	}
+
+	proxiesDB := make([]Proxy, 0)
+	DB.Select("link").Find(&proxiesDB)
+
+	for _, proxyDB := range proxiesDB {
+		if proxiesDB != nil {
+			proxies = append(proxies, getter.String2Proxy(proxyDB.Link))
+		}
+	}
+	return
+}
--- a/main.go
+++ b/main.go
@@ -3,17 +3,24 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/http"
+	_ "net/http/pprof"
 	"os"

 	"github.com/zu1k/proxypool/api"
 	"github.com/zu1k/proxypool/internal/app"
 	"github.com/zu1k/proxypool/internal/cron"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )

 var configFilePath = ""

 func main() {
+	go func() {
+		http.ListenAndServe("0.0.0.0:6060", nil)
+	}()
+
 	flag.StringVar(&configFilePath, "c", "", "path to config file: config.yaml")
 	flag.Parse()

@@ -28,6 +35,7 @@ func main() {
 		panic(err)
 	}

+	database.InitTables()
 	proxy.InitGeoIpDB()
 	fmt.Println("Do the first crawl...")
 	go app.CrawlGo()
--- a/pkg/provider/base.go
+++ b/pkg/provider/base.go
@@ -1,5 +1,11 @@
 package provider

+import (
+	"strings"
+
+	"github.com/zu1k/proxypool/pkg/proxy"
+)
+
 type Provider interface {
 	Provide() string
 }
@@ -12,3 +18,72 @@ func checkInList(list []string, item string) bool {
 	}
 	return false
 }
+
+type Base struct {
+	Proxies    *proxy.ProxyList `yaml:"proxies"`
+	Types      string           `yaml:"type"`
+	Country    string           `yaml:"country"`
+	NotCountry string           `yaml:"not_country"`
+}
+
+func (b *Base) preFilter() {
+	proxies := make(proxy.ProxyList, 0)
+
+	needFilterType := true
+	needFilterCountry := true
+	needFilterNotCountry := true
+	if b.Types == "" || b.Types == "all" {
+		needFilterType = false
+	}
+	if b.Country == "" || b.Country == "all" {
+		needFilterCountry = false
+	}
+	if b.NotCountry == "" {
+		needFilterNotCountry = false
+	}
+	types := strings.Split(b.Types, ",")
+	countries := strings.Split(b.Country, ",")
+	notCountries := strings.Split(b.NotCountry, ",")
+
+	bProxies := *b.Proxies
+	for _, p := range bProxies {
+		if needFilterType {
+			typeOk := false
+			for _, t := range types {
+				if p.TypeName() == t {
+					typeOk = true
+					break
+				}
+			}
+			if !typeOk {
+				goto exclude
+			}
+		}
+
+		if needFilterNotCountry {
+			for _, c := range notCountries {
+				if strings.Contains(p.BaseInfo().Name, c) {
+					goto exclude
+				}
+			}
+		}
+
+		if needFilterCountry {
+			countryOk := false
+			for _, c := range countries {
+				if strings.Contains(p.BaseInfo().Name, c) {
+					countryOk = true
+					break
+				}
+			}
+			if !countryOk {
+				goto exclude
+			}
+		}
+
+		proxies = append(proxies, p)
+	exclude:
+	}
+
+	b.Proxies = &proxies
+}
--- a/pkg/provider/clash.go
+++ b/pkg/provider/clash.go
@@ -7,15 +7,12 @@ import (
 )

 type Clash struct {
-	Proxies    proxy.ProxyList `yaml:"proxies"`
-	Types      string          `yaml:"type"`
-	Country    string          `yaml:"country"`
-	NotCountry string          `yaml:"not_country"`
+	Base
 }

 func (c Clash) CleanProxies() (proxies proxy.ProxyList) {
 	proxies = make(proxy.ProxyList, 0)
-	for _, p := range c.Proxies {
+	for _, p := range *c.Proxies {
 		if checkClashSupport(p) {
 			proxies = append(proxies, p)
 		}
@@ -24,68 +21,15 @@ func (c Clash) CleanProxies() (proxies proxy.ProxyList) {
 }

 func (c Clash) Provide() string {
+	c.preFilter()
+
 	var resultBuilder strings.Builder
 	resultBuilder.WriteString("proxies:\n")
-
-	needFilterType := true
-	needFilterCountry := true
-	needFilterNotCountry := true
-	if c.Types == "" || c.Types == "all" {
-		needFilterType = false
-	}
-	if c.Country == "" || c.Country == "all" {
-		needFilterCountry = false
-	}
-	if c.NotCountry == "" {
-		needFilterNotCountry = false
-	}
-	types := strings.Split(c.Types, ",")
-	countries := strings.Split(c.Country, ",")
-	notCountries := strings.Split(c.NotCountry, ",")
-
-	for _, p := range c.Proxies {
-		if !checkClashSupport(p) {
-			continue
+	for _, p := range *c.Proxies {
+		if checkClashSupport(p) {
+			resultBuilder.WriteString(p.ToClash() + "\n")
 		}
-
-		if needFilterType {
-			typeOk := false
-			for _, t := range types {
-				if p.TypeName() == t {
-					typeOk = true
-					break
-				}
-			}
-			if !typeOk {
-				goto exclude
-			}
-		}
-
-		if needFilterNotCountry {
-			for _, c := range notCountries {
-				if strings.Contains(p.BaseInfo().Name, c) {
-					goto exclude
-				}
-			}
-		}
-
-		if needFilterCountry {
-			countryOk := false
-			for _, c := range countries {
-				if strings.Contains(p.BaseInfo().Name, c) {
-					countryOk = true
-					break
-				}
-			}
-			if !countryOk {
-				goto exclude
-			}
-		}
-
-		resultBuilder.WriteString(p.ToClash() + "\n")
-	exclude:
 	}
-
 	return resultBuilder.String()
 }

--- a/pkg/provider/ssrsub.go
+++ b/pkg/provider/ssrsub.go
@@ -0,0 +1,21 @@
+package provider
+
+import (
+	"strings"
+
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+type SSRSub struct {
+	Base
+}
+
+func (sub SSRSub) Provide() string {
+	sub.Types = "ssr"
+	sub.preFilter()
+	var resultBuilder strings.Builder
+	for _, p := range *sub.Proxies {
+		resultBuilder.WriteString(p.Link() + "\n")
+	}
+	return tool.Base64EncodeString(resultBuilder.String(), false)
+}
--- a/pkg/provider/sssub.go
+++ b/pkg/provider/sssub.go
@@ -0,0 +1,45 @@
+package provider
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"github.com/zu1k/proxypool/pkg/proxy"
+)
+
+type SSSub struct {
+	Base
+}
+
+type ssJson struct {
+	Remarks    string                 `json:"remarks"`
+	Server     string                 `json:"server"`
+	ServerPort string                 `json:"server_port"`
+	Method     string                 `json:"method"`
+	Password   string                 `json:"password"`
+	Plugin     string                 `json:"plugin"`
+	PluginOpts map[string]interface{} `json:"plugin_opts"`
+}
+
+func (sub SSSub) Provide() string {
+	sub.Types = "ss"
+	sub.preFilter()
+	proxies := make([]ssJson, 0, sub.Proxies.Len())
+	for _, p := range *sub.Proxies {
+		pp := p.(*proxy.Shadowsocks)
+		proxies = append(proxies, ssJson{
+			Remarks:    pp.Name,
+			Server:     pp.Server,
+			ServerPort: strconv.Itoa(pp.Port),
+			Method:     pp.Cipher,
+			Password:   pp.Password,
+			Plugin:     pp.Plugin,
+			PluginOpts: pp.PluginOpts,
+		})
+	}
+	text, err := json.Marshal(proxies)
+	if err != nil {
+		return ""
+	}
+	return string(text)
+}
--- a/pkg/provider/surge.go
+++ b/pkg/provider/surge.go
@@ -7,18 +7,18 @@ import (
 )

 type Surge struct {
-	Proxies proxy.ProxyList `yaml:"proxies"`
+	Base
 }

 func (s Surge) Provide() string {
-	var resultBuilder strings.Builder
+	s.preFilter()

-	for _, p := range s.Proxies {
+	var resultBuilder strings.Builder
+	for _, p := range *s.Proxies {
 		if checkSurgeSupport(p) {
-			resultBuilder.WriteString(p.ToSurge() + "\n")
+			resultBuilder.WriteString(p.ToClash() + "\n")
 		}
 	}
-
 	return resultBuilder.String()
 }

--- a/pkg/provider/vmesssub.go
+++ b/pkg/provider/vmesssub.go
@@ -0,0 +1,21 @@
+package provider
+
+import (
+	"strings"
+
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+type VmessSub struct {
+	Base
+}
+
+func (sub VmessSub) Provide() string {
+	sub.Types = "vmess"
+	sub.preFilter()
+	var resultBuilder strings.Builder
+	for _, p := range *sub.Proxies {
+		resultBuilder.WriteString(p.Link() + "\n")
+	}
+	return tool.Base64EncodeString(resultBuilder.String(), false)
+}
--- a/pkg/proxy/base.go
+++ b/pkg/proxy/base.go
@@ -1,11 +1,13 @@
 package proxy

 type Base struct {
-	Name   string `yaml:"name" json:"name"`
-	Server string `yaml:"server" json:"server"`
-	Port   int    `yaml:"port" json:"port"`
-	Type   string `yaml:"type" json:"type"`
-	UDP    bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
+	Name    string `yaml:"name" json:"name" gorm:"index"`
+	Server  string `yaml:"server" json:"server" gorm:"index"`
+	Port    int    `yaml:"port" json:"port" gorm:"index"`
+	Type    string `yaml:"type" json:"type" gorm:"index"`
+	UDP     bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
+	Country string `yaml:"country,omitempty" json:"country,omitempty" gorm:"index"`
+	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"`
 }

 func (b *Base) TypeName() string {
@@ -32,14 +34,25 @@ func (b *Base) Clone() Base {
 	return c
 }

+func (b *Base) SetUseable(useable bool) {
+	b.Useable = useable
+}
+
+func (b *Base) SetCountry(country string) {
+	b.Country = country
+}
+
 type Proxy interface {
 	String() string
 	ToClash() string
 	ToSurge() string
+	Link() string
 	Identifier() string
 	SetName(name string)
 	SetIP(ip string)
 	TypeName() string
 	BaseInfo() *Base
 	Clone() Proxy
+	SetUseable(useable bool)
+	SetCountry(country string)
 }
--- a/pkg/proxy/check.go
+++ b/pkg/proxy/check.go
@@ -109,7 +109,10 @@ func CleanBadProxies(proxies []Proxy) (cproxies []Proxy) {
 	cproxies = make(ProxyList, 0, 500)
 	for _, p := range proxies {
 		if _, ok := okMap[p.Identifier()]; ok {
+			p.SetUseable(true)
 			cproxies = append(cproxies, p.Clone())
+		} else {
+			p.SetUseable(false)
 		}
 	}
 	return
--- a/pkg/proxy/link_test.go
+++ b/pkg/proxy/link_test.go
@@ -0,0 +1,62 @@
+package proxy
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestSSLink(t *testing.T) {
+	ss, err := ParseSSLink("ss://YWVzLTI1Ni1jZmI6ZUlXMERuazY5NDU0ZTZuU3d1c3B2OURtUzIwMXRRMERAMTcyLjEwNC4xNjEuNTQ6ODA5OQ==#翻墙党223.13新加坡")
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(ss)
+	fmt.Println(ss.Link())
+	ss, err = ParseSSLink(ss.Link())
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(ss)
+}
+
+func TestSSRLink(t *testing.T) {
+	ssr, err := ParseSSRLink("ssr://MTcyLjEwNC4xNjEuNTQ6ODA5OTpvcmlnaW46YWVzLTI1Ni1jZmI6cGxhaW46WlVsWE1FUnVhelk1TkRVMFpUWnVVM2QxYzNCMk9VUnRVekl3TVhSUk1FUT0vP29iZnNwYXJhbT0mcHJvdG9wYXJhbT0mcmVtYXJrcz01Ny03NWFLWjVZV2FNakl6TGpFejVwYXc1WXFnNVoyaCZncm91cD01cGF3NVlxZzVaMmg=")
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(ssr)
+	fmt.Println(ssr.Link())
+	ssr, err = ParseSSRLink(ssr.Link())
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(ssr)
+}
+
+func TestTrojanLink(t *testing.T) {
+	trojan, err := ParseTrojanLink("trojan://65474277@sqcu.hostmsu.ru:55551?allowinsecure=0&peer=mza.hkfq.xyz&mux=1&ws=0&wspath=&wshost=&ss=0&ssmethod=aes-128-gcm&sspasswd=&group=#%E9%A6%99%E6%B8%AFCN2-MZA%E8%8A%82%E7%82%B9-%E5%AE%BF%E8%BF%81%E8%81%94%E9%80%9A%E4%B8%AD%E8%BD%AC")
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(trojan)
+	fmt.Println(trojan.Link())
+	trojan, err = ParseTrojanLink(trojan.Link())
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(trojan)
+}
+
+func TestVmessLink(t *testing.T) {
+	v, err := ParseVmessLink("vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIuW+ruS/oeWFrOS8l+WPtyDlpJrlvannmoTlpKfljYPkuJbnlYwiLA0KICAiYWRkIjogInMyNzEuc25vZGUueHl6IiwNCiAgInBvcnQiOiAiNDQzIiwNCiAgImlkIjogIjZhOTAwZDYzLWNiOTItMzVhMC1hZWYwLTNhMGMxMWFhODUyMyIsDQogICJhaWQiOiAiMSIsDQogICJuZXQiOiAid3MiLA0KICAidHlwZSI6ICJub25lIiwNCiAgImhvc3QiOiAiczI3MS5zbm9kZS54eXoiLA0KICAicGF0aCI6ICIvcGFuZWwiLA0KICAidGxzIjogInRscyINCn0=")
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(v)
+	fmt.Println(v.Link())
+	v, err = ParseVmessLink(v.Link())
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(v)
+}
--- a/pkg/proxy/proxies.go
+++ b/pkg/proxy/proxies.go
@@ -76,6 +76,7 @@ func (ps ProxyList) NameAddCounrty() ProxyList {
 				country = "🏁 ZZ"
 			}
 			ps[ii].SetName(fmt.Sprintf("%s", country))
+			ps[ii].SetCountry(country)
 			//ps[ii].SetIP(ip)
 		}()
 	}
--- a/pkg/proxy/shadowsocks.go
+++ b/pkg/proxy/shadowsocks.go
@@ -65,6 +65,13 @@ func (ss Shadowsocks) Clone() Proxy {
 	return &ss
 }

+// https://shadowsocks.org/en/config/quick-guide.html
+func (ss Shadowsocks) Link() (link string) {
+	payload := fmt.Sprintf("%s:%s@%s:%d", ss.Cipher, ss.Password, ss.Server, ss.Port)
+	payload = tool.Base64EncodeString(payload, false)
+	return fmt.Sprintf("ss://%s#%s", payload, ss.Name)
+}
+
 func ParseSSLink(link string) (*Shadowsocks, error) {
 	if !strings.HasPrefix(link, "ss://") {
 		return nil, ErrorNotSSRLink
--- a/pkg/proxy/shadowsocksr.go
+++ b/pkg/proxy/shadowsocksr.go
@@ -3,6 +3,7 @@ package proxy
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"math/rand"
 	"net"
 	"net/url"
@@ -61,6 +62,19 @@ func (ssr ShadowsocksR) Clone() Proxy {
 	return &ssr
 }

+// https://github.com/HMBSbige/ShadowsocksR-Windows/wiki/SSR-QRcode-scheme
+func (ssr ShadowsocksR) Link() (link string) {
+	payload := fmt.Sprintf("%s:%d:%s:%s:%s:%s",
+		ssr.Server, ssr.Port, ssr.Protocol, ssr.Cipher, ssr.Obfs, tool.Base64EncodeString(ssr.Password, true))
+	query := url.Values{}
+	query.Add("obfsparam", tool.Base64EncodeString(ssr.ObfsParam, true))
+	query.Add("protoparam", tool.Base64EncodeString(ssr.ProtocolParam, true))
+	query.Add("remarks", tool.Base64EncodeString(ssr.Name, true))
+	query.Add("group", tool.Base64EncodeString("proxy.tgbot.co", true))
+	payload = tool.Base64EncodeString(fmt.Sprintf("%s/?%s", payload, query.Encode()), true)
+	return fmt.Sprintf("ssr://%s", payload)
+}
+
 func ParseSSRLink(link string) (*ShadowsocksR, error) {
 	if !strings.HasPrefix(link, "ssr") {
 		return nil, ErrorNotSSRLink
--- a/pkg/proxy/trojan.go
+++ b/pkg/proxy/trojan.go
@@ -66,6 +66,24 @@ func (t Trojan) Clone() Proxy {
 	return &t
 }

+// https://p4gefau1t.github.io/trojan-go/developer/url/
+func (t Trojan) Link() (link string) {
+	query := url.Values{}
+	if t.SNI != "" {
+		query.Set("sni", url.QueryEscape(t.SNI))
+	}
+
+	uri := url.URL{
+		Scheme:   "trojan",
+		User:     url.User(url.QueryEscape(t.Password)),
+		Host:     net.JoinHostPort(t.Server, strconv.Itoa(t.Port)),
+		RawQuery: query.Encode(),
+		Fragment: t.Name,
+	}
+
+	return uri.String()
+}
+
 func ParseTrojanLink(link string) (*Trojan, error) {
 	if !strings.HasPrefix(link, "trojan://") && !strings.HasPrefix(link, "trojan-go://") {
 		return nil, ErrorNotTrojanink
--- a/pkg/proxy/vmess.go
+++ b/pkg/proxy/vmess.go
@@ -86,6 +86,14 @@ func (v Vmess) Clone() Proxy {
 	return &v
 }

+func (v Vmess) Link() (link string) {
+	vjv, err := json.Marshal(v.toLinkJson())
+	if err != nil {
+		return
+	}
+	return fmt.Sprintf("vmess://%s", tool.Base64EncodeBytes(vjv))
+}
+
 type vmessLinkJson struct {
 	Add  string      `json:"add"`
 	V    string      `json:"v"`
@@ -100,6 +108,27 @@ type vmessLinkJson struct {
 	Tls  string      `json:"tls"`
 }

+func (v Vmess) toLinkJson() vmessLinkJson {
+	vj := vmessLinkJson{
+		Add:  v.Server,
+		Ps:   v.Name,
+		Port: v.Port,
+		Id:   v.UUID,
+		Aid:  strconv.Itoa(v.AlterID),
+		Net:  v.Network,
+		Path: v.WSPath,
+		Host: v.ServerName,
+		V:    "2",
+	}
+	if v.TLS {
+		vj.Tls = "tls"
+	}
+	if host, ok := v.WSHeaders["HOST"]; ok && host != "" {
+		vj.Host = host
+	}
+	return vj
+}
+
 func ParseVmessLink(link string) (*Vmess, error) {
 	if !strings.HasPrefix(link, "vmess") {
 		return nil, ErrorNotVmessLink
@@ -190,10 +219,11 @@ func ParseVmessLink(link string) (*Vmess, error) {
 		}
 		port := 443
 		portInterface := vmessJson.Port
-		if i, ok := portInterface.(int); ok {
-			port = i
-		} else if s, ok := portInterface.(string); ok {
-			port, _ = strconv.Atoi(s)
+		switch portInterface.(type) {
+		case int:
+			port = portInterface.(int)
+		case string:
+			port, _ = strconv.Atoi(portInterface.(string))
 		}

 		alterId, err := strconv.Atoi(vmessJson.Aid)
--- a/pkg/tool/base64.go
+++ b/pkg/tool/base64.go
@@ -26,3 +26,14 @@ func Base64DecodeString(src string) (dst string, err error) {
 	dst = string(dstbytes)
 	return
 }
+
+func Base64EncodeString(origin string, urlsafe bool) (result string) {
+	if urlsafe {
+		return base64.URLEncoding.EncodeToString([]byte(origin))
+	}
+	return base64.StdEncoding.EncodeToString([]byte(origin))
+}
+
+func Base64EncodeBytes(origin []byte) (result string) {
+	return base64.StdEncoding.EncodeToString([]byte(origin))
+}
Author	SHA1	Message	Date
zu1k	679c6fa0d4	fetch proxies from db every task & bump version	2020-09-06 09:17:50 +08:00
zu1k	864f8403ac	use miner patch size	2020-09-05 22:30:05 +08:00
zu1k	f2275c7bdc	manual gc every crawl	2020-09-05 22:16:06 +08:00
zu1k	5b5999834d	add debug pprof	2020-09-05 21:08:58 +08:00
zu1k	30e46c6ace	mod proxy model	2020-09-05 17:40:07 +08:00
zu1k	53e5dc4011	restore html when load	2020-09-05 12:23:53 +08:00
zu1k	c27411dc3d	bump version	2020-09-05 12:03:55 +08:00
zu1k	65ad07f579	add database url to config file	2020-09-05 12:03:22 +08:00
zu1k	5ef4144e72	expose country and useable‘	2020-09-05 11:24:09 +08:00
zu1k	a17b7f6fc9	specific goversion for heroku	2020-09-05 10:58:45 +08:00
zu1k	0d7c58716e	fix db	2020-09-05 10:40:30 +08:00
zu1k	9eca907b88	add db support	2020-09-05 10:30:20 +08:00
zu1k	49e1105763	try to fix ssr base64	2020-09-04 17:51:15 +08:00
zu1k	5fde0b6658	add link for all proxies	2020-09-04 17:13:19 +08:00
zu1k	6b769ef998	fix preFilter error & mod sssub	2020-09-04 17:08:43 +08:00
zu1k	458ba6dd35	fix bugs	2020-09-04 16:49:32 +08:00
zu1k	82f6e1dae8	add sub support	2020-09-04 16:40:50 +08:00
zu1k	9de327fb29	add link export for vmess	2020-09-04 16:40:50 +08:00
zu1k	0e88bb62b5	add link export support for ssr	2020-09-04 16:40:50 +08:00
zu1k	ef1b6eab6b	add link export for ss & trojan	2020-09-04 16:40:50 +08:00
dependabot[bot]	fb90213e33	Bump github.com/cloudflare/cloudflare-go from 0.13.1 to 0.13.2 Bumps [github.com/cloudflare/cloudflare-go](https://github.com/cloudflare/cloudflare-go) from 0.13.1 to 0.13.2. - [Release notes](https://github.com/cloudflare/cloudflare-go/releases) - [Commits](https://github.com/cloudflare/cloudflare-go/compare/v0.13.1...v0.13.2) Signed-off-by: dependabot[bot] <support@github.com>	2020-09-04 16:40:50 +08:00
zu1k	a87e298002	check all proxies's health every time	2020-08-30 15:02:19 +08:00