Compare commits

..

13 Commits

Author SHA1 Message Date
zu1k
a6387cfefd remove windows platform 2020-08-25 14:30:51 +08:00
zu1k
b54d2d4c01 fix linux file path 2020-08-25 14:29:14 +08:00
zu1k
933a2ea73b Merge pull request #10 from zu1k/dependabot/go_modules/github.com/cloudflare/cloudflare-go-0.13.1
Bump github.com/cloudflare/cloudflare-go from 0.13.0 to 0.13.1
2020-08-25 14:26:35 +08:00
zu1k
673da5b59a fix for github action 2020-08-25 14:25:32 +08:00
zu1k
0828209ae1 bump version 2020-08-25 14:25:32 +08:00
zu1k
386b302208 use bindata 2020-08-25 14:25:32 +08:00
dependabot[bot]
bbebdca897 Bump github.com/cloudflare/cloudflare-go from 0.13.0 to 0.13.1
Bumps [github.com/cloudflare/cloudflare-go](https://github.com/cloudflare/cloudflare-go) from 0.13.0 to 0.13.1.
- [Release notes](https://github.com/cloudflare/cloudflare-go/releases)
- [Commits](https://github.com/cloudflare/cloudflare-go/compare/v0.13.0...v0.13.1)

Signed-off-by: dependabot[bot] <support@github.com>
2020-08-25 06:21:26 +00:00
zu1k
88ad9e8ba1 Merge pull request #9 from zu1k/dependabot/github_actions/actions/checkout-v2.3.2
Bump actions/checkout from v2.3.1 to v2.3.2
2020-08-25 07:39:59 +08:00
zu1k
655d5facaa opti some func 2020-08-24 19:26:29 +08:00
dependabot[bot]
69025eb9c2 Bump actions/checkout from v2.3.1 to v2.3.2
Bumps [actions/checkout](https://github.com/actions/checkout) from v2.3.1 to v2.3.2.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v2.3.1...2036a08e25fa78bbd946711a407b529a0a1204bf)

Signed-off-by: dependabot[bot] <support@github.com>
2020-08-24 08:06:44 +00:00
zu1k
51f6f9f953 update readme 2020-08-24 13:28:40 +08:00
zu1k
ffda71e641 docker add tzdata dep 2020-08-24 13:26:18 +08:00
zu1k
2b22846028 add getters count show 2020-08-24 13:20:55 +08:00
25 changed files with 720 additions and 81 deletions

View File

@@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2.3.1
uses: actions/checkout@v2.3.2
- name: Build and push Docker images
uses: docker/build-push-action@v1.1.0

View File

@@ -12,7 +12,7 @@ jobs:
go-version: 1.14.x
- name: Check out code into the Go module directory
uses: actions/checkout@v2.3.1
uses: actions/checkout@v2.3.2
- name: Cache go module
uses: actions/cache@v2
@@ -26,6 +26,13 @@ jobs:
run: |
go test ./...
- name: gen go-bindata
if: startsWith(github.ref, 'refs/tags/')
run: |
go get -u github.com/go-bindata/go-bindata/...
go-bindata -o internal/bindata/geoip/geoip.go -pkg bingeoip assets/GeoLite2-City.mmdb
go-bindata -o internal/bindata/html/html.go -pkg binhtml assets/html/
- name: Build
if: startsWith(github.ref, 'refs/tags/')
env:

View File

@@ -9,7 +9,7 @@ RUN go mod download && \
FROM alpine:latest
RUN apk add --no-cache ca-certificates
RUN apk add --no-cache ca-certificates tzdata
WORKDIR /proxypool-src
COPY ./assets /proxypool-src/assets
COPY --from=builder /proxypool /proxypool-src/

View File

@@ -20,11 +20,8 @@ PLATFORM_LIST = \
freebsd-386 \
freebsd-amd64
WINDOWS_ARCH_LIST = \
windows-386 \
windows-amd64
all: linux-amd64 darwin-amd64 windows-amd64 # Most used
all: linux-amd64 darwin-amd64
docker:
$(GOBUILD) -o $(BINDIR)/$(NAME)-$@
@@ -74,24 +71,14 @@ freebsd-386:
freebsd-amd64:
GOARCH=amd64 GOOS=freebsd $(GOBUILD) -o $(BINDIR)/$(NAME)-$@
windows-386:
GOARCH=386 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe
windows-amd64:
GOARCH=amd64 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe
gz_releases=$(addsuffix .gz, $(PLATFORM_LIST))
zip_releases=$(addsuffix .zip, $(WINDOWS_ARCH_LIST))
$(gz_releases): %.gz : %
chmod +x $(BINDIR)/$(NAME)-$(basename $@)
gzip -f -S -$(VERSION).gz $(BINDIR)/$(NAME)-$(basename $@)
$(zip_releases): %.zip : %
zip -m -j $(BINDIR)/$(NAME)-$(basename $@)-$(VERSION).zip $(BINDIR)/$(NAME)-$(basename $@).exe
all-arch: $(PLATFORM_LIST)
all-arch: $(PLATFORM_LIST) $(WINDOWS_ARCH_LIST)
releases: $(gz_releases) $(zip_releases)
releases: $(gz_releases)
clean:
rm $(BINDIR)/*

View File

@@ -32,7 +32,7 @@
点击按钮进入部署页面,填写基本信息然后运行
其中 `DOMAIN` 需要填写为你需要绑定的域名,`CONFIG_FILE` 需要填写你的配置文件路径,配置文件模板见 config/source.yaml 文件
其中 `DOMAIN` 需要填写为你需要绑定的域名,`CONFIG_FILE` 需要填写你的配置文件路径,配置文件模板见 config/config.yaml 文件
`CF` 开头的选项暂不需要填写,不影响程序运行
@@ -53,6 +53,8 @@ $ go get -u -v github.com/zu1k/proxypool
从这里下载预编译好的程序 [release](https://github.com/zu1k/proxypool/releases)
除了编译好的二进制程序,你还需要下载仓库 `assets` 文件夹,放置在可执行文件同目录
### 使用docker
```sh
@@ -63,16 +65,16 @@ docker pull docker.pkg.github.com/zu1k/proxypool/proxypool:latest
### 修改配置文件
首先修改 source.yaml 中的必要配置信息cf开头的选项不需要填写
首先修改 config.yaml 中的必要配置信息cf开头的选项不需要填写
配置文件中定义了抓取源,需要定期手动维护更新
source.yaml 文件中定义了抓取源,需要定期手动维护更新
### 启动程序
使用 `-c` 参数指定配置文件路径支持http链接
```shell
proxypool -c source.yaml
proxypool -c config.yaml
```
## 截图

View File

@@ -1,28 +1,36 @@
package api
import (
"html/template"
"net/http"
"os"
"github.com/zu1k/proxypool/config"
"github.com/gin-gonic/gin"
_ "github.com/heroku/x/hmetrics/onload"
"github.com/zu1k/proxypool/config"
binhtml "github.com/zu1k/proxypool/internal/bindata/html"
"github.com/zu1k/proxypool/internal/cache"
"github.com/zu1k/proxypool/pkg/provider"
)
const version = "v0.3.0"
const version = "v0.3.2"
var router *gin.Engine
func setupRouter() {
router = gin.Default()
router.LoadHTMLGlob("assets/html/*")
gin.SetMode(gin.ReleaseMode)
router = gin.New()
router.Use(gin.Logger(), gin.Recovery())
temp, err := loadTemplate()
if err != nil {
panic(err)
}
router.SetHTMLTemplate(temp)
router.GET("/", func(c *gin.Context) {
c.HTML(http.StatusOK, "index.html", gin.H{
c.HTML(http.StatusOK, "assets/html/index.html", gin.H{
"domain": config.Config.Domain,
"getters_count": cache.GettersCount,
"all_proxies_count": cache.AllProxiesCount,
"ss_proxies_count": cache.SSProxiesCount,
"ssr_proxies_count": cache.SSRProxiesCount,
@@ -35,25 +43,25 @@ func setupRouter() {
})
router.GET("/clash", func(c *gin.Context) {
c.HTML(http.StatusOK, "clash.html", gin.H{
c.HTML(http.StatusOK, "assets/html/clash.html", gin.H{
"domain": config.Config.Domain,
})
})
router.GET("/surge", func(c *gin.Context) {
c.HTML(http.StatusOK, "surge.html", gin.H{
c.HTML(http.StatusOK, "assets/html/surge.html", gin.H{
"domain": config.Config.Domain,
})
})
router.GET("/clash/config", func(c *gin.Context) {
c.HTML(http.StatusOK, "clash-config.yaml", gin.H{
c.HTML(http.StatusOK, "assets/html/clash-config.yaml", gin.H{
"domain": config.Config.Domain,
})
})
router.GET("/surge/config", func(c *gin.Context) {
c.HTML(http.StatusOK, "surge.conf", gin.H{
c.HTML(http.StatusOK, "assets/html/surge.conf", gin.H{
"domain": config.Config.Domain,
})
})
@@ -101,3 +109,15 @@ func Run() {
}
router.Run(":" + port)
}
func loadTemplate() (t *template.Template, err error) {
t = template.New("")
for _, fileName := range binhtml.AssetNames() {
data := binhtml.MustAsset(fileName)
t, err = t.New(fileName).Parse(string(data))
if err != nil {
return nil, err
}
}
return t, nil
}

View File

@@ -50,6 +50,19 @@ proxy-groups:
- provider
url: 'http://www.gstatic.com/generate_204'
interval: 300
- name: 网址 https://proxy.tgbot.co
type: select
proxies:
- DIRECT
- name: 欢迎Star https://github.com/zu1k/proxypool
type: select
proxies:
- DIRECT
- name: TG频道 @peekfun
type: select
proxies:
- DIRECT
proxy-providers:
provider:

View File

@@ -106,7 +106,7 @@
<div class='section friendly'>
<h1><strong>免费节点</strong></h1>
<div class='article'>
<p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess、trojan节点信息聚合去重后提供节点列表每15分钟更新</p>
<p>自动抓取tg频道、订阅地址、公开互联网上的ss、ssr、vmess、trojan节点信息聚合去重后提供节点列表每15分钟更新,目前共有{{.getters_count}}个抓取源</p>
<p>汇总节点数量:{{ .all_proxies_count }}</p>
<p>ss节点数量{{ .ss_proxies_count }}</p>
<p>ssr节点数量{{ .ssr_proxies_count }}</p>
@@ -130,6 +130,7 @@
</p>
<br>
{{- /* 所有使用本代码提供服务的禁止删除该行*/}}
<p>欢迎关注tg频道<a href="https://t.me/peekfun">@peekfun</a></p>
<p>抓取程序已开源:<a href="https://github.com/zu1k/proxypool">https://github.com/zu1k/proxypool</a> {{ .version }}</p>
</div>
</div>

5
go.mod
View File

@@ -9,14 +9,16 @@ require (
github.com/antchfx/htmlquery v1.2.3 // indirect
github.com/antchfx/xmlquery v1.2.4 // indirect
github.com/antchfx/xpath v1.1.8 // indirect
github.com/cloudflare/cloudflare-go v0.13.0
github.com/cloudflare/cloudflare-go v0.13.1
github.com/ghodss/yaml v1.0.0
github.com/gin-gonic/gin v1.6.3
github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect
github.com/go-playground/validator/v10 v10.3.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gocolly/colly v1.2.0
github.com/golang/protobuf v1.4.2 // indirect
github.com/heroku/x v0.0.25
github.com/ivpusic/grpool v1.0.0
github.com/jasonlvhit/gocron v0.0.1
github.com/json-iterator/go v1.1.10 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
@@ -28,7 +30,6 @@ require (
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/temoto/robotstxt v1.1.1 // indirect
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect
golang.org/x/text v0.3.3 // indirect
google.golang.org/appengine v1.6.6 // indirect

9
go.sum
View File

@@ -28,6 +28,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/cloudflare-go v0.13.0 h1:Mx2gNz/r4H0eHETENq5ZPtc/kOX/uzEfT2UFuA+/XAU=
github.com/cloudflare/cloudflare-go v0.13.0/go.mod h1:6rZ6s/XWxP8WIWMATgDM7aUop0Z0ZOAfcm/4rEd0lOY=
github.com/cloudflare/cloudflare-go v0.13.1 h1:8i8E5CubfAlD/2zWM7VGKoIPS2/x/lO/nRnqDl0j5Dk=
github.com/cloudflare/cloudflare-go v0.13.1/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM=
github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -45,6 +47,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/go-bindata/go-bindata v1.0.0 h1:DZ34txDXWn1DyWa+vQf7V9ANc2ILTtrEjtlsdJRF26M=
github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE=
github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
@@ -120,6 +125,8 @@ github.com/heroku/x v0.0.25/go.mod h1:qE/I0jp6rIeTBBosrPYV4ygRX3OMhqmC/A6x8ewodJ
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ivpusic/grpool v1.0.0 h1:+FCiCo3GhfsvzfXuJWnpJUNb/VaqyYVgG8C+qvh07Rc=
github.com/ivpusic/grpool v1.0.0/go.mod h1:WPmiAI5ExAn06vg+0JzyPzXMQutJmpb7TrBtyLJkOHQ=
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -264,6 +271,8 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

View File

@@ -4,6 +4,8 @@ import (
"errors"
"fmt"
"github.com/zu1k/proxypool/internal/cache"
"github.com/ghodss/yaml"
"github.com/zu1k/proxypool/config"
@@ -48,4 +50,5 @@ func initGetters(sourceFiles []string) {
}
}
fmt.Println("Getter count:", len(Getters))
cache.GettersCount = len(Getters)
}

View File

@@ -33,19 +33,28 @@ func CrawlGo() {
proxies = proxies.Deduplication()
log.Println("CrawlGo node count:", len(proxies))
proxies = provider.Clash{Proxies: proxies}.CleanProxies()
proxies.NameAddCounrty().Sort().NameAddIndex()
log.Println("CrawlGo cleaned node count:", len(proxies))
proxies.NameAddCounrty().Sort().NameAddIndex().NameAddTG()
log.Println("Proxy rename DONE!")
cache.SetProxies("allproxies", proxies)
cache.AllProxiesCount = proxies.Len()
log.Println("AllProxiesCount:", cache.AllProxiesCount)
cache.SSProxiesCount = proxies.TypeLen("ss")
log.Println("SSProxiesCount:", cache.SSProxiesCount)
cache.SSRProxiesCount = proxies.TypeLen("ssr")
log.Println("SSRProxiesCount:", cache.SSRProxiesCount)
cache.VmessProxiesCount = proxies.TypeLen("vmess")
log.Println("VmessProxiesCount:", cache.VmessProxiesCount)
cache.TrojanProxiesCount = proxies.TypeLen("trojan")
log.Println("TrojanProxiesCount:", cache.TrojanProxiesCount)
cache.LastCrawlTime = time.Now().In(location).Format("2006-01-02 15:04:05")
// 可用性检测
proxies = proxy.CleanBadProxies(proxies)
log.Println("CrawlGo clash useable node count:", len(proxies))
proxies.NameAddCounrty().Sort().NameAddIndex()
log.Println("Now check proxy health...")
proxies = proxy.CleanBadProxiesWithGrpool(proxies)
log.Println("CrawlGo clash usefull node count:", len(proxies))
proxies.NameReIndex()
cache.SetProxies("proxies", proxies)
cache.UsefullProxiesCount = proxies.Len()

View File

@@ -0,0 +1,192 @@
// Code generated for package bingeoip by go-bindata DO NOT EDIT. (@generated)
// sources:
// assets/GeoLite2-City.mmdb
package bingeoip
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
// bindataRead reads the given file from disk. It returns an error on failure.
func bindataRead(path, name string) ([]byte, error) {
buf, err := ioutil.ReadFile(path)
if err != nil {
err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err)
}
return buf, err
}
type asset struct {
bytes []byte
info os.FileInfo
}
// assetsGeolite2CityMmdb reads file data from disk. It returns an error on failure.
func assetsGeolite2CityMmdb() (*asset, error) {
path := "assets/GeoLite2-City.mmdb"
name := "assets/GeoLite2-City.mmdb"
bytes, err := bindataRead(path, name)
if err != nil {
return nil, err
}
fi, err := os.Stat(path)
if err != nil {
err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
}
a := &asset{bytes: bytes, info: fi}
return a, err
}
// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
}
return a.bytes, nil
}
return nil, fmt.Errorf("Asset %s not found", name)
}
// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
a, err := Asset(name)
if err != nil {
panic("asset: Asset(" + name + "): " + err.Error())
}
return a
}
// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
}
return a.info, nil
}
return nil, fmt.Errorf("AssetInfo %s not found", name)
}
// AssetNames returns the names of the assets.
func AssetNames() []string {
names := make([]string, 0, len(_bindata))
for name := range _bindata {
names = append(names, name)
}
return names
}
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
"assets/GeoLite2-City.mmdb": assetsGeolite2CityMmdb,
}
// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
// data/
// foo.txt
// img/
// a.png
// b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
node := _bintree
if len(name) != 0 {
cannonicalName := strings.Replace(name, "\\", "/", -1)
pathList := strings.Split(cannonicalName, "/")
for _, p := range pathList {
node = node.Children[p]
if node == nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
}
}
if node.Func != nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
rv := make([]string, 0, len(node.Children))
for childName := range node.Children {
rv = append(rv, childName)
}
return rv, nil
}
type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"assets": &bintree{nil, map[string]*bintree{
"GeoLite2-City.mmdb": &bintree{assetsGeolite2CityMmdb, map[string]*bintree{}},
}},
}}
// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
data, err := Asset(name)
if err != nil {
return err
}
info, err := AssetInfo(name)
if err != nil {
return err
}
err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
if err != nil {
return err
}
err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
if err != nil {
return err
}
err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
if err != nil {
return err
}
return nil
}
// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
children, err := AssetDir(name)
// File
if err != nil {
return RestoreAsset(dir, name)
}
// Dir
for _, child := range children {
err = RestoreAssets(dir, filepath.Join(name, child))
if err != nil {
return err
}
}
return nil
}
func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

View File

@@ -0,0 +1,278 @@
// Code generated for package binhtml by go-bindata DO NOT EDIT. (@generated)
// sources:
// assets/html/clash-config.yaml
// assets/html/clash.html
// assets/html/index.html
// assets/html/surge.conf
// assets/html/surge.html
package binhtml
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
// bindataRead reads the given file from disk. It returns an error on failure.
func bindataRead(path, name string) ([]byte, error) {
buf, err := ioutil.ReadFile(path)
if err != nil {
err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err)
}
return buf, err
}
type asset struct {
bytes []byte
info os.FileInfo
}
// assetsHtmlClashConfigYaml reads file data from disk. It returns an error on failure.
func assetsHtmlClashConfigYaml() (*asset, error) {
path := "assets/html/clash-config.yaml"
name := "assets/html/clash-config.yaml"
bytes, err := bindataRead(path, name)
if err != nil {
return nil, err
}
fi, err := os.Stat(path)
if err != nil {
err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
}
a := &asset{bytes: bytes, info: fi}
return a, err
}
// assetsHtmlClashHtml reads file data from disk. It returns an error on failure.
func assetsHtmlClashHtml() (*asset, error) {
path := "assets/html/clash.html"
name := "assets/html/clash.html"
bytes, err := bindataRead(path, name)
if err != nil {
return nil, err
}
fi, err := os.Stat(path)
if err != nil {
err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
}
a := &asset{bytes: bytes, info: fi}
return a, err
}
// assetsHtmlIndexHtml reads file data from disk. It returns an error on failure.
func assetsHtmlIndexHtml() (*asset, error) {
path := "assets/html/index.html"
name := "assets/html/index.html"
bytes, err := bindataRead(path, name)
if err != nil {
return nil, err
}
fi, err := os.Stat(path)
if err != nil {
err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
}
a := &asset{bytes: bytes, info: fi}
return a, err
}
// assetsHtmlSurgeConf reads file data from disk. It returns an error on failure.
func assetsHtmlSurgeConf() (*asset, error) {
path := "assets/html/surge.conf"
name := "assets/html/surge.conf"
bytes, err := bindataRead(path, name)
if err != nil {
return nil, err
}
fi, err := os.Stat(path)
if err != nil {
err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
}
a := &asset{bytes: bytes, info: fi}
return a, err
}
// assetsHtmlSurgeHtml reads file data from disk. It returns an error on failure.
func assetsHtmlSurgeHtml() (*asset, error) {
path := "assets/html/surge.html"
name := "assets/html/surge.html"
bytes, err := bindataRead(path, name)
if err != nil {
return nil, err
}
fi, err := os.Stat(path)
if err != nil {
err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
}
a := &asset{bytes: bytes, info: fi}
return a, err
}
// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
}
return a.bytes, nil
}
return nil, fmt.Errorf("Asset %s not found", name)
}
// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
a, err := Asset(name)
if err != nil {
panic("asset: Asset(" + name + "): " + err.Error())
}
return a
}
// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
}
return a.info, nil
}
return nil, fmt.Errorf("AssetInfo %s not found", name)
}
// AssetNames returns the names of the assets.
func AssetNames() []string {
names := make([]string, 0, len(_bindata))
for name := range _bindata {
names = append(names, name)
}
return names
}
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
"assets/html/clash-config.yaml": assetsHtmlClashConfigYaml,
"assets/html/clash.html": assetsHtmlClashHtml,
"assets/html/index.html": assetsHtmlIndexHtml,
"assets/html/surge.conf": assetsHtmlSurgeConf,
"assets/html/surge.html": assetsHtmlSurgeHtml,
}
// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
// data/
// foo.txt
// img/
// a.png
// b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
node := _bintree
if len(name) != 0 {
cannonicalName := strings.Replace(name, "\\", "/", -1)
pathList := strings.Split(cannonicalName, "/")
for _, p := range pathList {
node = node.Children[p]
if node == nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
}
}
if node.Func != nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
rv := make([]string, 0, len(node.Children))
for childName := range node.Children {
rv = append(rv, childName)
}
return rv, nil
}
type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"assets": &bintree{nil, map[string]*bintree{
"html": &bintree{nil, map[string]*bintree{
"clash-config.yaml": &bintree{assetsHtmlClashConfigYaml, map[string]*bintree{}},
"clash.html": &bintree{assetsHtmlClashHtml, map[string]*bintree{}},
"index.html": &bintree{assetsHtmlIndexHtml, map[string]*bintree{}},
"surge.conf": &bintree{assetsHtmlSurgeConf, map[string]*bintree{}},
"surge.html": &bintree{assetsHtmlSurgeHtml, map[string]*bintree{}},
}},
}},
}}
// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
data, err := Asset(name)
if err != nil {
return err
}
info, err := AssetInfo(name)
if err != nil {
return err
}
err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
if err != nil {
return err
}
err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
if err != nil {
return err
}
err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
if err != nil {
return err
}
return nil
}
// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
children, err := AssetDir(name)
// File
if err != nil {
return RestoreAsset(dir, name)
}
// Dir
for _, child := range children {
err = RestoreAssets(dir, filepath.Join(name, child))
if err != nil {
return err
}
}
return nil
}
func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

View File

@@ -1,7 +1,6 @@
package cache
import (
"log"
"time"
"github.com/patrickmn/go-cache"
@@ -13,10 +12,8 @@ var c = cache.New(cache.NoExpiration, 10*time.Minute)
func GetProxies(key string) proxy.ProxyList {
result, found := c.Get(key)
if found {
log.Println("found cache for:", key, "length:", len(result.(proxy.ProxyList)))
return result.(proxy.ProxyList)
}
log.Println("cache not found:", key)
return nil
}

View File

@@ -1,6 +1,8 @@
package cache
var (
GettersCount = 0
AllProxiesCount = 0
SSRProxiesCount = 0
SSProxiesCount = 0

View File

@@ -40,7 +40,7 @@ func NewTGChannelGetter(options tool.Options) (getter Getter, err error) {
return nil, err
}
return &TGChannelGetter{
c: colly.NewCollector(),
c: tool.GetColly(),
NumNeeded: t,
Url: "https://t.me/s/" + url,
}, nil

View File

@@ -16,9 +16,9 @@ func init() {
}
type WebFanqiangdang struct {
c *colly.Collector
Url string
results proxy.ProxyList
c *colly.Collector
Url string
results proxy.ProxyList
}
func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
@@ -29,8 +29,8 @@ func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
return nil, err
}
return &WebFanqiangdang{
c: colly.NewCollector(),
Url: url,
c: colly.NewCollector(),
Url: url,
}, nil
}
return nil, ErrorUrlNotFound
@@ -70,11 +70,10 @@ func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
}
}
type WebFanqiangdangRSS struct {
c *colly.Collector
Url string
results []string
c *colly.Collector
Url string
results []string
}
func NewWebFanqiangdangRSSGetter(options tool.Options) (getter Getter, err error) {
@@ -85,8 +84,8 @@ func NewWebFanqiangdangRSSGetter(options tool.Options) (getter Getter, err error
return nil, err
}
return &WebFanqiangdangRSS{
c: colly.NewCollector(),
Url: url,
c: tool.GetColly(),
Url: url,
}, nil
}
return nil, ErrorUrlNotFound

View File

@@ -19,6 +19,10 @@ func (b *Base) SetName(name string) {
b.Name = name
}
func (b *Base) SetIP(ip string) {
b.Server = ip
}
func (b *Base) BaseInfo() *Base {
return b
}
@@ -34,6 +38,7 @@ type Proxy interface {
ToSurge() string
Identifier() string
SetName(name string)
SetIP(ip string)
TypeName() string
BaseInfo() *Base
Clone() Proxy

View File

@@ -4,8 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/ivpusic/grpool"
"github.com/Dreamacro/clash/adapters/outbound"
)
@@ -35,21 +38,75 @@ func testDelay(p Proxy) (delay uint16, err error) {
return delay, err
}
func CleanBadProxiesWithGrpool(proxies []Proxy) (cproxies []Proxy) {
pool := grpool.NewPool(500, 200)
c := make(chan checkResult)
defer close(c)
pool.WaitCount(len(proxies))
go func() {
for _, p := range proxies {
pp := p
pool.JobQueue <- func() {
defer pool.JobDone()
delay, err := testDelay(pp)
if err == nil {
c <- checkResult{
name: pp.Identifier(),
delay: delay,
}
}
}
}
}()
done := make(chan struct{})
defer close(done)
go func() {
pool.WaitAll()
pool.Release()
done <- struct{}{}
}()
okMap := make(map[string]struct{})
for {
select {
case r := <-c:
if r.delay > 0 {
okMap[r.name] = struct{}{}
}
case <-done:
cproxies = make(ProxyList, 0, 500)
for _, p := range proxies {
if _, ok := okMap[p.Identifier()]; ok {
cproxies = append(cproxies, p.Clone())
}
}
return
}
}
}
func CleanBadProxies(proxies []Proxy) (cproxies []Proxy) {
c := make(chan checkResult, 40)
defer close(c)
wg := &sync.WaitGroup{}
wg.Add(len(proxies))
for _, p := range proxies {
go testProxyDelayToChan(p, c)
go testProxyDelayToChan(p, c, wg)
}
go func() {
wg.Wait()
close(c)
}()
okMap := make(map[string]struct{})
size := len(proxies)
for i := 0; i < size; i++ {
r := <-c
for r := range c {
if r.delay > 0 {
okMap[r.name] = struct{}{}
}
}
cproxies = make([]Proxy, 0)
cproxies = make(ProxyList, 0, 500)
for _, p := range proxies {
if _, ok := okMap[p.Identifier()]; ok {
cproxies = append(cproxies, p.Clone())
@@ -63,17 +120,13 @@ type checkResult struct {
delay uint16
}
func testProxyDelayToChan(p Proxy, c chan checkResult) {
func testProxyDelayToChan(p Proxy, c chan checkResult, wg *sync.WaitGroup) {
defer wg.Done()
delay, err := testDelay(p)
if err != nil {
if err == nil {
c <- checkResult{
name: p.Identifier(),
delay: 0,
delay: delay,
}
return
}
c <- checkResult{
name: p.Identifier(),
delay: delay,
}
}

View File

@@ -6,11 +6,16 @@ import (
"os"
"github.com/oschwald/geoip2-golang"
bingeoip "github.com/zu1k/proxypool/internal/bindata/geoip"
)
var geoIp GeoIP
func InitGeoIpDB() {
err := bingeoip.RestoreAsset("", "assets/GeoLite2-City.mmdb")
if err != nil {
panic(err)
}
geoIp = NewGeoIP("assets/GeoLite2-City.mmdb")
}
@@ -37,15 +42,15 @@ func NewGeoIP(filePath string) (geoip GeoIP) {
}
// find ip info
func (g GeoIP) Find(ipORdomain string) (country string, err error) {
func (g GeoIP) Find(ipORdomain string) (ip, country string, err error) {
ips, err := net.LookupIP(ipORdomain)
if err != nil {
return "", err
return "", "", err
}
ipData := net.ParseIP(ips[0].String())
record, err := g.db.City(ipData)
if err != nil {
return "", err
return "", "", err
}
return record.Country.IsoCode, nil
return ips[0].String(), record.Country.IsoCode, nil
}

View File

@@ -3,6 +3,7 @@ package proxy
import (
"fmt"
"sort"
"sync"
)
type ProxyList []Proxy
@@ -63,13 +64,21 @@ func (ps ProxyList) Sort() ProxyList {
func (ps ProxyList) NameAddCounrty() ProxyList {
num := len(ps)
wg := &sync.WaitGroup{}
wg.Add(num)
for i := 0; i < num; i++ {
country, err := geoIp.Find(ps[i].BaseInfo().Server)
if err != nil || country == "" {
country = "Earth"
}
ps[i].SetName(fmt.Sprintf("%s", country))
ii := i
go func() {
defer wg.Done()
_, country, err := geoIp.Find(ps[ii].BaseInfo().Server)
if err != nil || country == "" {
country = "ZZ"
}
ps[ii].SetName(fmt.Sprintf("%s", country))
//ps[ii].SetIP(ip)
}()
}
wg.Wait()
return ps
}
@@ -81,6 +90,24 @@ func (ps ProxyList) NameAddIndex() ProxyList {
return ps
}
func (ps ProxyList) NameReIndex() ProxyList {
num := len(ps)
for i := 0; i < num; i++ {
originName := ps[i].BaseInfo().Name
country := string([]rune(originName)[:2])
ps[i].SetName(fmt.Sprintf("%s_%d", country, i+1))
}
return ps
}
func (ps ProxyList) NameAddTG() ProxyList {
num := len(ps)
for i := 0; i < num; i++ {
ps[i].SetName(fmt.Sprintf("%s %s", ps[i].BaseInfo().Name, "@peekfun"))
}
return ps
}
func Deduplication(src ProxyList) ProxyList {
result := make(ProxyList, 0, len(src))
temp := map[string]struct{}{}

View File

@@ -34,7 +34,7 @@ type ShadowsocksR struct {
}
func (ssr ShadowsocksR) Identifier() string {
return net.JoinHostPort(ssr.Server, strconv.Itoa(ssr.Port)) + ssr.ProtocolParam
return net.JoinHostPort(ssr.Server, strconv.Itoa(ssr.Port)) + ssr.Password + ssr.ProtocolParam
}
func (ssr ShadowsocksR) String() string {

27
pkg/tool/colly.go Normal file
View File

@@ -0,0 +1,27 @@
package tool
import (
"net"
"net/http"
"time"
"github.com/gocolly/colly"
)
func GetColly() *colly.Collector {
c := colly.NewCollector(
colly.UserAgent(UserAgent),
)
c.WithTransport(&http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 10 * time.Second, // 超时时间
KeepAlive: 10 * time.Second, // keepAlive 超时时间
}).DialContext,
MaxIdleConns: 100, // 最大空闲连接数
IdleConnTimeout: 20 * time.Second, // 空闲连接超时
TLSHandshakeTimeout: 10 * time.Second, // TLS 握手超时
ExpectContinueTimeout: 10 * time.Second,
})
return c
}

View File

@@ -6,6 +6,8 @@ import (
"time"
)
const UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36"
type HttpClient struct {
*http.Client
}
@@ -28,7 +30,7 @@ func (c *HttpClient) Get(url string) (resp *http.Response, err error) {
return nil, err
}
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
req.Header.Set("User-Agent", UserAgent)
return c.Do(req)
}
@@ -38,6 +40,6 @@ func (c *HttpClient) Post(url string, body io.Reader) (resp *http.Response, err
return nil, err
}
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
req.Header.Set("User-Agent", UserAgent)
return c.Do(req)
}