Merge branch 'master' of github.com:zu1k/proxypool

Bump docker/build-push-action from v1.1.0 to v2

.github/workflows/docker.yml

@@ -12,7 +12,7 @@ jobs:
         uses: actions/checkout@v2.3.2
 
       - name: Build and push Docker images
-        uses: docker/build-push-action@v1.1.0
+        uses: docker/build-push-action@v2
         with:
           username: zu1k
           password: ${{ secrets.GITHUB_TOKEN }}

api/router.go

@@ -14,7 +14,7 @@ import (
 	"github.com/zu1k/proxypool/pkg/provider"
 )
 
-const version = "v0.3.6"
+const version = "v0.3.9"
 
 var router *gin.Engine
 
@@ -182,6 +182,18 @@ func setupRouter() {
 		}
 		c.String(200, vmessSub.Provide())
 	})
+
+	router.GET("/sip002/sub", func(c *gin.Context) {
+		proxies := cache.GetProxies("proxies")
+		sip002Sub := provider.SIP002Sub{
+			provider.Base{
+				Proxies: &proxies,
+				Types:   "ss",
+			},
+		}
+		c.String(200, sip002Sub.Provide())
+	})
+
 	router.GET("/link/:id", func(c *gin.Context) {
 		idx := c.Param("id")
 		proxies := cache.GetProxies("allproxies")
@@ -206,6 +218,7 @@ func Run() {
 }
 
 func loadTemplate() (t *template.Template, err error) {
+	_ = binhtml.RestoreAssets("", "assets/html")
 	t = template.New("")
 	for _, fileName := range binhtml.AssetNames() {
 		data := binhtml.MustAsset(fileName)

internal/app/task.go

@@ -5,9 +5,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/zu1k/proxypool/internal/database"
-
 	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/provider"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
@@ -22,6 +21,7 @@ func CrawlGo() {
 		go g.Get2Chan(pc, wg)
 	}
 	proxies := cache.GetProxies("allproxies")
+	proxies = append(proxies, database.GetAllProxies()...)
 	go func() {
 		wg.Wait()
 		close(pc)
@@ -31,8 +31,8 @@ func CrawlGo() {
 			proxies = append(proxies, node)
 		}
 	}
-	// 节点去重
-	proxies = proxies.Deduplication()
+	// 节点衍生并去重
+	proxies = proxies.Deduplication().Derive()
 	log.Println("CrawlGo node count:", len(proxies))
 	proxies = provider.Clash{
 		provider.Base{
@@ -40,7 +40,7 @@ func CrawlGo() {
 		},
 	}.CleanProxies()
 	log.Println("CrawlGo cleaned node count:", len(proxies))
-	proxies.NameAddCounrty().Sort().NameAddIndex().NameAddTG()
+	proxies.NameSetCounrty().Sort().NameAddIndex().NameAddTG()
 	log.Println("Proxy rename DONE!")
 
 	// 全节点存储到数据库

internal/cron/cron.go

@@ -1,6 +1,8 @@
 package cron
 
 import (
+	"runtime"
+
 	"github.com/jasonlvhit/gocron"
 	"github.com/zu1k/proxypool/internal/app"
 )
@@ -13,4 +15,6 @@ func Cron() {
 func crawlTask() {
 	_ = app.InitConfigAndGetters("")
 	app.CrawlGo()
+	app.Getters = nil
+	runtime.GC()
 }

internal/database/db.go

@@ -8,6 +8,7 @@ import (
 
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 )
 
 var DB *gorm.DB
@@ -20,7 +21,9 @@ func connect() (err error) {
 	if url := os.Getenv("DATABASE_URL"); url != "" {
 		dsn = url
 	}
-	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
 	if err == nil {
 		fmt.Println("DB connect success: ", DB.Name())
 	}

internal/database/proxy.go

@@ -7,9 +7,9 @@ import (
 
 type Proxy struct {
 	gorm.Model
-	proxy.Base `gorm:"index"`
+	proxy.Base
 	Link       string
-	Identifier string `gorm:"primaryKey"`
+	Identifier string `gorm:"unique"`
 }
 
 func InitTables() {
@@ -25,14 +25,55 @@ func InitTables() {
 	}
 }
 
+const roundSize = 100
+
 func SaveProxyList(pl proxy.ProxyList) {
-	proxies := make([]Proxy, pl.Len())
-	for i, p := range pl {
-		proxies[i] = Proxy{
+	if DB == nil {
+		return
+	}
+
+	// TODO 批量插入因为是生成一个sql，如果插入失败（重复）就全都没了
+	//size := pl.Len()
+	//round := (size + roundSize - 1) / roundSize
+	//
+	//for r := 0; r < round; r++ {
+	//	proxies := make([]Proxy, 0, roundSize)
+	//	for i, j := r*roundSize, (r+1)*roundSize-1; i < j && i < size; i++ {
+	//		p := pl[i]
+	//		proxies = append(proxies, Proxy{
+	//			Base:       *p.BaseInfo(),
+	//			Link:       p.Link(),
+	//			Identifier: p.Identifier(),
+	//		})
+	//	}
+	//	DB.Create(&proxies)
+	//}
+
+	for _, p := range pl {
+		DB.Create(&Proxy{
 			Base:       *p.BaseInfo(),
 			Link:       p.Link(),
 			Identifier: p.Identifier(),
+		})
+	}
+}
+
+func GetAllProxies() (proxies proxy.ProxyList) {
+	proxies = make(proxy.ProxyList, 0)
+	if DB == nil {
+		return
+	}
+
+	proxiesDB := make([]Proxy, 0)
+	DB.Select("link").Find(&proxiesDB)
+
+	for _, proxyDB := range proxiesDB {
+		if proxiesDB != nil {
+			p, err := proxy.ParseProxyFromLink(proxyDB.Link)
+			if err == nil && p != nil {
+				proxies = append(proxies, p)
+			}
 		}
 	}
-	DB.Create(&proxies)
+	return
 }

main.go

@@ -3,19 +3,24 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
-	"github.com/zu1k/proxypool/internal/database"
-
 	"github.com/zu1k/proxypool/api"
 	"github.com/zu1k/proxypool/internal/app"
 	"github.com/zu1k/proxypool/internal/cron"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
 var configFilePath = ""
 
 func main() {
+	go func() {
+		http.ListenAndServe("0.0.0.0:6060", nil)
+	}()
+
 	flag.StringVar(&configFilePath, "c", "", "path to config file: config.yaml")
 	flag.Parse()
 

pkg/getter/base.go

@@ -2,7 +2,6 @@ package getter
 
 import (
 	"errors"
-	"strings"
 	"sync"
 
 	"github.com/zu1k/proxypool/pkg/proxy"
@@ -30,28 +29,13 @@ func NewGetter(sourceType string, options tool.Options) (getter Getter, err erro
 	return nil, ErrorCreaterNotSupported
 }
 
-func String2Proxy(link string) proxy.Proxy {
-	var err error
-	var data proxy.Proxy
-	if strings.HasPrefix(link, "ssr://") {
-		data, err = proxy.ParseSSRLink(link)
-	} else if strings.HasPrefix(link, "vmess://") {
-		data, err = proxy.ParseVmessLink(link)
-	} else if strings.HasPrefix(link, "ss://") {
-		data, err = proxy.ParseSSLink(link)
-	} else if strings.HasPrefix(link, "trojan://") {
-		data, err = proxy.ParseTrojanLink(link)
-	}
-	if err != nil {
-		return nil
-	}
-	return data
-}
-
 func StringArray2ProxyArray(origin []string) proxy.ProxyList {
 	results := make(proxy.ProxyList, 0)
 	for _, link := range origin {
-		results = append(results, String2Proxy(link))
+		p, err := proxy.ParseProxyFromLink(link)
+		if err == nil && p != nil {
+			results = append(results, p)
+		}
 	}
 	return results
 }

pkg/provider/base.go

@@ -10,15 +10,6 @@ type Provider interface {
 	Provide() string
 }
 
-func checkInList(list []string, item string) bool {
-	for _, i := range list {
-		if item == i {
-			return true
-		}
-	}
-	return false
-}
-
 type Base struct {
 	Proxies    *proxy.ProxyList `yaml:"proxies"`
 	Types      string           `yaml:"type"`

pkg/provider/clash.go

@@ -3,6 +3,8 @@ package provider
 import (
 	"strings"
 
+	"github.com/zu1k/proxypool/pkg/tool"
+
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
@@ -37,17 +39,17 @@ func checkClashSupport(p proxy.Proxy) bool {
 	switch p.TypeName() {
 	case "ssr":
 		ssr := p.(*proxy.ShadowsocksR)
-		if checkInList(ssrCipherList, ssr.Cipher) && checkInList(ssrProtocolList, ssr.Protocol) && checkInList(ssrObfsList, ssr.Obfs) {
+		if tool.CheckInList(proxy.SSRCipherList, ssr.Cipher) && tool.CheckInList(ssrProtocolList, ssr.Protocol) && tool.CheckInList(ssrObfsList, ssr.Obfs) {
 			return true
 		}
 	case "vmess":
 		vmess := p.(*proxy.Vmess)
-		if checkInList(vmessCipherList, vmess.Cipher) {
+		if tool.CheckInList(vmessCipherList, vmess.Cipher) {
 			return true
 		}
 	case "ss":
 		ss := p.(*proxy.Shadowsocks)
-		if checkInList(ssCipherList, ss.Cipher) {
+		if tool.CheckInList(proxy.SSCipherList, ss.Cipher) {
 			return true
 		}
 	case "trojan":
@@ -58,30 +60,6 @@ func checkClashSupport(p proxy.Proxy) bool {
 	return false
 }
 
-var ssrCipherList = []string{
-	"aes-128-cfb",
-	"aes-192-cfb",
-	"aes-256-cfb",
-	"aes-128-ctr",
-	"aes-192-ctr",
-	"aes-256-ctr",
-	"aes-128-ofb",
-	"aes-192-ofb",
-	"aes-256-ofb",
-	"des-cfb",
-	"bf-cfb",
-	"cast5-cfb",
-	"rc4-md5",
-	"chacha20-ietf",
-	"salsa20",
-	"camellia-128-cfb",
-	"camellia-192-cfb",
-	"camellia-256-cfb",
-	"idea-cfb",
-	"rc2-cfb",
-	"seed-cfb",
-}
-
 var ssrObfsList = []string{
 	"plain",
 	"http_simple",
@@ -110,20 +88,3 @@ var vmessCipherList = []string{
 	"chacha20-poly1305",
 	"none",
 }
-
-var ssCipherList = []string{
-	"aes-128-gcm",
-	"aes-192-gcm",
-	"aes-256-gcm",
-	"aes-128-cfb",
-	"aes-192-cfb",
-	"aes-256-cfb",
-	"aes-128-ctr",
-	"aes-192-ctr",
-	"aes-256-ctr",
-	"rc4-md5",
-	"chacha20-ietf",
-	"xchacha20",
-	"chacha20-ietf-poly1305",
-	"xchacha20-ietf-poly1305",
-}

pkg/provider/sssub.go

@@ -3,6 +3,9 @@ package provider
 import (
 	"encoding/json"
 	"strconv"
+	"strings"
+
+	"github.com/zu1k/proxypool/pkg/tool"
 
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
@@ -43,3 +46,17 @@ func (sub SSSub) Provide() string {
 	}
 	return string(text)
 }
+
+type SIP002Sub struct {
+	Base
+}
+
+func (sub SIP002Sub) Provide() string {
+	sub.Types = "ss"
+	sub.preFilter()
+	var resultBuilder strings.Builder
+	for _, p := range *sub.Proxies {
+		resultBuilder.WriteString(p.Link() + "\n")
+	}
+	return tool.Base64EncodeString(resultBuilder.String(), false)
+}

pkg/provider/surge.go

@@ -3,6 +3,8 @@ package provider
 import (
 	"strings"
 
+	"github.com/zu1k/proxypool/pkg/tool"
+
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
@@ -16,7 +18,7 @@ func (s Surge) Provide() string {
 	var resultBuilder strings.Builder
 	for _, p := range *s.Proxies {
 		if checkSurgeSupport(p) {
-			resultBuilder.WriteString(p.ToClash() + "\n")
+			resultBuilder.WriteString(p.ToSurge() + "\n")
 		}
 	}
 	return resultBuilder.String()
@@ -30,7 +32,7 @@ func checkSurgeSupport(p proxy.Proxy) bool {
 		return true
 	case *proxy.Shadowsocks:
 		ss := p.(*proxy.Shadowsocks)
-		if checkInList(ssCipherList, ss.Cipher) {
+		if tool.CheckInList(proxy.SSCipherList, ss.Cipher) {
 			return true
 		}
 	default:

pkg/proxy/base.go

@@ -1,13 +1,18 @@
 package proxy
 
+import (
+	"errors"
+	"strings"
+)
+
 type Base struct {
-	Name    string `yaml:"name" json:"name"`
-	Server  string `yaml:"server" json:"server"`
-	Port    int    `yaml:"port" json:"port"`
-	Type    string `yaml:"type" json:"type"`
+	Name    string `yaml:"name" json:"name" gorm:"index"`
+	Server  string `yaml:"server" json:"server" gorm:"index"`
+	Port    int    `yaml:"port" json:"port" gorm:"index"`
+	Type    string `yaml:"type" json:"type" gorm:"index"`
 	UDP     bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
-	Country string `yaml:"country,omitempty" json:"country,omitempty"`
-	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty"`
+	Country string `yaml:"country,omitempty" json:"country,omitempty" gorm:"index"`
+	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"`
 }
 
 func (b *Base) TypeName() string {
@@ -56,3 +61,28 @@ type Proxy interface {
 	SetUseable(useable bool)
 	SetCountry(country string)
 }
+
+func ParseProxyFromLink(link string) (p Proxy, err error) {
+	if strings.HasPrefix(link, "ssr://") {
+		p, err = ParseSSRLink(link)
+	} else if strings.HasPrefix(link, "vmess://") {
+		p, err = ParseVmessLink(link)
+	} else if strings.HasPrefix(link, "ss://") {
+		p, err = ParseSSLink(link)
+	} else if strings.HasPrefix(link, "trojan://") {
+		p, err = ParseTrojanLink(link)
+	}
+	if err != nil || p == nil {
+		return nil, errors.New("link parse failed")
+	}
+	ip, country, err := geoIp.Find(p.BaseInfo().Server)
+	if err != nil {
+		country = "🏁 ZZ"
+	}
+	p.SetCountry(country)
+	// trojan依赖域名？
+	if p.TypeName() != "trojan" {
+		p.SetIP(ip)
+	}
+	return
+}

pkg/proxy/convert.go

@@ -0,0 +1,100 @@
+package proxy
+
+import (
+	"errors"
+
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+var ErrorTypeCanNotConvert = errors.New("type not support")
+
+// Convert2SS convert proxy to ShadowsocksR if possible
+func Convert2SSR(p Proxy) (ssr *ShadowsocksR, err error) {
+	if p.TypeName() == "ss" {
+		ss := p.(*Shadowsocks)
+		if ss == nil {
+			return nil, errors.New("ss is nil")
+		}
+		if !tool.CheckInList(SSRCipherList, ss.Cipher) {
+			return nil, errors.New("cipher not support")
+		}
+		base := ss.Base
+		base.Type = "ssr"
+		return &ShadowsocksR{
+			Base:     base,
+			Password: ss.Password,
+			Cipher:   ss.Cipher,
+			Protocol: "origin",
+			Obfs:     "plain",
+			Group:    "proxy.tgbot.co",
+		}, nil
+	}
+	return nil, ErrorTypeCanNotConvert
+}
+
+// Convert2SS convert proxy to Shadowsocks if possible
+func Convert2SS(p Proxy) (ss *Shadowsocks, err error) {
+	if p.TypeName() == "ss" {
+		ssr := p.(*ShadowsocksR)
+		if ssr == nil {
+			return nil, errors.New("ssr is nil")
+		}
+		if !tool.CheckInList(SSCipherList, ssr.Cipher) {
+			return nil, errors.New("cipher not support")
+		}
+		if ssr.Protocol != "origin" || ssr.Obfs != "plain" {
+			return nil, errors.New("protocol or obfs not allowed")
+		}
+		base := ssr.Base
+		base.Type = "ss"
+		return &Shadowsocks{
+			Base:       base,
+			Password:   ssr.Password,
+			Cipher:     ssr.Cipher,
+			Plugin:     "",
+			PluginOpts: nil,
+		}, nil
+	}
+	return nil, ErrorTypeCanNotConvert
+}
+
+var SSRCipherList = []string{
+	"aes-128-cfb",
+	"aes-192-cfb",
+	"aes-256-cfb",
+	"aes-128-ctr",
+	"aes-192-ctr",
+	"aes-256-ctr",
+	"aes-128-ofb",
+	"aes-192-ofb",
+	"aes-256-ofb",
+	"des-cfb",
+	"bf-cfb",
+	"cast5-cfb",
+	"rc4-md5",
+	"chacha20-ietf",
+	"salsa20",
+	"camellia-128-cfb",
+	"camellia-192-cfb",
+	"camellia-256-cfb",
+	"idea-cfb",
+	"rc2-cfb",
+	"seed-cfb",
+}
+
+var SSCipherList = []string{
+	"aes-128-gcm",
+	"aes-192-gcm",
+	"aes-256-gcm",
+	"aes-128-cfb",
+	"aes-192-cfb",
+	"aes-256-cfb",
+	"aes-128-ctr",
+	"aes-192-ctr",
+	"aes-256-ctr",
+	"rc4-md5",
+	"chacha20-ietf",
+	"xchacha20",
+	"chacha20-ietf-poly1305",
+	"xchacha20-ietf-poly1305",
+}

pkg/proxy/proxies.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
 )
 
 type ProxyList []Proxy
@@ -63,24 +62,11 @@ func (ps ProxyList) Sort() ProxyList {
 	return ps
 }
 
-func (ps ProxyList) NameAddCounrty() ProxyList {
+func (ps ProxyList) NameSetCounrty() ProxyList {
 	num := len(ps)
-	wg := &sync.WaitGroup{}
-	wg.Add(num)
 	for i := 0; i < num; i++ {
-		ii := i
-		go func() {
-			defer wg.Done()
-			_, country, err := geoIp.Find(ps[ii].BaseInfo().Server)
-			if err != nil {
-				country = "🏁 ZZ"
-			}
-			ps[ii].SetName(fmt.Sprintf("%s", country))
-			ps[ii].SetCountry(country)
-			//ps[ii].SetIP(ip)
-		}()
+		ps[i].SetName(ps[i].BaseInfo().Country)
 	}
-	wg.Wait()
 	return ps
 }
 
@@ -110,20 +96,6 @@ func (ps ProxyList) NameAddTG() ProxyList {
 	return ps
 }
 
-func Deduplication(src ProxyList) ProxyList {
-	result := make(ProxyList, 0, len(src))
-	temp := map[string]struct{}{}
-	for _, item := range src {
-		if item != nil {
-			if _, ok := temp[item.Identifier()]; !ok {
-				temp[item.Identifier()] = struct{}{}
-				result = append(result, item)
-			}
-		}
-	}
-	return result
-}
-
 func (ps ProxyList) Clone() ProxyList {
 	result := make(ProxyList, 0, len(ps))
 	for _, pp := range ps {
@@ -133,3 +105,25 @@ func (ps ProxyList) Clone() ProxyList {
 	}
 	return result
 }
+
+// Derive 将原有节点中的ss和ssr互相转换进行衍生
+func (ps ProxyList) Derive() ProxyList {
+	proxies := ps
+	for _, p := range ps {
+		if p == nil {
+			continue
+		}
+		if p.TypeName() == "ss" {
+			ssr, err := Convert2SSR(p)
+			if err == nil {
+				proxies = append(proxies, ssr)
+			}
+		} else if p.TypeName() == "ssr" {
+			ss, err := Convert2SS(p)
+			if err == nil {
+				proxies = append(proxies, ss)
+			}
+		}
+	}
+	return proxies.Deduplication()
+}

pkg/tool/check.go

@@ -0,0 +1,10 @@
+package tool
+
+func CheckInList(list []string, item string) bool {
+	for _, i := range list {
+		if item == i {
+			return true
+		}
+	}
+	return false
+}