Merge branch 'master' of github.com:zu1k/proxypool

Bump docker/build-push-action from v1.1.0 to v2

.github/workflows/docker.yml

@@ -12,7 +12,7 @@ jobs:
         uses: actions/checkout@v2.3.2
 
       - name: Build and push Docker images
-        uses: docker/build-push-action@v1.1.0
+        uses: docker/build-push-action@v2
         with:
           username: zu1k
           password: ${{ secrets.GITHUB_TOKEN }}

api/router.go

@@ -14,7 +14,7 @@ import (
 	"github.com/zu1k/proxypool/pkg/provider"
 )
 
-const version = "v0.3.8"
+const version = "v0.3.9"
 
 var router *gin.Engine
 
@@ -182,6 +182,18 @@ func setupRouter() {
 		}
 		c.String(200, vmessSub.Provide())
 	})
+
+	router.GET("/sip002/sub", func(c *gin.Context) {
+		proxies := cache.GetProxies("proxies")
+		sip002Sub := provider.SIP002Sub{
+			provider.Base{
+				Proxies: &proxies,
+				Types:   "ss",
+			},
+		}
+		c.String(200, sip002Sub.Provide())
+	})
+
 	router.GET("/link/:id", func(c *gin.Context) {
 		idx := c.Param("id")
 		proxies := cache.GetProxies("allproxies")

internal/app/task.go

@@ -31,8 +31,8 @@ func CrawlGo() {
 			proxies = append(proxies, node)
 		}
 	}
-	// 节点去重
-	proxies = proxies.Deduplication()
+	// 节点衍生并去重
+	proxies = proxies.Deduplication().Derive()
 	log.Println("CrawlGo node count:", len(proxies))
 	proxies = provider.Clash{
 		provider.Base{
@@ -40,7 +40,7 @@ func CrawlGo() {
 		},
 	}.CleanProxies()
 	log.Println("CrawlGo cleaned node count:", len(proxies))
-	proxies.NameAddCounrty().Sort().NameAddIndex().NameAddTG()
+	proxies.NameSetCounrty().Sort().NameAddIndex().NameAddTG()
 	log.Println("Proxy rename DONE!")
 
 	// 全节点存储到数据库

internal/database/proxy.go

@@ -1,7 +1,6 @@
 package database
 
 import (
-	"github.com/zu1k/proxypool/pkg/getter"
 	"github.com/zu1k/proxypool/pkg/proxy"
 	"gorm.io/gorm"
 )
@@ -33,20 +32,29 @@ func SaveProxyList(pl proxy.ProxyList) {
 		return
 	}
 
-	size := pl.Len()
-	round := (size + roundSize - 1) / roundSize
+	// TODO 批量插入因为是生成一个sql，如果插入失败（重复）就全都没了
+	//size := pl.Len()
+	//round := (size + roundSize - 1) / roundSize
+	//
+	//for r := 0; r < round; r++ {
+	//	proxies := make([]Proxy, 0, roundSize)
+	//	for i, j := r*roundSize, (r+1)*roundSize-1; i < j && i < size; i++ {
+	//		p := pl[i]
+	//		proxies = append(proxies, Proxy{
+	//			Base:       *p.BaseInfo(),
+	//			Link:       p.Link(),
+	//			Identifier: p.Identifier(),
+	//		})
+	//	}
+	//	DB.Create(&proxies)
+	//}
 
-	for r := 0; r < round; r++ {
-		proxies := make([]Proxy, 0, roundSize)
-		for i, j := r*roundSize, (r+1)*roundSize-1; i < j && i < size; i++ {
-			p := pl[i]
-			proxies = append(proxies, Proxy{
-				Base:       *p.BaseInfo(),
-				Link:       p.Link(),
-				Identifier: p.Identifier(),
-			})
-		}
-		DB.Create(&proxies)
+	for _, p := range pl {
+		DB.Create(&Proxy{
+			Base:       *p.BaseInfo(),
+			Link:       p.Link(),
+			Identifier: p.Identifier(),
+		})
 	}
 }
 
@@ -61,7 +69,10 @@ func GetAllProxies() (proxies proxy.ProxyList) {
 
 	for _, proxyDB := range proxiesDB {
 		if proxiesDB != nil {
-			proxies = append(proxies, getter.String2Proxy(proxyDB.Link))
+			p, err := proxy.ParseProxyFromLink(proxyDB.Link)
+			if err == nil && p != nil {
+				proxies = append(proxies, p)
+			}
 		}
 	}
 	return

pkg/getter/base.go

@@ -2,7 +2,6 @@ package getter
 
 import (
 	"errors"
-	"strings"
 	"sync"
 
 	"github.com/zu1k/proxypool/pkg/proxy"
@@ -30,28 +29,13 @@ func NewGetter(sourceType string, options tool.Options) (getter Getter, err erro
 	return nil, ErrorCreaterNotSupported
 }
 
-func String2Proxy(link string) proxy.Proxy {
-	var err error
-	var data proxy.Proxy
-	if strings.HasPrefix(link, "ssr://") {
-		data, err = proxy.ParseSSRLink(link)
-	} else if strings.HasPrefix(link, "vmess://") {
-		data, err = proxy.ParseVmessLink(link)
-	} else if strings.HasPrefix(link, "ss://") {
-		data, err = proxy.ParseSSLink(link)
-	} else if strings.HasPrefix(link, "trojan://") {
-		data, err = proxy.ParseTrojanLink(link)
-	}
-	if err != nil {
-		return nil
-	}
-	return data
-}
-
 func StringArray2ProxyArray(origin []string) proxy.ProxyList {
 	results := make(proxy.ProxyList, 0)
 	for _, link := range origin {
-		results = append(results, String2Proxy(link))
+		p, err := proxy.ParseProxyFromLink(link)
+		if err == nil && p != nil {
+			results = append(results, p)
+		}
 	}
 	return results
 }

pkg/provider/ssrsub.go

@@ -3,8 +3,6 @@ package provider
 import (
 	"strings"
 
-	"github.com/zu1k/proxypool/pkg/proxy"
-
 	"github.com/zu1k/proxypool/pkg/tool"
 )
 
@@ -13,18 +11,11 @@ type SSRSub struct {
 }
 
 func (sub SSRSub) Provide() string {
-	sub.Types = "ssr,ss"
+	sub.Types = "ssr"
 	sub.preFilter()
 	var resultBuilder strings.Builder
 	for _, p := range *sub.Proxies {
-		if p.TypeName() == "ssr" {
-			resultBuilder.WriteString(p.Link() + "\n")
-		} else if p.TypeName() == "ss" {
-			ssr, err := proxy.SS2SSR(p.(*proxy.Shadowsocks))
-			if err == nil {
-				resultBuilder.WriteString(ssr.Link() + "\n")
-			}
-		}
+		resultBuilder.WriteString(p.Link() + "\n")
 	}
 	return tool.Base64EncodeString(resultBuilder.String(), false)
 }

pkg/provider/sssub.go

@@ -3,6 +3,9 @@ package provider
 import (
 	"encoding/json"
 	"strconv"
+	"strings"
+
+	"github.com/zu1k/proxypool/pkg/tool"
 
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
@@ -22,22 +25,11 @@ type ssJson struct {
 }
 
 func (sub SSSub) Provide() string {
-	sub.Types = "ss,ssr"
+	sub.Types = "ss"
 	sub.preFilter()
 	proxies := make([]ssJson, 0, sub.Proxies.Len())
 	for _, p := range *sub.Proxies {
-		var pp *proxy.Shadowsocks
-
-		if p.TypeName() == "ssr" {
-			var err error
-			pp, err = proxy.SSR2SS(p.(*proxy.ShadowsocksR))
-			if err != nil {
-				continue
-			}
-		} else if p.TypeName() == "ss" {
-			pp = p.(*proxy.Shadowsocks)
-		}
-
+		pp := p.(*proxy.Shadowsocks)
 		proxies = append(proxies, ssJson{
 			Remarks:    pp.Name,
 			Server:     pp.Server,
@@ -54,3 +46,17 @@ func (sub SSSub) Provide() string {
 	}
 	return string(text)
 }
+
+type SIP002Sub struct {
+	Base
+}
+
+func (sub SIP002Sub) Provide() string {
+	sub.Types = "ss"
+	sub.preFilter()
+	var resultBuilder strings.Builder
+	for _, p := range *sub.Proxies {
+		resultBuilder.WriteString(p.Link() + "\n")
+	}
+	return tool.Base64EncodeString(resultBuilder.String(), false)
+}

pkg/provider/surge.go

@@ -18,7 +18,7 @@ func (s Surge) Provide() string {
 	var resultBuilder strings.Builder
 	for _, p := range *s.Proxies {
 		if checkSurgeSupport(p) {
-			resultBuilder.WriteString(p.ToClash() + "\n")
+			resultBuilder.WriteString(p.ToSurge() + "\n")
 		}
 	}
 	return resultBuilder.String()

pkg/proxy/base.go

@@ -1,5 +1,10 @@
 package proxy
 
+import (
+	"errors"
+	"strings"
+)
+
 type Base struct {
 	Name    string `yaml:"name" json:"name" gorm:"index"`
 	Server  string `yaml:"server" json:"server" gorm:"index"`
@@ -56,3 +61,28 @@ type Proxy interface {
 	SetUseable(useable bool)
 	SetCountry(country string)
 }
+
+func ParseProxyFromLink(link string) (p Proxy, err error) {
+	if strings.HasPrefix(link, "ssr://") {
+		p, err = ParseSSRLink(link)
+	} else if strings.HasPrefix(link, "vmess://") {
+		p, err = ParseVmessLink(link)
+	} else if strings.HasPrefix(link, "ss://") {
+		p, err = ParseSSLink(link)
+	} else if strings.HasPrefix(link, "trojan://") {
+		p, err = ParseTrojanLink(link)
+	}
+	if err != nil || p == nil {
+		return nil, errors.New("link parse failed")
+	}
+	ip, country, err := geoIp.Find(p.BaseInfo().Server)
+	if err != nil {
+		country = "🏁 ZZ"
+	}
+	p.SetCountry(country)
+	// trojan依赖域名？
+	if p.TypeName() != "trojan" {
+		p.SetIP(ip)
+	}
+	return
+}

pkg/proxy/convert.go

@@ -6,44 +6,56 @@ import (
 	"github.com/zu1k/proxypool/pkg/tool"
 )
 
-func SS2SSR(ss *Shadowsocks) (ssr *ShadowsocksR, err error) {
-	if ss == nil {
-		return nil, errors.New("ss is nil")
+var ErrorTypeCanNotConvert = errors.New("type not support")
+
+// Convert2SS convert proxy to ShadowsocksR if possible
+func Convert2SSR(p Proxy) (ssr *ShadowsocksR, err error) {
+	if p.TypeName() == "ss" {
+		ss := p.(*Shadowsocks)
+		if ss == nil {
+			return nil, errors.New("ss is nil")
+		}
+		if !tool.CheckInList(SSRCipherList, ss.Cipher) {
+			return nil, errors.New("cipher not support")
+		}
+		base := ss.Base
+		base.Type = "ssr"
+		return &ShadowsocksR{
+			Base:     base,
+			Password: ss.Password,
+			Cipher:   ss.Cipher,
+			Protocol: "origin",
+			Obfs:     "plain",
+			Group:    "proxy.tgbot.co",
+		}, nil
 	}
-	if !tool.CheckInList(SSRCipherList, ss.Cipher) {
-		return nil, errors.New("cipher not support")
-	}
-	base := ss.Base
-	base.Type = "ssr"
-	return &ShadowsocksR{
-		Base:     base,
-		Password: ss.Password,
-		Cipher:   ss.Cipher,
-		Protocol: "origin",
-		Obfs:     "plain",
-		Group:    "proxy.tgbot.co",
-	}, nil
+	return nil, ErrorTypeCanNotConvert
 }
 
-func SSR2SS(ssr *ShadowsocksR) (ss *Shadowsocks, err error) {
-	if ssr == nil {
-		return nil, errors.New("ssr is nil")
+// Convert2SS convert proxy to Shadowsocks if possible
+func Convert2SS(p Proxy) (ss *Shadowsocks, err error) {
+	if p.TypeName() == "ss" {
+		ssr := p.(*ShadowsocksR)
+		if ssr == nil {
+			return nil, errors.New("ssr is nil")
+		}
+		if !tool.CheckInList(SSCipherList, ssr.Cipher) {
+			return nil, errors.New("cipher not support")
+		}
+		if ssr.Protocol != "origin" || ssr.Obfs != "plain" {
+			return nil, errors.New("protocol or obfs not allowed")
+		}
+		base := ssr.Base
+		base.Type = "ss"
+		return &Shadowsocks{
+			Base:       base,
+			Password:   ssr.Password,
+			Cipher:     ssr.Cipher,
+			Plugin:     "",
+			PluginOpts: nil,
+		}, nil
 	}
-	if !tool.CheckInList(SSCipherList, ssr.Cipher) {
-		return nil, errors.New("cipher not support")
-	}
-	if ssr.Protocol != "origin" || ssr.Obfs != "plain" {
-		return nil, errors.New("protocol or obfs not allowed")
-	}
-	base := ssr.Base
-	base.Type = "ss"
-	return &Shadowsocks{
-		Base:       base,
-		Password:   ssr.Password,
-		Cipher:     ssr.Cipher,
-		Plugin:     "",
-		PluginOpts: nil,
-	}, nil
+	return nil, ErrorTypeCanNotConvert
 }
 
 var SSRCipherList = []string{

pkg/proxy/proxies.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
 )
 
 type ProxyList []Proxy
@@ -63,24 +62,11 @@ func (ps ProxyList) Sort() ProxyList {
 	return ps
 }
 
-func (ps ProxyList) NameAddCounrty() ProxyList {
+func (ps ProxyList) NameSetCounrty() ProxyList {
 	num := len(ps)
-	wg := &sync.WaitGroup{}
-	wg.Add(num)
 	for i := 0; i < num; i++ {
-		ii := i
-		go func() {
-			defer wg.Done()
-			_, country, err := geoIp.Find(ps[ii].BaseInfo().Server)
-			if err != nil {
-				country = "🏁 ZZ"
-			}
-			ps[ii].SetName(fmt.Sprintf("%s", country))
-			ps[ii].SetCountry(country)
-			//ps[ii].SetIP(ip)
-		}()
+		ps[i].SetName(ps[i].BaseInfo().Country)
 	}
-	wg.Wait()
 	return ps
 }
 
@@ -110,20 +96,6 @@ func (ps ProxyList) NameAddTG() ProxyList {
 	return ps
 }
 
-func Deduplication(src ProxyList) ProxyList {
-	result := make(ProxyList, 0, len(src))
-	temp := map[string]struct{}{}
-	for _, item := range src {
-		if item != nil {
-			if _, ok := temp[item.Identifier()]; !ok {
-				temp[item.Identifier()] = struct{}{}
-				result = append(result, item)
-			}
-		}
-	}
-	return result
-}
-
 func (ps ProxyList) Clone() ProxyList {
 	result := make(ProxyList, 0, len(ps))
 	for _, pp := range ps {
@@ -133,3 +105,25 @@ func (ps ProxyList) Clone() ProxyList {
 	}
 	return result
 }
+
+// Derive 将原有节点中的ss和ssr互相转换进行衍生
+func (ps ProxyList) Derive() ProxyList {
+	proxies := ps
+	for _, p := range ps {
+		if p == nil {
+			continue
+		}
+		if p.TypeName() == "ss" {
+			ssr, err := Convert2SSR(p)
+			if err == nil {
+				proxies = append(proxies, ssr)
+			}
+		} else if p.TypeName() == "ssr" {
+			ss, err := Convert2SS(p)
+			if err == nil {
+				proxies = append(proxies, ss)
+			}
+		}
+	}
+	return proxies.Deduplication()
+}