fix null pointer bug

api/router.go

@@ -14,7 +14,7 @@ import (
 	"github.com/zu1k/proxypool/pkg/provider"
 )
 
-const version = "v0.3.6"
+const version = "v0.3.8"
 
 var router *gin.Engine
 
@@ -206,6 +206,7 @@ func Run() {
 }
 
 func loadTemplate() (t *template.Template, err error) {
+	_ = binhtml.RestoreAssets("", "assets/html")
 	t = template.New("")
 	for _, fileName := range binhtml.AssetNames() {
 		data := binhtml.MustAsset(fileName)

internal/app/task.go

@@ -5,9 +5,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/zu1k/proxypool/internal/database"
-
 	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/provider"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
@@ -22,6 +21,7 @@ func CrawlGo() {
 		go g.Get2Chan(pc, wg)
 	}
 	proxies := cache.GetProxies("allproxies")
+	proxies = append(proxies, database.GetAllProxies()...)
 	go func() {
 		wg.Wait()
 		close(pc)

internal/cron/cron.go

@@ -1,6 +1,8 @@
 package cron
 
 import (
+	"runtime"
+
 	"github.com/jasonlvhit/gocron"
 	"github.com/zu1k/proxypool/internal/app"
 )
@@ -13,4 +15,6 @@ func Cron() {
 func crawlTask() {
 	_ = app.InitConfigAndGetters("")
 	app.CrawlGo()
+	app.Getters = nil
+	runtime.GC()
 }

internal/database/db.go

@@ -8,6 +8,7 @@ import (
 
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 )
 
 var DB *gorm.DB
@@ -20,7 +21,9 @@ func connect() (err error) {
 	if url := os.Getenv("DATABASE_URL"); url != "" {
 		dsn = url
 	}
-	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
 	if err == nil {
 		fmt.Println("DB connect success: ", DB.Name())
 	}

internal/database/proxy.go

@@ -1,15 +1,16 @@
 package database
 
 import (
+	"github.com/zu1k/proxypool/pkg/getter"
 	"github.com/zu1k/proxypool/pkg/proxy"
 	"gorm.io/gorm"
 )
 
 type Proxy struct {
 	gorm.Model
-	proxy.Base `gorm:"index"`
+	proxy.Base
 	Link       string
-	Identifier string `gorm:"primaryKey"`
+	Identifier string `gorm:"unique"`
 }
 
 func InitTables() {
@@ -25,14 +26,43 @@ func InitTables() {
 	}
 }
 
+const roundSize = 100
+
 func SaveProxyList(pl proxy.ProxyList) {
-	proxies := make([]Proxy, pl.Len())
-	for i, p := range pl {
-		proxies[i] = Proxy{
-			Base:       *p.BaseInfo(),
-			Link:       p.Link(),
-			Identifier: p.Identifier(),
+	if DB == nil {
+		return
+	}
+
+	size := pl.Len()
+	round := (size + roundSize - 1) / roundSize
+
+	for r := 0; r < round; r++ {
+		proxies := make([]Proxy, 0, roundSize)
+		for i, j := r*roundSize, (r+1)*roundSize-1; i < j && i < size; i++ {
+			p := pl[i]
+			proxies = append(proxies, Proxy{
+				Base:       *p.BaseInfo(),
+				Link:       p.Link(),
+				Identifier: p.Identifier(),
+			})
+		}
+		DB.Create(&proxies)
+	}
+}
+
+func GetAllProxies() (proxies proxy.ProxyList) {
+	proxies = make(proxy.ProxyList, 0)
+	if DB == nil {
+		return
+	}
+
+	proxiesDB := make([]Proxy, 0)
+	DB.Select("link").Find(&proxiesDB)
+
+	for _, proxyDB := range proxiesDB {
+		if proxiesDB != nil {
+			proxies = append(proxies, getter.String2Proxy(proxyDB.Link))
 		}
 	}
-	DB.Create(&proxies)
+	return
 }

main.go

@@ -3,19 +3,24 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
-	"github.com/zu1k/proxypool/internal/database"
-
 	"github.com/zu1k/proxypool/api"
 	"github.com/zu1k/proxypool/internal/app"
 	"github.com/zu1k/proxypool/internal/cron"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
 var configFilePath = ""
 
 func main() {
+	go func() {
+		http.ListenAndServe("0.0.0.0:6060", nil)
+	}()
+
 	flag.StringVar(&configFilePath, "c", "", "path to config file: config.yaml")
 	flag.Parse()
 

pkg/provider/base.go

@@ -10,15 +10,6 @@ type Provider interface {
 	Provide() string
 }
 
-func checkInList(list []string, item string) bool {
-	for _, i := range list {
-		if item == i {
-			return true
-		}
-	}
-	return false
-}
-
 type Base struct {
 	Proxies    *proxy.ProxyList `yaml:"proxies"`
 	Types      string           `yaml:"type"`

pkg/provider/clash.go

@@ -3,6 +3,8 @@ package provider
 import (
 	"strings"
 
+	"github.com/zu1k/proxypool/pkg/tool"
+
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
@@ -37,17 +39,17 @@ func checkClashSupport(p proxy.Proxy) bool {
 	switch p.TypeName() {
 	case "ssr":
 		ssr := p.(*proxy.ShadowsocksR)
-		if checkInList(ssrCipherList, ssr.Cipher) && checkInList(ssrProtocolList, ssr.Protocol) && checkInList(ssrObfsList, ssr.Obfs) {
+		if tool.CheckInList(proxy.SSRCipherList, ssr.Cipher) && tool.CheckInList(ssrProtocolList, ssr.Protocol) && tool.CheckInList(ssrObfsList, ssr.Obfs) {
 			return true
 		}
 	case "vmess":
 		vmess := p.(*proxy.Vmess)
-		if checkInList(vmessCipherList, vmess.Cipher) {
+		if tool.CheckInList(vmessCipherList, vmess.Cipher) {
 			return true
 		}
 	case "ss":
 		ss := p.(*proxy.Shadowsocks)
-		if checkInList(ssCipherList, ss.Cipher) {
+		if tool.CheckInList(proxy.SSCipherList, ss.Cipher) {
 			return true
 		}
 	case "trojan":
@@ -58,30 +60,6 @@ func checkClashSupport(p proxy.Proxy) bool {
 	return false
 }
 
-var ssrCipherList = []string{
-	"aes-128-cfb",
-	"aes-192-cfb",
-	"aes-256-cfb",
-	"aes-128-ctr",
-	"aes-192-ctr",
-	"aes-256-ctr",
-	"aes-128-ofb",
-	"aes-192-ofb",
-	"aes-256-ofb",
-	"des-cfb",
-	"bf-cfb",
-	"cast5-cfb",
-	"rc4-md5",
-	"chacha20-ietf",
-	"salsa20",
-	"camellia-128-cfb",
-	"camellia-192-cfb",
-	"camellia-256-cfb",
-	"idea-cfb",
-	"rc2-cfb",
-	"seed-cfb",
-}
-
 var ssrObfsList = []string{
 	"plain",
 	"http_simple",
@@ -110,20 +88,3 @@ var vmessCipherList = []string{
 	"chacha20-poly1305",
 	"none",
 }
-
-var ssCipherList = []string{
-	"aes-128-gcm",
-	"aes-192-gcm",
-	"aes-256-gcm",
-	"aes-128-cfb",
-	"aes-192-cfb",
-	"aes-256-cfb",
-	"aes-128-ctr",
-	"aes-192-ctr",
-	"aes-256-ctr",
-	"rc4-md5",
-	"chacha20-ietf",
-	"xchacha20",
-	"chacha20-ietf-poly1305",
-	"xchacha20-ietf-poly1305",
-}

pkg/provider/ssrsub.go

@@ -3,6 +3,8 @@ package provider
 import (
 	"strings"
 
+	"github.com/zu1k/proxypool/pkg/proxy"
+
 	"github.com/zu1k/proxypool/pkg/tool"
 )
 
@@ -11,11 +13,18 @@ type SSRSub struct {
 }
 
 func (sub SSRSub) Provide() string {
-	sub.Types = "ssr"
+	sub.Types = "ssr,ss"
 	sub.preFilter()
 	var resultBuilder strings.Builder
 	for _, p := range *sub.Proxies {
-		resultBuilder.WriteString(p.Link() + "\n")
+		if p.TypeName() == "ssr" {
+			resultBuilder.WriteString(p.Link() + "\n")
+		} else if p.TypeName() == "ss" {
+			ssr, err := proxy.SS2SSR(p.(*proxy.Shadowsocks))
+			if err == nil {
+				resultBuilder.WriteString(ssr.Link() + "\n")
+			}
+		}
 	}
 	return tool.Base64EncodeString(resultBuilder.String(), false)
 }

pkg/provider/sssub.go

@@ -22,11 +22,22 @@ type ssJson struct {
 }
 
 func (sub SSSub) Provide() string {
-	sub.Types = "ss"
+	sub.Types = "ss,ssr"
 	sub.preFilter()
 	proxies := make([]ssJson, 0, sub.Proxies.Len())
 	for _, p := range *sub.Proxies {
-		pp := p.(*proxy.Shadowsocks)
+		var pp *proxy.Shadowsocks
+
+		if p.TypeName() == "ssr" {
+			var err error
+			pp, err = proxy.SSR2SS(p.(*proxy.ShadowsocksR))
+			if err != nil {
+				continue
+			}
+		} else if p.TypeName() == "ss" {
+			pp = p.(*proxy.Shadowsocks)
+		}
+
 		proxies = append(proxies, ssJson{
 			Remarks:    pp.Name,
 			Server:     pp.Server,

pkg/provider/surge.go

@@ -3,6 +3,8 @@ package provider
 import (
 	"strings"
 
+	"github.com/zu1k/proxypool/pkg/tool"
+
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
@@ -30,7 +32,7 @@ func checkSurgeSupport(p proxy.Proxy) bool {
 		return true
 	case *proxy.Shadowsocks:
 		ss := p.(*proxy.Shadowsocks)
-		if checkInList(ssCipherList, ss.Cipher) {
+		if tool.CheckInList(proxy.SSCipherList, ss.Cipher) {
 			return true
 		}
 	default:

pkg/proxy/base.go

@@ -1,13 +1,13 @@
 package proxy
 
 type Base struct {
-	Name    string `yaml:"name" json:"name"`
-	Server  string `yaml:"server" json:"server"`
-	Port    int    `yaml:"port" json:"port"`
-	Type    string `yaml:"type" json:"type"`
+	Name    string `yaml:"name" json:"name" gorm:"index"`
+	Server  string `yaml:"server" json:"server" gorm:"index"`
+	Port    int    `yaml:"port" json:"port" gorm:"index"`
+	Type    string `yaml:"type" json:"type" gorm:"index"`
 	UDP     bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
-	Country string `yaml:"country,omitempty" json:"country,omitempty"`
-	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty"`
+	Country string `yaml:"country,omitempty" json:"country,omitempty" gorm:"index"`
+	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"`
 }
 
 func (b *Base) TypeName() string {

pkg/proxy/convert.go

@@ -0,0 +1,88 @@
+package proxy
+
+import (
+	"errors"
+
+	"github.com/zu1k/proxypool/pkg/tool"
+)
+
+func SS2SSR(ss *Shadowsocks) (ssr *ShadowsocksR, err error) {
+	if ss == nil {
+		return nil, errors.New("ss is nil")
+	}
+	if !tool.CheckInList(SSRCipherList, ss.Cipher) {
+		return nil, errors.New("cipher not support")
+	}
+	base := ss.Base
+	base.Type = "ssr"
+	return &ShadowsocksR{
+		Base:     base,
+		Password: ss.Password,
+		Cipher:   ss.Cipher,
+		Protocol: "origin",
+		Obfs:     "plain",
+		Group:    "proxy.tgbot.co",
+	}, nil
+}
+
+func SSR2SS(ssr *ShadowsocksR) (ss *Shadowsocks, err error) {
+	if ssr == nil {
+		return nil, errors.New("ssr is nil")
+	}
+	if !tool.CheckInList(SSCipherList, ssr.Cipher) {
+		return nil, errors.New("cipher not support")
+	}
+	if ssr.Protocol != "origin" || ssr.Obfs != "plain" {
+		return nil, errors.New("protocol or obfs not allowed")
+	}
+	base := ssr.Base
+	base.Type = "ss"
+	return &Shadowsocks{
+		Base:       base,
+		Password:   ssr.Password,
+		Cipher:     ssr.Cipher,
+		Plugin:     "",
+		PluginOpts: nil,
+	}, nil
+}
+
+var SSRCipherList = []string{
+	"aes-128-cfb",
+	"aes-192-cfb",
+	"aes-256-cfb",
+	"aes-128-ctr",
+	"aes-192-ctr",
+	"aes-256-ctr",
+	"aes-128-ofb",
+	"aes-192-ofb",
+	"aes-256-ofb",
+	"des-cfb",
+	"bf-cfb",
+	"cast5-cfb",
+	"rc4-md5",
+	"chacha20-ietf",
+	"salsa20",
+	"camellia-128-cfb",
+	"camellia-192-cfb",
+	"camellia-256-cfb",
+	"idea-cfb",
+	"rc2-cfb",
+	"seed-cfb",
+}
+
+var SSCipherList = []string{
+	"aes-128-gcm",
+	"aes-192-gcm",
+	"aes-256-gcm",
+	"aes-128-cfb",
+	"aes-192-cfb",
+	"aes-256-cfb",
+	"aes-128-ctr",
+	"aes-192-ctr",
+	"aes-256-ctr",
+	"rc4-md5",
+	"chacha20-ietf",
+	"xchacha20",
+	"chacha20-ietf-poly1305",
+	"xchacha20-ietf-poly1305",
+}

pkg/tool/check.go

@@ -0,0 +1,10 @@
+package tool
+
+func CheckInList(list []string, item string) bool {
+	for _, i := range list {
+		if item == i {
+			return true
+		}
+	}
+	return false
+}