fetch proxies from db every task & bump version

api/router.go

@@ -14,7 +14,7 @@ import (
 	"github.com/zu1k/proxypool/pkg/provider"
 )
 
-const version = "v0.3.6"
+const version = "v0.3.7"
 
 var router *gin.Engine
 
@@ -206,6 +206,7 @@ func Run() {
 }
 
 func loadTemplate() (t *template.Template, err error) {
+	_ = binhtml.RestoreAssets("", "assets/html")
 	t = template.New("")
 	for _, fileName := range binhtml.AssetNames() {
 		data := binhtml.MustAsset(fileName)

internal/app/task.go

@@ -5,9 +5,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/zu1k/proxypool/internal/database"
-
 	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/provider"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
@@ -22,6 +21,7 @@ func CrawlGo() {
 		go g.Get2Chan(pc, wg)
 	}
 	proxies := cache.GetProxies("allproxies")
+	proxies = append(proxies, database.GetAllProxies()...)
 	go func() {
 		wg.Wait()
 		close(pc)

internal/cron/cron.go

@@ -1,6 +1,8 @@
 package cron
 
 import (
+	"runtime"
+
 	"github.com/jasonlvhit/gocron"
 	"github.com/zu1k/proxypool/internal/app"
 )
@@ -13,4 +15,6 @@ func Cron() {
 func crawlTask() {
 	_ = app.InitConfigAndGetters("")
 	app.CrawlGo()
+	app.Getters = nil
+	runtime.GC()
 }

internal/database/db.go

@@ -8,6 +8,7 @@ import (
 
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 )
 
 var DB *gorm.DB
@@ -20,7 +21,9 @@ func connect() (err error) {
 	if url := os.Getenv("DATABASE_URL"); url != "" {
 		dsn = url
 	}
-	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
 	if err == nil {
 		fmt.Println("DB connect success: ", DB.Name())
 	}

internal/database/proxy.go

@@ -1,15 +1,16 @@
 package database
 
 import (
+	"github.com/zu1k/proxypool/pkg/getter"
 	"github.com/zu1k/proxypool/pkg/proxy"
 	"gorm.io/gorm"
 )
 
 type Proxy struct {
 	gorm.Model
-	proxy.Base `gorm:"index"`
+	proxy.Base
 	Link       string
-	Identifier string `gorm:"primaryKey"`
+	Identifier string `gorm:"unique"`
 }
 
 func InitTables() {
@@ -25,14 +26,43 @@ func InitTables() {
 	}
 }
 
+const roundSize = 100
+
 func SaveProxyList(pl proxy.ProxyList) {
-	proxies := make([]Proxy, pl.Len())
-	for i, p := range pl {
-		proxies[i] = Proxy{
-			Base:       *p.BaseInfo(),
-			Link:       p.Link(),
-			Identifier: p.Identifier(),
+	if DB == nil {
+		return
+	}
+
+	size := pl.Len()
+	round := (size + roundSize - 1) / roundSize
+
+	for r := 0; r < round; r++ {
+		proxies := make([]Proxy, 0, roundSize)
+		for i, j := r*roundSize, (r+1)*roundSize-1; i < j && i < size; i++ {
+			p := pl[i]
+			proxies = append(proxies, Proxy{
+				Base:       *p.BaseInfo(),
+				Link:       p.Link(),
+				Identifier: p.Identifier(),
+			})
+		}
+		DB.Create(&proxies)
+	}
+}
+
+func GetAllProxies() (proxies proxy.ProxyList) {
+	proxies = make(proxy.ProxyList, 0)
+	if DB == nil {
+		return
+	}
+
+	proxiesDB := make([]Proxy, 0)
+	DB.Select("link").Find(&proxiesDB)
+
+	for _, proxyDB := range proxiesDB {
+		if proxiesDB != nil {
+			proxies = append(proxies, getter.String2Proxy(proxyDB.Link))
 		}
 	}
-	DB.Create(&proxies)
+	return
 }

main.go

@@ -3,19 +3,24 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
-	"github.com/zu1k/proxypool/internal/database"
-
 	"github.com/zu1k/proxypool/api"
 	"github.com/zu1k/proxypool/internal/app"
 	"github.com/zu1k/proxypool/internal/cron"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )
 
 var configFilePath = ""
 
 func main() {
+	go func() {
+		http.ListenAndServe("0.0.0.0:6060", nil)
+	}()
+
 	flag.StringVar(&configFilePath, "c", "", "path to config file: config.yaml")
 	flag.Parse()
 

pkg/proxy/base.go

@@ -1,13 +1,13 @@
 package proxy
 
 type Base struct {
-	Name    string `yaml:"name" json:"name"`
-	Server  string `yaml:"server" json:"server"`
-	Port    int    `yaml:"port" json:"port"`
-	Type    string `yaml:"type" json:"type"`
+	Name    string `yaml:"name" json:"name" gorm:"index"`
+	Server  string `yaml:"server" json:"server" gorm:"index"`
+	Port    int    `yaml:"port" json:"port" gorm:"index"`
+	Type    string `yaml:"type" json:"type" gorm:"index"`
 	UDP     bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
-	Country string `yaml:"country,omitempty" json:"country,omitempty"`
-	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty"`
+	Country string `yaml:"country,omitempty" json:"country,omitempty" gorm:"index"`
+	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"`
 }
 
 func (b *Base) TypeName() string {