add source config test

app/config_test.go

@@ -0,0 +1,36 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/zu1k/proxypool/config"
+	"github.com/zu1k/proxypool/getter"
+)
+
+func TestConfigFile(t *testing.T) {
+	c, err := config.Parse("../source.yaml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if c == nil {
+		t.Error(errors.New("no sources"))
+		return
+	}
+	for idx, source := range c.Sources {
+		g, err := getter.NewGetter(source.Type, source.Options)
+		if err != nil {
+			t.Error(err, idx)
+			fmt.Println(source)
+			return
+		}
+		if g == nil {
+			t.Error(errors.New("getter is nil:" + strconv.Itoa(idx)))
+			fmt.Println(source)
+			return
+		}
+	}
+}

app/getter.go

@@ -1,8 +1,8 @@
 package app
 
 import (
+	"errors"
 	"fmt"
-	"os"
 
 	"github.com/zu1k/proxypool/config"
 	"github.com/zu1k/proxypool/getter"
@@ -10,24 +10,23 @@ import (
 
 var Getters = make([]getter.Getter, 0)
 
-func InitConfigAndGetters(path string) {
+func InitConfigAndGetters(path string) (err error) {
 	c, err := config.Parse(path)
 	if err != nil {
-		fmt.Println("Error: ", err.Error())
-		os.Exit(1)
+		return err
 	}
 	if c == nil {
-		fmt.Println("Error: no sources")
-		os.Exit(2)
+		return errors.New("no sources")
 	}
 	InitGetters(c.Sources)
+	return nil
 }
 
 func InitGetters(sources []config.Source) {
 	Getters = make([]getter.Getter, 0)
 	for _, source := range sources {
-		g := getter.NewGetter(source.Type, source.Options)
-		if g != nil {
+		g, err := getter.NewGetter(source.Type, source.Options)
+		if err == nil && g != nil {
 			Getters = append(Getters, g)
 			fmt.Println("init getter:", source.Type, source.Options)
 		}

getter/base.go

@@ -1,6 +1,7 @@
 package getter
 
 import (
+	"errors"
 	"strings"
 	"sync"
 
@@ -13,7 +14,7 @@ type Getter interface {
 	Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup)
 }
 
-type creator func(options tool.Options) Getter
+type creator func(options tool.Options) (getter Getter, err error)
 
 var creatorMap = make(map[string]creator)
 
@@ -21,12 +22,12 @@ func Register(sourceType string, c creator) {
 	creatorMap[sourceType] = c
 }
 
-func NewGetter(sourceType string, options tool.Options) Getter {
+func NewGetter(sourceType string, options tool.Options) (getter Getter, err error) {
 	c, ok := creatorMap[sourceType]
 	if ok {
 		return c(options)
 	}
-	return nil
+	return nil, ErrorCreaterNotSupported
 }
 
 func String2Proxy(link string) proxy.Proxy {
@@ -63,3 +64,21 @@ func GrepLinksFromString(text string) []string {
 func FuzzParseProxyFromString(text string) []proxy.Proxy {
 	return StringArray2ProxyArray(GrepLinksFromString(text))
 }
+
+var (
+	ErrorUrlNotFound         = errors.New("url should be specified")
+	ErrorCreaterNotSupported = errors.New("type not supported")
+)
+
+func AssertTypeStringNotNull(i interface{}) (str string, err error) {
+	switch i.(type) {
+	case string:
+		str = i.(string)
+		if str == "" {
+			return "", errors.New("string is null")
+		}
+	default:
+		return "", errors.New("type is not string")
+	}
+	return "", nil
+}

getter/subscribe.go

@@ -46,12 +46,16 @@ func (s *Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	}
 }
 
-func NewSubscribe(options tool.Options) Getter {
-	url, found := options["url"]
+func NewSubscribe(options tool.Options) (getter Getter, err error) {
+	urlInterface, found := options["url"]
 	if found {
-		return &Subscribe{
-			Url: url.(string),
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
 		}
+		return &Subscribe{
+			Url: url,
+		}, nil
 	}
-	return nil
+	return nil, ErrorUrlNotFound
 }

getter/tgchannel.go

@@ -20,7 +20,7 @@ type TGChannelGetter struct {
 	Url       string
 }
 
-func NewTGChannelGetter(options tool.Options) Getter {
+func NewTGChannelGetter(options tool.Options) (getter Getter, err error) {
 	num, found := options["num"]
 	t := 200
 	switch num.(type) {
@@ -33,15 +33,19 @@ func NewTGChannelGetter(options tool.Options) Getter {
 	if !found || t <= 0 {
 		t = 200
 	}
-	url, found := options["channel"]
+	urlInterface, found := options["channel"]
 	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
 		return &TGChannelGetter{
 			c:         colly.NewCollector(),
 			NumNeeded: t,
-			Url:       "https://t.me/s/" + url.(string),
-		}
+			Url:       "https://t.me/s/" + url,
+		}, nil
 	}
-	return nil
+	return nil, ErrorUrlNotFound
 }
 
 func (g *TGChannelGetter) Get() []proxy.Proxy {

getter/web_fanqiangdang.go

@@ -20,7 +20,7 @@ type WebFanqiangdang struct {
 	results   []string
 }
 
-func NewWebFanqiangdangGetter(options tool.Options) Getter {
+func NewWebFanqiangdangGetter(options tool.Options) (getter Getter, err error) {
 	num, found := options["num"]
 
 	t := 200
@@ -34,15 +34,19 @@ func NewWebFanqiangdangGetter(options tool.Options) Getter {
 	if !found || t <= 0 {
 		t = 200
 	}
-	url, found := options["url"]
+	urlInterface, found := options["url"]
 	if found {
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
 		return &WebFanqiangdang{
 			c:         colly.NewCollector(),
 			NumNeeded: t,
-			Url:       url.(string),
-		}
+			Url:       url,
+		}, nil
 	}
-	return nil
+	return nil, ErrorUrlNotFound
 }
 
 func (w *WebFanqiangdang) Get() []proxy.Proxy {

getter/web_free_ssr_xyz.go

@@ -21,8 +21,8 @@ const (
 type WebFreessrXyz struct {
 }
 
-func NewWebFreessrxyzGetter(options tool.Options) Getter {
-	return &WebFreessrXyz{}
+func NewWebFreessrxyzGetter(options tool.Options) (getter Getter, err error) {
+	return &WebFreessrXyz{}, nil
 }
 
 func (w *WebFreessrXyz) Get() []proxy.Proxy {

getter/web_fuzz.go

@@ -37,10 +37,14 @@ func (w *WebFuzz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	}
 }
 
-func NewWebFuzzGetter(options tool.Options) Getter {
-	url, found := options["url"]
+func NewWebFuzzGetter(options tool.Options) (getter Getter, err error) {
+	urlInterface, found := options["url"]
 	if found {
-		return &WebFuzz{Url: url.(string)}
+		url, err := AssertTypeStringNotNull(urlInterface)
+		if err != nil {
+			return nil, err
+		}
+		return &WebFuzz{Url: url}, nil
 	}
-	return nil
+	return nil, ErrorUrlNotFound
 }

getter/web_lucn_org.go

@@ -19,8 +19,8 @@ const lucnorgSsrLink = "https://lncn.org/api/ssrList"
 type WebLucnOrg struct {
 }
 
-func NewWebLucnorg(options tool.Options) Getter {
-	return &WebLucnOrg{}
+func NewWebLucnorg(options tool.Options) (getter Getter, err error) {
+	return &WebLucnOrg{}, nil
 }
 
 func (w *WebLucnOrg) Get() []proxy.Proxy {

main.go

@@ -18,7 +18,10 @@ func main() {
 	if *filePath == "" {
 		app.NeedFetchNewConfigFile = true
 	} else {
-		app.InitConfigAndGetters(*filePath)
+		err := app.InitConfigAndGetters(*filePath)
+		if err != nil {
+			fmt.Println(err)
+		}
 	}
 	go app.Cron()
 	fmt.Println("Do the first crawl...")