Yasso更新大改动，更新扫描方式，去除不常用功能，增加指纹和协议识别，修补bug等

pkg/webscan/dismap.go

@@ -0,0 +1,571 @@
+package webscan
+
+import (
+	"Yasso/config"
+	"Yasso/core/logger"
+	"fmt"
+	"regexp"
+	"runtime"
+	"strconv"
+	"time"
+)
+
+type IdentifyResult struct {
+	Type     string
+	RespCode string
+	Result   string
+	ResultNc string
+	Url      string
+	Title    string
+}
+
+func DisMapConn(host string, port int, timeout time.Duration) bool {
+	url := ParseUrl(host, strconv.Itoa(port))
+	for _, r := range Identify(url, timeout) {
+		if r.RespCode != "" {
+			logger.Success(fmt.Sprintf("%v %v %v [%v]", r.Url, r.RespCode, r.Result, r.Title))
+		}
+	}
+	return true
+}
+
+func Identify(url string, timeout time.Duration) []IdentifyResult {
+	var DefaultFavicon string
+	var CustomFavicon string
+	var DefaultTarget string
+	var CustomTarget string
+	var Favicon string
+	var RequestRule string
+	var RespTitle string
+	var RespBody string
+	var RespHeader string
+	var RespCode string
+	var DefaultRespTitle string
+	var DefaultRespBody string
+	var DefaultRespHeader string
+	var DefaultRespCode string
+	var CustomRespTitle string
+	var CustomRespBody string
+	var CustomRespHeader string
+	var CustomRespCode string
+	for _, resp := range DefaultRequests(url, timeout) { // Default Request
+		DefaultRespBody = resp.RespBody
+		DefaultRespHeader = resp.RespHeader
+		DefaultRespCode = resp.RespStatusCode
+		DefaultRespTitle = resp.RespTitle
+		DefaultTarget = resp.Url
+		DefaultFavicon = resp.FaviconMd5
+	}
+	// start identify
+	var identifyData []string
+	var successType string
+	for _, rule := range config.RuleData {
+		if rule.Http.ReqMethod != "" { // Custom Request Result
+			for _, resp := range CustomRequests(url, timeout, rule.Http.ReqMethod, rule.Http.ReqPath, rule.Http.ReqHeader, rule.Http.ReqBody) {
+				CustomRespBody = resp.RespBody
+				CustomRespHeader = resp.RespHeader
+				CustomRespCode = resp.RespStatusCode
+				CustomRespTitle = resp.RespTitle
+				CustomTarget = resp.Url
+				CustomFavicon = resp.FaviconMd5
+			}
+			url = CustomTarget
+			Favicon = CustomFavicon
+			RespBody = CustomRespBody
+			RespHeader = CustomRespHeader
+			RespCode = CustomRespCode
+			RespTitle = CustomRespTitle
+			// If the http request fails, then RespBody and RespHeader are both null
+			// At this time, it is considered that the url does not exist
+			if RespBody == RespHeader {
+				continue
+			}
+			if rule.Mode == "" {
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "CustomRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "or" {
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "and" {
+				index := 0
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						index = index + 1
+					}
+				}
+				if index == 2 {
+					identifyData = append(identifyData, rule.Name)
+					RequestRule = "CustomRequest"
+				}
+			}
+			if rule.Mode == "and|and" {
+				index := 0
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						index = index + 1
+					}
+				}
+				if index == 3 {
+					identifyData = append(identifyData, rule.Name)
+					RequestRule = "CustomRequest"
+				}
+			}
+			if rule.Mode == "or|or" {
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "and|or" {
+				grep := regexp.MustCompile("(.*)\\|(.*)\\|(.*)")
+				all_type := grep.FindStringSubmatch(rule.Type)
+				//
+				//Println(all_type)
+				if len(regexp.MustCompile("header").FindAllStringIndex(all_type[1], -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(all_type[1], -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(all_type[1], -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "or|and" {
+				grep := regexp.MustCompile("(.*)\\|(.*)\\|(.*)")
+				all_type := grep.FindStringSubmatch(rule.Type)
+				//Println(all_type)
+				if len(regexp.MustCompile("header").FindAllStringIndex(all_type[3], -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(all_type[3], -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(all_type[3], -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+		} else { // Default Request Result
+			url = DefaultTarget
+			Favicon = DefaultFavicon
+			RespBody = DefaultRespBody
+			RespHeader = DefaultRespHeader
+			RespCode = DefaultRespCode
+			RespTitle = DefaultRespTitle
+			// If the http request fails, then RespBody and RespHeader are both null
+			// At this time, it is considered that the url does not exist
+			if RespBody == RespHeader {
+				continue
+			}
+			if rule.Mode == "" {
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "or" {
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "and" {
+				index := 0
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						index = index + 1
+					}
+				}
+				if index == 2 {
+					identifyData = append(identifyData, rule.Name)
+					RequestRule = "DefaultRequest"
+				}
+			}
+			if rule.Mode == "and|and" {
+				index := 0
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						index = index + 1
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						index = index + 1
+					}
+				}
+				if index == 3 {
+					identifyData = append(identifyData, rule.Name)
+					RequestRule = "DefaultRequest"
+				}
+			}
+			if rule.Mode == "or|or" {
+				if len(regexp.MustCompile("header").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(rule.Type, -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == true {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "and|or" {
+				grep := regexp.MustCompile("(.*)\\|(.*)\\|(.*)")
+				allType := grep.FindStringSubmatch(rule.Type)
+				//Println(all_type)
+				if len(regexp.MustCompile("header").FindAllStringIndex(allType[1], -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(allType[1], -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(allType[1], -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+			if rule.Mode == "or|and" {
+				grep := regexp.MustCompile("(.*)\\|(.*)\\|(.*)")
+				all_type := grep.FindStringSubmatch(rule.Type)
+				//Println(all_type)
+				if len(regexp.MustCompile("header").FindAllStringIndex(all_type[3], -1)) == 1 {
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+					if checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("body").FindAllStringIndex(all_type[3], -1)) == 1 {
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+					if checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) == checkFavicon(Favicon, rule.Rule.InIcoMd5) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+				if len(regexp.MustCompile("ico").FindAllStringIndex(all_type[3], -1)) == 1 {
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkHeader(url, RespHeader, rule.Rule.InHeader, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+					if checkFavicon(Favicon, rule.Rule.InIcoMd5) == checkBody(url, RespBody, rule.Rule.InBody, rule.Name, RespTitle, RespCode) {
+						identifyData = append(identifyData, rule.Name)
+						RequestRule = "DefaultRequest"
+						successType = rule.Type
+						continue
+					}
+				}
+			}
+		}
+	}
+	// identify
+	if RequestRule == "DefaultRequest" {
+		RespBody = DefaultRespBody
+		RespHeader = DefaultRespHeader
+		RespCode = DefaultRespCode
+		RespTitle = DefaultRespTitle
+		url = DefaultTarget
+	} else if RequestRule == "CustomRequest" {
+		url = CustomTarget
+		RespBody = CustomRespBody
+		RespHeader = CustomRespHeader
+		RespCode = CustomRespCode
+		RespTitle = CustomRespTitle
+	}
+	var identifyResult string
+	var identifyResultNocolor string
+	for _, result := range identifyData {
+		if runtime.GOOS == "windows" {
+			identifyResult += "[" + result + "]" + " "
+		} else {
+			identifyResult += "[" + result + "]" + " "
+		}
+	}
+	for _, result := range identifyData {
+		identifyResultNocolor += "[" + result + "]" + " "
+	}
+
+	Result := []IdentifyResult{
+		{successType, RespCode, identifyResult, identifyResultNocolor, url, RespTitle},
+	}
+	return Result
+}
+
+func checkHeader(url, responseHeader string, ruleHeader string, name string, title string, RespCode string) bool {
+	grep := regexp.MustCompile("(?i)" + ruleHeader)
+	if len(grep.FindStringSubmatch(responseHeader)) != 0 {
+		//fmt.Print("[header] ")
+		return true
+	} else {
+		return false
+	}
+}
+
+func checkBody(url, responseBody string, ruleBody string, name string, title string, RespCode string) bool {
+	grep := regexp.MustCompile("(?i)" + ruleBody)
+	if len(grep.FindStringSubmatch(responseBody)) != 0 {
+		//fmt.Print("[body] ")
+		return true
+	} else {
+		return false
+	}
+}
+
+func checkFavicon(Favicon, ruleFaviconMd5 string) bool {
+	grep := regexp.MustCompile("(?i)" + ruleFaviconMd5)
+	if len(grep.FindStringSubmatch(Favicon)) != 0 {
+		// fmt.Print("url")
+		return true
+	} else {
+		return false
+	}
+}

pkg/webscan/http.go

@@ -0,0 +1,340 @@
+package webscan
+
+import (
+	"Yasso/config"
+	"bytes"
+	"crypto/md5"
+	"crypto/tls"
+	"fmt"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+//TODO: dismap RespLab
+
+type RespLab struct {
+	Url            string
+	RespBody       string
+	RespHeader     string
+	RespStatusCode string
+	RespTitle      string
+	FaviconMd5     string
+}
+
+func FaviconMd5(Url string, timeout time.Duration, Path string) string {
+	client := &http.Client{
+		Timeout: time.Duration(timeout),
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	Url = Url + "/favicon.ico"
+	req, err := http.NewRequest("GET", Url, nil)
+	if err != nil {
+		return ""
+	}
+	for key, value := range config.DefaultHeader {
+		req.Header.Set(key, value)
+	}
+	//req.Header.Set("Accept-Language", "zh,zh-TW;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6")
+	//req.Header.Set("User-agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36")
+	//req.Header.Set("Cookie", "rememberMe=int")
+	resp, err := client.Do(req)
+	if err != nil {
+		return ""
+	}
+	defer resp.Body.Close()
+	body_bytes, err := ioutil.ReadAll(resp.Body)
+	hash := md5.Sum(body_bytes)
+	md5 := fmt.Sprintf("%x", hash)
+	return md5
+}
+
+func DefaultRequests(Url string, timeout time.Duration) []RespLab {
+
+	var redirect_url string
+	var resp_title string
+	var response_header string
+	var response_body string
+	var response_status_code string
+	var res []string
+
+	// 设置http请求客户端
+	client := &http.Client{
+		Timeout: time.Duration(timeout),
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	req, err := http.NewRequest("GET", Url, nil)
+	if err != nil {
+		return nil
+	}
+	// 设置默认请求头
+	for key, value := range config.DefaultHeader {
+		req.Header.Set(key, value)
+	}
+	// 做http请求
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+
+	// 获取请求的状态马
+	var status_code = resp.StatusCode
+	response_status_code = strconv.Itoa(status_code)
+
+	//TODO: 根据请求来拦截状态码，如果是30x则需要拦截url进行重定向
+
+	if len(regexp.MustCompile("30").FindAllStringIndex(response_status_code, -1)) == 1 {
+		// 进行重定向
+		redirect_path := resp.Header.Get("Location") // 拦截url进行重定向请求
+		if len(regexp.MustCompile("http").FindAllStringIndex(redirect_path, -1)) == 1 {
+			redirect_url = redirect_path
+		} else {
+			redirect_url = Url + redirect_path
+		}
+		client = &http.Client{
+			Timeout: time.Duration(timeout),
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		}
+		// 设置重定向请求
+		req, err := http.NewRequest("GET", redirect_url, nil)
+		if err != nil {
+			return nil
+		}
+		for key, value := range config.DefaultHeader {
+			req.Header.Set(key, value)
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			return nil
+		}
+		defer resp.Body.Close()
+
+		//TODO: 解决两次的30x跳转问题
+		var twoStatusCode = resp.StatusCode
+		responseStatusCodeTwo := strconv.Itoa(twoStatusCode)
+		if len(regexp.MustCompile("30").FindAllStringIndex(responseStatusCodeTwo, -1)) == 1 {
+			redirectPath := resp.Header.Get("Location")
+			if len(regexp.MustCompile("http").FindAllStringIndex(redirectPath, -1)) == 1 {
+				redirect_url = redirectPath
+			} else {
+				redirect_url = Url + redirectPath
+			}
+			client = &http.Client{
+				Timeout: time.Duration(timeout),
+				Transport: &http.Transport{
+					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+				},
+				CheckRedirect: func(req *http.Request, via []*http.Request) error {
+					return http.ErrUseLastResponse
+				},
+			}
+			req, err := http.NewRequest("GET", redirect_url, nil)
+			if err != nil {
+				return nil
+			}
+			for key, value := range config.DefaultHeader {
+				req.Header.Set(key, value)
+			}
+			resp, err := client.Do(req)
+			if err != nil {
+				return nil
+			}
+			defer resp.Body.Close()
+			// get response body for string
+			bodyBytes, err := ioutil.ReadAll(resp.Body)
+			response_body = string(bodyBytes)
+			// Solve the problem of garbled body codes with unmatched numbers
+			if !utf8.Valid(bodyBytes) {
+				data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(bodyBytes)
+				response_body = string(data)
+			}
+			// Get Response title
+			grepTitle := regexp.MustCompile("<title>(.*)</title>")
+			if len(grepTitle.FindStringSubmatch(response_body)) != 0 {
+				resp_title = grepTitle.FindStringSubmatch(response_body)[1]
+			} else {
+				resp_title = "None"
+			}
+			for name, values := range resp.Header {
+				for _, value := range values {
+					res = append(res, fmt.Sprintf("%s: %s", name, value))
+				}
+			}
+			for _, re := range res {
+				response_header += re + "\n"
+			}
+			favicon5 := FaviconMd5(Url, timeout, "")
+			RespData := []RespLab{
+				{redirect_url, response_body, response_header, response_status_code, resp_title, favicon5},
+			}
+			return RespData
+		}
+		// get response body for string
+		body_bytes, err := ioutil.ReadAll(resp.Body)
+		response_body = string(body_bytes)
+		// Solve the problem of garbled body codes with unmatched numbers
+		if !utf8.Valid(body_bytes) {
+			data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(body_bytes)
+			response_body = string(data)
+		}
+		// Get Response title
+		grep_title := regexp.MustCompile("<title>(.*)</title>")
+		if len(grep_title.FindStringSubmatch(response_body)) != 0 {
+			resp_title = grep_title.FindStringSubmatch(response_body)[1]
+		} else {
+			resp_title = "None"
+		}
+		// get response header for string
+		for name, values := range resp.Header {
+			for _, value := range values {
+				res = append(res, fmt.Sprintf("%s: %s", name, value))
+			}
+		}
+		for _, re := range res {
+			response_header += re + "\n"
+		}
+		favicon5 := FaviconMd5(Url, timeout, "")
+		RespData := []RespLab{
+			{redirect_url, response_body, response_header, response_status_code, resp_title, favicon5},
+		}
+		return RespData
+	}
+	// get response body for string
+	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	response_body = string(bodyBytes)
+	// Solve the problem of garbled body codes with unmatched numbers
+	if !utf8.Valid(bodyBytes) {
+		data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(bodyBytes)
+		response_body = string(data)
+	}
+
+	// Get Response title
+	grep_title := regexp.MustCompile("<title>(.*)</title>")
+	if len(grep_title.FindStringSubmatch(response_body)) != 0 {
+		resp_title = grep_title.FindStringSubmatch(response_body)[1]
+	} else {
+		resp_title = "None"
+	}
+	// get response header for string
+	for name, values := range resp.Header {
+		for _, value := range values {
+			res = append(res, fmt.Sprintf("%s: %s", name, value))
+		}
+	}
+	for _, re := range res {
+		response_header += re + "\n"
+	}
+	faviconmd5 := FaviconMd5(Url, timeout, "")
+	RespData := []RespLab{
+		{Url, response_body, response_header, response_status_code, resp_title, faviconmd5},
+	}
+	return RespData
+}
+
+func CustomRequests(Url string, timeout time.Duration, Method string, Path string, Header []string, Body string) []RespLab {
+	var respTitle string
+	// Splicing Custom Path
+	u, err := url.Parse(Url)
+	u.Path = path.Join(u.Path, Path)
+	Url = u.String()
+	if strings.HasSuffix(Path, "/") {
+		Url = Url + "/"
+	}
+	client := &http.Client{
+		Timeout: time.Duration(timeout),
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+	// Send Http requests
+	body_byte := bytes.NewBuffer([]byte(Body))
+	req, err := http.NewRequest(Method, Url, body_byte)
+	if err != nil {
+		return nil
+	}
+
+	// Set Requests Headers
+	for _, header := range Header {
+		grep_key := regexp.MustCompile("(.*): ")
+		var header_key = grep_key.FindStringSubmatch(header)[1]
+		grep_value := regexp.MustCompile(": (.*)")
+		var header_value = grep_value.FindStringSubmatch(header)[1]
+		req.Header.Set(header_key, header_value)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	// Get Response Body for string
+	body_bytes, err := ioutil.ReadAll(resp.Body)
+	var response_body = string(body_bytes)
+	// Solve the problem of garbled body codes with unmatched numbers
+	if !utf8.Valid(body_bytes) {
+		data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(body_bytes)
+		response_body = string(data)
+	}
+	// Get Response title
+	grep_title := regexp.MustCompile("<title>(.*)</title>")
+	if len(grep_title.FindStringSubmatch(response_body)) != 0 {
+		respTitle = grep_title.FindStringSubmatch(response_body)[1]
+	} else {
+		respTitle = "None"
+	}
+	// Get Response Header for string
+	var res []string
+	for name, values := range resp.Header {
+		for _, value := range values {
+			res = append(res, fmt.Sprintf("%s: %s", name, value))
+		}
+	}
+	var response_header string
+	for _, re := range res {
+		response_header += re + "\n"
+	}
+	// get response status code
+	var status_code = resp.StatusCode
+	response_status_code := strconv.Itoa(status_code)
+	RespData := []RespLab{
+		{Url, response_body, response_header, response_status_code, respTitle, ""},
+	}
+	return RespData
+}
+
+//dismap 解析IP
+
+func ParseUrl(host string, port string) string {
+	if port == "80" {
+		return "http://" + host
+	} else if port == "443" {
+		return "https://" + host
+	} else if len(regexp.MustCompile("443").FindAllStringIndex(port, -1)) == 1 {
+		return "https://" + host + ":" + port
+	} else {
+		return "http://" + host + ":" + port
+	}
+}