package webscan import ( "Yasso/config" "bytes" "crypto/md5" "crypto/tls" "fmt" "golang.org/x/text/encoding/simplifiedchinese" "io/ioutil" "net/http" "net/url" "path" "regexp" "strconv" "strings" "time" "unicode/utf8" ) //TODO: dismap RespLab type RespLab struct { Url string RespBody string RespHeader string RespStatusCode string RespTitle string FaviconMd5 string } func FaviconMd5(Url string, timeout time.Duration, Path string) string { client := &http.Client{ Timeout: time.Duration(timeout), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } Url = Url + "/favicon.ico" req, err := http.NewRequest("GET", Url, nil) if err != nil { return "" } for key, value := range config.DefaultHeader { req.Header.Set(key, value) } //req.Header.Set("Accept-Language", "zh,zh-TW;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6") //req.Header.Set("User-agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36") //req.Header.Set("Cookie", "rememberMe=int") resp, err := client.Do(req) if err != nil { return "" } defer resp.Body.Close() body_bytes, err := ioutil.ReadAll(resp.Body) hash := md5.Sum(body_bytes) md5 := fmt.Sprintf("%x", hash) return md5 } func DefaultRequests(Url string, timeout time.Duration) []RespLab { var redirect_url string var resp_title string var response_header string var response_body string var response_status_code string var res []string // 设置http请求客户端 client := &http.Client{ Timeout: time.Duration(timeout), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } req, err := http.NewRequest("GET", Url, nil) if err != nil { return nil } // 设置默认请求头 for key, value := range config.DefaultHeader { req.Header.Set(key, value) } // 做http请求 resp, err := client.Do(req) if err != nil { return nil } defer resp.Body.Close() // 获取请求的状态马 var status_code = resp.StatusCode response_status_code = strconv.Itoa(status_code) //TODO: 根据请求来拦截状态码,如果是30x则需要拦截url进行重定向 if len(regexp.MustCompile("30").FindAllStringIndex(response_status_code, -1)) == 1 { // 进行重定向 redirect_path := resp.Header.Get("Location") // 拦截url进行重定向请求 if len(regexp.MustCompile("http").FindAllStringIndex(redirect_path, -1)) == 1 { redirect_url = redirect_path } else { redirect_url = Url + redirect_path } client = &http.Client{ Timeout: time.Duration(timeout), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } // 设置重定向请求 req, err := http.NewRequest("GET", redirect_url, nil) if err != nil { return nil } for key, value := range config.DefaultHeader { req.Header.Set(key, value) } resp, err := client.Do(req) if err != nil { return nil } defer resp.Body.Close() //TODO: 解决两次的30x跳转问题 var twoStatusCode = resp.StatusCode responseStatusCodeTwo := strconv.Itoa(twoStatusCode) if len(regexp.MustCompile("30").FindAllStringIndex(responseStatusCodeTwo, -1)) == 1 { redirectPath := resp.Header.Get("Location") if len(regexp.MustCompile("http").FindAllStringIndex(redirectPath, -1)) == 1 { redirect_url = redirectPath } else { redirect_url = Url + redirectPath } client = &http.Client{ Timeout: time.Duration(timeout), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } req, err := http.NewRequest("GET", redirect_url, nil) if err != nil { return nil } for key, value := range config.DefaultHeader { req.Header.Set(key, value) } resp, err := client.Do(req) if err != nil { return nil } defer resp.Body.Close() // get response body for string bodyBytes, err := ioutil.ReadAll(resp.Body) response_body = string(bodyBytes) // Solve the problem of garbled body codes with unmatched numbers if !utf8.Valid(bodyBytes) { data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(bodyBytes) response_body = string(data) } // Get Response title grepTitle := regexp.MustCompile("(.*)") if len(grepTitle.FindStringSubmatch(response_body)) != 0 { resp_title = grepTitle.FindStringSubmatch(response_body)[1] } else { resp_title = "None" } for name, values := range resp.Header { for _, value := range values { res = append(res, fmt.Sprintf("%s: %s", name, value)) } } for _, re := range res { response_header += re + "\n" } favicon5 := FaviconMd5(Url, timeout, "") RespData := []RespLab{ {redirect_url, response_body, response_header, response_status_code, resp_title, favicon5}, } return RespData } // get response body for string body_bytes, err := ioutil.ReadAll(resp.Body) response_body = string(body_bytes) // Solve the problem of garbled body codes with unmatched numbers if !utf8.Valid(body_bytes) { data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(body_bytes) response_body = string(data) } // Get Response title grep_title := regexp.MustCompile("(.*)") if len(grep_title.FindStringSubmatch(response_body)) != 0 { resp_title = grep_title.FindStringSubmatch(response_body)[1] } else { resp_title = "None" } // get response header for string for name, values := range resp.Header { for _, value := range values { res = append(res, fmt.Sprintf("%s: %s", name, value)) } } for _, re := range res { response_header += re + "\n" } favicon5 := FaviconMd5(Url, timeout, "") RespData := []RespLab{ {redirect_url, response_body, response_header, response_status_code, resp_title, favicon5}, } return RespData } // get response body for string bodyBytes, err := ioutil.ReadAll(resp.Body) response_body = string(bodyBytes) // Solve the problem of garbled body codes with unmatched numbers if !utf8.Valid(bodyBytes) { data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(bodyBytes) response_body = string(data) } // Get Response title grep_title := regexp.MustCompile("(.*)") if len(grep_title.FindStringSubmatch(response_body)) != 0 { resp_title = grep_title.FindStringSubmatch(response_body)[1] } else { resp_title = "None" } // get response header for string for name, values := range resp.Header { for _, value := range values { res = append(res, fmt.Sprintf("%s: %s", name, value)) } } for _, re := range res { response_header += re + "\n" } faviconmd5 := FaviconMd5(Url, timeout, "") RespData := []RespLab{ {Url, response_body, response_header, response_status_code, resp_title, faviconmd5}, } return RespData } func CustomRequests(Url string, timeout time.Duration, Method string, Path string, Header []string, Body string) []RespLab { var respTitle string // Splicing Custom Path u, err := url.Parse(Url) u.Path = path.Join(u.Path, Path) Url = u.String() if strings.HasSuffix(Path, "/") { Url = Url + "/" } client := &http.Client{ Timeout: time.Duration(timeout), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } // Send Http requests body_byte := bytes.NewBuffer([]byte(Body)) req, err := http.NewRequest(Method, Url, body_byte) if err != nil { return nil } // Set Requests Headers for _, header := range Header { grep_key := regexp.MustCompile("(.*): ") var header_key = grep_key.FindStringSubmatch(header)[1] grep_value := regexp.MustCompile(": (.*)") var header_value = grep_value.FindStringSubmatch(header)[1] req.Header.Set(header_key, header_value) } resp, err := client.Do(req) if err != nil { return nil } defer resp.Body.Close() // Get Response Body for string body_bytes, err := ioutil.ReadAll(resp.Body) var response_body = string(body_bytes) // Solve the problem of garbled body codes with unmatched numbers if !utf8.Valid(body_bytes) { data, _ := simplifiedchinese.GBK.NewDecoder().Bytes(body_bytes) response_body = string(data) } // Get Response title grep_title := regexp.MustCompile("(.*)") if len(grep_title.FindStringSubmatch(response_body)) != 0 { respTitle = grep_title.FindStringSubmatch(response_body)[1] } else { respTitle = "None" } // Get Response Header for string var res []string for name, values := range resp.Header { for _, value := range values { res = append(res, fmt.Sprintf("%s: %s", name, value)) } } var response_header string for _, re := range res { response_header += re + "\n" } // get response status code var status_code = resp.StatusCode response_status_code := strconv.Itoa(status_code) RespData := []RespLab{ {Url, response_body, response_header, response_status_code, respTitle, ""}, } return RespData } //dismap 解析IP func ParseUrl(host string, port string) string { if port == "80" { return "http://" + host } else if port == "443" { return "https://" + host } else if len(regexp.MustCompile("443").FindAllStringIndex(port, -1)) == 1 { return "https://" + host + ":" + port } else { return "http://" + host + ":" + port } }