package lxhttp import ( "bytes" "compress/gzip" "crypto/tls" "encoding/base64" "encoding/json" "git.listensoft.net/tool/jspkit/logger" "go.uber.org/zap" "io" "net/http" "net/http/cookiejar" "net/url" "strings" "time" "git.listensoft.net/tool/jspkit/taxerr" "golang.org/x/net/publicsuffix" ) // POSTForm 发送Form表单请求 func POSTForm(c *http.Client, requrl string, params map[string]string, headers map[string]string) ([]byte, error) { form := url.Values{} for k, v := range params { form.Add(k, v) } var bys []byte req, err := http.NewRequest("POST", requrl, strings.NewReader(form.Encode())) if err != nil { return bys, err } for k, v := range headers { req.Header.Add(k, v) } resp, err := c.Do(req) if err != nil { return bys, err } defer resp.Body.Close() return io.ReadAll(resp.Body) } // GET 发送get请求 func GET(c *http.Client, requrl string, params map[string]string, headers map[string]string) ([]byte, error) { form := url.Values{} for k, v := range params { form.Add(k, v) } var bys []byte req, err := http.NewRequest("GET", requrl, strings.NewReader(form.Encode())) if err != nil { return bys, err } for k, v := range headers { req.Header.Add(k, v) } resp, err := c.Do(req) if err != nil { return bys, err } defer resp.Body.Close() return io.ReadAll(resp.Body) } func GZIPDecode(in []byte) ([]byte, error) { reader, err := gzip.NewReader(bytes.NewReader(in)) if err != nil { var out []byte return out, err } defer reader.Close() return io.ReadAll(reader) } // POSTJson 发送json请求 func POSTJson1(c *http.Client, requrl string, params map[string]any, headers map[string]string) ([]byte, error) { bytesData, _ := json.Marshal(params) request, err := http.NewRequest("POST", requrl, bytes.NewReader(bytesData)) if err != nil { return nil, err } for k, v := range headers { request.Header.Add(k, v) } request.Header.Set("Content-Type", "application/json;charset=UTF-8") request.Header.Set("Accept-Encoding", "gzip, deflate, br") resp, err := c.Do(request) if err != nil { return nil, err } defer resp.Body.Close() bytes, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if strings.Contains(resp.Header.Get(`Content-Encoding`), "gzip") { return GZIPDecode(bytes) } return bytes, nil } // POSTJson 发送json请求 func POSTJsonAny(c *http.Client, requrl string, params any, headers map[string]string) ([]byte, error) { bytesData, _ := json.Marshal(params) request, err := http.NewRequest("POST", requrl, bytes.NewReader(bytesData)) if err != nil { return nil, err } for k, v := range headers { request.Header.Add(k, v) } request.Header.Set("Content-Type", "application/json;charset=UTF-8") request.Header.Set("Accept-Encoding", "gzip, deflate, br") resp, err := c.Do(request) if err != nil { return nil, err } defer resp.Body.Close() bytes, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if strings.Contains(resp.Header.Get(`Content-Encoding`), "gzip") { return GZIPDecode(bytes) } return bytes, nil } // POSTJson 发送json请求 func POSTStrReader(c *http.Client, requrl, params string, headers map[string]string) ([]byte, error) { request, err := http.NewRequest("POST", requrl, strings.NewReader(params)) if err != nil { return nil, err } for k, v := range headers { request.Header.Add(k, v) } request.Header.Set("Accept-Encoding", "gzip, deflate, br") resp, err := c.Do(request) if err != nil { return nil, err } defer resp.Body.Close() bytes, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if strings.Contains(resp.Header.Get(`Content-Encoding`), "gzip") { return GZIPDecode(bytes) } return bytes, nil } // POSTJson 发送json请求 func POSTJson(c *http.Client, requrl string, params map[string]string, headers map[string]string) ([]byte, error) { bytesData, _ := json.Marshal(params) request, err := http.NewRequest("POST", requrl, bytes.NewReader(bytesData)) if err != nil { return nil, err } for k, v := range headers { request.Header.Add(k, v) } request.Header.Set("Content-Type", "application/json;charset=UTF-8") request.Header.Set("Accept-Encoding", "gzip, deflate, br") resp, err := c.Do(request) if err != nil { return nil, err } if resp.StatusCode == 504 && strings.Contains(requrl, "keepalive") { // 超时 return nil, taxerr.KeepTimeOut } defer resp.Body.Close() bytes, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if strings.Contains(resp.Header.Get(`Content-Encoding`), "gzip") { return GZIPDecode(bytes) } return bytes, nil } // Get http get请求 func Get(c *http.Client, url string) ([]byte, error) { resp, err := c.Get(url) if err != nil { return nil, err } defer resp.Body.Close() return io.ReadAll(resp.Body) } // 发出Post请求 请求数据和返回数据都是json格式 func PostJson(url string, data map[string]interface{}, v any) error { bytesData, _ := json.Marshal(data) request, err := http.NewRequest("POST", url, bytes.NewReader(bytesData)) if err != nil { logger.GetLogger().Info("err", zap.Error(err)) return err } request.Header.Set("Content-Type", "application/json;charset=UTF-8") client := NewHttpClient() resp, err := client.Do(request) if err != nil { return err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return err } if err = json.Unmarshal(body, v); err != nil { return err } return nil } // NewHttpClient 生成一个htppclient func NewHttpClient(tt ...int) *http.Client { if len(tt) == 0 { tt = append(tt, 60) } tr := &http.Transport{ // Proxy: http.ProxyFromEnvironment, // 从环境变量中获取代理 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // 忽略证书校验错误 } var c = &http.Client{ Timeout: time.Duration(tt[0]) * time.Second, Transport: tr, } options := cookiejar.Options{ PublicSuffixList: publicsuffix.List, } curCookieJar, _ := cookiejar.New(&options) c.Jar = curCookieJar return c } // NewHttpClient 生成一个htppclient 请求等待时间加长 func NewLongHttpClient() *http.Client { tr := &http.Transport{ // Proxy: http.ProxyFromEnvironment, // 从环境变量中获取代理 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // 忽略证书校验错误 } var c = &http.Client{ Timeout: time.Duration(600 * time.Second), Transport: tr, } options := cookiejar.Options{ PublicSuffixList: publicsuffix.List, } curCookieJar, _ := cookiejar.New(&options) c.Jar = curCookieJar return c } // NewHttpClient 生成一个代理htppclient func NewHttpClientForProxy(proxy string) *http.Client { var tr *http.Transport if proxy == "" { tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //忽略证书校验 } } else { p, _ := url.Parse(proxy) tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //忽略证书校验 Proxy: http.ProxyURL(p), } } var c = &http.Client{ Timeout: time.Duration(20 * time.Second), Transport: tr, } options := cookiejar.Options{ PublicSuffixList: publicsuffix.List, } curCookieJar, _ := cookiejar.New(&options) c.Jar = curCookieJar return c } // NewHttpClient 生成一个待认证的htppclient (仅适用西藏) func NewHttpClientForXizang(ip, auth string) *http.Client { p, _ := url.Parse(ip) tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //忽略证书校验 Proxy: http.ProxyURL(p), } if auth != "" { tr.ProxyConnectHeader = http.Header{ "Proxy-Authorization": []string{"Basic " + base64.StdEncoding.EncodeToString([]byte(auth))}, } } var c = &http.Client{ Timeout: time.Duration(20 * time.Second), Transport: tr, } options := cookiejar.Options{ PublicSuffixList: publicsuffix.List, } curCookieJar, _ := cookiejar.New(&options) c.Jar = curCookieJar return c } // NewHttpClientForLocalProxy 针对本地代理 需要账号认证的 不认证的也可以用,后面全用这个 func NewHttpClientForLocalProxy(ip, auth string) *http.Client { if ip == "" { return NewHttpClient() } p, _ := url.Parse(ip) tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //忽略证书校验 Proxy: http.ProxyURL(p), } if auth != "" { tr.ProxyConnectHeader = http.Header{ "Proxy-Authorization": []string{"Basic " + base64.StdEncoding.EncodeToString([]byte(auth))}, } } var c = &http.Client{ Timeout: time.Duration(60 * time.Second), Transport: tr, } options := cookiejar.Options{ PublicSuffixList: publicsuffix.List, } curCookieJar, _ := cookiejar.New(&options) c.Jar = curCookieJar return c } // CreateFormReader 将 map 转换成 header func CreateFormReader(data map[string]string) io.Reader { form := url.Values{} for k, v := range data { form.Add(k, v) } return strings.NewReader(form.Encode()) } func HttpClientPost(client *http.Client, URL, contentType string, body io.Reader) (res []byte, err error) { resp, err := client.Post(URL, contentType, body) if err != nil { return } defer resp.Body.Close() return io.ReadAll(resp.Body) }