94 lines
2.2 KiB
Go
94 lines
2.2 KiB
Go
package http
|
|
|
|
import (
|
|
"errors"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
// ErrNoCookieJar is the error type for missing cookie jar
|
|
ErrNoCookieJar = errors.New("cookie jar is not available")
|
|
)
|
|
|
|
// Client is a small wrapper around *http.Client to provide new methods.
|
|
type Client struct {
|
|
*http.Client
|
|
}
|
|
|
|
// NewClient creates http.Client with modified values for typical web scraper
|
|
func NewClient() *Client {
|
|
client := &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 30 * time.Second,
|
|
KeepAlive: 30 * time.Second,
|
|
DualStack: true,
|
|
}).DialContext,
|
|
MaxIdleConns: 0, // Default: 100
|
|
MaxIdleConnsPerHost: 1000, // Default: 2
|
|
IdleConnTimeout: 90 * time.Second,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
ExpectContinueTimeout: 1 * time.Second,
|
|
},
|
|
Timeout: time.Second * 180, // Google's timeout
|
|
}
|
|
return &Client{Client: client}
|
|
}
|
|
|
|
// SetCookies handles the receipt of the cookies in a reply for the given URL
|
|
func (c *Client) SetCookies(URL string, cookies []*http.Cookie) error {
|
|
if c.Jar == nil {
|
|
return ErrNoCookieJar
|
|
}
|
|
u, err := url.Parse(URL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c.Jar.SetCookies(u, cookies)
|
|
return nil
|
|
}
|
|
|
|
// Cookies returns the cookies to send in a request for the given URL.
|
|
func (c *Client) Cookies(URL string) []*http.Cookie {
|
|
if c.Jar == nil {
|
|
return nil
|
|
}
|
|
parsedURL, err := url.Parse(URL)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return c.Jar.Cookies(parsedURL)
|
|
}
|
|
|
|
// SetDefaultHeader sets header if not exists before
|
|
func SetDefaultHeader(header http.Header, key string, value string) http.Header {
|
|
if header.Get(key) == "" {
|
|
header.Set(key, value)
|
|
}
|
|
return header
|
|
}
|
|
|
|
// ConvertHeaderToMap converts http.Header to map[string]interface{}
|
|
func ConvertHeaderToMap(header http.Header) map[string]interface{} {
|
|
m := make(map[string]interface{})
|
|
for key, values := range header {
|
|
for _, value := range values {
|
|
m[key] = value
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// ConvertMapToHeader converts map[string]interface{} to http.Header
|
|
func ConvertMapToHeader(m map[string]interface{}) http.Header {
|
|
header := http.Header{}
|
|
for k, v := range m {
|
|
header.Set(k, v.(string))
|
|
}
|
|
return header
|
|
}
|