package internal import ( "errors" "net" "net/http" "net/http/cookiejar" "net/url" "time" ) var ( // ErrNoCookieJar is the error type for missing cookie jar ErrNoCookieJar = errors.New("cookie jar is not available") ) // Client is a small wrapper around *http.Client to provide new methods. type Client struct { *http.Client } // NewClient creates http.Client with modified values for typical web scraper func NewClient(cookiesDisabled bool) *Client { client := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, DualStack: true, }).DialContext, MaxIdleConns: 0, // Default: 100 MaxIdleConnsPerHost: 1000, // Default: 2 IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, Timeout: time.Second * 180, // Google's timeout } if !cookiesDisabled { client.Jar, _ = cookiejar.New(nil) } return &Client{Client: client} } // SetCookies handles the receipt of the cookies in a reply for the given URL func (c *Client) SetCookies(URL string, cookies []*http.Cookie) error { if c.Jar == nil { return ErrNoCookieJar } u, err := url.Parse(URL) if err != nil { return err } c.Jar.SetCookies(u, cookies) return nil } // Cookies returns the cookies to send in a request for the given URL. func (c *Client) Cookies(URL string) []*http.Cookie { if c.Jar == nil { return nil } parsedURL, err := url.Parse(URL) if err != nil { return nil } return c.Jar.Cookies(parsedURL) } // SetDefaultHeader sets header if not exists before func SetDefaultHeader(header http.Header, key string, value string) http.Header { if header.Get(key) == "" { header.Set(key, value) } return header }