36 lines
886 B
Go
36 lines
886 B
Go
package internal
|
|
|
|
import (
|
|
"net"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// NewClient creates http.Client with modified values for typical web scraper
|
|
func NewClient() *http.Client {
|
|
return &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 30 * time.Second,
|
|
KeepAlive: 30 * time.Second,
|
|
DualStack: true,
|
|
}).DialContext,
|
|
MaxIdleConns: 0, // Default: 100
|
|
MaxIdleConnsPerHost: 1000, // Default: 2
|
|
IdleConnTimeout: 90 * time.Second,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
ExpectContinueTimeout: 1 * time.Second,
|
|
},
|
|
Timeout: time.Second * 180, // Google's timeout
|
|
}
|
|
}
|
|
|
|
// SetDefaultHeader sets header if not exists before
|
|
func SetDefaultHeader(header http.Header, key string, value string) http.Header {
|
|
if header.Get(key) == "" {
|
|
header.Set(key, value)
|
|
}
|
|
return header
|
|
}
|