HTTP Client can be changed now. Docs updated.

internal/http.go

@@ -1,93 +0,0 @@
-package internal
-
-import (
-	"errors"
-	"net"
-	"net/http"
-	"net/url"
-	"time"
-)
-
-var (
-	// ErrNoCookieJar is the error type for missing cookie jar
-	ErrNoCookieJar = errors.New("cookie jar is not available")
-)
-
-// Client is a small wrapper around *http.Client to provide new methods.
-type Client struct {
-	*http.Client
-}
-
-// NewClient creates http.Client with modified values for typical web scraper
-func NewClient() *Client {
-	client := &http.Client{
-		Transport: &http.Transport{
-			Proxy: http.ProxyFromEnvironment,
-			DialContext: (&net.Dialer{
-				Timeout:   30 * time.Second,
-				KeepAlive: 30 * time.Second,
-				DualStack: true,
-			}).DialContext,
-			MaxIdleConns:          0,    // Default: 100
-			MaxIdleConnsPerHost:   1000, // Default: 2
-			IdleConnTimeout:       90 * time.Second,
-			TLSHandshakeTimeout:   10 * time.Second,
-			ExpectContinueTimeout: 1 * time.Second,
-		},
-		Timeout: time.Second * 180, // Google's timeout
-	}
-	return &Client{Client: client}
-}
-
-// SetCookies handles the receipt of the cookies in a reply for the given URL
-func (c *Client) SetCookies(URL string, cookies []*http.Cookie) error {
-	if c.Jar == nil {
-		return ErrNoCookieJar
-	}
-	u, err := url.Parse(URL)
-	if err != nil {
-		return err
-	}
-	c.Jar.SetCookies(u, cookies)
-	return nil
-}
-
-// Cookies returns the cookies to send in a request for the given URL.
-func (c *Client) Cookies(URL string) []*http.Cookie {
-	if c.Jar == nil {
-		return nil
-	}
-	parsedURL, err := url.Parse(URL)
-	if err != nil {
-		return nil
-	}
-	return c.Jar.Cookies(parsedURL)
-}
-
-// SetDefaultHeader sets header if not exists before
-func SetDefaultHeader(header http.Header, key string, value string) http.Header {
-	if header.Get(key) == "" {
-		header.Set(key, value)
-	}
-	return header
-}
-
-// ConvertHeaderToMap converts http.Header to map[string]interface{}
-func ConvertHeaderToMap(header http.Header) map[string]interface{} {
-	m := make(map[string]interface{})
-	for key, values := range header {
-		for _, value := range values {
-			m[key] = value
-		}
-	}
-	return m
-}
-
-// ConvertMapToHeader converts map[string]interface{} to http.Header
-func ConvertMapToHeader(m map[string]interface{}) http.Header {
-	header := make(map[string][]string)
-	for k, v := range m {
-		header[k] = []string{v.(string)}
-	}
-	return header
-}