http package renamed to client to reduce cunfusion

client/client.go

@@ -0,0 +1,93 @@
+package client
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+var (
+	// ErrNoCookieJar is the error type for missing cookie jar
+	ErrNoCookieJar = errors.New("cookie jar is not available")
+)
+
+// Client is a small wrapper around *http.Client to provide new methods.
+type Client struct {
+	*http.Client
+}
+
+// NewClient creates http.Client with modified values for typical web scraper
+func NewClient() *Client {
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          0,    // Default: 100
+			MaxIdleConnsPerHost:   1000, // Default: 2
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+		Timeout: time.Second * 180, // Google's timeout
+	}
+	return &Client{Client: client}
+}
+
+// SetCookies handles the receipt of the cookies in a reply for the given URL
+func (c *Client) SetCookies(URL string, cookies []*http.Cookie) error {
+	if c.Jar == nil {
+		return ErrNoCookieJar
+	}
+	u, err := url.Parse(URL)
+	if err != nil {
+		return err
+	}
+	c.Jar.SetCookies(u, cookies)
+	return nil
+}
+
+// Cookies returns the cookies to send in a request for the given URL.
+func (c *Client) Cookies(URL string) []*http.Cookie {
+	if c.Jar == nil {
+		return nil
+	}
+	parsedURL, err := url.Parse(URL)
+	if err != nil {
+		return nil
+	}
+	return c.Jar.Cookies(parsedURL)
+}
+
+// SetDefaultHeader sets header if not exists before
+func SetDefaultHeader(header http.Header, key string, value string) http.Header {
+	if header.Get(key) == "" {
+		header.Set(key, value)
+	}
+	return header
+}
+
+// ConvertHeaderToMap converts http.Header to map[string]interface{}
+func ConvertHeaderToMap(header http.Header) map[string]interface{} {
+	m := make(map[string]interface{})
+	for key, values := range header {
+		for _, value := range values {
+			m[key] = value
+		}
+	}
+	return m
+}
+
+// ConvertMapToHeader converts map[string]interface{} to http.Header
+func ConvertMapToHeader(m map[string]interface{}) http.Header {
+	header := http.Header{}
+	for k, v := range m {
+		header.Set(k, v.(string))
+	}
+	return header
+}

client/request.go

@@ -0,0 +1,30 @@
+package client
+
+import (
+	"io"
+	"net/http"
+)
+
+// Request is a small wrapper around *http.Request that contains Metadata and Rendering option
+type Request struct {
+	*http.Request
+	Meta         map[string]interface{}
+	Synchronized bool
+	Rendered     bool
+	Cancelled    bool
+}
+
+// Cancel request
+func (r *Request) Cancel() {
+	r.Cancelled = true
+}
+
+// NewRequest returns a new Request given a method, URL, and optional body.
+func NewRequest(method, url string, body io.Reader) (*Request, error) {
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Request{Request: req}, nil
+}

client/response.go

@@ -0,0 +1,40 @@
+package client
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// Response type wraps http.Response
+// Contains parsed response data and Geziyor functions.
+type Response struct {
+	*http.Response
+	Body    []byte
+	HTMLDoc *goquery.Document
+	Meta    map[string]interface{}
+	Request *Request
+}
+
+// JoinURL joins base response URL and provided relative URL.
+func (r *Response) JoinURL(relativeURL string) string {
+	parsedRelativeURL, err := url.Parse(relativeURL)
+	if err != nil {
+		return ""
+	}
+
+	joinedURL := r.Response.Request.URL.ResolveReference(parsedRelativeURL)
+	return joinedURL.String()
+}
+
+// IsHTML checks if response content is HTML by looking to content-type header
+func (r *Response) IsHTML() bool {
+	contentType := r.Header.Get("Content-Type")
+	for _, htmlContentType := range []string{"text/html", "application/xhtml+xml", "application/vnd.wap.xhtml+xml"} {
+		if strings.Contains(contentType, htmlContentType) {
+			return true
+		}
+	}
+	return false
+}