Middlewares and some string util functions refactored. Added partial Documentation.

internal/http.go

@@ -0,0 +1,35 @@
+package internal
+
+import (
+	"net"
+	"net/http"
+	"time"
+)
+
+// NewClient creates http.Client with modified values for typical web scraper
+func NewClient() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          0,    // Default: 100
+			MaxIdleConnsPerHost:   1000, // Default: 2
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+		Timeout: time.Second * 180, // Google's timeout
+	}
+}
+
+// SetDefaultHeader sets header if not exists before
+func SetDefaultHeader(header http.Header, key string, value string) http.Header {
+	if header.Get(key) == "" {
+		header.Set(key, value)
+	}
+	return header
+}

internal/strings.go

@@ -0,0 +1,19 @@
+package internal
+
+// PreferFirst returns first non-empty string
+func PreferFirst(first string, second string) string {
+	if first != "" {
+		return first
+	}
+	return second
+}
+
+// Contains checks whether []string Contains string
+func Contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}