From f5b3b0d04975897b5081facf3bc179f062f02b1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Musab=20G=C3=BCltekin?= <gultekinmusab@gmail.com>
Date: Fri, 14 Jun 2019 16:10:24 +0300
Subject: [PATCH] Fixed race conditions on exporters. MaxIdleConns limit
 disabled to support unlimited requests to all hosts. MaxIdleConnsPerHost
 limit increased to speed up requests to same host.

---
 exporter/csv.go  | 12 ++++++++----
 exporter/json.go | 14 ++++++++------
 geziyor.go       | 17 ++++++++++++++++-
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/exporter/csv.go b/exporter/csv.go
index 38103d4..4bf0a4a 100644
--- a/exporter/csv.go
+++ b/exporter/csv.go
@@ -15,7 +15,7 @@ type CSVExporter struct {
 	FileName string
 
 	once   sync.Once
-	file   *os.File
+	mut    sync.Mutex
 	writer *csv.Writer
 }
 
@@ -33,8 +33,7 @@ func (e *CSVExporter) Export(response *geziyor.Response) {
 			fmt.Fprintf(os.Stderr, "output file creation error: %v", err)
 			return
 		}
-		e.file = newFile
-		e.writer = csv.NewWriter(e.file)
+		e.writer = csv.NewWriter(newFile)
 	})
 
 	// Export data as responses came
@@ -59,9 +58,14 @@ func (e *CSVExporter) Export(response *geziyor.Response) {
 		}
 
 		// Write to file
+		e.mut.Lock()
 		if err := e.writer.Write(values); err != nil {
 			log.Printf("CSV writing error on exporter: %v\n", err)
 		}
-		e.writer.Flush()
+		e.mut.Unlock()
 	}
+
+	e.mut.Lock()
+	e.writer.Flush()
+	e.mut.Unlock()
 }
diff --git a/exporter/json.go b/exporter/json.go
index f3bea61..535fa2b 100644
--- a/exporter/json.go
+++ b/exporter/json.go
@@ -14,8 +14,9 @@ type JSONExporter struct {
 	FileName   string
 	EscapeHTML bool
 
-	once sync.Once
-	file *os.File
+	once    sync.Once
+	mut     sync.Mutex
+	encoder *json.Encoder
 }
 
 // Export exports response data as JSON streaming file
@@ -33,15 +34,16 @@ func (e *JSONExporter) Export(response *geziyor.Response) {
 			fmt.Fprintf(os.Stderr, "output file creation error: %v", err)
 			return
 		}
-		e.file = newFile
+		e.encoder = json.NewEncoder(newFile)
+		e.encoder.SetEscapeHTML(e.EscapeHTML)
 	})
 
 	// Export data as responses came
 	for res := range response.Exports {
-		encoder := json.NewEncoder(e.file)
-		encoder.SetEscapeHTML(e.EscapeHTML)
-		if err := encoder.Encode(res); err != nil {
+		e.mut.Lock()
+		if err := e.encoder.Encode(res); err != nil {
 			log.Printf("JSON encoding error on exporter: %v\n", err)
 		}
+		e.mut.Unlock()
 	}
 }
diff --git a/geziyor.go b/geziyor.go
index f389951..a86ea29 100644
--- a/geziyor.go
+++ b/geziyor.go
@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"log"
 	"math/rand"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -52,13 +53,27 @@ func init() {
 func NewGeziyor(opt Options) *Geziyor {
 	geziyor := &Geziyor{
 		client: &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				DialContext: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+					DualStack: true,
+				}).DialContext,
+				MaxIdleConns:          0,    // Default: 100
+				MaxIdleConnsPerHost:   1000, // Default: 2
+				IdleConnTimeout:       90 * time.Second,
+				TLSHandshakeTimeout:   10 * time.Second,
+				ExpectContinueTimeout: 1 * time.Second,
+			},
 			Timeout: time.Second * 180, // Google's timeout
 		},
 		Opt: opt,
 	}
 
 	if opt.Cache != nil {
-		geziyor.client.Transport = httpcache.NewTransport(opt.Cache)
+		geziyor.client.Transport = &httpcache.Transport{
+			Transport: geziyor.client.Transport, Cache: opt.Cache, MarkCachedResponses: true}
 	}
 	if opt.Timeout != 0 {
 		geziyor.client.Timeout = opt.Timeout