Fixed race conditions on exporters.

MaxIdleConns limit disabled to support unlimited requests to all hosts.
MaxIdleConnsPerHost limit increased to speed up requests to same host.

exporter/csv.go

@@ -15,7 +15,7 @@ type CSVExporter struct {
 	FileName string
 
 	once   sync.Once
-	file   *os.File
+	mut    sync.Mutex
 	writer *csv.Writer
 }
 
@@ -33,8 +33,7 @@ func (e *CSVExporter) Export(response *geziyor.Response) {
 			fmt.Fprintf(os.Stderr, "output file creation error: %v", err)
 			return
 		}
-		e.file = newFile
-		e.writer = csv.NewWriter(e.file)
+		e.writer = csv.NewWriter(newFile)
 	})
 
 	// Export data as responses came
@@ -59,9 +58,14 @@ func (e *CSVExporter) Export(response *geziyor.Response) {
 		}
 
 		// Write to file
+		e.mut.Lock()
 		if err := e.writer.Write(values); err != nil {
 			log.Printf("CSV writing error on exporter: %v\n", err)
 		}
-		e.writer.Flush()
+		e.mut.Unlock()
 	}
+
+	e.mut.Lock()
+	e.writer.Flush()
+	e.mut.Unlock()
 }

exporter/json.go

@@ -14,8 +14,9 @@ type JSONExporter struct {
 	FileName   string
 	EscapeHTML bool
 
-	once sync.Once
-	file *os.File
+	once    sync.Once
+	mut     sync.Mutex
+	encoder *json.Encoder
 }
 
 // Export exports response data as JSON streaming file
@@ -33,15 +34,16 @@ func (e *JSONExporter) Export(response *geziyor.Response) {
 			fmt.Fprintf(os.Stderr, "output file creation error: %v", err)
 			return
 		}
-		e.file = newFile
+		e.encoder = json.NewEncoder(newFile)
+		e.encoder.SetEscapeHTML(e.EscapeHTML)
 	})
 
 	// Export data as responses came
 	for res := range response.Exports {
-		encoder := json.NewEncoder(e.file)
-		encoder.SetEscapeHTML(e.EscapeHTML)
-		if err := encoder.Encode(res); err != nil {
+		e.mut.Lock()
+		if err := e.encoder.Encode(res); err != nil {
 			log.Printf("JSON encoding error on exporter: %v\n", err)
 		}
+		e.mut.Unlock()
 	}
 }

geziyor.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"log"
 	"math/rand"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -52,13 +53,27 @@ func init() {
 func NewGeziyor(opt Options) *Geziyor {
 	geziyor := &Geziyor{
 		client: &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				DialContext: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+					DualStack: true,
+				}).DialContext,
+				MaxIdleConns:          0,    // Default: 100
+				MaxIdleConnsPerHost:   1000, // Default: 2
+				IdleConnTimeout:       90 * time.Second,
+				TLSHandshakeTimeout:   10 * time.Second,
+				ExpectContinueTimeout: 1 * time.Second,
+			},
 			Timeout: time.Second * 180, // Google's timeout
 		},
 		Opt: opt,
 	}
 
 	if opt.Cache != nil {
-		geziyor.client.Transport = httpcache.NewTransport(opt.Cache)
+		geziyor.client.Transport = &httpcache.Transport{
+			Transport: geziyor.client.Transport, Cache: opt.Cache, MarkCachedResponses: true}
 	}
 	if opt.Timeout != 0 {
 		geziyor.client.Timeout = opt.Timeout