From a9aaf86df3ccc6e7a2410e5b572631a31f259003 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Musab=20G=C3=BCltekin?= <gultekinmusab@gmail.com>
Date: Sun, 9 Jun 2019 10:46:32 +0300
Subject: [PATCH] Automatic determining response and decoding it.

---
 geziyor.go | 18 ++++++++++++++----
 go.mod     |  2 ++
 go.sum     |  3 +++
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/geziyor.go b/geziyor.go
index 49b5578..423ef17 100644
--- a/geziyor.go
+++ b/geziyor.go
@@ -2,9 +2,9 @@ package geziyor
 
 import (
 	"bytes"
-	"fmt"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/fpfeng/httpcache"
+	"golang.org/x/net/html/charset"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -89,6 +89,9 @@ func (g *Geziyor) Do(req *http.Request) {
 	// Log
 	log.Println("Fetching: ", req.URL.String())
 
+	// Modify Request
+	req.Header.Set("Accept-Charset", "utf-8")
+
 	// Do request
 	resp, err := g.client.Do(req)
 	if resp != nil {
@@ -99,10 +102,17 @@ func (g *Geziyor) Do(req *http.Request) {
 		return
 	}
 
-	// Read body
-	body, err := ioutil.ReadAll(resp.Body)
+	// Start reading body and determine encoding
+	reader, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "reading body error: %v\n", err)
+		log.Printf("Determine encoding error: %v\n", err)
+		return
+	}
+
+	// Continue reading body
+	body, err := ioutil.ReadAll(reader)
+	if err != nil {
+		log.Printf("Reading Body error: %v\n", err)
 		return
 	}
 
diff --git a/go.mod b/go.mod
index c1c814d..7f5b3ba 100644
--- a/go.mod
+++ b/go.mod
@@ -5,4 +5,6 @@ go 1.12
 require (
 	github.com/PuerkitoBio/goquery v1.5.0
 	github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3
+	golang.org/x/net v0.0.0-20181114220301-adae6a3d119a
+	golang.org/x/text v0.3.2 // indirect
 )
diff --git a/go.sum b/go.sum
index c8a6c16..6c455f9 100644
--- a/go.sum
+++ b/go.sum
@@ -7,3 +7,6 @@ github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3/go.mod h1:QThlC5q
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=