gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Automatic determining response and decoding it.

Browse Source
This commit is contained in:
Musab Gültekin 2019-06-09 10:46:32 +03:00
parent f2fba85266
commit a9aaf86df3
3 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
geziyor.go
View File

@ -2,9 +2,9 @@ package geziyor
import ( import (
"bytes" "bytes"
"fmt"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/fpfeng/httpcache" "github.com/fpfeng/httpcache"
"golang.org/x/net/html/charset"
"io/ioutil" "io/ioutil"
"log" "log"
"net/http" "net/http"
@ -89,6 +89,9 @@ func (g *Geziyor) Do(req *http.Request) {
// Log // Log
log.Println("Fetching: ", req.URL.String()) log.Println("Fetching: ", req.URL.String())
// Modify Request
req.Header.Set("Accept-Charset", "utf-8")
// Do request // Do request
resp, err := g.client.Do(req) resp, err := g.client.Do(req)
if resp != nil { if resp != nil {
@ -99,10 +102,17 @@ func (g *Geziyor) Do(req *http.Request) {
return return
} }
// Read body // Start reading body and determine encoding
body, err := ioutil.ReadAll(resp.Body) reader, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "reading body error: %v\n", err) log.Printf("Determine encoding error: %v\n", err)
return
}
// Continue reading body
body, err := ioutil.ReadAll(reader)
if err != nil {
log.Printf("Reading Body error: %v\n", err)
return return
} }

2
go.mod
View File

@ -5,4 +5,6 @@ go 1.12
require ( require (
github.com/PuerkitoBio/goquery v1.5.0 github.com/PuerkitoBio/goquery v1.5.0
github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3 github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a
golang.org/x/text v0.3.2 // indirect
) )

3
go.sum
View File

@ -7,3 +7,6 @@ github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3/go.mod h1:QThlC5q
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=