gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

http package renamed to client to reduce cunfusion

Browse Source
This commit is contained in:
Musab Gültekin
2019-06-29 14:18:31 +03:00
parent 1e109c555d
commit bd6466a5f2
8 changed files with 73 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
geziyor.go
View File

@ -7,7 +7,7 @@ import (
"github.com/chromedp/cdproto/network"
"github.com/chromedp/chromedp"
"github.com/fpfeng/httpcache"
"github.com/geziyor/geziyor/http"
"github.com/geziyor/geziyor/client"
"github.com/geziyor/geziyor/metrics"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
@ -15,7 +15,7 @@ import (
"io"
"io/ioutil"
"log"
stdhttp "net/http"
"net/http"
"net/http/cookiejar"
"net/url"
"sync"
@ -36,7 +36,7 @@ type Exporter interface {
// Geziyor is our main scraper type
type Geziyor struct {
Opt *Options
Client *http.Client
Client *client.Client
Exports chan interface{}
metrics *metrics.Metrics
@ -55,7 +55,7 @@ type Geziyor struct {
// If options provided, options
func NewGeziyor(opt *Options) *Geziyor {
geziyor := &Geziyor{
Client: http.NewClient(),
Client: client.NewClient(),
Opt: opt,
Exports: make(chan interface{}),
requestMiddlewares: []RequestMiddleware{
@ -114,10 +114,10 @@ func (g *Geziyor) Start() {
// Metrics
if g.Opt.MetricsType == metrics.Prometheus {
metricsServer := &stdhttp.Server{Addr: ":2112"}
metricsServer := &http.Server{Addr: ":2112"}
defer metricsServer.Close()
go func() {
stdhttp.Handle("/metrics", promhttp.Handler())
http.Handle("/metrics", promhttp.Handler())
metricsServer.ListenAndServe()
}()
}
@ -149,39 +149,39 @@ func (g *Geziyor) Start() {
}
// Get issues a GET to the specified URL.
func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *http.Response)) {
req, err := stdhttp.NewRequest("GET", url, nil)
func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *client.Response)) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("Request creating error %v\n", err)
return
}
g.Do(&http.Request{Request: req}, callback)
g.Do(&client.Request{Request: req}, callback)
}
// GetRendered issues GET request using headless browser
// Opens up a new Chrome instance, makes request, waits for 1 second to render HTML DOM and closed.
// Rendered requests only supported for GET requests.
func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *http.Response)) {
req, err := stdhttp.NewRequest("GET", url, nil)
func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Response)) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("Request creating error %v\n", err)
return
}
g.Do(&http.Request{Request: req, Rendered: true}, callback)
g.Do(&client.Request{Request: req, Rendered: true}, callback)
}
// Head issues a HEAD to the specified URL
func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *http.Response)) {
req, err := stdhttp.NewRequest("HEAD", url, nil)
func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *client.Response)) {
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
log.Printf("Request creating error %v\n", err)
return
}
g.Do(&http.Request{Request: req}, callback)
g.Do(&client.Request{Request: req}, callback)
}
// Do sends an HTTP request
func (g *Geziyor) Do(req *http.Request, callback func(g *Geziyor, r *http.Response)) {
func (g *Geziyor) Do(req *client.Request, callback func(g *Geziyor, r *client.Response)) {
if req.Synchronized {
g.do(req, callback)
} else {
@ -191,7 +191,7 @@ func (g *Geziyor) Do(req *http.Request, callback func(g *Geziyor, r *http.Respon
}
// Do sends an HTTP request
func (g *Geziyor) do(req *http.Request, callback func(g *Geziyor, r *http.Response)) {
func (g *Geziyor) do(req *client.Request, callback func(g *Geziyor, r *client.Response)) {
g.acquireSem(req)
defer g.releaseSem(req)
if !req.Synchronized {
@ -207,7 +207,7 @@ func (g *Geziyor) do(req *http.Request, callback func(g *Geziyor, r *http.Respon
}
// Do request normal or Chrome and read response
var response *http.Response
var response *client.Response
var err error
if !req.Rendered {
response, err = g.doRequestClient(req)
@ -233,7 +233,7 @@ func (g *Geziyor) do(req *http.Request, callback func(g *Geziyor, r *http.Respon
}
}
func (g *Geziyor) doRequestClient(req *http.Request) (*http.Response, error) {
func (g *Geziyor) doRequestClient(req *client.Request) (*client.Response, error) {
// Do request
resp, err := g.Client.Do(req.Request)
@ -260,7 +260,7 @@ func (g *Geziyor) doRequestClient(req *http.Request) (*http.Response, error) {
return nil, errors.Wrap(err, "Reading body error")
}
response := http.Response{
response := client.Response{
Response: resp,
Body: body,
Meta: req.Meta,
@ -270,7 +270,7 @@ func (g *Geziyor) doRequestClient(req *http.Request) (*http.Response, error) {
return &response, nil
}
func (g *Geziyor) doRequestChrome(req *http.Request) (*http.Response, error) {
func (g *Geziyor) doRequestChrome(req *client.Request) (*client.Response, error) {
var body string
var reqID network.RequestID
var res *network.Response
@ -280,7 +280,7 @@ func (g *Geziyor) doRequestChrome(req *http.Request) (*http.Response, error) {
if err := chromedp.Run(ctx,
network.Enable(),
network.SetExtraHTTPHeaders(network.Headers(http.ConvertHeaderToMap(req.Header))),
network.SetExtraHTTPHeaders(network.Headers(client.ConvertHeaderToMap(req.Header))),
chromedp.ActionFunc(func(ctx context.Context) error {
chromedp.ListenTarget(ctx, func(ev interface{}) {
switch ev.(type) {
@ -317,11 +317,11 @@ func (g *Geziyor) doRequestChrome(req *http.Request) (*http.Response, error) {
// Set new URL in case of redirection
req.URL, _ = url.Parse(res.URL)
response := http.Response{
Response: &stdhttp.Response{
response := client.Response{
Response: &http.Response{
Request: req.Request,
StatusCode: int(res.Status),
Header: http.ConvertMapToHeader(res.Headers),
Header: client.ConvertMapToHeader(res.Headers),
},
Body: []byte(body),
Meta: req.Meta,
@ -331,7 +331,7 @@ func (g *Geziyor) doRequestChrome(req *http.Request) (*http.Response, error) {
return &response, nil
}
func (g *Geziyor) acquireSem(req *http.Request) {
func (g *Geziyor) acquireSem(req *client.Request) {
if g.Opt.ConcurrentRequests != 0 {
g.semGlobal <- struct{}{}
}
@ -349,7 +349,7 @@ func (g *Geziyor) acquireSem(req *http.Request) {
}
}
func (g *Geziyor) releaseSem(req *http.Request) {
func (g *Geziyor) releaseSem(req *client.Request) {
if g.Opt.ConcurrentRequests != 0 {
<-g.semGlobal
}