gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added response joinURL test and updated chromedp.

Browse Source
This commit is contained in:
Musab Gültekin 2019-09-13 14:34:29 +03:00
parent 3264057679
commit 9b8a3837bd
5 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.travis.yml
View File

@ -4,6 +4,7 @@ go:
- 1.10.x - 1.10.x
- 1.11.x - 1.11.x
- 1.12.x - 1.12.x
- 1.13.x
- tip - tip
env: env:

11
README.md
View File

@ -162,14 +162,3 @@ BenchmarkRequests-8 200000 108710 ns/op
PASS PASS
ok github.com/geziyor/geziyor 22.861s ok github.com/geziyor/geziyor 22.861s
``` ```
## Roadmap
If you're interested in helping this project, please consider these features:
- Command line tool for: pausing and resuming scraper etc. (like [this](https://docs.scrapy.org/en/latest/topics/commands.html))
- ~~Automatically exporting extracted data to multiple places (AWS, FTP, DB, JSON, CSV etc)~~
- Downloading media (Images, Videos etc) (like [this](https://docs.scrapy.org/en/latest/topics/media-pipeline.html))
- ~~Realtime metrics (Prometheus etc.)~~

15
client/response_test.go Normal file
View File

@ -0,0 +1,15 @@
package client
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestResponse_JoinURL(t *testing.T) {
req, _ := NewRequest("GET", "https://localhost.com/test/a.html", nil)
resp := Response{
Response: nil,
Request: req,
}
assert.Equal(t, "https://localhost.com/source", resp.JoinURL("/source"))
}

4
go.mod
View File

@ -5,8 +5,8 @@ go 1.12
require ( require (
github.com/PuerkitoBio/goquery v1.5.0 github.com/PuerkitoBio/goquery v1.5.0
github.com/VividCortex/gohistogram v1.0.0 // indirect github.com/VividCortex/gohistogram v1.0.0 // indirect
github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4 github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d
github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1
github.com/fortytw2/leaktest v1.3.0 github.com/fortytw2/leaktest v1.3.0
github.com/go-kit/kit v0.8.0 github.com/go-kit/kit v0.8.0
github.com/google/btree v1.0.0 // indirect github.com/google/btree v1.0.0 // indirect

6
go.sum
View File

@ -11,8 +11,12 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4 h1:xe2O+jIdrblRgMr8wMHGcR9ZUQwfp3Vx105jmTpcpKU= github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4 h1:xe2O+jIdrblRgMr8wMHGcR9ZUQwfp3Vx105jmTpcpKU=
github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d h1:00kLGv5nKzpFchNhGDXDRbKtYx/WoT983Ka2t8/pzRE=
github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa h1:rUwJ1xjoJMkE2Tucw8MsdDV+XGSLg3o/fhCEGxFNsXo= github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa h1:rUwJ1xjoJMkE2Tucw8MsdDV+XGSLg3o/fhCEGxFNsXo=
github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa/go.mod h1:zmXqbhGztPqJGTOgYSg9SnkUwjASK0lFHWPfHAiHZrk= github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa/go.mod h1:zmXqbhGztPqJGTOgYSg9SnkUwjASK0lFHWPfHAiHZrk=
github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1 h1:X4XQDbZ9CbAXr8T0aWyhvgrSbX0gJcxRzo+71lboh2g=
github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1/go.mod h1:DC3QUn4mJ24dwjcaGQLoZrhm4X/uPHZ6spDbS2uFhm4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -103,6 +107,8 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=