gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初始化

Browse Source
This commit is contained in:
Administrator
2024-09-04 16:48:42 +08:00
parent 229b8ca83a
commit 688c516c9f
26 changed files with 126 additions and 1936 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
geziyor.go
View File

@@ -2,13 +2,13 @@ package geziyor
import (
"github.com/chromedp/chromedp"
"github.com/geziyor/geziyor/cache"
"github.com/geziyor/geziyor/client"
"github.com/geziyor/geziyor/export"
"github.com/geziyor/geziyor/internal"
"github.com/geziyor/geziyor/metrics"
"github.com/geziyor/geziyor/middleware"
"golang.org/x/time/rate"
"softdown.com/shusou/geziyor/cache"
"softdown.com/shusou/geziyor/client"
"softdown.com/shusou/geziyor/export"
"softdown.com/shusou/geziyor/internal"
"softdown.com/shusou/geziyor/metrics"
"softdown.com/shusou/geziyor/middleware"
"io"
"io/ioutil"
@@ -162,7 +162,7 @@ func (g *Geziyor) Start() {
g.Opt.StartRequestsFunc(g)
} else {
for _, startURL := range g.Opt.StartURLs {
g.Get(startURL, g.Opt.ParseFunc)
g.Get(startURL, g.Opt.Encoding, g.Opt.ParseFunc)
}
}
@@ -174,8 +174,8 @@ func (g *Geziyor) Start() {
}
// Get issues a GET to the specified URL.
func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("GET", url, nil)
func (g *Geziyor) Get(url, encoding string, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("GET", url, encoding, nil)
if err != nil {
internal.Logger.Printf("Request creating error %v\n", err)
return
@@ -186,8 +186,8 @@ func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *client.Response))
// GetRendered issues GET request using headless browser
// Opens up a new Chrome instance, makes request, waits for rendering HTML DOM and closed.
// Rendered requests only supported for GET requests.
func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("GET", url, nil)
func (g *Geziyor) GetRendered(url, encoding string, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("GET", url, encoding, nil)
if err != nil {
internal.Logger.Printf("Request creating error %v\n", err)
return
@@ -198,7 +198,7 @@ func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Re
// Head issues a HEAD to the specified URL
func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("HEAD", url, nil)
req, err := client.NewRequest("HEAD", url, "", nil)
if err != nil {
internal.Logger.Printf("Request creating error %v\n", err)
return
@@ -208,7 +208,7 @@ func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *client.Response)
// Post issues a POST to the specified URL
func (g *Geziyor) Post(url string, body io.Reader, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("POST", url, body)
req, err := client.NewRequest("POST", url, "", body)
if err != nil {
internal.Logger.Printf("Request creating error %v\n", err)
return