Feature: Implement Geziyor.Post which wraps the httpClient(POST)

1. Implement Geziyor.Post by the same style of Geziyor.Head
2. Add two examples in geziyor_test (TestPostJson, TestPostFormUrlEncoded)

issue #38
This commit is contained in:
walker088 2021-10-21 09:06:34 -03:00
parent 6415a775f4
commit b1e4683037
2 changed files with 65 additions and 3 deletions

View File

@ -9,6 +9,8 @@ import (
"github.com/geziyor/geziyor/metrics"
"github.com/geziyor/geziyor/middleware"
"golang.org/x/time/rate"
"io"
"io/ioutil"
"net/http/cookiejar"
"os"
@ -203,6 +205,16 @@ func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *client.Response)
g.Do(req, callback)
}
// Post issues a POST to the specified URL
func (g *Geziyor) Post(url string, body io.Reader, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("POST", url, body)
if err != nil {
internal.Logger.Printf("Request creating error %v\n", err)
return
}
g.Do(req, callback)
}
// Do sends an HTTP request
func (g *Geziyor) Do(req *client.Request, callback func(g *Geziyor, r *client.Response)) {
if g.shutdown {

View File

@ -1,8 +1,15 @@
package geziyor_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/PuerkitoBio/goquery"
"github.com/elazarl/goproxy"
"github.com/fortytw2/leaktest"
@ -14,9 +21,6 @@ import (
"github.com/geziyor/geziyor/internal"
"github.com/geziyor/geziyor/metrics"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"testing"
)
func TestSimple(t *testing.T) {
@ -180,6 +184,52 @@ func TestHEADRequest(t *testing.T) {
}).Start()
}
type PostBody struct {
UserName string `json:"user_name"`
Message string `json:"message"`
}
func TestPostJson(t *testing.T) {
postBody := &PostBody{
UserName: "Juan Valdez",
Message: "Best coffee in town",
}
payloadBuf := new(bytes.Buffer)
json.NewEncoder(payloadBuf).Encode(postBody)
geziyor.NewGeziyor(&geziyor.Options{
StartRequestsFunc: func(g *geziyor.Geziyor) {
g.Post("https://reqbin.com/echo/post/json", payloadBuf, g.Opt.ParseFunc)
},
ParseFunc: func(g *geziyor.Geziyor, r *client.Response) {
fmt.Println(string(r.Body))
g.Exports <- string(r.Body)
},
Exporters: []export.Exporter{&export.JSON{FileName: "post_json.json"}},
}).Start()
}
func TestPostFormUrlEncoded(t *testing.T) {
postForm := url.Values{}
postForm.Set("user_name", "Juan Valdez")
postForm.Set("message", "Enjoy a good coffee!")
geziyor.NewGeziyor(&geziyor.Options{
StartRequestsFunc: func(g *geziyor.Geziyor) {
g.Post("https://reqbin.com/echo/post/form", strings.NewReader(postForm.Encode()), g.Opt.ParseFunc)
},
ParseFunc: func(g *geziyor.Geziyor, r *client.Response) {
fmt.Println(string(r.Body))
g.Exports <- map[string]interface{}{
"host": r.Request.Host,
"h1": r.HTMLDoc.Find("h1").Text(),
"entire_response": string(r.Body),
}
},
Exporters: []export.Exporter{&export.JSON{FileName: "post_form.json"}},
}).Start()
}
func TestCookies(t *testing.T) {
geziyor.NewGeziyor(&geziyor.Options{
StartURLs: []string{"http://quotes.toscrape.com/login"},