Metrics Server support added for expvar. Refactored some methods.
This commit is contained in:
parent
ec4551a8a0
commit
7c383b175f
23
geziyor.go
23
geziyor.go
@ -5,10 +5,8 @@ import (
|
|||||||
"github.com/fpfeng/httpcache"
|
"github.com/fpfeng/httpcache"
|
||||||
"github.com/geziyor/geziyor/client"
|
"github.com/geziyor/geziyor/client"
|
||||||
"github.com/geziyor/geziyor/metrics"
|
"github.com/geziyor/geziyor/metrics"
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
|
||||||
"net/http/cookiejar"
|
"net/http/cookiejar"
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
@ -106,13 +104,9 @@ func (g *Geziyor) Start() {
|
|||||||
log.Println("Scraping Started")
|
log.Println("Scraping Started")
|
||||||
|
|
||||||
// Metrics
|
// Metrics
|
||||||
if g.Opt.MetricsType == metrics.Prometheus {
|
if g.Opt.MetricsType == metrics.Prometheus || g.Opt.MetricsType == metrics.ExpVar {
|
||||||
metricsServer := &http.Server{Addr: ":2112"}
|
metricsServer := metrics.StartMetricsServer(g.Opt.MetricsType)
|
||||||
defer metricsServer.Close()
|
defer metricsServer.Close()
|
||||||
go func() {
|
|
||||||
http.Handle("/metrics", promhttp.Handler())
|
|
||||||
metricsServer.ListenAndServe()
|
|
||||||
}()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start Exporters
|
// Start Exporters
|
||||||
@ -150,34 +144,35 @@ func (g *Geziyor) Start() {
|
|||||||
|
|
||||||
// Get issues a GET to the specified URL.
|
// Get issues a GET to the specified URL.
|
||||||
func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *client.Response)) {
|
func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *client.Response)) {
|
||||||
req, err := http.NewRequest("GET", url, nil)
|
req, err := client.NewRequest("GET", url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Request creating error %v\n", err)
|
log.Printf("Request creating error %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
g.Do(&client.Request{Request: req}, callback)
|
g.Do(req, callback)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetRendered issues GET request using headless browser
|
// GetRendered issues GET request using headless browser
|
||||||
// Opens up a new Chrome instance, makes request, waits for 1 second to render HTML DOM and closed.
|
// Opens up a new Chrome instance, makes request, waits for 1 second to render HTML DOM and closed.
|
||||||
// Rendered requests only supported for GET requests.
|
// Rendered requests only supported for GET requests.
|
||||||
func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Response)) {
|
func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Response)) {
|
||||||
req, err := http.NewRequest("GET", url, nil)
|
req, err := client.NewRequest("GET", url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Request creating error %v\n", err)
|
log.Printf("Request creating error %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
g.Do(&client.Request{Request: req, Rendered: true}, callback)
|
req.Rendered = true
|
||||||
|
g.Do(req, callback)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Head issues a HEAD to the specified URL
|
// Head issues a HEAD to the specified URL
|
||||||
func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *client.Response)) {
|
func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *client.Response)) {
|
||||||
req, err := http.NewRequest("HEAD", url, nil)
|
req, err := client.NewRequest("HEAD", url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Request creating error %v\n", err)
|
log.Printf("Request creating error %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
g.Do(&client.Request{Request: req}, callback)
|
g.Do(req, callback)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Do sends an HTTP request
|
// Do sends an HTTP request
|
||||||
|
@ -6,6 +6,8 @@ import (
|
|||||||
"github.com/go-kit/kit/metrics/expvar"
|
"github.com/go-kit/kit/metrics/expvar"
|
||||||
"github.com/go-kit/kit/metrics/prometheus"
|
"github.com/go-kit/kit/metrics/prometheus"
|
||||||
stdprometheus "github.com/prometheus/client_golang/prometheus"
|
stdprometheus "github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
"net/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Type represents metrics Types
|
// Type represents metrics Types
|
||||||
@ -66,3 +68,17 @@ func NewMetrics(metricsType Type) *Metrics {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StartMetricsServer starts server that handles metrics
|
||||||
|
// Prometheus: http://localhost:2112/metrics
|
||||||
|
// Expvar : http://localhost:2112/debug/vars
|
||||||
|
func StartMetricsServer(metricsType Type) *http.Server {
|
||||||
|
if metricsType == Prometheus {
|
||||||
|
http.Handle("/metrics", promhttp.Handler())
|
||||||
|
}
|
||||||
|
server := &http.Server{Addr: ":2112"}
|
||||||
|
go func() {
|
||||||
|
server.ListenAndServe()
|
||||||
|
}()
|
||||||
|
return server
|
||||||
|
}
|
||||||
|
@ -76,7 +76,10 @@ func delayMiddleware(g *Geziyor, r *client.Request) {
|
|||||||
|
|
||||||
// logMiddleware logs requests
|
// logMiddleware logs requests
|
||||||
func logMiddleware(g *Geziyor, r *client.Request) {
|
func logMiddleware(g *Geziyor, r *client.Request) {
|
||||||
log.Println("Fetching: ", r.URL.String())
|
// LogDisabled check is not necessary, but done here for performance reasons
|
||||||
|
if !g.Opt.LogDisabled {
|
||||||
|
log.Println("Fetching: ", r.URL.String())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// metricsRequestMiddleware sets stats
|
// metricsRequestMiddleware sets stats
|
||||||
|
Loading…
x
Reference in New Issue
Block a user