diff --git a/geziyor.go b/geziyor.go index efeeac3..7aa9a1f 100644 --- a/geziyor.go +++ b/geziyor.go @@ -5,10 +5,8 @@ import ( "github.com/fpfeng/httpcache" "github.com/geziyor/geziyor/client" "github.com/geziyor/geziyor/metrics" - "github.com/prometheus/client_golang/prometheus/promhttp" "io/ioutil" "log" - "net/http" "net/http/cookiejar" "sync" ) @@ -106,13 +104,9 @@ func (g *Geziyor) Start() { log.Println("Scraping Started") // Metrics - if g.Opt.MetricsType == metrics.Prometheus { - metricsServer := &http.Server{Addr: ":2112"} + if g.Opt.MetricsType == metrics.Prometheus || g.Opt.MetricsType == metrics.ExpVar { + metricsServer := metrics.StartMetricsServer(g.Opt.MetricsType) defer metricsServer.Close() - go func() { - http.Handle("/metrics", promhttp.Handler()) - metricsServer.ListenAndServe() - }() } // Start Exporters @@ -150,34 +144,35 @@ func (g *Geziyor) Start() { // Get issues a GET to the specified URL. func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *client.Response)) { - req, err := http.NewRequest("GET", url, nil) + req, err := client.NewRequest("GET", url, nil) if err != nil { log.Printf("Request creating error %v\n", err) return } - g.Do(&client.Request{Request: req}, callback) + g.Do(req, callback) } // GetRendered issues GET request using headless browser // Opens up a new Chrome instance, makes request, waits for 1 second to render HTML DOM and closed. // Rendered requests only supported for GET requests. func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Response)) { - req, err := http.NewRequest("GET", url, nil) + req, err := client.NewRequest("GET", url, nil) if err != nil { log.Printf("Request creating error %v\n", err) return } - g.Do(&client.Request{Request: req, Rendered: true}, callback) + req.Rendered = true + g.Do(req, callback) } // Head issues a HEAD to the specified URL func (g *Geziyor) Head(url string, callback func(g *Geziyor, r *client.Response)) { - req, err := http.NewRequest("HEAD", url, nil) + req, err := client.NewRequest("HEAD", url, nil) if err != nil { log.Printf("Request creating error %v\n", err) return } - g.Do(&client.Request{Request: req}, callback) + g.Do(req, callback) } // Do sends an HTTP request diff --git a/metrics/metrics.go b/metrics/metrics.go index 7f2f0e6..d864fb2 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -6,6 +6,8 @@ import ( "github.com/go-kit/kit/metrics/expvar" "github.com/go-kit/kit/metrics/prometheus" stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" ) // Type represents metrics Types @@ -66,3 +68,17 @@ func NewMetrics(metricsType Type) *Metrics { return nil } } + +// StartMetricsServer starts server that handles metrics +// Prometheus: http://localhost:2112/metrics +// Expvar : http://localhost:2112/debug/vars +func StartMetricsServer(metricsType Type) *http.Server { + if metricsType == Prometheus { + http.Handle("/metrics", promhttp.Handler()) + } + server := &http.Server{Addr: ":2112"} + go func() { + server.ListenAndServe() + }() + return server +} diff --git a/middleware.go b/middleware.go index 4c8832d..e1a2cf2 100644 --- a/middleware.go +++ b/middleware.go @@ -76,7 +76,10 @@ func delayMiddleware(g *Geziyor, r *client.Request) { // logMiddleware logs requests func logMiddleware(g *Geziyor, r *client.Request) { - log.Println("Fetching: ", r.URL.String()) + // LogDisabled check is not necessary, but done here for performance reasons + if !g.Opt.LogDisabled { + log.Println("Fetching: ", r.URL.String()) + } } // metricsRequestMiddleware sets stats