From a26894dac89abc9c54c2746ab30b5627929d02ea Mon Sep 17 00:00:00 2001 From: Daan Sprenkels Date: Sun, 16 May 2021 20:21:44 +0200 Subject: [PATCH 1/3] Refactor metric collection --- metrics.go | 81 +++++++++++++++++++++++++++++++++++------------------- router.go | 5 +++- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/metrics.go b/metrics.go index 2c58732..0df6acf 100644 --- a/metrics.go +++ b/metrics.go @@ -14,42 +14,37 @@ import ( const metricNamespace = "rushlink" -var metricRequestsTotalCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: metricNamespace, - Subsystem: "http", - Name: "requests_total", - Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", -}, []string{"code", "method"}) +// metricURLsTotalGauge counts the number of requests that are handled by +// the application, partitioned by status code and HTTP method. +// +// This counter is updated by throughout the application. +var metricRequestsTotalCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Subsystem: "http", + Name: "requests_total", + Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", + }, []string{"code", "method"}) -func metricURLsTotal(database *db.Database) float64 { - var metric float64 - if err := database.Transaction(func(tx *db.Database) error { - var count int64 - if err := database.Model(&db.Paste{}).Count(&count).Error; err != nil { - return err - } - metric = float64(count) - return nil - }); err != nil { - log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric")) - return 0 - } - return metric -} +// metricURLsTotalGauge measures the amount of pastes stored in the database, +// partitioned by type and state. +// +// Its values are computed on the fly by updateMetrics(). +var metricURLsTotalGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Subsystem: "pastes", + Name: "urls_total", + Help: "The current amount of pastes in the database, partitioned by state and type.", + }, []string{"state", "type"}) // StartMetricsServer starts sering Prometheus metrics exports on addr func StartMetricsServer(addr string, database *db.Database, fs *db.FileStore) { prometheus.MustRegister(metricRequestsTotalCounter) - - prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Namespace: metricNamespace, - Subsystem: "pastes", - Name: "urls_total", - Help: "The current amount of pastes in the database.", - }, func() float64 { return metricURLsTotal(database) })) + prometheus.MustRegister(metricURLsTotalGauge) router := mux.NewRouter() - router.Handle("/metrics", promhttp.Handler()).Methods("GET") + router.Handle("/metrics", &MetricsHandler{database}).Methods("GET") srv := &http.Server{ Handler: router, Addr: addr, @@ -58,3 +53,31 @@ func StartMetricsServer(addr string, database *db.Database, fs *db.FileStore) { } log.Fatal(srv.ListenAndServe()) } + +type MetricsHandler struct { + db *db.Database +} + +func (mh *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + mh.updateMetrics() + promhttp.Handler().ServeHTTP(w, r) +} + +func (mh *MetricsHandler) updateMetrics() { + // Update metricURLsTotalGauge + for state := db.PasteStateUndef; state <= db.PasteStateDeleted; state++ { + for ty := db.PasteTypeUndef; ty <= db.PasteTypeFileUpload; ty++ { + var count int64 + query := mh.db.Unscoped().Model(&db.Paste{}).Where("type = ? AND state = ?", ty, state).Count(&count) + if err := query.Error; err != nil { + log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric")) + return + } + labels := map[string]string{ + "state": state.String(), + "type": ty.String(), + } + metricURLsTotalGauge.With(labels).Set(float64(count)) + } + } +} diff --git a/router.go b/router.go index 0cad611..d9e5df4 100644 --- a/router.go +++ b/router.go @@ -60,8 +60,11 @@ func (rl *rushlink) metricsMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srw := statusResponseWriter{Inner: w} next.ServeHTTP(&srw, r) + status := strconv.Itoa(srw.StatusCode) - metricRequestsTotalCounter.WithLabelValues(status, r.Method).Inc() + labels := map[string]string{"code": status, "method": r.Method} + // Update requests counter metric + metricRequestsTotalCounter.With(labels).Inc() }) } -- 2.44.0 From c4ff0ab1b7501a66732cd923f999032f4f3ffe32 Mon Sep 17 00:00:00 2001 From: Daan Sprenkels Date: Sun, 16 May 2021 20:24:00 +0200 Subject: [PATCH 2/3] Add request_duration_seconds metric --- metrics.go | 24 +++++++++++++++++++++++- router.go | 5 +++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/metrics.go b/metrics.go index 0df6acf..d3b629e 100644 --- a/metrics.go +++ b/metrics.go @@ -17,7 +17,7 @@ const metricNamespace = "rushlink" // metricURLsTotalGauge counts the number of requests that are handled by // the application, partitioned by status code and HTTP method. // -// This counter is updated by throughout the application. +// This counter is updated by the router. var metricRequestsTotalCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, @@ -26,6 +26,27 @@ var metricRequestsTotalCounter = prometheus.NewCounterVec( Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", }, []string{"code", "method"}) +// metricRequestsLatencyNanoSeconds keeps track of the request latencies for +// each http request. +// +// This historogram is updated by the router. +var metricRequestsLatencyNanoSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricNamespace, + Subsystem: "http", + Name: "request_duration_seconds", + Buckets: []float64{ + float64(500e-6), + float64(1e-3), + float64(2e-3), + float64(5e-3), + float64(10e-3), + float64(20e-3), + float64(50e-3), + }, + Help: "The latency of each HTTP request, partitioned by status code and HTTP method.", + }, []string{"code", "method"}) + // metricURLsTotalGauge measures the amount of pastes stored in the database, // partitioned by type and state. // @@ -41,6 +62,7 @@ var metricURLsTotalGauge = prometheus.NewGaugeVec( // StartMetricsServer starts sering Prometheus metrics exports on addr func StartMetricsServer(addr string, database *db.Database, fs *db.FileStore) { prometheus.MustRegister(metricRequestsTotalCounter) + prometheus.MustRegister(metricRequestsLatencyNanoSeconds) prometheus.MustRegister(metricURLsTotalGauge) router := mux.NewRouter() diff --git a/router.go b/router.go index d9e5df4..6e888a3 100644 --- a/router.go +++ b/router.go @@ -58,13 +58,18 @@ func (rl *rushlink) recoveryMiddleware(next http.Handler) http.Handler { func (rl *rushlink) metricsMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tick := time.Now() srw := statusResponseWriter{Inner: w} next.ServeHTTP(&srw, r) + tock := time.Now() status := strconv.Itoa(srw.StatusCode) labels := map[string]string{"code": status, "method": r.Method} // Update requests counter metric metricRequestsTotalCounter.With(labels).Inc() + // Update request latency metric + elapsed := tock.Sub(tick) + metricRequestsLatencyNanoSeconds.With(labels).Observe(elapsed.Seconds()) }) } -- 2.44.0 From 306705cb28a45169d9da0e72951e45d70eb7d158 Mon Sep 17 00:00:00 2001 From: Daan Sprenkels Date: Sun, 16 May 2021 20:40:36 +0200 Subject: [PATCH 3/3] Optimize query for updating metricURLsTotalGauge --- metrics.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/metrics.go b/metrics.go index d3b629e..82de3dc 100644 --- a/metrics.go +++ b/metrics.go @@ -87,19 +87,22 @@ func (mh *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (mh *MetricsHandler) updateMetrics() { // Update metricURLsTotalGauge - for state := db.PasteStateUndef; state <= db.PasteStateDeleted; state++ { - for ty := db.PasteTypeUndef; ty <= db.PasteTypeFileUpload; ty++ { - var count int64 - query := mh.db.Unscoped().Model(&db.Paste{}).Where("type = ? AND state = ?", ty, state).Count(&count) - if err := query.Error; err != nil { - log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric")) - return - } - labels := map[string]string{ - "state": state.String(), - "type": ty.String(), - } - metricURLsTotalGauge.With(labels).Set(float64(count)) + results := make([](struct { + Type db.PasteType + State db.PasteState + Count float64 + }), 0) + query := mh.db.Unscoped().Model(&db.Paste{}).Select("type", "state", "COUNT(*) as count").Group("type, state").Find(&results) + if err := query.Error; err != nil { + log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric")) + return + } + metricURLsTotalGauge.Reset() + for _, r := range results { + labels := map[string]string{ + "type": r.Type.String(), + "state": r.State.String(), } + metricURLsTotalGauge.With(labels).Set(r.Count) } } -- 2.44.0