In this post I'd like to give a short overview on the parts needed to add monitoring for a GO API with Prometheus/Grafana in Kubernetes(K8).
Prerequisites
- K8 cluster that you can deploy to locally
- Intermediate knowledge of K8, Docker and GO
- Basic knowledge of what Prometheus and Grafana is used for
Summary
- Create a GO API with Prometheus
/metricsendpoint - Deploy API, service, service monitor to K8
- Deploy Prometheus to K8
- Access Grafana dashboards
Create a GO API with Prometheus /metrics endpoint
Here's a simple GO API that'll have an welcome page at / and a metrics page at /metrics, which will display metrics from the API. The /metrics will later be used to by Prometheus Operator to scrape data about the API
package main
import (
"log"
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
type prometheusHTTPMetric struct {
Prefix string
ClientConnected prometheus.Gauge
TransactionTotal *prometheus.CounterVec
ResponseTimeHistogram *prometheus.HistogramVec
Buckets []float64
}
func initPrometheusHTTPMetric(prefix string, buckets []float64) *prometheusHTTPMetric {
phm := prometheusHTTPMetric{
Prefix: prefix,
ClientConnected: promauto.NewGauge(prometheus.GaugeOpts{
Name: prefix + "_client_connected",
Help: "Number of active client connections",
}),
TransactionTotal: promauto.NewCounterVec(prometheus.CounterOpts{
Name: prefix + "_requests_total",
Help: "total HTTP requests processed",
}, []string{"code", "method", "type", "action"},
),
ResponseTimeHistogram: promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: prefix + "_response_time",
Help: "Histogram of response time for handler",
Buckets: buckets,
}, []string{"type", "action", "method"}),
}
return &phm
}
func (phm *prometheusHTTPMetric) wrapHandler(typeLabel string, actionLabel string, handlerFunc http.HandlerFunc) http.Handler {
handle := http.HandlerFunc(handlerFunc)
wrappedHandler := promhttp.InstrumentHandlerInFlight(phm.ClientConnected,
promhttp.InstrumentHandlerCounter(phm.TransactionTotal.MustCurryWith(prometheus.Labels{"type": typeLabel, "action": actionLabel}),
promhttp.InstrumentHandlerDuration(phm.ResponseTimeHistogram.MustCurryWith(prometheus.Labels{"type": typeLabel, "action": actionLabel}),
handle),
),
)
return wrappedHandler
}
func index(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("GO API is up"))
}
func main() {
phm := initPrometheusHTTPMetric("go_api", prometheus.LinearBuckets(0, 5, 20))
http.Handle("/metrics", promhttp.Handler())
http.Handle("/", phm.wrapHandler("Index", "GET", index))
port := ":8080"
print("API running on http://localhost" + port)
log.Fatal(http.ListenAndServe(port, nil))
}
Deploy API, service, service monitor to K8
YAML file to deploy the API, service and service monitor to K8
apiVersion: v1
kind: Namespace
metadata:
name: prom-go-api
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prom-go-api
namespace: prom-go-api
spec:
selector:
matchLabels:
app: prom-go-api
template:
metadata:
labels:
app: prom-go-api
spec:
containers:
- name: prom-go-api
image: $PROM_GO_API_IMAGE_NAME
imagePullPolicy: Always
resources:
limits:
memory: "128Mi"
cpu: "500m"
ports:
- containerPort: 8080
name: api
---
apiVersion: v1
kind: Service
metadata:
name: prom-go-api
namespace: prom-go-api
labels:
app: prom-go-api
spec:
selector:
app: prom-go-api
ports:
- port: 8080
targetPort: api
name: api
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: prom-go-api-servicemonitor
namespace: prom-go-api
labels:
app: prom-go-api
spec:
selector:
matchLabels:
app: prom-go-api
namespaceSelector:
matchNames:
- prom-go-api
endpoints:
- port: api
path: /metrics
Create a local docker image of the GO API and replace IMG with the docker image name to add the above YAML to K8
# Replace ${IMG} with your local docker image name
cat ./manifests/deployment.yaml | PROM_GO_API_IMAGE_NAME=$IMG envsubst | kubectl apply -f -
kubectl apply -f ./manifests/service.yaml
Deploy Prometheus to K8
Install the prometheus-operator helm chart
# prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false means all serviceMonitors are discovered not just
# those deployed by the helm chart itself
helm install prom-test-api stable/prometheus-operator --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false
Check it's running
kubectl port-forward service/prom-azure-databricks-operator-grafana 8090:80 --namespace="default"
Access Grafana dashboards
Access Grafana on http://localhost:8090 with the credentials below
Username: admin
Password: prom-operator
Create charts with metrics such as
increase(go_api_requests_total[1m])
Top comments (1)
nice 👍👍