1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

package main

import (
	"log"
	"net/http"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "cpu_temperature_celsius",
		Help: "Current temperature of the CPU.",
	})
	hdFailures = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "hd_errors_total",
			Help: "Number of hard-disk errors.",
		},
		[]string{"device"},
	)
)

func init() {
	// Metrics have to be registered to be exposed:
	prometheus.MustRegister(cpuTemp)
	prometheus.MustRegister(hdFailures)
}

func main() {
	cpuTemp.Set(65.3)
	hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc()

	// The Handler function provides a default handler to expose metrics
	// via an HTTP server. "/metrics" is the usual endpoint for that.
	http.Handle("/metrics", promhttp.Handler())
	log.Fatal(http.ListenAndServe(":8080", nil))
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

const (
	// DefMaxAge is the default duration for which observations stay
	// relevant.
	DefMaxAge time.Duration = 10 * time.Minute
	// DefAgeBuckets is the default number of buckets used to calculate the
	// age of observations.
	DefAgeBuckets = 5
	// DefBufCap is the standard buffer size for collecting Summary observations.
	DefBufCap = 500
)

1

const ExemplarMaxRunes = 64

1
2
3
4

var (
	DefaultRegisterer Registerer = defaultRegistry
	DefaultGatherer   Gatherer   = defaultRegistry
)

1
2
3

var (
	DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
)

1

func BuildFQName(namespace, subsystem, name string) string

1

func DescribeByCollect(c Collector, descs chan<- *Desc)

1
2
3

func (c customCollector) Describe(ch chan<- *Desc) {
	DescribeByCollect(c, ch)
}

1

func ExponentialBuckets(start, factor float64, count int) []float64

1

func LinearBuckets(start, width float64, count int) []float64

1

func MustRegister(cs ...Collector)

1

func Register(c Collector) error

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

// Imagine you have a worker pool and want to count the tasks completed.
taskCounter := prometheus.NewCounter(prometheus.CounterOpts{
	Subsystem: "worker_pool",
	Name:      "completed_tasks_total",
	Help:      "Total number of tasks completed.",
})
// This will register fine.
if err := prometheus.Register(taskCounter); err != nil {
	fmt.Println(err)
} else {
	fmt.Println("taskCounter registered.")
}
// Don't forget to tell the HTTP server about the Prometheus handler.
// (In a real program, you still need to start the HTTP server...)
http.Handle("/metrics", promhttp.Handler())

// Now you can start workers and give every one of them a pointer to
// taskCounter and let it increment it whenever it completes a task.
taskCounter.Inc() // This has to happen somewhere in the worker code.

// But wait, you want to see how individual workers perform. So you need
// a vector of counters, with one element for each worker.
taskCounterVec := prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Subsystem: "worker_pool",
		Name:      "completed_tasks_total",
		Help:      "Total number of tasks completed.",
	},
	[]string{"worker_id"},
)

// Registering will fail because we already have a metric of that name.
if err := prometheus.Register(taskCounterVec); err != nil {
	fmt.Println("taskCounterVec not registered:", err)
} else {
	fmt.Println("taskCounterVec registered.")
}

// To fix, first unregister the old taskCounter.
if prometheus.Unregister(taskCounter) {
	fmt.Println("taskCounter unregistered.")
}

// Try registering taskCounterVec again.
if err := prometheus.Register(taskCounterVec); err != nil {
	fmt.Println("taskCounterVec not registered:", err)
} else {
	fmt.Println("taskCounterVec registered.")
}
// Bummer! Still doesn't work.

// Prometheus will not allow you to ever export metrics with
// inconsistent help strings or label names. After unregistering, the
// unregistered metrics will cease to show up in the /metrics HTTP
// response, but the registry still remembers that those metrics had
// been exported before. For this example, we will now choose a
// different name. (In a real program, you would obviously not export
// the obsolete metric in the first place.)
taskCounterVec = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Subsystem: "worker_pool",
		Name:      "completed_tasks_by_id",
		Help:      "Total number of tasks completed.",
	},
	[]string{"worker_id"},
)
if err := prometheus.Register(taskCounterVec); err != nil {
	fmt.Println("taskCounterVec not registered:", err)
} else {
	fmt.Println("taskCounterVec registered.")
}
// Finally it worked!

// The workers have to tell taskCounterVec their id to increment the
// right element in the metric vector.
taskCounterVec.WithLabelValues("42").Inc() // Code from worker 42.

// Each worker could also keep a reference to their own counter element
// around. Pick the counter at initialization time of the worker.
myCounter := taskCounterVec.WithLabelValues("42") // From worker 42 initialization code.
myCounter.Inc()                                   // Somewhere in the code of that worker.

// Note that something like WithLabelValues("42", "spurious arg") would
// panic (because you have provided too many label values). If you want
// to get an error instead, use GetMetricWithLabelValues(...) instead.
notMyCounter, err := taskCounterVec.GetMetricWithLabelValues("42", "spurious arg")
if err != nil {
	fmt.Println("Worker initialization failed:", err)
}
if notMyCounter == nil {
	fmt.Println("notMyCounter is nil.")
}

// A different (and somewhat tricky) approach is to use
// ConstLabels. ConstLabels are pairs of label names and label values
// that never change. Each worker creates and registers an own Counter
// instance where the only difference is in the value of the
// ConstLabels. Those Counters can all be registered because the
// different ConstLabel values guarantee that each worker will increment
// a different Counter metric.
counterOpts := prometheus.CounterOpts{
	Subsystem:   "worker_pool",
	Name:        "completed_tasks",
	Help:        "Total number of tasks completed.",
	ConstLabels: prometheus.Labels{"worker_id": "42"},
}
taskCounterForWorker42 := prometheus.NewCounter(counterOpts)
if err := prometheus.Register(taskCounterForWorker42); err != nil {
	fmt.Println("taskCounterVForWorker42 not registered:", err)
} else {
	fmt.Println("taskCounterForWorker42 registered.")
}
// Obviously, in real code, taskCounterForWorker42 would be a member
// variable of a worker struct, and the "42" would be retrieved with a
// GetId() method or something. The Counter would be created and
// registered in the initialization code of the worker.

// For the creation of the next Counter, we can recycle
// counterOpts. Just change the ConstLabels.
counterOpts.ConstLabels = prometheus.Labels{"worker_id": "2001"}
taskCounterForWorker2001 := prometheus.NewCounter(counterOpts)
if err := prometheus.Register(taskCounterForWorker2001); err != nil {
	fmt.Println("taskCounterVForWorker2001 not registered:", err)
} else {
	fmt.Println("taskCounterForWorker2001 registered.")
}

taskCounterForWorker2001.Inc()
taskCounterForWorker42.Inc()
taskCounterForWorker2001.Inc()

// Yet another approach would be to turn the workers themselves into
// Collectors and register them. See the Collector example for details.

1
2
3
4
5
6
7
8
9

taskCounter registered.
taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [worker_id]} has different label names or a different help string
taskCounter unregistered.
taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [worker_id]} has different label names or a different help string
taskCounterVec registered.
Worker initialization failed: inconsistent label cardinality: expected 1 label values but got 2 in []string{"42", "spurious arg"}
notMyCounter is nil.
taskCounterForWorker42 registered.
taskCounterForWorker2001 registered.

1

func Unregister(c Collector) bool

1

func WriteToTextfile(filename string, g Gatherer) error

1
2
3

type AlreadyRegisteredError struct {
	ExistingCollector, NewCollector Collector
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

reqCounter := prometheus.NewCounter(prometheus.CounterOpts{
	Name: "requests_total",
	Help: "The total number of requests served.",
})
if err := prometheus.Register(reqCounter); err != nil {
	if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
		// A counter for that metric has been registered before.
		// Use the old counter from now on.
		reqCounter = are.ExistingCollector.(prometheus.Counter)
	} else {
		// Something else went wrong!
		panic(err)
	}
}
reqCounter.Inc()

1

func (err AlreadyRegisteredError) Error() string

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

type Collector interface {
	// Describe sends the super-set of all possible descriptors of metrics
	// collected by this Collector to the provided channel and returns once
	// the last descriptor has been sent. The sent descriptors fulfill the
	// consistency and uniqueness requirements described in the Desc
	// documentation.
	//
	// It is valid if one and the same Collector sends duplicate
	// descriptors. Those duplicates are simply ignored. However, two
	// different Collectors must not send duplicate descriptors.
	//
	// Sending no descriptor at all marks the Collector as “unchecked”,
	// i.e. no checks will be performed at registration time, and the
	// Collector may yield any Metric it sees fit in its Collect method.
	//
	// This method idempotently sends the same descriptors throughout the
	// lifetime of the Collector. It may be called concurrently and
	// therefore must be implemented in a concurrency safe way.
	//
	// If a Collector encounters an error while executing this method, it
	// must send an invalid descriptor (created with NewInvalidDesc) to
	// signal the error to the registry.
	Describe(chan<- *Desc)
	// Collect is called by the Prometheus registry when collecting
	// metrics. The implementation sends each collected metric via the
	// provided channel and returns once the last metric has been sent. The
	// descriptor of each sent metric is one of those returned by Describe
	// (unless the Collector is unchecked, see above). Returned metrics that
	// share the same descriptor must differ in their variable label
	// values.
	//
	// This method may be called concurrently and must therefore be
	// implemented in a concurrency safe way. Blocking occurs at the expense
	// of total performance of rendering all registered metrics. Ideally,
	// Collector implementations support concurrent readers.
	Collect(chan<- Metric)
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

package prometheus_test

import (
	"log"
	"net/http"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

// ClusterManager is an example for a system that might have been built without
// Prometheus in mind. It models a central manager of jobs running in a
// cluster. Thus, we implement a custom Collector called
// ClusterManagerCollector, which collects information from a ClusterManager
// using its provided methods and turns them into Prometheus Metrics for
// collection.
//
// An additional challenge is that multiple instances of the ClusterManager are
// run within the same binary, each in charge of a different zone. We need to
// make use of wrapping Registerers to be able to register each
// ClusterManagerCollector instance with Prometheus.
type ClusterManager struct {
	Zone string
	// Contains many more fields not listed in this example.
}

// ReallyExpensiveAssessmentOfTheSystemState is a mock for the data gathering a
// real cluster manager would have to do. Since it may actually be really
// expensive, it must only be called once per collection. This implementation,
// obviously, only returns some made-up data.
func (c *ClusterManager) ReallyExpensiveAssessmentOfTheSystemState() (
	oomCountByHost map[string]int, ramUsageByHost map[string]float64,
) {
	// Just example fake data.
	oomCountByHost = map[string]int{
		"foo.example.org": 42,
		"bar.example.org": 2001,
	}
	ramUsageByHost = map[string]float64{
		"foo.example.org": 6.023e23,
		"bar.example.org": 3.14,
	}
	return
}

// ClusterManagerCollector implements the Collector interface.
type ClusterManagerCollector struct {
	ClusterManager *ClusterManager
}

// Descriptors used by the ClusterManagerCollector below.
var (
	oomCountDesc = prometheus.NewDesc(
		"clustermanager_oom_crashes_total",
		"Number of OOM crashes.",
		[]string{"host"}, nil,
	)
	ramUsageDesc = prometheus.NewDesc(
		"clustermanager_ram_usage_bytes",
		"RAM usage as reported to the cluster manager.",
		[]string{"host"}, nil,
	)
)

// Describe is implemented with DescribeByCollect. That's possible because the
// Collect method will always return the same two metrics with the same two
// descriptors.
func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
	prometheus.DescribeByCollect(cc, ch)
}

// Collect first triggers the ReallyExpensiveAssessmentOfTheSystemState. Then it
// creates constant metrics for each host on the fly based on the returned data.
//
// Note that Collect could be called concurrently, so we depend on
// ReallyExpensiveAssessmentOfTheSystemState to be concurrency-safe.
func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
	oomCountByHost, ramUsageByHost := cc.ClusterManager.ReallyExpensiveAssessmentOfTheSystemState()
	for host, oomCount := range oomCountByHost {
		ch <- prometheus.MustNewConstMetric(
			oomCountDesc,
			prometheus.CounterValue,
			float64(oomCount),
			host,
		)
	}
	for host, ramUsage := range ramUsageByHost {
		ch <- prometheus.MustNewConstMetric(
			ramUsageDesc,
			prometheus.GaugeValue,
			ramUsage,
			host,
		)
	}
}

// NewClusterManager first creates a Prometheus-ignorant ClusterManager
// instance. Then, it creates a ClusterManagerCollector for the just created
// ClusterManager. Finally, it registers the ClusterManagerCollector with a
// wrapping Registerer that adds the zone as a label. In this way, the metrics
// collected by different ClusterManagerCollectors do not collide.
func NewClusterManager(zone string, reg prometheus.Registerer) *ClusterManager {
	c := &ClusterManager{
		Zone: zone,
	}
	cc := ClusterManagerCollector{ClusterManager: c}
	prometheus.WrapRegistererWith(prometheus.Labels{"zone": zone}, reg).MustRegister(cc)
	return c
}

func ExampleCollector() {
	// Since we are dealing with custom Collector implementations, it might
	// be a good idea to try it out with a pedantic registry.
	reg := prometheus.NewPedanticRegistry()

	// Construct cluster managers. In real code, we would assign them to
	// variables to then do something with them.
	NewClusterManager("db", reg)
	NewClusterManager("ca", reg)

	// Add the standard process and Go metrics to the custom registry.
	reg.MustRegister(
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
		prometheus.NewGoCollector(),
	)

	http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
	log.Fatal(http.ListenAndServe(":8080", nil))
}

1

func NewBuildInfoCollector() Collector

1

func NewExpvarCollector(exports map[string]*Desc) Collector

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

expvarCollector := prometheus.NewExpvarCollector(map[string]*prometheus.Desc{
	"memstats": prometheus.NewDesc(
		"expvar_memstats",
		"All numeric memstats as one metric family. Not a good role-model, actually... ;-)",
		[]string{"type"}, nil,
	),
	"lone-int": prometheus.NewDesc(
		"expvar_lone_int",
		"Just an expvar int as an example.",
		nil, nil,
	),
	"http-request-map": prometheus.NewDesc(
		"expvar_http_request_total",
		"How many http requests processed, partitioned by status code and http method.",
		[]string{"code", "method"}, nil,
	),
})
prometheus.MustRegister(expvarCollector)

// The Prometheus part is done here. But to show that this example is
// doing anything, we have to manually export something via expvar.  In
// real-life use-cases, some library would already have exported via
// expvar what we want to re-export as Prometheus metrics.
expvar.NewInt("lone-int").Set(42)
expvarMap := expvar.NewMap("http-request-map")
var (
	expvarMap1, expvarMap2                             expvar.Map
	expvarInt11, expvarInt12, expvarInt21, expvarInt22 expvar.Int
)
expvarMap1.Init()
expvarMap2.Init()
expvarInt11.Set(3)
expvarInt12.Set(13)
expvarInt21.Set(11)
expvarInt22.Set(212)
expvarMap1.Set("POST", &expvarInt11)
expvarMap1.Set("GET", &expvarInt12)
expvarMap2.Set("POST", &expvarInt21)
expvarMap2.Set("GET", &expvarInt22)
expvarMap.Set("404", &expvarMap1)
expvarMap.Set("200", &expvarMap2)
// Results in the following expvar map:
// "http-request-count": {"200": {"POST": 11, "GET": 212}, "404": {"POST": 3, "GET": 13}}

// Let's see what the scrape would yield, but exclude the memstats metrics.
metricStrings := []string{}
metric := dto.Metric{}
metricChan := make(chan prometheus.Metric)
go func() {
	expvarCollector.Collect(metricChan)
	close(metricChan)
}()
for m := range metricChan {
	if !strings.Contains(m.Desc().String(), "expvar_memstats") {
		metric.Reset()
		m.Write(&metric)
		metricStrings = append(metricStrings, metric.String())
	}
}
sort.Strings(metricStrings)
for _, s := range metricStrings {
	fmt.Println(strings.TrimRight(s, " "))
}

1
2
3
4
5

label:<name:"code" value:"200" > label:<name:"method" value:"GET" > untyped:<value:212 >
label:<name:"code" value:"200" > label:<name:"method" value:"POST" > untyped:<value:11 >
label:<name:"code" value:"404" > label:<name:"method" value:"GET" > untyped:<value:13 >
label:<name:"code" value:"404" > label:<name:"method" value:"POST" > untyped:<value:3 >
untyped:<value:42 >

1

func NewGoCollector() Collector

1

func NewProcessCollector(opts ProcessCollectorOpts) Collector

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

type Counter interface {
	Metric
	Collector

	// Inc increments the counter by 1. Use Add to increment it by arbitrary
	// non-negative values.
	Inc()
	// Add adds the given value to the counter. It panics if the value is <
	// 0.
	Add(float64)
}

1

func NewCounter(opts CounterOpts) Counter

1
2
3
4

type CounterFunc interface {
	Metric
	Collector
}

1

func NewCounterFunc(opts CounterOpts, function func() float64) CounterFunc

1

type CounterOpts Opts

1
2
3

type CounterVec struct {
	// contains filtered or unexported fields
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

httpReqs := prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "http_requests_total",
		Help: "How many HTTP requests processed, partitioned by status code and HTTP method.",
	},
	[]string{"code", "method"},
)
prometheus.MustRegister(httpReqs)

httpReqs.WithLabelValues("404", "POST").Add(42)

// If you have to access the same set of labels very frequently, it
// might be good to retrieve the metric only once and keep a handle to
// it. But beware of deletion of that metric, see below!
m := httpReqs.WithLabelValues("200", "GET")
for i := 0; i < 1000000; i++ {
	m.Inc()
}
// Delete a metric from the vector. If you have previously kept a handle
// to that metric (as above), future updates via that handle will go
// unseen (even if you re-create a metric with the same label set
// later).
httpReqs.DeleteLabelValues("200", "GET")
// Same thing with the more verbose Labels syntax.
httpReqs.Delete(prometheus.Labels{"method": "GET", "code": "200"})

1

func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec

1

func (m CounterVec) Collect(ch chan<- Metric)

1

func (v *CounterVec) CurryWith(labels Labels) (*CounterVec, error)

1

func (m CounterVec) Delete(labels Labels) bool

1

func (m CounterVec) DeleteLabelValues(lvs ...string) bool

1

func (m CounterVec) Describe(ch chan<- *Desc)

1

func (v *CounterVec) GetMetricWith(labels Labels) (Counter, error)

1

func (v *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error)

1

func (v *CounterVec) MustCurryWith(labels Labels) *CounterVec

1

func (m CounterVec) Reset()

1

func (v *CounterVec) With(labels Labels) Counter

1

myVec.WithLabelValues("404", "GET").Add(42)

1

func (v *CounterVec) WithLabelValues(lvs ...string) Counter

1

myVec.WithLabelValues("404", "GET").Add(42)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

type Gauge interface {
	Metric
	Collector

	// Set sets the Gauge to an arbitrary value.
	Set(float64)
	// Inc increments the Gauge by 1. Use Add to increment it by arbitrary
	// values.
	Inc()
	// Dec decrements the Gauge by 1. Use Sub to decrement it by arbitrary
	// values.
	Dec()
	// Add adds the given value to the Gauge. (The value can be negative,
	// resulting in a decrease of the Gauge.)
	Add(float64)
	// Sub subtracts the given value from the Gauge. (The value can be
	// negative, resulting in an increase of the Gauge.)
	Sub(float64)

	// SetToCurrentTime sets the Gauge to the current Unix time in seconds.
	SetToCurrentTime()
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

opsQueued := prometheus.NewGauge(prometheus.GaugeOpts{
	Namespace: "our_company",
	Subsystem: "blob_storage",
	Name:      "ops_queued",
	Help:      "Number of blob storage operations waiting to be processed.",
})
prometheus.MustRegister(opsQueued)

// 10 operations queued by the goroutine managing incoming requests.
opsQueued.Add(10)
// A worker goroutine has picked up a waiting operation.
opsQueued.Dec()
// And once more...
opsQueued.Dec()

1

func NewGauge(opts GaugeOpts) Gauge

1
2
3
4

type GaugeFunc interface {
	Metric
	Collector
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

// primaryDB and secondaryDB represent two example *sql.DB connections we want to instrument.
var primaryDB, secondaryDB interface {
	Stats() struct{ OpenConnections int }
}

if err := prometheus.Register(prometheus.NewGaugeFunc(
	prometheus.GaugeOpts{
		Namespace:   "mysql",
		Name:        "connections_open",
		Help:        "Number of mysql connections open.",
		ConstLabels: prometheus.Labels{"destination": "primary"},
	},
	func() float64 { return float64(primaryDB.Stats().OpenConnections) },
)); err == nil {
	fmt.Println(`GaugeFunc 'connections_open' for primary DB connection registered with labels {destination="primary"}`)
}

if err := prometheus.Register(prometheus.NewGaugeFunc(
	prometheus.GaugeOpts{
		Namespace:   "mysql",
		Name:        "connections_open",
		Help:        "Number of mysql connections open.",
		ConstLabels: prometheus.Labels{"destination": "secondary"},
	},
	func() float64 { return float64(secondaryDB.Stats().OpenConnections) },
)); err == nil {
	fmt.Println(`GaugeFunc 'connections_open' for secondary DB connection registered with labels {destination="secondary"}`)
}

// Note that we can register more than once GaugeFunc with same metric name
// as long as their const labels are consistent.

1
2

GaugeFunc 'connections_open' for primary DB connection registered with labels {destination="primary"}
GaugeFunc 'connections_open' for secondary DB connection registered with labels {destination="secondary"}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

if err := prometheus.Register(prometheus.NewGaugeFunc(
	prometheus.GaugeOpts{
		Subsystem: "runtime",
		Name:      "goroutines_count",
		Help:      "Number of goroutines that currently exist.",
	},
	func() float64 { return float64(runtime.NumGoroutine()) },
)); err == nil {
	fmt.Println("GaugeFunc 'goroutines_count' registered.")
}
// Note that the count of goroutines is a gauge (and not a counter) as
// it can go up and down.

1

GaugeFunc 'goroutines_count' registered.

1

func NewGaugeFunc(opts GaugeOpts, function func() float64) GaugeFunc

1

type GaugeOpts Opts

1
2
3

type GaugeVec struct {
	// contains filtered or unexported fields
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

opsQueued := prometheus.NewGaugeVec(
	prometheus.GaugeOpts{
		Namespace: "our_company",
		Subsystem: "blob_storage",
		Name:      "ops_queued",
		Help:      "Number of blob storage operations waiting to be processed, partitioned by user and type.",
	},
	[]string{
		// Which user has requested the operation?
		"user",
		// Of what type is the operation?
		"type",
	},
)
prometheus.MustRegister(opsQueued)

// Increase a value using compact (but order-sensitive!) WithLabelValues().
opsQueued.WithLabelValues("bob", "put").Add(4)
// Increase a value with a map using WithLabels. More verbose, but order
// doesn't matter anymore.
opsQueued.With(prometheus.Labels{"type": "delete", "user": "alice"}).Inc()

1

func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec

1

func (m GaugeVec) Collect(ch chan<- Metric)

1

func (v *GaugeVec) CurryWith(labels Labels) (*GaugeVec, error)

1

func (m GaugeVec) Delete(labels Labels) bool

1

func (m GaugeVec) DeleteLabelValues(lvs ...string) bool

1

func (m GaugeVec) Describe(ch chan<- *Desc

1

func (v *GaugeVec) GetMetricWith(labels Labels) (Gauge, error)

1

func (v *GaugeVec) GetMetricWithLabelValues(lvs ...string) (Gauge, error)

1

func (v *GaugeVec) MustCurryWith(labels Labels) *GaugeVec

1

func (m GaugeVec) Reset()

1

func (v *GaugeVec) With(labels Labels) Gauge

1

myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42)

1

func (v *GaugeVec) WithLabelValues(lvs ...string) Gauge

1

myVec.WithLabelValues("404", "GET").Add(42)

1
2
3
4
5
6
7

type Histogram interface {
	Metric
	Collector

	// Observe adds a single observation to the histogram.
	Observe(float64)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

temps := prometheus.NewHistogram(prometheus.HistogramOpts{
	Name:    "pond_temperature_celsius",
	Help:    "The temperature of the frog pond.", // Sorry, we can't measure how badly it smells.
	Buckets: prometheus.LinearBuckets(20, 5, 5),  // 5 buckets, each 5 centigrade wide.
})

// Simulate some observations.
for i := 0; i < 1000; i++ {
	temps.Observe(30 + math.Floor(120*math.Sin(float64(i)*0.1))/10)
}

// Just for demonstration, let's check the state of the histogram by
// (ab)using its Write method (which is usually only used by Prometheus
// internally).
metric := &dto.Metric{}
temps.Write(metric)
fmt.Println(proto.MarshalTextString(metric))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

histogram: <
  sample_count: 1000
  sample_sum: 29969.50000000001
  bucket: <
    cumulative_count: 192
    upper_bound: 20
  >
  bucket: <
    cumulative_count: 366
    upper_bound: 25
  >
  bucket: <
    cumulative_count: 501
    upper_bound: 30
  >
  bucket: <
    cumulative_count: 638
    upper_bound: 35
  >
  bucket: <
    cumulative_count: 816
    upper_bound: 40
  >
>

1

func NewHistogram(opts HistogramOpts) Histogram

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

type HistogramOpts struct {
	// Namespace, Subsystem, and Name are components of the fully-qualified
	// name of the Histogram (created by joining these components with
	// "_"). Only Name is mandatory, the others merely help structuring the
	// name. Note that the fully-qualified name of the Histogram must be a
	// valid Prometheus metric name.
	Namespace string
	Subsystem string
	Name      string

	// Help provides information about this Histogram.
	//
	// Metrics with the same fully-qualified name must have the same Help
	// string.
	Help string

	// ConstLabels are used to attach fixed labels to this metric. Metrics
	// with the same fully-qualified name must have the same label names in
	// their ConstLabels.
	//
	// ConstLabels are only used rarely. In particular, do not use them to
	// attach the same labels to all your metrics. Those use cases are
	// better covered by target labels set by the scraping Prometheus
	// server, or by one specific metric (e.g. a build_info or a
	// machine_role metric). See also
	// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
	ConstLabels Labels

	// Buckets defines the buckets into which observations are counted. Each
	// element in the slice is the upper inclusive bound of a bucket. The
	// values must be sorted in strictly increasing order. There is no need
	// to add a highest bucket with +Inf bound, it will be added
	// implicitly. The default value is DefBuckets.
	Buckets []float64
}

1
2
3

type HistogramVec struct {
	// contains filtered or unexported fields
}

1

func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec

1

func (m HistogramVec) Collect(ch chan<- Metric)

1

func (v *HistogramVec) CurryWith(labels Labels) (ObserverVec, error)

1

func (m HistogramVec) Delete(labels Labels) bool

1

func (m HistogramVec) DeleteLabelValues(lvs ...string) bool

1

func (m HistogramVec) Describe(ch chan<- *Desc)

1

func (v *HistogramVec) GetMetricWith(labels Labels) (Observer, error)

1

func (v *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Observer, error)

1

func (v *HistogramVec) MustCurryWith(labels Labels) ObserverVec

1

func (m HistogramVec) Reset()

1

func (v *HistogramVec) With(labels Labels) Observer

1

myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Observe(42.21)

1

func (v *HistogramVec) WithLabelValues(lvs ...string) Observer

1

myVec.WithLabelValues("404", "GET").Observe(42.21)

1
2
3
4
5
6
7

type Summary interface {
	Metric
	Collector

	// Observe adds a single observation to the summary.
	Observe(float64)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

temps := prometheus.NewSummary(prometheus.SummaryOpts{
	Name:       "pond_temperature_celsius",
	Help:       "The temperature of the frog pond.",
	Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
})

// Simulate some observations.
for i := 0; i < 1000; i++ {
	temps.Observe(30 + math.Floor(120*math.Sin(float64(i)*0.1))/10)
}

// Just for demonstration, let's check the state of the summary by
// (ab)using its Write method (which is usually only used by Prometheus
// internally).
metric := &dto.Metric{}
temps.Write(metric)
fmt.Println(proto.MarshalTextString(metric))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

summary: <
  sample_count: 1000
  sample_sum: 29969.50000000001
  quantile: <
    quantile: 0.5
    value: 31.1
  >
  quantile: <
    quantile: 0.9
    value: 41.3
  >
  quantile: <
    quantile: 0.99
    value: 41.9
  >

1

func NewSummary(opts SummaryOpts) Summary

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

type SummaryOpts struct {
	// Namespace, Subsystem, and Name are components of the fully-qualified
	// name of the Summary (created by joining these components with
	// "_"). Only Name is mandatory, the others merely help structuring the
	// name. Note that the fully-qualified name of the Summary must be a
	// valid Prometheus metric name.
	Namespace string
	Subsystem string
	Name      string

	// Help provides information about this Summary.
	//
	// Metrics with the same fully-qualified name must have the same Help
	// string.
	Help string

	// ConstLabels are used to attach fixed labels to this metric. Metrics
	// with the same fully-qualified name must have the same label names in
	// their ConstLabels.
	//
	// Due to the way a Summary is represented in the Prometheus text format
	// and how it is handled by the Prometheus server internally, “quantile”
	// is an illegal label name. Construction of a Summary or SummaryVec
	// will panic if this label name is used in ConstLabels.
	//
	// ConstLabels are only used rarely. In particular, do not use them to
	// attach the same labels to all your metrics. Those use cases are
	// better covered by target labels set by the scraping Prometheus
	// server, or by one specific metric (e.g. a build_info or a
	// machine_role metric). See also
	// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels,-not-static-scraped-labels
	ConstLabels Labels

	// Objectives defines the quantile rank estimates with their respective
	// absolute error. If Objectives[q] = e, then the value reported for q
	// will be the φ-quantile value for some φ between q-e and q+e.  The
	// default value is an empty map, resulting in a summary without
	// quantiles.
	Objectives map[float64]float64

	// MaxAge defines the duration for which an observation stays relevant
	// for the summary. Must be positive. The default value is DefMaxAge.
	MaxAge time.Duration

	// AgeBuckets is the number of buckets used to exclude observations that
	// are older than MaxAge from the summary. A higher number has a
	// resource penalty, so only increase it if the higher resolution is
	// really required. For very high observation rates, you might want to
	// reduce the number of age buckets. With only one age bucket, you will
	// effectively see a complete reset of the summary each time MaxAge has
	// passed. The default value is DefAgeBuckets.
	AgeBuckets uint32

	// BufCap defines the default sample stream buffer size.  The default
	// value of DefBufCap should suffice for most uses. If there is a need
	// to increase the value, a multiple of 500 is recommended (because that
	// is the internal buffer size of the underlying package
	// "github.com/bmizerany/perks/quantile").
	BufCap uint32
}

1
2
3

type SummaryVec struct {
	// contains filtered or unexported fields
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

temps := prometheus.NewSummaryVec(
	prometheus.SummaryOpts{
		Name:       "pond_temperature_celsius",
		Help:       "The temperature of the frog pond.",
		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
	},
	[]string{"species"},
)

// Simulate some observations.
for i := 0; i < 1000; i++ {
	temps.WithLabelValues("litoria-caerulea").Observe(30 + math.Floor(120*math.Sin(float64(i)*0.1))/10)
	temps.WithLabelValues("lithobates-catesbeianus").Observe(32 + math.Floor(100*math.Cos(float64(i)*0.11))/10)
}

// Create a Summary without any observations.
temps.WithLabelValues("leiopelma-hochstetteri")

// Just for demonstration, let's check the state of the summary vector
// by registering it with a custom registry and then let it collect the
// metrics.
reg := prometheus.NewRegistry()
reg.MustRegister(temps)

metricFamilies, err := reg.Gather()
if err != nil || len(metricFamilies) != 1 {
	panic("unexpected behavior of custom test registry")
}
fmt.Println(proto.MarshalTextString(metricFamilies[0]))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

name: "pond_temperature_celsius"
help: "The temperature of the frog pond."
type: SUMMARY
metric: <
  label: <
    name: "species"
    value: "leiopelma-hochstetteri"
  >
  summary: <
    sample_count: 0
    sample_sum: 0
    quantile: <
      quantile: 0.5
      value: nan
    >
    quantile: <
      quantile: 0.9
      value: nan
    >
    quantile: <
      quantile: 0.99
      value: nan
    >
  >
>
metric: <
  label: <
    name: "species"
    value: "lithobates-catesbeianus"
  >
  summary: <
    sample_count: 1000
    sample_sum: 31956.100000000017
    quantile: <
      quantile: 0.5
      value: 32.4
    >
    quantile: <
      quantile: 0.9
      value: 41.4
    >
    quantile: <
      quantile: 0.99
      value: 41.9
    >
  >
>
metric: <
  label: <
    name: "species"
    value: "litoria-caerulea"
  >
  summary: <
    sample_count: 1000
    sample_sum: 29969.50000000001
    quantile: <
      quantile: 0.5
      value: 31.1
    >
    quantile: <
      quantile: 0.9
      value: 41.3
    >
    quantile: <
      quantile: 0.99
      value: 41.9
    >
  >
>

1

func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec