mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-04-20 16:09:25 +00:00
docs: update OTEL guide (#7887)
* simplify wording
* update styles
* remove extra info about go application details. The details are likely
not needed and we didn't have details for rolling-dice app anyway. So
keep it simple for consstency and brevity.
* update navigation for simplicity sake
* fix typos
follow-up after
40b47601d1
Signed-off-by: hagen1778 <roman@victoriametrics.com>
This commit is contained in:
Roman Khavronenko
f41gh7
parent
b3939de3f8
commit
8cfaf7592b
2 changed files with 99 additions and 334 deletions
docs/guides/getting-started-with-opentelemetry
|
|
@ -1,13 +1,16 @@
|
|||
VictoriaMetrics and VictoriaLogs support ingestion of OpenTelemetry [metrics](https://docs.victoriametrics.com/single-server-victoriametrics/#sending-data-via-opentelemetry) and [logs](https://docs.victoriametrics.com/victorialogs/data-ingestion/opentelemetry/) respectively.
|
||||
This guide covers data ingestion via [opentelemetry-collector](https://opentelemetry.io/docs/collector/) and direct metrics and logs push from application.
|
||||
VictoriaMetrics and VictoriaLogs support ingestion [metrics](https://docs.victoriametrics.com/single-server-victoriametrics/#sending-data-via-opentelemetry)
|
||||
and [logs](https://docs.victoriametrics.com/victorialogs/data-ingestion/opentelemetry/) in OpenTelemetry format.
|
||||
This guide covers examples of using [opentelemetry-collector](https://opentelemetry.io/docs/collector/) and direct pushing of metrics and logs from the Go application.
|
||||
|
||||
## Pre-Requirements
|
||||
## Pre-Requirements
|
||||
|
||||
* [kubernetes cluster](https://kubernetes.io/docs/tasks/tools/#kind)
|
||||
* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
|
||||
* [helm](https://helm.sh/docs/intro/install/)
|
||||
|
||||
### Install VictoriaMetrics and VictoriaLogs
|
||||
## Installation
|
||||
|
||||
### VictoriaMetrics
|
||||
|
||||
Install VictoriaMetrics helm repo:
|
||||
```sh
|
||||
|
|
@ -15,7 +18,7 @@ helm repo add vm https://victoriametrics.github.io/helm-charts/
|
|||
helm repo update
|
||||
```
|
||||
|
||||
Add VictoriaMetrics chart values to sanitize OTEL metrics:
|
||||
Add VictoriaMetrics chart values to convert OTEL metric names to Prometheus canonical format:
|
||||
```sh
|
||||
cat << EOF > vm-values.yaml
|
||||
server:
|
||||
|
|
@ -24,13 +27,12 @@ server:
|
|||
EOF
|
||||
```
|
||||
|
||||
Install VictoriaMetrics single-server version
|
||||
Install VictoriaMetrics single-server version:
|
||||
```sh
|
||||
helm install victoria-metrics vm/victoria-metrics-single -f vm-values.yaml
|
||||
```
|
||||
|
||||
Verify it's up and running:
|
||||
|
||||
```sh
|
||||
kubectl get pods
|
||||
# NAME READY STATUS RESTARTS AGE
|
||||
|
|
@ -38,7 +40,6 @@ kubectl get pods
|
|||
```
|
||||
|
||||
VictoriaMetrics helm chart provides the following URL for writing data:
|
||||
|
||||
```text
|
||||
Write URL inside the kubernetes cluster:
|
||||
http://victoria-metrics-victoria-metrics-single-server.default.svc.cluster.local.:8428/<protocol-specific-write-endpoint>
|
||||
|
|
@ -51,13 +52,14 @@ For OpenTelemetry VictoriaMetrics write endpoint is:
|
|||
http://victoria-metrics-victoria-metrics-single-server.default.svc.cluster.local.:8428/opentelemetry/v1/metrics
|
||||
```
|
||||
|
||||
Install VictoriaLogs single-server version
|
||||
### VictoriaLogs
|
||||
|
||||
Install VictoriaLogs:
|
||||
```sh
|
||||
helm install victoria-logs vm/victoria-logs-single
|
||||
```
|
||||
|
||||
Verify it's up and running:
|
||||
|
||||
```sh
|
||||
kubectl get pods
|
||||
# NAME READY STATUS RESTARTS AGE
|
||||
|
|
@ -65,7 +67,6 @@ kubectl get pods
|
|||
```
|
||||
|
||||
VictoriaLogs helm chart provides the following URL for writing data:
|
||||
|
||||
```text
|
||||
Write URL inside the kubernetes cluster:
|
||||
http://victoria-logs-victoria-logs-single-server.default.svc.cluster.local.:9428/<protocol-specific-write-endpoint>
|
||||
|
|
@ -78,19 +79,18 @@ For OpenTelemetry VictoriaLogs write endpoint is:
|
|||
http://victoria-logs-victoria-logs-single-server.default.svc.cluster.local.:9428/insert/opentelemetry/v1/logs
|
||||
```
|
||||
|
||||
## Using opentelemetry-collector with VictoriaMetrics and VictoriaLogs
|
||||
## OpenTelemetry collector with VictoriaMetrics and VictoriaLogs
|
||||
|
||||
|
||||
{width="500"}
|
||||
|
||||
### Deploy opentelemetry-collector and configure metrics and logs forwarding
|
||||
|
||||
Add OpenTelemetry helm repo
|
||||
Add OpenTelemetry helm repo:
|
||||
```sh
|
||||
helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
|
||||
helm repo update
|
||||
```
|
||||
|
||||
Add OTEL Collector values
|
||||
Add OpenTelemetry Collector values:
|
||||
```sh
|
||||
cat << EOF > otel-values.yaml
|
||||
mode: deployment
|
||||
|
|
@ -102,7 +102,8 @@ presets:
|
|||
logsCollection:
|
||||
enabled: true
|
||||
config:
|
||||
# deltatocumulative processor is needed only to support metrics with delta temporality, which is not supported by VictoriaMetrics
|
||||
# deltatocumulative processor is needed to convert metrics with delta temporality to cumulative temporality.
|
||||
# VictoriaMetrics doesn't support delta temporality. Skip this processor if you don't use delta temporality.
|
||||
processors:
|
||||
deltatocumulative:
|
||||
max_stale: 5m
|
||||
|
|
@ -135,337 +136,99 @@ config:
|
|||
EOF
|
||||
```
|
||||
|
||||
Install OTEL Collector helm chart
|
||||
Install OpenTelemetry Collector helm chart:
|
||||
```sh
|
||||
helm upgrade -i otel open-telemetry/opentelemetry-collector -f otel-values.yaml
|
||||
```
|
||||
|
||||
Check if OTEL Collector pod is healthy
|
||||
```
|
||||
Check if OpenTelemetry Collector pod is up and running:
|
||||
```sh
|
||||
kubectl get pod
|
||||
# NAME READY STATUS RESTARTS AGE
|
||||
# otel-opentelemetry-collector-7467bbb559-2pq2n 1/1 Running 0 23m
|
||||
```
|
||||
|
||||
Forward VictoriaMetrics port to local machine to verify metrics are ingested
|
||||
Forward VictoriaMetrics port to local machine to explore metrics ingested by the collector:
|
||||
```sh
|
||||
kubectl port-forward svc/victoria-metrics-victoria-metrics-single-server 8428
|
||||
```
|
||||
|
||||
Check metric `k8s_container_ready` via browser `http://localhost:8428/vmui/#/?g0.expr=k8s_container_ready`
|
||||
Visit [http://localhost:8428/vmui/#/?g0.expr=k8s_container_ready](http://localhost:8428/vmui/#/?g0.expr=k8s_container_ready) to check if metric `k8s_container_ready` is present.
|
||||
Check other available metrics by visiting [cardinality explorer](https://docs.victoriametrics.com/#cardinality-explorer) page.
|
||||
|
||||
Forward VictoriaMetrics port to local machine to verify metrics are ingested
|
||||
Forward VictoriaLogs port to local machine to explore logs ingested by the collector:
|
||||
```sh
|
||||
kubectl port-forward svc/victoria-logs-victoria-logs-single-server 9428
|
||||
```
|
||||
|
||||
Check ingested logs in browser at `http://localhost:9428/select/vmui`
|
||||
Visit [http://localhost:9428/select/vmui](http://localhost:9428/select/vmui) to check if logs ingested by collector are present.
|
||||
|
||||
The full version of possible configuration options can be found in [OpenTelemetry docs](https://opentelemetry.io/docs/collector/configuration/).
|
||||
The full version of possible configuration options for the collector can be found in [OpenTelemetry docs](https://opentelemetry.io/docs/collector/configuration/).
|
||||
|
||||
## Sending to VictoriaMetrics and VictoriaLogs via OpenTelemetry
|
||||
Metrics and logs can be sent to VictoriaMetrics and VictoriaLogs via OpenTelemetry instrumentation libraries. You can use any compatible OpenTelemetry instrumentation [clients](https://opentelemetry.io/docs/languages/).
|
||||
In our example, we'll create a WEB server in [Golang](https://go.dev/) and instrument it with metrics and logs.
|
||||
## Sending metrics and logs from Go application
|
||||
|
||||
### Building the Go application instrumented with metrics and logs
|
||||
Copy the go file from [here](app.go-collector.example). This will give you a basic implementation of a dice roll WEB server with the urls for opentelemetry-collector pointing to localhost:4318.
|
||||
In the same directory run the following command to create the `go.mod` file:
|
||||
Metrics and logs can be sent via OpenTelemetry instrumentation libraries. You can use any compatible OpenTelemetry
|
||||
instrumentation [clients](https://opentelemetry.io/docs/languages/).
|
||||
In our example, we'll create a WEB server in [Golang](https://go.dev/), instrument it with metrics and logs and configure
|
||||
it to send telemetry data to OpenTelemetry collector. The collector will then forward received data to
|
||||
VictoriaMetrics or VictoriaLogs.
|
||||
|
||||
### Sending to OpenTelemetry collector
|
||||
|
||||
Create file `main.go` from [example](app.go-collector.example) that implements a dice roll WEB server instrumented with
|
||||
OpenTelemetry SDK and is configured to send data to OpenTelemetry collector at http://localhost:4318 address.
|
||||
See how to setup and run OpenTelemetry collector [here](#OpenTelemetry-collector-with-VictoriaMetrics-and-VictoriaLogs).
|
||||
|
||||
In the same directory with the file create the `go.mod` file and execute following commands:
|
||||
```sh
|
||||
go mod init vm/otel
|
||||
```
|
||||
|
||||
For demo purposes, we'll add the following dependencies to `go.mod` file:
|
||||
```go
|
||||
require (
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.7.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0
|
||||
go.opentelemetry.io/otel v1.32.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0
|
||||
go.opentelemetry.io/otel/log v0.8.0
|
||||
go.opentelemetry.io/otel/metric v1.32.0
|
||||
go.opentelemetry.io/otel/sdk v1.32.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.8.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.32.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.32.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.4.0 // indirect
|
||||
golang.org/x/net v0.32.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
|
||||
google.golang.org/grpc v1.68.1 // indirect
|
||||
google.golang.org/protobuf v1.35.2 // indirect
|
||||
)
|
||||
```
|
||||
|
||||
Once you have these in your `go.mod` file, you can run the following command to download the dependencies:
|
||||
```sh
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Now you can run the application:
|
||||
Now try running the application:
|
||||
```sh
|
||||
go run .
|
||||
```
|
||||
|
||||
### Test ingestion
|
||||
By default, the application will be available at `localhost:8080`. You can start sending requests to /rolldice endpoint to generate metrics. The following command will send 20 requests to the /rolldice endpoint:
|
||||
By default, the application from example is listening at `http://localhost:8080`. Start sending requests
|
||||
to http://localhost:8080/rolldice endpoint to generate some metrics. The following command will send 20 requests:
|
||||
```sh
|
||||
for i in `seq 1 20`; do curl http://localhost:8080/rolldice; done
|
||||
```
|
||||
|
||||
After a few seconds you should start to see metrics sent to VictoriaMetrics by visiting `http://localhost:8428/vmui/#/?g0.expr=dice_rolls_total` in your browser or by querying the metric `dice_rolls_total` in the UI interface.
|
||||
After a few seconds you should start seeing metrics sent to VictoriaMetrics by visiting [http://localhost:8428/vmui/#/?g0.expr=dice_rolls_total](http://localhost:8428/vmui/#/?g0.expr=dice_rolls_total)
|
||||
in your browser or by querying the metric `dice_rolls_total` in the UI interface.
|
||||
|
||||
|
||||
Logs should be available by visiting `http://localhost:9428/select/vmui` using query `service.name: unknown_service:otel`.
|
||||
Logs should be available by visiting [http://localhost:9428/select/vmui](http://localhost:9428/select/vmui)
|
||||
using query `service.name: unknown_service:otel`.
|
||||
|
||||
|
||||
## Direct metrics and logs push
|
||||
### Sending without OpenTelemetry collector
|
||||
|
||||
Metrics and logs can be ingested into VictoriaMetrics directly with HTTP requests. You can use any compatible OpenTelemetry
|
||||
instrumentation [clients](https://opentelemetry.io/docs/languages/).
|
||||
In our example, we'll create a WEB server in [Golang](https://go.dev/) and instrument it with metrics and logs.
|
||||
Metrics and logs can be ingested into VictoriaMetrics and VictoriaLogs directly via HTTP requests.
|
||||
Use any compatible OpenTelemetry instrumentation [clients](https://opentelemetry.io/docs/languages/).
|
||||
|
||||
|
||||
{width="500"}
|
||||
|
||||
In our example, we'll create a WEB server in [Golang](https://go.dev/), instrument it with metrics and logs and configure
|
||||
it to send this telemetry data to VictoriaMetrics and VictoriaLogs.
|
||||
|
||||
### Building the Go application instrumented with metrics and logs
|
||||
|
||||
See the full source code of the example [here](app.go.example).
|
||||
|
||||
The list of OpenTelemetry dependencies for `go.mod` is the following:
|
||||
|
||||
```go
|
||||
go 1.23.4
|
||||
|
||||
require (
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.7.0
|
||||
go.opentelemetry.io/otel v1.32.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0
|
||||
go.opentelemetry.io/otel/log v0.8.0
|
||||
go.opentelemetry.io/otel/metric v1.32.0
|
||||
go.opentelemetry.io/otel/sdk v1.32.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.8.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.32.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.32.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.4.0 // indirect
|
||||
golang.org/x/net v0.32.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
|
||||
google.golang.org/grpc v1.68.1 // indirect
|
||||
google.golang.org/protobuf v1.35.2 // indirect
|
||||
)
|
||||
Create file `main.go` from [example](app.go.example). In the same directory with the file create the `go.mod` file and execute following commands:
|
||||
```sh
|
||||
go mod init vm/otel
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Let's create a new file `main.go` with basic implementation of the WEB server:
|
||||
```go
|
||||
package main
|
||||
|
||||
func main() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/api/fast", func(writer http.ResponseWriter, request *http.Request) {
|
||||
logger.InfoContext(ctx, "Anonymous access to fast endpoint")
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
writer.Write([]byte(`fast ok`))
|
||||
})
|
||||
mux.HandleFunc("/api/slow", func(writer http.ResponseWriter, request *http.Request) {
|
||||
time.Sleep(time.Second * 2)
|
||||
logger.InfoContext(ctx, "Anonymous access to slow endpoint")
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
writer.Write([]byte(`slow ok`))
|
||||
})
|
||||
mw, err := newMiddleware(ctx, mux)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("cannot build middleware: %q", err))
|
||||
}
|
||||
mustStop := make(chan os.Signal, 1)
|
||||
signal.Notify(mustStop, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
http.ListenAndServe("localhost:8081", mw)
|
||||
}()
|
||||
log.Printf("web server started at localhost:8081.")
|
||||
<-mustStop
|
||||
log.Println("receive shutdown signal, stopping webserver")
|
||||
|
||||
for _, shutdown := range mw.onShutdown {
|
||||
if err := shutdown(ctx); err != nil {
|
||||
log.Println("cannot shutdown metric provider ", err)
|
||||
}
|
||||
}
|
||||
log.Printf("Done!")
|
||||
}
|
||||
The example implements WEB server with two HTTP handlers: `/api/slow` and `/api/fast`. Start the application:
|
||||
```sh
|
||||
go run main.go
|
||||
2024/03/25 19:27:41 Starting web server...
|
||||
2024/03/25 19:27:41 web server started at localhost:8081.
|
||||
```
|
||||
|
||||
In the code above, we used `newMiddleware` function to create a `handler` for our server.
|
||||
Let's define it below:
|
||||
```go
|
||||
type middleware struct {
|
||||
ctx context.Context
|
||||
h http.Handler
|
||||
requestsCount metric.Int64Counter
|
||||
requestsLatency metric.Float64Histogram
|
||||
activeRequests int64
|
||||
onShutdown []func(ctx context.Context) error
|
||||
}
|
||||
|
||||
func newMiddleware(ctx context.Context, h http.Handler) (*middleware, error) {
|
||||
mw := &middleware{
|
||||
ctx: ctx,
|
||||
h: h,
|
||||
}
|
||||
|
||||
lp, err := newLoggerProvider(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot build logs provider: %w", err)
|
||||
}
|
||||
global.SetLoggerProvider(lp)
|
||||
|
||||
mp, err := newMeterProvider(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot build metrics provider: %w", err)
|
||||
}
|
||||
otel.SetMeterProvider(mp)
|
||||
meter := mp.Meter("")
|
||||
|
||||
mw.requestsLatency, err = meter.Float64Histogram("http.requests.latency")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create histogram: %w", err)
|
||||
}
|
||||
mw.requestsCount, err = meter.Int64Counter("http.requests")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create int64 counter: %w", err)
|
||||
}
|
||||
cb := func(c context.Context, o metric.Int64Observer) error {
|
||||
o.Observe(atomic.LoadInt64(&mw.activeRequests))
|
||||
return nil
|
||||
}
|
||||
_, err = meter.Int64ObservableGauge("http.requests.active", metric.WithInt64Callback(cb))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create Int64ObservableGauge: %w", err)
|
||||
}
|
||||
mw.onShutdown = append(mw.onShutdown, mp.Shutdown, lp.Shutdown)
|
||||
|
||||
return mw, nil
|
||||
}
|
||||
```
|
||||
|
||||
Also you can find there `logger`, which is `otel` logger:
|
||||
|
||||
```go
|
||||
var (
|
||||
logger = otelslog.NewLogger("rolldice")
|
||||
)
|
||||
```
|
||||
|
||||
and initialized in a `newLoggerProvider`
|
||||
```go
|
||||
func newLoggerProvider(ctx context.Context) (*sdklog.LoggerProvider, error) {
|
||||
exporter, err := otlploghttp.New(ctx, otlploghttp.WithEndpointURL(*logsEndpoint))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
provider := sdklog.NewLoggerProvider(
|
||||
sdklog.WithProcessor(sdklog.NewBatchProcessor(exporter)),
|
||||
)
|
||||
return provider, nil
|
||||
}
|
||||
```
|
||||
|
||||
The new type `middleware` is instrumented with 3 [metrics](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#timeseries-model)
|
||||
initialized in `newMiddleware` method:
|
||||
* counter `http.requests`
|
||||
* histogram `http.requests.latency`
|
||||
* gauge `http.requests.active`
|
||||
|
||||
Let's implement http.Handler interface for `middleWare` by adding `ServeHTTP` method:
|
||||
```go
|
||||
func (m *middleWare) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
t := time.Now()
|
||||
path := r.URL.Path
|
||||
m.requestsCount.Add(m.ctx, 1, metric.WithAttributes(attribute.String("path", path)))
|
||||
atomic.AddInt64(&m.activeRequests, 1)
|
||||
defer func() {
|
||||
atomic.AddInt64(&m.activeRequests, -1)
|
||||
m.requestsLatency.Record(m.ctx, time.Since(t).Seconds(), metric.WithAttributes(attribute.String("path", path)))
|
||||
}()
|
||||
|
||||
m.h.ServeHTTP(w, r)
|
||||
}
|
||||
```
|
||||
|
||||
In method above, our middleware processes received HTTP requests and updates metrics with each new request.
|
||||
But for these metrics to be shipped we need to add a new method `newMeterProvider` to organize metrics collection:
|
||||
```go
|
||||
func newMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) {
|
||||
exporter, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithEndpointURL(*metricsEndpoint))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create otlphttp exporter: %w", err)
|
||||
}
|
||||
|
||||
res, err := resource.New(ctx,
|
||||
resource.WithAttributes(
|
||||
attribute.String("job", *jobName),
|
||||
attribute.String("instance", *instanceName),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create meter resource: %w", err)
|
||||
}
|
||||
expView := sdkmetric.NewView(
|
||||
sdkmetric.Instrument{
|
||||
Name: "http.requests.latency",
|
||||
Kind: sdkmetric.InstrumentKindHistogram,
|
||||
},
|
||||
sdkmetric.Stream{
|
||||
Name: "http.requests.latency.exp",
|
||||
Aggregation: sdkmetric.AggregationBase2ExponentialHistogram{
|
||||
MaxSize: 160,
|
||||
MaxScale: 20,
|
||||
},
|
||||
},
|
||||
)
|
||||
return sdkmetric.NewMeterProvider(
|
||||
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter, sdkmetric.WithInterval(*pushInterval))),
|
||||
sdkmetric.WithResource(res),
|
||||
sdkmetric.WithView(expView),
|
||||
), nil
|
||||
}
|
||||
```
|
||||
|
||||
This controller will collect and push collected metrics to VictoriaMetrics address with interval of `10s`.
|
||||
|
||||
See the full source code of the example [here](app.go.example).
|
||||
|
||||
### Test ingestion
|
||||
|
||||
In order to push metrics and logs of our WEB server to VictoriaMetrics and VictoriaLogs it is necessary to ensure that both services are available locally.
|
||||
In previous steps we already deployed a single-server VictoriaMetrics and VictoriaLogs, so let's make them available locally:
|
||||
|
||||
Make sure that VictoriaMetrics and VictoriaLogs are available locally at their default ports:
|
||||
```sh
|
||||
# port-forward victoriametrics to ingest metrics
|
||||
kubectl port-forward victoria-metrics-victoria-metrics-single-server-0 8428
|
||||
|
|
@ -473,27 +236,22 @@ kubectl port-forward victoria-metrics-victoria-metrics-single-server-0 8428
|
|||
kubectl port-forward victoria-logs-victoria-logs-single-server-0 9428
|
||||
```
|
||||
|
||||
Now let's run our WEB server and call its APIs:
|
||||
```sh
|
||||
# build and run the app
|
||||
go run main.go
|
||||
2024/03/25 19:27:41 Starting web server...
|
||||
2024/03/25 19:27:41 web server started at localhost:8081.
|
||||
Visit application links [http://localhost:8081/api/fast](http://localhost:8081/api/fast) or [http://localhost:8081/api/slow](http://localhost:8081/api/slow)
|
||||
couple of times. The application will generate metrics and logs and will send them to VictoriaMetrics and VictoriaLogs.
|
||||
|
||||
# execute few queries with curl
|
||||
curl http://localhost:8081/api/fast
|
||||
curl http://localhost:8081/api/slow
|
||||
```
|
||||
|
||||
Open VictoriaMetrics at `http://localhost:8428/vmui` and query `http_requests_total` or `http_requests_active`
|
||||
After a few seconds you should start seeing metrics sent to VictoriaMetrics by visiting
|
||||
[http://localhost:8428/vmui/#/?g0.expr=http_requests_total](http://localhost:8428/vmui/#/?g0.expr=http_requests_total).
|
||||
|
||||
|
||||
|
||||
Open VictoriaLogs UI at `http://localhost:9428/select/vmui` and query `service.name: unknown_service:otel`
|
||||
Check other available metrics by visiting [cardinality explorer](https://docs.victoriametrics.com/#cardinality-explorer) page.
|
||||
|
||||
Logs should be available by visiting [http://localhost:9428/select/vmui](http://localhost:9428/select/vmui)
|
||||
using query `service.name: unknown_service:otel`.
|
||||
|
||||
|
||||
|
||||
## Limitations
|
||||
|
||||
* VictoriaMetrics and VictoriaLogs do not support experimental JSON encoding [format](https://github.com/open-telemetry/opentelemetry-proto/blob/main/examples/metrics.json).
|
||||
* VictoriaMetrics supports only `AggregationTemporalityCumulative` type for [histogram](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#histogram) and [summary](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#summary-legacy). Either consider using cumulative temporality temporality or try [`delta-to-cumulative processor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) to make conversion to cumulative temporality in OTEL Collector.
|
||||
* VictoriaMetrics supports only `AggregationTemporalityCumulative` type for [histogram](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#histogram) and [summary](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#summary-legacy). Either consider using cumulative temporality or try [`delta-to-cumulative processor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) to make conversion to cumulative temporality in OTEL Collector.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
|
@ -27,20 +26,20 @@ import (
|
|||
var (
|
||||
metricsEndpoint = flag.String("vm.endpoint", "http://localhost:8428/opentelemetry/v1/metrics", "VictoriaMetrics endpoint")
|
||||
logsEndpoint = flag.String("vl.endpoint", "http://localhost:9428/insert/opentelemetry/v1/logs", "VictoriaLogs endpoint")
|
||||
pushInterval = flag.Duration("vm.pushInterval", 10*time.Second, "how often push samples, aka scrapeInterval at pull model")
|
||||
jobName = flag.String("metrics.jobName", "otlp", "job name for web-application")
|
||||
instanceName = flag.String("metrics.instance", "localhost", "hostname of web-application instance")
|
||||
pushInterval = flag.Duration("vm.pushInterval", 10*time.Second, "How often to push collected metrics samples to vm.endpoint")
|
||||
)
|
||||
|
||||
var (
|
||||
logger = otelslog.NewLogger("rolldice")
|
||||
logger = otelslog.NewLogger("demo-app")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
log.Printf("Starting web server...")
|
||||
fmt.Println("Starting web server...")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/api/fast", func(writer http.ResponseWriter, request *http.Request) {
|
||||
logger.InfoContext(ctx, "Anonymous access to fast endpoint")
|
||||
|
|
@ -57,21 +56,25 @@ func main() {
|
|||
if err != nil {
|
||||
panic(fmt.Sprintf("cannot build middleware: %q", err))
|
||||
}
|
||||
|
||||
mustStop := make(chan os.Signal, 1)
|
||||
signal.Notify(mustStop, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
http.ListenAndServe("localhost:8081", mw)
|
||||
err := http.ListenAndServe("localhost:8081", mw)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
log.Printf("web server started at localhost:8081.")
|
||||
<-mustStop
|
||||
log.Println("receive shutdown signal, stopping webserver")
|
||||
|
||||
fmt.Println("web server started at http://localhost:8081")
|
||||
<-mustStop
|
||||
fmt.Println("receive shutdown signal, stopping webserver")
|
||||
for _, shutdown := range mw.onShutdown {
|
||||
if err := shutdown(ctx); err != nil {
|
||||
log.Println("cannot shutdown metric provider ", err)
|
||||
fmt.Printf("cannot shutdown metric provider: %s", err)
|
||||
}
|
||||
}
|
||||
log.Printf("Done!")
|
||||
fmt.Println("Done!")
|
||||
}
|
||||
|
||||
func newMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) {
|
||||
|
|
@ -82,18 +85,22 @@ func newMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) {
|
|||
|
||||
res, err := resource.New(ctx,
|
||||
resource.WithAttributes(
|
||||
attribute.String("job", *jobName),
|
||||
attribute.String("instance", *instanceName),
|
||||
attribute.String("job", "otlp"),
|
||||
attribute.String("instance", "localhost"),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create meter resource: %w", err)
|
||||
}
|
||||
|
||||
// define histograms for measuring requests latency
|
||||
expView := sdkmetric.NewView(
|
||||
// classic histogram
|
||||
sdkmetric.Instrument{
|
||||
Name: "http.requests.latency",
|
||||
Kind: sdkmetric.InstrumentKindHistogram,
|
||||
},
|
||||
// exponential histogram
|
||||
sdkmetric.Stream{
|
||||
Name: "http.requests.latency.exp",
|
||||
Aggregation: sdkmetric.AggregationBase2ExponentialHistogram{
|
||||
|
|
@ -128,24 +135,24 @@ func newMiddleware(ctx context.Context, h http.Handler) (*middleware, error) {
|
|||
|
||||
lp, err := newLoggerProvider(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot build logs provider: %w", err)
|
||||
return nil, fmt.Errorf("cannot create logs provider: %w", err)
|
||||
}
|
||||
global.SetLoggerProvider(lp)
|
||||
|
||||
mp, err := newMeterProvider(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot build metrics provider: %w", err)
|
||||
return nil, fmt.Errorf("cannot create metrics provider: %w", err)
|
||||
}
|
||||
otel.SetMeterProvider(mp)
|
||||
meter := mp.Meter("")
|
||||
|
||||
mw.requestsLatency, err = meter.Float64Histogram("http.requests.latency")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create histogram: %w", err)
|
||||
return nil, fmt.Errorf("cannot create Float64Histogram: %w", err)
|
||||
}
|
||||
mw.requestsCount, err = meter.Int64Counter("http.requests")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create int64 counter: %w", err)
|
||||
return nil, fmt.Errorf("cannot create Int64Counter: %w", err)
|
||||
}
|
||||
cb := func(c context.Context, o metric.Int64Observer) error {
|
||||
o.Observe(atomic.LoadInt64(&mw.activeRequests))
|
||||
|
|
@ -180,4 +187,4 @@ func (m *middleware) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
}()
|
||||
|
||||
m.h.ServeHTTP(w, r)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue