Middlewares and tools to integrate axum + tracing + opentelemetry.
For examples, you can look at:
```rust //... use axumtracingopentelemetry::opentelemetrytracinglayer;
fn inittracing() -> Result<(), axum::BoxError> { use tracingsubscriber::filter::EnvFilter; use tracingsubscriber::fmt::format::FmtSpan; use tracingsubscriber::layer::SubscriberExt;
let subscriber = tracing_subscriber::registry();
// register opentelemetry tracer layer
let otel_layer = {
use axum_tracing_opentelemetry::{
init_propagator, //stdio,
resource::DetectResource,
otlp,
};
let otel_rsrc = DetectResource::default()
.with_fallback_service_name(env!("CARGO_PKG_NAME"))
.with_fallback_service_version(env!("CARGO_PKG_VERSION"))
.with_println()
.build();
let otel_tracer = otlp::init_tracer(otel_rsrc, otlp::identity)?;
// to not send trace somewhere, but continue to create and propagate,...
// then send them to `axum_tracing_opentelemetry::stdio::WriteNoWhere::default()`
// or to `std::io::stdout()` to print
//
// let otel_tracer =
// stdio::init_tracer(otel_rsrc, stdio::identity, stdio::WriteNoWhere::default())?;
init_propagator()?;
tracing_opentelemetry::layer().with_tracer(otel_tracer)
};
let subscriber = subscriber.with(otel_layer);
// filter what is output on log (fmt), but not what is send to trace (opentelemetry collector)
// std::env::set_var("RUST_LOG", "info,kube=trace");
std::env::set_var(
"RUST_LOG",
std::env::var("RUST_LOG")
.or_else(|_| std::env::var("OTEL_LOG_LEVEL"))
.unwrap_or_else(|_| "info".to_string()),
);
let subscriber = subscriber.with(EnvFilter::from_default_env());
if cfg!(debug_assertions) {
let fmt_layer = tracing_subscriber::fmt::layer()
.pretty()
.with_line_number(true)
.with_thread_names(true)
.with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
.with_timer(tracing_subscriber::fmt::time::uptime());
let subscriber = subscriber.with(fmt_layer);
tracing::subscriber::set_global_default(subscriber)?;
} else {
let fmt_layer = tracing_subscriber::fmt::layer()
.json()
.with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
.with_timer(tracing_subscriber::fmt::time::uptime());
let subscriber = subscriber.with(fmt_layer);
tracing::subscriber::set_global_default(subscriber)?;
};
Ok(())
}
async fn main() -> Result<(), axum::BoxError> {
inittracing()?;
let app = app();
// run it
let addr = &"0.0.0.0:3000".parse::
fn app() -> Router {
// build our application with a route
Router::new()
.route("/", get(health)) // request processed inside span
// opentelemetrytracinglayer setup TraceLayer, that is provided by tower-http so you have to add that as a dependency.
.layer(opentelemetrytracinglayer())
.route("/health", get(health)) // request processed without span / trace
}
async fn shutdownsignal() { //... opentelemetry::global::shutdowntracer_provider(); } ```
To retrieve the current trace_id (eg to add it into error message (as header or attributes))
rust
let trace_id = axum_tracing_opentelemetry::find_current_trace_id();
json!({ "error" : "xxxxxx", "trace_id": trace_id})
To also inject the trace id into the response (could be useful for debugging) uses the layer response_with_trace_layer
rust
// build our application with a route
Router::new()
...
// include trace context as header into the response
.layer(response_with_trace_layer())
To ease setup and compliancy with Opentelemetry SDK configuration, the configuration can be done with the following environment variables (see sample init_tracing() above):
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT fallback to OTEL_EXPORTER_OTLP_ENDPOINT for the url of the exporter / collectorOTEL_EXPORTER_OTLP_TRACES_PROTOCOL fallback to OTEL_EXPORTER_OTLP_PROTOCOL, fallback to auto-detection based on ENDPOINT portOTEL_SERVICE_NAME for the name of the serviceOTEL_PROPAGATORS for the configuration of propagatorIn the context of kubernetes, the above environment variable can be injected by the Opentelemetry operator (via inject-sdk):
yaml
apiVersion: apps/v1
kind: Deployment
spec:
template:
metadata:
annotations:
# to inject environment variables only by opentelemetry-operator
instrumentation.opentelemetry.io/inject-sdk: "opentelemetry-operator/instrumentation"
instrumentation.opentelemetry.io/container-names: "app"
containers:
- name: app
Or if you don't setup inject-sdk, you can manually set the environment variable eg
yaml
apiVersion: apps/v1
kind: Deployment
spec:
template:
metadata:
containers:
- name: app
env:
- name: OTEL_SERVICE_NAME
value: "app"
- name: OTEL_EXPORTER_OTLP_PROTOCOL
value: "grpc"
# for otel collector in `deployment` mode, use the name of the service
# - name: OTEL_EXPORTER_OTLP_ENDPOINT
# value: "http://opentelemetry-collector.opentelemetry-collector:4317"
# for otel collector in sidecar mode (imply to deploy a sidecar CR per namespace)
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://localhost:4317"
# for `daemonset` mode: need to use the local daemonset (value interpolated by k8s: `$(...)`)
# - name: OTEL_EXPORTER_OTLP_ENDPOINT
# value: "http://$(HOST_IP):4317"
# - name: HOST_IP
# valueFrom:
# fieldRef:
# fieldPath: status.hostIP
examples/otlpIn a terminal, run
``sh
❯ cd examples/otlp
❯ cargo run
Compiling examples-otlp v0.1.0 (/home/david/src/github.com/davidB/axum-tracing-opentelemetry/examples/otlp)
Finished dev [unoptimized + debuginfo] target(s) in 2.96s
Runningtarget/debug/examples-otlp`
0.000170750s WARN examples_otlp: listening on 0.0.0.0:3003
at src/main.rs:70 on main
0.000203401s INFO examples_otlp: try to call `curl -i http://127.0.0.1:3003/` (with trace)
at src/main.rs:71 on main
0.000213920s INFO examples_otlp: try to call `curl -i http://127.0.0.1:3003/heatlh` (with NO trace)
at src/main.rs:72 on main
... ```
Into an other terminal, call the / (endpoint with opentelemetry_tracing_layer and response_with_trace_layer)
```sh ❯ curl -i http://127.0.0.1:3003/ HTTP/1.1 200 OK content-type: application/json content-length: 50 traceparent: 00-b2611246a58fd7ea623d2264c5a1e226-b2c9b811f2f424af-01 tracestate: date: Wed, 28 Dec 2022 17:04:59 GMT
{"mytraceid":"b2611246a58fd7ea623d2264c5a1e226"} ```
call the /health (endpoint with NO layer)
```sh ❯ curl -i http://127.0.0.1:3003/health HTTP/1.1 200 OK content-type: application/json content-length: 15 date: Wed, 28 Dec 2022 17:14:07 GMT
{"status":"UP"} ```
To collect and visualize trace on local, one of the simplest solution:
```sh
nerdctl run -d --name jaeger \ -e COLLECTORZIPKINHOSTPORT=:9411 \ -e COLLECTOROTLP_ENABLED=true \ -p 6831:6831/udp \ -p 6832:6832/udp \ -p 5778:5778 \ -p 16686:16686 \ -p 4317:4317 \ -p 4318:4318 \ -p 14250:14250 \ -p 14268:14268 \ -p 14269:14269 \ -p 9411:9411 \ jaegertracing/all-in-one:1.41
open http://localhost:16686 ```
Then :
| axum | axum-tracing-opentelemetry | |------|----------------------------| | 0.6 | latest - 0.6 | | 0.5 | 0.1 - 0.5 |
DetectResource builder to help detection for Resource Semantic Conventions | OpenTelemetryinit_propagator to configure the global propagator based on content of the env variable OTEL_PROPAGATORSresponse_with_trace_layer to have traceparent injected into responseOTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, OTEL_EXPORTER_OTLP_PROTOCOL/dev/null) to allow to have trace_id
and the opentelemetry span & metadata on log and http response (without collector)