feat: improved tracing

GitOrigin-RevId: 98ae16a6f1de4414538c7692758af850aa0caaac
This commit is contained in:
Arne Luenser 2025-11-28 11:43:57 +01:00 committed by ory-bot
parent 8482dd5de0
commit 46c1028e2b
14 changed files with 90 additions and 74 deletions

View File

@ -10,13 +10,11 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
"github.com/urfave/negroni"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"golang.org/x/sync/errgroup"
"github.com/ory/graceful"
"github.com/ory/kratos/driver"
"github.com/ory/x/configx"
"github.com/ory/x/otelx"
"github.com/ory/x/prometheusx"
"github.com/ory/x/reqlog"
)
@ -67,10 +65,6 @@ func ServeMetrics(ctx context.Context, r driver.Registry, port int) error {
n.UseHandler(router)
var handler http.Handler = n
if tracer := r.Tracer(ctx); tracer.IsLoaded() {
tp := tracer.Provider()
handler = otelx.NewHandler(handler, "cmd.courier.ServeMetrics", otelhttp.WithTracerProvider(tp))
}
//#nosec G112 -- the correct settings are set by graceful.WithDefaults
server := graceful.WithDefaults(&http.Server{

View File

@ -70,7 +70,7 @@ func servePublic(ctx context.Context, r *driver.RegistryDefault, cmd *cobra.Comm
n.Use(sqa(ctx, cmd, r))
router := x.NewRouterPublic(r)
csrf := nosurfx.NewCSRFHandler(router, r)
csrf := nosurfx.NewCSRFHandler(otelx.SpanNameRecorderMiddleware(router), r)
// we need to always load the CORS middleware even if it is disabled, to allow hot-enabling CORS
n.UseFunc(func(w http.ResponseWriter, req *http.Request, next http.HandlerFunc) {
@ -84,7 +84,7 @@ func servePublic(ctx context.Context, r *driver.RegistryDefault, cmd *cobra.Comm
n.UseFunc(x.CleanPath) // Prevent double slashes from breaking CSRF.
r.WithCSRFHandler(csrf)
n.UseHandler(http.MaxBytesHandler(r.CSRFHandler(), 5*1024*1024 /* 5 MB */))
n.UseHandler(r.CSRFHandler())
// Disable CSRF for these endpoints
csrf.DisablePath(healthx.AliveCheckPath)
@ -96,9 +96,13 @@ func servePublic(ctx context.Context, r *driver.RegistryDefault, cmd *cobra.Comm
var handler http.Handler = n
if tracer := r.Tracer(ctx); tracer.IsLoaded() {
handler = otelx.TraceHandler(handler, otelhttp.WithTracerProvider(tracer.Provider()))
handler = otelx.NewMiddleware(handler, "servePublic",
otelhttp.WithTracerProvider(tracer.Provider()),
)
}
handler = http.MaxBytesHandler(handler, 5*1024*1024 /* 5 MB */) // Important: this must be the outermost handler or our tracing breaks
certFunc, err := cfg.TLS.GetCertFunc(ctx, l, "public")
if err != nil {
return nil, err
@ -161,18 +165,19 @@ func serveAdmin(ctx context.Context, r *driver.RegistryDefault, cmd *cobra.Comma
router := x.NewRouterAdmin(r)
r.RegisterAdminRoutes(ctx, router)
n.UseHandler(http.MaxBytesHandler(router, 5*1024*1024 /* 5 MB */))
n.UseHandler(router)
n.UseFunc(otelx.SpanNameRecorderNegroniFunc)
var handler http.Handler = n
if tracer := r.Tracer(ctx); tracer.IsLoaded() {
handler = otelx.TraceHandler(handler,
handler = otelx.NewMiddleware(handler, "serveAdmin",
otelhttp.WithTracerProvider(tracer.Provider()),
otelhttp.WithFilter(func(req *http.Request) bool {
return req.URL.Path != x.AdminPrefix+prometheus.MetricsPrometheusPath
}),
)
}
handler = http.MaxBytesHandler(handler, 5*1024*1024 /* 5 MB */) // Important: this must be the outermost handler or our tracing breaks
certFunc, err := cfg.TLS.GetCertFunc(ctx, l, "admin")
if err != nil {
return nil, err

2
go.mod
View File

@ -49,7 +49,7 @@ require (
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf
github.com/jarcoal/httpmock v1.3.1
github.com/jmoiron/sqlx v1.4.0
github.com/julienschmidt/httprouter v1.3.0
github.com/julienschmidt/httprouter v1.3.1-0.20240130105656-484018016424
github.com/knadh/koanf/parsers/json v0.1.0
github.com/laher/mergefs v0.1.2-0.20230223191438-d16611b2f4e7 // indirect
github.com/lestrrat-go/jwx/v2 v2.1.1

4
go.sum
View File

@ -471,8 +471,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/julienschmidt/httprouter v1.3.1-0.20240130105656-484018016424 h1:KsUAkP+Y6n+542zpxWiQDUvOqfh3n429HYleEvq/V7M=
github.com/julienschmidt/httprouter v1.3.1-0.20240130105656-484018016424/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

View File

@ -37,6 +37,7 @@ import (
"golang.org/x/crypto/scrypt"
"github.com/ory/kratos/driver/config"
"github.com/ory/x/otelx"
)
var ErrUnknownHashAlgorithm = errors.New("unknown hash algorithm")
@ -137,9 +138,9 @@ var supportedHashers = []SupportedHasher{
},
}
func Compare(ctx context.Context, password, hash []byte) error {
func Compare(ctx context.Context, password, hash []byte) (err error) {
ctx, span := otel.GetTracerProvider().Tracer(tracingComponent).Start(ctx, "hash.Compare")
defer span.End()
defer otelx.End(span, &err)
for _, h := range supportedHashers {
if h.Is(hash) {

View File

@ -20,6 +20,7 @@ import (
"golang.org/x/crypto/argon2"
"github.com/ory/kratos/driver/config"
"github.com/ory/x/otelx"
)
var (
@ -45,14 +46,14 @@ func toKB(mem bytesize.ByteSize) uint32 {
return uint32(mem / bytesize.KB)
}
func (h *Argon2) Generate(ctx context.Context, password []byte) ([]byte, error) {
func (h *Argon2) Generate(ctx context.Context, password []byte) (_ []byte, err error) {
conf := h.c.Config().HasherArgon2(ctx)
_, span := otel.GetTracerProvider().Tracer(tracingComponent).Start(ctx, "hash.Generate", trace.WithAttributes(
attribute.String("hash.type", "argon2id"),
attribute.String("hash.config", fmt.Sprintf("%#v", conf)),
))
defer span.End()
defer otelx.End(span, &err)
salt := make([]byte, conf.SaltLength)
if _, err := rand.Read(salt); err != nil {

View File

@ -8,6 +8,7 @@ import (
"fmt"
"github.com/ory/kratos/text"
"github.com/ory/x/otelx"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
@ -32,14 +33,14 @@ func NewHasherBcrypt(c BcryptConfiguration) *Bcrypt {
return &Bcrypt{c: c}
}
func (h *Bcrypt) Generate(ctx context.Context, password []byte) ([]byte, error) {
func (h *Bcrypt) Generate(ctx context.Context, password []byte) (_ []byte, err error) {
conf := h.c.Config().HasherBcrypt(ctx)
_, span := otel.GetTracerProvider().Tracer(tracingComponent).Start(ctx, "hash.Generate", trace.WithAttributes(
attribute.String("hash.type", "bcrypt"),
attribute.String("hash.config", fmt.Sprintf("%#v", conf)),
))
defer span.End()
defer otelx.End(span, &err)
if err := validateBcryptPasswordLength(password); err != nil {
return nil, err

View File

@ -14,6 +14,7 @@ import (
"fmt"
"hash"
"github.com/ory/x/otelx"
"github.com/pkg/errors"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
@ -30,12 +31,12 @@ type Pbkdf2 struct {
KeyLength uint32
}
func (h *Pbkdf2) Generate(ctx context.Context, password []byte) ([]byte, error) {
func (h *Pbkdf2) Generate(ctx context.Context, password []byte) (_ []byte, err error) {
_, span := otel.GetTracerProvider().Tracer(tracingComponent).Start(ctx, "hash.Generate", trace.WithAttributes(
attribute.String("hash.type", "pbkdf2"),
attribute.String("hash.config", fmt.Sprintf("%#v", h)),
))
defer span.End()
defer otelx.End(span, &err)
salt := make([]byte, h.SaltLength)
if _, err := rand.Read(salt); err != nil {

View File

@ -33,7 +33,7 @@ require (
github.com/jackc/pgx/v4 v4.18.3
github.com/jackc/puddle/v2 v2.2.2
github.com/jmoiron/sqlx v1.4.0
github.com/julienschmidt/httprouter v1.3.0
github.com/julienschmidt/httprouter v1.3.1-0.20240130105656-484018016424
github.com/knadh/koanf/maps v0.1.2
github.com/knadh/koanf/parsers/json v0.1.0
github.com/knadh/koanf/parsers/toml v0.1.0

View File

@ -304,8 +304,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/julienschmidt/httprouter v1.3.1-0.20240130105656-484018016424 h1:KsUAkP+Y6n+542zpxWiQDUvOqfh3n429HYleEvq/V7M=
github.com/julienschmidt/httprouter v1.3.1-0.20240130105656-484018016424/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

View File

@ -3,11 +3,11 @@ package hasherx
import (
"context"
"github.com/ory/x/otelx"
"github.com/pkg/errors"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"golang.org/x/crypto/bcrypt"
)
@ -35,22 +35,18 @@ func NewHasherBcrypt(c BCryptConfigurator) *Bcrypt {
}
// Generate generates a hash for the given password.
func (h *Bcrypt) Generate(ctx context.Context, password []byte) ([]byte, error) {
func (h *Bcrypt) Generate(ctx context.Context, password []byte) (hash []byte, err error) {
ctx, span := otel.GetTracerProvider().Tracer(tracingComponent).Start(ctx, "hash.Bcrypt.Generate")
defer span.End()
defer otelx.End(span, &err)
if err := validateBcryptPasswordLength(password); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, err
}
cost := int(h.c.HasherBcryptConfig(ctx).Cost)
span.SetAttributes(attribute.Int("bcrypt.cost", cost))
hash, err := bcrypt.GenerateFromPassword(password, cost)
hash, err = bcrypt.GenerateFromPassword(password, cost)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, err
}

View File

@ -11,6 +11,7 @@ import (
"fmt"
"hash"
"github.com/ory/x/otelx"
"github.com/pkg/errors"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
@ -48,9 +49,9 @@ func NewHasherPBKDF2(c PBKDF2Configurator) *PBKDF2 {
}
// Generate generates a hash for the given password.
func (h *PBKDF2) Generate(ctx context.Context, password []byte) ([]byte, error) {
_, span := otel.GetTracerProvider().Tracer("").Start(ctx, "hash.PBKDF2.Generate")
defer span.End()
func (h *PBKDF2) Generate(ctx context.Context, password []byte) (hash []byte, err error) {
ctx, span := otel.GetTracerProvider().Tracer("").Start(ctx, "hash.PBKDF2.Generate")
defer otelx.End(span, &err)
conf := h.c.HasherPBKDF2Config(ctx)
salt := make([]byte, conf.SaltLength)

View File

@ -4,48 +4,68 @@
package otelx
import (
"cmp"
"context"
"net/http"
"strings"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
func isHealthFilter(r *http.Request) bool {
path := r.URL.Path
return !strings.HasPrefix(path, "/health/")
var WithDefaultFilters otelhttp.Option = otelhttp.WithFilter(func(r *http.Request) bool {
return !(strings.HasPrefix(r.URL.Path, "/health") ||
strings.HasPrefix(r.URL.Path, "/admin/health") ||
strings.HasPrefix(r.URL.Path, "/metrics") ||
strings.HasPrefix(r.URL.Path, "/admin/metrics"))
})
type contextKey int
const callbackContextKey contextKey = iota
func SpanNameRecorderMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
cb, _ := r.Context().Value(callbackContextKey).(func(string))
if cb == nil {
return
}
if r.Pattern != "" {
cb(r.Pattern)
}
}()
next.ServeHTTP(w, r)
})
}
func isAdminHealthFilter(r *http.Request) bool {
path := r.URL.Path
return !strings.HasPrefix(path, "/admin/health/")
func SpanNameRecorderNegroniFunc(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
defer func() {
cb, _ := r.Context().Value(callbackContextKey).(func(string))
if cb == nil {
return
}
if r.Pattern != "" {
cb(r.Pattern)
}
}()
next(w, r)
}
func filterOpts() []otelhttp.Option {
filters := []otelhttp.Filter{
isHealthFilter,
isAdminHealthFilter,
}
opts := []otelhttp.Option{}
for _, f := range filters {
opts = append(opts, otelhttp.WithFilter(f))
}
return opts
}
// NewHandler returns a wrapped otelhttp.NewHandler with our request filters.
func NewHandler(handler http.Handler, operation string, opts ...otelhttp.Option) http.Handler {
opts = append(filterOpts(), opts...)
return otelhttp.NewHandler(handler, operation, opts...)
}
// TraceHandler wraps otelx.NewHandler, passing the URL path as the span name.
func TraceHandler(h http.Handler, opts ...otelhttp.Option) http.Handler {
// Use a span formatter to set the span name to the URL path, rather than passing in the operation to NewHandler.
// This allows us to use the same handler for multiple routes.
middlewareOpts := []otelhttp.Option{
func NewMiddleware(next http.Handler, operation string, opts ...otelhttp.Option) http.Handler {
myOpts := []otelhttp.Option{
WithDefaultFilters,
otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string {
return r.URL.Path
return cmp.Or(r.Pattern, operation, r.Method+" "+r.URL.Path)
}),
}
return NewHandler(h, "", append(middlewareOpts, opts...)...)
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callback := func(s string) {
r.Pattern = cmp.Or(r.Pattern, s)
}
ctx := context.WithValue(r.Context(), callbackContextKey, callback)
r2 := r.WithContext(ctx)
next.ServeHTTP(w, r2)
r.Pattern = cmp.Or(r2.Pattern, r.Pattern) // best-effort in case callback never is called
})
return otelhttp.NewHandler(handler, operation, append(myOpts, opts...)...)
}

View File

@ -8,8 +8,6 @@ import (
"crypto/hmac"
"crypto/sha512"
"fmt"
"go.opentelemetry.io/otel/trace"
)
func (p *Persister) hmacValue(ctx context.Context, value string) string {
@ -17,8 +15,6 @@ func (p *Persister) hmacValue(ctx context.Context, value string) string {
}
func hmacValueWithSecret(ctx context.Context, value string, secret []byte) string {
_, span := trace.SpanFromContext(ctx).TracerProvider().Tracer("").Start(ctx, "persistence.sql.hmacValueWithSecret")
defer span.End()
h := hmac.New(sha512.New512_256, secret)
_, _ = h.Write([]byte(value))
return fmt.Sprintf("%x", h.Sum(nil))