Merge pull request #3 from equinixmetal/all-the-traces

ensure otel tracing is passed through all event handlers and clients

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @equinixmetal/governor-metal-identity

cmd/serve.go

@@ -9,6 +9,7 @@ import (
 	"go.infratographer.com/x/otelx"
 	"go.infratographer.com/x/versionx"
 	"go.infratographer.com/x/viperx"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/config"
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal"
@@ -68,6 +69,7 @@ func serve(cmd *cobra.Command, _ []string) {
 		}
 
 		permHTTPClient = oauth2x.NewClient(cmd.Context(), tokenSrc)
+		permHTTPClient.Transport = otelhttp.NewTransport(permHTTPClient.Transport)
 	}
 
 	perms, err := permissions.NewClient("",

go.mod

@@ -3,14 +3,15 @@ module go.equinixmetal.net/infra9-metal-bridge
 go 1.20
 
 require (
-	github.com/ThreeDotsLabs/watermill v1.2.0
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/nats-io/nats.go v1.27.1
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
-	go.infratographer.com/x v0.3.3
+	go.infratographer.com/x v0.3.4
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
 	go.opentelemetry.io/otel v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0
 	go.uber.org/zap v1.24.0
@@ -19,11 +20,13 @@ require (
 
 require (
 	github.com/MicahParks/keyfunc/v2 v2.0.3 // indirect
+	github.com/ThreeDotsLabs/watermill v1.2.0 // indirect
 	github.com/ThreeDotsLabs/watermill-nats/v2 v2.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/garsue/watermillzap v1.2.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -53,7 +56,6 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect

go.sum

@@ -81,6 +81,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
+github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
@@ -292,8 +294,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.infratographer.com/x v0.3.3 h1:dTaLEp75RgL0JxKJhrcuQTP4a2x/MrevvZ3OdtkEhCs=
+go.infratographer.com/x v0.3.4 h1:K7azcoiLZPRdOnr4M7DMyB2DjZzXRVcfr7G6FeQd16o=
-go.infratographer.com/x v0.3.3/go.mod h1:pXXSdeJBisAK3AdED5EFj7Yo8z8td7fOWDkNl4Dkp0s=
+go.infratographer.com/x v0.3.4/go.mod h1:pXXSdeJBisAK3AdED5EFj7Yo8z8td7fOWDkNl4Dkp0s=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -302,6 +304,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.42.0 h1:sYefIhrd/A3fO8rmr0vy2tgCLoR8CsbMqwbcUa70x00=
 go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.42.0/go.mod h1:5Ll2ndRzg9UNUrj1n+v4ZCcrD/SYy7BnVrlCQXECowA=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8=
 go.opentelemetry.io/contrib/propagators/b3 v1.17.0 h1:ImOVvHnku8jijXqkwCSyYKRDt2YrnGXD4BbhcpfbfJo=
 go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
 go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=

internal/metal/providers/emapi/client.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/providers"
@@ -28,6 +29,7 @@ const (
 // DefaultHTTPClient is the default http client used if no client is provided.
 var DefaultHTTPClient = &http.Client{
 	Timeout:   defaultHTTPTimeout,
+	Transport: otelhttp.NewTransport(http.DefaultTransport),
 }
 
 var _ providers.Provider = &Client{}

internal/metal/providers/emapi/options.go

@@ -1,6 +1,8 @@
 package emapi
 
 import (
+	"net/http"
+
 	"go.uber.org/zap"
 )
 
@@ -15,3 +17,12 @@ func WithLogger(logger *zap.SugaredLogger) Option {
 		return nil
 	}
 }
+
+// WithHTTPClient sets the http client to be used by the client.
+func WithHTTPClient(httpClient *http.Client) Option {
+	return func(c *Client) error {
+		c.httpClient = httpClient
+
+		return nil
+	}
+}

internal/metal/providers/emgql/client.go

@@ -5,6 +5,7 @@ import (
 	"net/url"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/providers"
@@ -17,6 +18,7 @@ const (
 // DefaultHTTPClient is the default http client used if no client is provided.
 var DefaultHTTPClient = &http.Client{
 	Timeout:   defaultHTTPTimeout,
+	Transport: otelhttp.NewTransport(http.DefaultTransport),
 }
 
 var _ providers.Provider = &Client{}

internal/metal/providers/emgql/options.go

@@ -1,6 +1,7 @@
 package emgql
 
 import (
+	"net/http"
 	"net/url"
 
 	"go.uber.org/zap"
@@ -32,6 +33,15 @@ func WithBaseURL(baseURL string) Option {
 	}
 }
 
+// WithHTTPClient sets the http client to be used by the client.
+func WithHTTPClient(httpClient *http.Client) Option {
+	return func(c *Client) error {
+		c.httpClient = httpClient
+
+		return nil
+	}
+}
+
 // WithConfig applies all configurations defined in the config.
 func WithConfig(config Config) Option {
 	return func(c *Client) error {

internal/permissions/client.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/labstack/echo/v4"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 )
 
@@ -33,6 +34,7 @@ const (
 // DefaultHTTPClient is the default HTTP client for the Permissions Client.
 var DefaultHTTPClient = &http.Client{
 	Timeout:   defaultHTTPClientTimeout,
+	Transport: otelhttp.NewTransport(http.DefaultTransport),
 }
 
 // Client defines the Permissions API client interface.

internal/pubsub/subscriber.go

@@ -142,7 +142,9 @@ func (s *Subscriber) processEvent(msg *changeEvent) error {
 		"event.type", msg.EventType,
 	)
 
-	ctx, span := tracer.Start(context.Background(), "pubsub.receive", trace.WithAttributes(attribute.String("pubsub.subject", msg.SubjectID.String())))
+	ctx := events.TraceContextFromChangeMessage(context.Background(), msg.ChangeMessage)
+
+	ctx, span := tracer.Start(ctx, "pubsub.receive", trace.WithAttributes(attribute.String("pubsub.subject", msg.SubjectID.String())))
 
 	defer span.End()
 

internal/service/organizations.go

@@ -5,6 +5,8 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/models"
 )
@@ -58,6 +60,10 @@ func (s *service) IsOrganizationID(id gidx.PrefixedID) bool {
 
 // TouchOrganization initializes a sync for the provided organization id for relationships and memberships.
 func (s *service) TouchOrganization(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "TouchOrganization", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	logger := s.logger.With("organization.id", id.String())
 
 	org, err := s.metal.GetOrganizationDetails(ctx, id)
@@ -89,6 +95,10 @@ func (s *service) TouchOrganization(ctx context.Context, id gidx.PrefixedID) err
 
 // DeleteOrganization deletes the provided organization id.
 func (s *service) DeleteOrganization(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "DeleteOrganization", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	err := s.publisher.PublishChange(ctx, organizationEvent, events.ChangeMessage{
 		SubjectID: id,
 		EventType: string(events.DeleteChangeType),

internal/service/process_memberships.go

@@ -5,6 +5,8 @@ import (
 	"strings"
 
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/exp/slices"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/permissions"
@@ -13,6 +15,15 @@ import (
 // processMemberships determines the changes between what is wanted and what is live and executes on the differences.
 // If skipDeletions is true, no deletes will be executed.
 func (s *service) processMemberships(ctx context.Context, relationships Relationships, skipDeletions bool) (int, int) {
+	ctx, span := tracer.Start(ctx, "processMemberships",
+		trace.WithAttributes(
+			attribute.String("resource.id", relationships.Resource.PrefixedID().String()),
+			attribute.Int("resource.memberships", len(relationships.Memberships)),
+		),
+	)
+
+	defer span.End()
+
 	if len(relationships.Memberships) == 0 {
 		return 0, 0
 	}

internal/service/process_relationships.go

@@ -5,6 +5,8 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/permissions"
 )
@@ -13,6 +15,15 @@ import (
 // Relationship creations use events.
 // Relationship deletions use the api, as delete events delete all related resources and not just the provided ones.
 func (s *service) processRelationships(ctx context.Context, eventType string, relationships Relationships) int {
+	ctx, span := tracer.Start(ctx, "processRelationships",
+		trace.WithAttributes(
+			attribute.String("resource.id", relationships.Resource.PrefixedID().String()),
+			attribute.Int("resource.subject_relationships", len(relationships.SubjectRelationships)),
+		),
+	)
+
+	defer span.End()
+
 	rlogger := s.logger.With("resource.id", relationships.Resource.PrefixedID())
 
 	wantParentRelationship, wantSubjectRelationships := s.mapRelationWants(relationships)

internal/service/projects.go

@@ -5,6 +5,8 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/models"
 )
@@ -51,6 +53,10 @@ func (s *service) IsProjectID(id gidx.PrefixedID) bool {
 
 // TouchProject initializes a sync for the provided project id for relationships and memberships.
 func (s *service) TouchProject(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "TouchProject", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	logger := s.logger.With("project.id", id.String())
 
 	project, err := s.metal.GetProjectDetails(ctx, id)
@@ -82,6 +88,10 @@ func (s *service) TouchProject(ctx context.Context, id gidx.PrefixedID) error {
 
 // DeleteProject deletes the provided project id.
 func (s *service) DeleteProject(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "DeleteProject", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	err := s.publisher.PublishChange(ctx, projectEvent, events.ChangeMessage{
 		SubjectID: id,
 		EventType: string(events.DeleteChangeType),

internal/service/service.go

@@ -5,6 +5,7 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal"
@@ -22,6 +23,8 @@ const (
 	TypeUser ObjectType = "user"
 )
 
+var tracer = otel.Tracer("go.equinixmetal.net/infra9-metal-bridge/internal/service")
+
 // DefaultPrefixMap is the default id prefix to type relationship.
 var DefaultPrefixMap = map[string]ObjectType{
 	TypeOrganization.Prefix(): TypeOrganization,

internal/service/users.go

@@ -4,6 +4,8 @@ import (
 	"context"
 
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // IsUser checks the provided id has the metal user prefix.
@@ -29,8 +31,27 @@ func (s *service) IsAssignableResource(id gidx.PrefixedID) bool {
 	return false
 }
 
+func stringIDs(ids []gidx.PrefixedID) []string {
+	result := make([]string, len(ids))
+
+	for i, id := range ids {
+		result[i] = id.String()
+	}
+
+	return result
+}
+
 // Assignuser assigns the provided users to the given resource ids.
 func (s *service) AssignUser(ctx context.Context, userID gidx.PrefixedID, resourceIDs ...gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "AssignUser",
+		trace.WithAttributes(
+			attribute.String("user.id", userID.String()),
+			attribute.StringSlice("user.resource_assignments", stringIDs(resourceIDs)),
+		),
+	)
+
+	defer span.End()
+
 	var totalResources, rolesChanged, assignmentsChanged int
 
 	mlogger := s.logger.With("member.id", userID.String())
@@ -74,6 +95,15 @@ func (s *service) AssignUser(ctx context.Context, userID gidx.PrefixedID, resour
 
 // UnassignUser removes the assignment for the provided user id to the given resources.
 func (s *service) UnassignUser(ctx context.Context, userID gidx.PrefixedID, resourceIDs ...gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "UnassignUser",
+		trace.WithAttributes(
+			attribute.String("user.id", userID.String()),
+			attribute.StringSlice("user.resource_assignments", stringIDs(resourceIDs)),
+		),
+	)
+
+	defer span.End()
+
 	for _, resourceID := range resourceIDs {
 		rlogger := s.logger.With("user.id", userID, "resource.id", resourceID)