graphql testin'

ensure otel tracing is passed through all event handlers and clients

.envrc

@@ -0,0 +1,3 @@
+source_url "https://raw.githubusercontent.com/cachix/devenv/d1f7b48e35e6dee421cfd0f51481d17f77586997/direnvrc" "sha256-YBzqskFZxmNb3kYVoKD9ZixoPXJh1C9ZvTLGFRkauZ0="
+
+use devenv

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @equinixmetal/governor-metal-identity

cmd/serve.go

@@ -9,6 +9,7 @@ import (
 	"go.infratographer.com/x/otelx"
 	"go.infratographer.com/x/versionx"
 	"go.infratographer.com/x/viperx"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/config"
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal"
@@ -68,6 +69,7 @@ func serve(cmd *cobra.Command, _ []string) {
 		}
 
 		permHTTPClient = oauth2x.NewClient(cmd.Context(), tokenSrc)
+		permHTTPClient.Transport = otelhttp.NewTransport(permHTTPClient.Transport)
 	}
 
 	perms, err := permissions.NewClient("",

devenv.lock

@@ -0,0 +1,156 @@
+{
+  "nodes": {
+    "devenv": {
+      "locked": {
+        "dir": "src/modules",
+        "lastModified": 1690391043,
+        "narHash": "sha256-8x5DkOaejES6C2JYX2A3riebbJHGFHBmJ+LwxXUIIVw=",
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "05240861ef3ae1bbe65b1acc88e2fad7dd24a84b",
+        "type": "github"
+      },
+      "original": {
+        "dir": "src/modules",
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1685518550,
+        "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1690327932,
+        "narHash": "sha256-Fv7PYZxN4eo0K6zXhHG/vOc+e2iuqQ5ywDrh0yeRjP0=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a9b47d85504bdd199e90846622c76aa0bfeabfac",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1685801374,
+        "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "c37ca420157f4abc31e26f436c1145f8951ff373",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1689668210,
+        "narHash": "sha256-XAATwDkaUxH958yXLs1lcEOmU6pSEIkatY3qjqk8X0E=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "eb433bff05b285258be76513add6f6c57b441775",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": "pre-commit-hooks"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

devenv.nix

@@ -0,0 +1,5 @@
+{ pkgs, ... }:
+
+{
+	languages.go.enable = true;
+}

devenv.yaml

@@ -0,0 +1,3 @@
+inputs:
+  nixpkgs:
+    url: github:NixOS/nixpkgs/nixpkgs-unstable

go.mod

@@ -3,14 +3,15 @@ module go.equinixmetal.net/infra9-metal-bridge
 go 1.20
 
 require (
-	github.com/ThreeDotsLabs/watermill v1.2.0
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/nats-io/nats.go v1.27.1
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
-	go.infratographer.com/x v0.3.3
+	go.infratographer.com/x v0.3.4
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
 	go.opentelemetry.io/otel v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0
 	go.uber.org/zap v1.24.0
@@ -19,11 +20,13 @@ require (
 
 require (
 	github.com/MicahParks/keyfunc/v2 v2.0.3 // indirect
+	github.com/ThreeDotsLabs/watermill v1.2.0 // indirect
 	github.com/ThreeDotsLabs/watermill-nats/v2 v2.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/garsue/watermillzap v1.2.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -53,7 +56,6 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect

go.sum

@@ -81,6 +81,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
+github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
@@ -292,8 +294,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.infratographer.com/x v0.3.3 h1:dTaLEp75RgL0JxKJhrcuQTP4a2x/MrevvZ3OdtkEhCs=
-go.infratographer.com/x v0.3.3/go.mod h1:pXXSdeJBisAK3AdED5EFj7Yo8z8td7fOWDkNl4Dkp0s=
+go.infratographer.com/x v0.3.4 h1:K7azcoiLZPRdOnr4M7DMyB2DjZzXRVcfr7G6FeQd16o=
+go.infratographer.com/x v0.3.4/go.mod h1:pXXSdeJBisAK3AdED5EFj7Yo8z8td7fOWDkNl4Dkp0s=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -302,6 +304,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.42.0 h1:sYefIhrd/A3fO8rmr0vy2tgCLoR8CsbMqwbcUa70x00=
 go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.42.0/go.mod h1:5Ll2ndRzg9UNUrj1n+v4ZCcrD/SYy7BnVrlCQXECowA=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8=
 go.opentelemetry.io/contrib/propagators/b3 v1.17.0 h1:ImOVvHnku8jijXqkwCSyYKRDt2YrnGXD4BbhcpfbfJo=
 go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
 go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=

internal/metal/models/users.go

@@ -12,7 +12,7 @@ const (
 	MetalUserIssuer = "https://auth.equinix.com/"
 
 	// MetalUserIssuerIDPrefix is the issuer id prefix added by the issuer.
-	MetalUserIssuerIDPrefix = "auth|"
+	MetalUserIssuerIDPrefix = "auth0|"
 )
 
 // UserDetails contains the user information.

internal/metal/providers/emapi/client.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/providers"
@@ -28,6 +29,7 @@ const (
 // DefaultHTTPClient is the default http client used if no client is provided.
 var DefaultHTTPClient = &http.Client{
 	Timeout:   defaultHTTPTimeout,
+	Transport: otelhttp.NewTransport(http.DefaultTransport),
 }
 
 var _ providers.Provider = &Client{}

internal/metal/providers/emapi/options.go

@@ -1,6 +1,8 @@
 package emapi
 
 import (
+	"net/http"
+
 	"go.uber.org/zap"
 )
 
@@ -15,3 +17,12 @@ func WithLogger(logger *zap.SugaredLogger) Option {
 		return nil
 	}
 }
+
+// WithHTTPClient sets the http client to be used by the client.
+func WithHTTPClient(httpClient *http.Client) Option {
+	return func(c *Client) error {
+		c.httpClient = httpClient
+
+		return nil
+	}
+}

internal/metal/providers/emgql/client.go

@@ -1,10 +1,16 @@
 package emgql
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/providers"
@@ -17,6 +23,7 @@ const (
 // DefaultHTTPClient is the default http client used if no client is provided.
 var DefaultHTTPClient = &http.Client{
 	Timeout:   defaultHTTPTimeout,
+	Transport: otelhttp.NewTransport(http.DefaultTransport),
 }
 
 var _ providers.Provider = &Client{}
@@ -48,3 +55,41 @@ func New(options ...Option) (*Client, error) {
 
 	return client, nil
 }
+
+func (c *Client) graphquery(ctx context.Context, q string, vars map[string]string) (io.ReadCloser, error) {
+	qObj := struct {
+		Query     string            `json:"query"`
+		Variables map[string]string `json:"variables"`
+	}{
+		Query:     q,
+		Variables: vars,
+	}
+
+	res, err := json.Marshal(qObj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal request object: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL.String(), bytes.NewBuffer(res))
+	if err != nil {
+		return nil, fmt.Errorf("failed to build post request: %w", err)
+	}
+
+	req.Header.Add("content-type", "application/json")
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make post request: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		b, err2 := io.ReadAll(resp.Body)
+		if err2 != nil {
+			return nil, fmt.Errorf("got unexpected response code [%d]: unable to parse request body: %v", resp.StatusCode, err)
+		}
+
+		return nil, fmt.Errorf("got unexpected response code [%d]: %s", resp.StatusCode, string(b))
+	}
+	return resp.Body, nil
+
+}

internal/metal/providers/emgql/mapping.go

@@ -0,0 +1,85 @@
+package emgql
+
+import "go.equinixmetal.net/infra9-metal-bridge/internal/metal/models"
+
+type organization struct {
+	ID       string    `json:"id"`
+	Name     string    `json:"name"`
+	Projects []project `json:"projects"`
+	Users    []user    `json:"users"`
+}
+
+func (o organization) ToOrganizationDetails() models.OrganizationDetails {
+	orgDetails := models.OrganizationDetails{
+		ID:   o.ID,
+		Name: o.Name,
+	}
+
+	memberships := []*models.Membership[models.OrganizationDetails]{}
+	for _, u := range o.Users {
+		m := u.ToOrganizationMembership()
+		memberships = append(memberships, &m)
+	}
+
+	pDetails := []*models.ProjectDetails{}
+	for _, p := range o.Projects {
+		details := p.ToProjectDetails()
+		details.Organization = &orgDetails
+		pDetails = append(pDetails, &details)
+	}
+
+	orgDetails.Memberships = memberships
+	orgDetails.Projects = pDetails
+
+	return orgDetails
+
+}
+
+type project struct {
+	ID           string `json:"id"`
+	Name         string `json:"name"`
+	Users        []user `json:"users"`
+	Organization struct {
+		ID string `json:"id"`
+	} `json:"organization"`
+}
+
+func (p project) ToProjectDetails() models.ProjectDetails {
+	memberships := []*models.Membership[models.ProjectDetails]{}
+	for _, u := range p.Users {
+		m := u.ToProjectMembership()
+		memberships = append(memberships, &m)
+	}
+
+	return models.ProjectDetails{
+		ID:          p.ID,
+		Name:        p.Name,
+		Memberships: memberships,
+		Organization: &models.OrganizationDetails{
+			ID: p.Organization.ID,
+		},
+	}
+}
+
+type user struct {
+	ID    string   `json:"id"`
+	Roles []string `json:"roles"`
+}
+
+func (u user) ToOrganizationMembership() models.Membership[models.OrganizationDetails] {
+	return models.Membership[models.OrganizationDetails]{
+		User: &models.UserDetails{
+			ID: u.ID,
+		},
+		Roles: u.Roles,
+	}
+}
+
+func (u user) ToProjectMembership() models.Membership[models.ProjectDetails] {
+	return models.Membership[models.ProjectDetails]{
+		User: &models.UserDetails{
+			ID: u.ID,
+		},
+		Roles: u.Roles,
+	}
+}

internal/metal/providers/emgql/options.go

@@ -1,6 +1,7 @@
 package emgql
 
 import (
+	"net/http"
 	"net/url"
 
 	"go.uber.org/zap"
@@ -32,6 +33,15 @@ func WithBaseURL(baseURL string) Option {
 	}
 }
 
+// WithHTTPClient sets the http client to be used by the client.
+func WithHTTPClient(httpClient *http.Client) Option {
+	return func(c *Client) error {
+		c.httpClient = httpClient
+
+		return nil
+	}
+}
+
 // WithConfig applies all configurations defined in the config.
 func WithConfig(config Config) Option {
 	return func(c *Client) error {

internal/metal/providers/emgql/organizations.go

@@ -2,6 +2,9 @@ package emgql
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"io"
 
 	"go.infratographer.com/x/gidx"
 
@@ -10,5 +13,56 @@ import (
 
 // GetOrganizationDetails fetches the organization id provided with its memberships.
 func (c *Client) GetOrganizationDetails(ctx context.Context, id gidx.PrefixedID) (*models.OrganizationDetails, error) {
-	return nil, nil
+	q := `
+query ($orgId: OrgId!){
+    organization(id: $orgId) {
+        id
+        name
+        users {
+          id
+          roles
+        }
+        projects {
+            id
+            name
+        }
+    }
+}
+`
+	variables := map[string]string{
+		"orgId": id.String(),
+	}
+
+	res, err := c.graphquery(ctx, q, variables)
+	if err != nil {
+		return nil, fmt.Errorf("emgql: error loading organization details: %w", err)
+	}
+
+	org, err := organizationFromResponse(res)
+	if err != nil {
+		return nil, fmt.Errorf("emgql: failed to parse response: %w", err)
+	}
+
+	out := org.ToOrganizationDetails()
+	return &out, nil
+}
+
+func organizationFromResponse(r io.ReadCloser) (organization, error) {
+	out := struct {
+		Data struct {
+			Organization organization `json:"organization"`
+		} `json:"data"`
+	}{}
+
+	body, err := io.ReadAll(r)
+	if err != nil {
+		return out.Data.Organization, fmt.Errorf("emgql: %w", err)
+	}
+
+	err = json.Unmarshal(body, &out)
+	if err != nil {
+		return out.Data.Organization, fmt.Errorf("emgql: %w", err)
+	}
+
+	return out.Data.Organization, nil
 }

internal/metal/providers/emgql/projects.go

@@ -2,6 +2,9 @@ package emgql
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"io"
 
 	"go.infratographer.com/x/gidx"
 
@@ -10,5 +13,55 @@ import (
 
 // GetProjectDetails fetchs the provided project id with membership information.
 func (c *Client) GetProjectDetails(ctx context.Context, id gidx.PrefixedID) (*models.ProjectDetails, error) {
-	return nil, nil
+	q := `
+query ($projectId: ProjectId!){
+    project(id: $projectId) {
+        id
+        name
+        users {
+          id
+          roles
+        }
+        organization {
+            id
+        }
+    }
+}
+`
+	variables := map[string]string{
+		"projectId": id.String(),
+	}
+
+	res, err := c.graphquery(ctx, q, variables)
+	if err != nil {
+		return nil, fmt.Errorf("error loading project details: %w", err)
+	}
+
+	prj, err := projectFromResponse(res)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse response: %w", err)
+	}
+
+	out := prj.ToProjectDetails()
+	return &out, nil
+}
+
+func projectFromResponse(r io.ReadCloser) (project, error) {
+	out := struct {
+		Data struct {
+			Project project `json:"project"`
+		} `json:"data"`
+	}{}
+
+	body, err := io.ReadAll(r)
+	if err != nil {
+		return out.Data.Project, fmt.Errorf("emgql: %w", err)
+	}
+
+	err = json.Unmarshal(body, &out)
+	if err != nil {
+		return out.Data.Project, fmt.Errorf("emgql: %w", err)
+	}
+
+	return out.Data.Project, nil
 }

internal/permissions/assignments.go

@@ -27,7 +27,7 @@ type roleAssignmentData struct {
 
 // AssignRole assigns the provided member ID to the given role ID.
 func (c *client) AssignRole(ctx context.Context, roleID gidx.PrefixedID, memberID gidx.PrefixedID) error {
-	path := fmt.Sprintf("/api/v1/roles/%s/assignments", roleID.String())
+	path := fmt.Sprintf(permsPathRoleAssignmentsFormat, roleID.String())
 
 	body, err := encodeJSON(RoleAssign{
 		SubjectID: memberID.String(),
@@ -51,7 +51,7 @@ func (c *client) AssignRole(ctx context.Context, roleID gidx.PrefixedID, memberI
 
 // UnassignRole removes the provided member ID from the given role ID.
 func (c *client) UnassignRole(ctx context.Context, roleID gidx.PrefixedID, memberID gidx.PrefixedID) error {
-	path := fmt.Sprintf("/api/v1/roles/%s/assignments", roleID.String())
+	path := fmt.Sprintf(permsPathRoleAssignmentsFormat, roleID.String())
 
 	body, err := encodeJSON(RoleAssign{
 		SubjectID: memberID.String(),
@@ -75,7 +75,7 @@ func (c *client) UnassignRole(ctx context.Context, roleID gidx.PrefixedID, membe
 
 // ListRoleAssignments lists all assignments for the given role.
 func (c *client) ListRoleAssignments(ctx context.Context, roleID gidx.PrefixedID) ([]gidx.PrefixedID, error) {
-	path := fmt.Sprintf("/api/v1/roles/%s/assignments", roleID.String())
+	path := fmt.Sprintf(permsPathRoleAssignmentsFormat, roleID.String())
 
 	var response roleAssignmentData
 

internal/permissions/client.go

@@ -13,18 +13,28 @@ import (
 
 	"github.com/labstack/echo/v4"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 )
 
 const (
-	defaultPermissionsURL = "https://permissions-api.hollow-a.sv15.metalkube.net"
-
 	defaultHTTPClientTimeout = 5 * time.Second
+
+	defaultPermissionsURL = "https://permissions-api.hollow-a.sv15.metalkube.net"
+	permsPathAllow        = "/api/v1/allow"
+
+	permsPathResourceRelationshipsFormat = "/api/v1/resources/%s/relationships"
+	permsPathResourceRelationshipsFrom   = "/api/v1/relationships/from/"
+	permsPathResourceRelationshipsTo     = "/api/v1/relationships/to/"
+	permsPathResourceRolesFormat         = "/api/v1/resources/%s/roles"
+
+	permsPathRoleAssignmentsFormat = "/api/v1/roles/%s/assignments"
 )
 
 // DefaultHTTPClient is the default HTTP client for the Permissions Client.
 var DefaultHTTPClient = &http.Client{
 	Timeout:   defaultHTTPClientTimeout,
+	Transport: otelhttp.NewTransport(http.DefaultTransport),
 }
 
 // Client defines the Permissions API client interface.

internal/permissions/relationships.go

@@ -34,7 +34,7 @@ type ResourceRelationshipDeleteResponse struct {
 
 // DeleteResourceRelationship deletes the provided resources relationship to the given subject id.
 func (c *client) DeleteResourceRelationship(ctx context.Context, resourceID gidx.PrefixedID, relation string, relatedResourceID gidx.PrefixedID) error {
-	path := fmt.Sprintf("/api/v1/resources/%s/relationships", resourceID.String())
+	path := fmt.Sprintf(permsPathResourceRelationshipsFormat, resourceID.String())
 
 	body, err := encodeJSON(ResourceRelationshipRequest{
 		Relation:  relation,
@@ -63,7 +63,7 @@ func (c *client) ListResourceRelationshipsFrom(ctx context.Context, resourceID g
 		Data []resourceRelationship `json:"data"`
 	}
 
-	if _, err := c.DoRequest(ctx, http.MethodGet, fmt.Sprintf("/api/v1/relationships/from/%s", resourceID.String()), nil, &response); err != nil { // nolint:bodyclose // closed by Do on json decode.
+	if _, err := c.DoRequest(ctx, http.MethodGet, permsPathResourceRelationshipsFrom+resourceID.String(), nil, &response); err != nil { // nolint:bodyclose // closed by Do on json decode.
 		return nil, err
 	}
 
@@ -90,7 +90,7 @@ func (c *client) ListResourceRelationshipsTo(ctx context.Context, resourceID gid
 		Data []resourceRelationship `json:"data"`
 	}
 
-	if _, err := c.DoRequest(ctx, http.MethodGet, fmt.Sprintf("/api/v1/relationships/to/%s", resourceID.String()), nil, &response); err != nil { // nolint:bodyclose // closed by Do on json decode.
+	if _, err := c.DoRequest(ctx, http.MethodGet, permsPathResourceRelationshipsTo+resourceID.String(), nil, &response); err != nil { // nolint:bodyclose // closed by Do on json decode.
 		return nil, err
 	}
 

internal/permissions/roles.go

@@ -35,7 +35,7 @@ type ResourceRole struct {
 
 // CreateRole creates a role on the given resource id with the provided actions.
 func (c *client) CreateRole(ctx context.Context, resourceID gidx.PrefixedID, actions []string) (gidx.PrefixedID, error) {
-	path := fmt.Sprintf("/api/v1/resources/%s/roles", resourceID.String())
+	path := fmt.Sprintf(permsPathResourceRolesFormat, resourceID.String())
 
 	body, err := encodeJSON(ResourceRoleCreate{
 		Actions: actions,
@@ -77,7 +77,7 @@ func (c *client) DeleteRole(ctx context.Context, roleID gidx.PrefixedID) error {
 
 // ListResourceRoles fetches all roles assigned to the provided resource.
 func (c *client) ListResourceRoles(ctx context.Context, resourceID gidx.PrefixedID) (ResourceRoles, error) {
-	path := fmt.Sprintf("/api/v1/resources/%s/roles", resourceID.String())
+	path := fmt.Sprintf(permsPathResourceRolesFormat, resourceID.String())
 
 	var response struct {
 		Data ResourceRoles `json:"data"`

internal/pubsub/nats.go

@@ -51,7 +51,7 @@ func (s *subscriber) SubscribeChanges(ctx context.Context, topic string) (<-chan
 
 	opts := s.subscriptionOptions
 
-	opts = append(opts, nats.Durable(s.durableName(subject)))
+	opts = append(opts, nats.Durable(s.durableName(subject)), nats.AckExplicit(), nats.ManualAck())
 
 	msgCh := make(chan *changeEvent, subscriptionBufferSize)
 

internal/pubsub/subscriber.go

@@ -142,7 +142,9 @@ func (s *Subscriber) processEvent(msg *changeEvent) error {
 		"event.type", msg.EventType,
 	)
 
-	ctx, span := tracer.Start(context.Background(), "pubsub.receive", trace.WithAttributes(attribute.String("pubsub.subject", msg.SubjectID.String())))
+	ctx := events.TraceContextFromChangeMessage(context.Background(), msg.ChangeMessage)
+
+	ctx, span := tracer.Start(ctx, "pubsub.receive", trace.WithAttributes(attribute.String("pubsub.subject", msg.SubjectID.String())))
 
 	defer span.End()
 

internal/service/options.go

@@ -3,6 +3,7 @@ package service
 import (
 	"go.infratographer.com/x/gidx"
 	"go.uber.org/zap"
+	"golang.org/x/exp/slices"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal"
 	"go.equinixmetal.net/infra9-metal-bridge/internal/permissions"
@@ -64,7 +65,13 @@ func WithRootTenant(sid string) Option {
 // WithRoles defines the role to action mapping.
 func WithRoles(roles map[string][]string) Option {
 	return func(s *service) error {
-		s.roles = roles
+		s.roles = make(map[string][]string, len(roles))
+
+		for role, actions := range roles {
+			slices.Sort(actions)
+
+			s.roles[role] = actions
+		}
 
 		return nil
 	}

internal/service/organizations.go

@@ -5,6 +5,8 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/models"
 )
@@ -58,6 +60,10 @@ func (s *service) IsOrganizationID(id gidx.PrefixedID) bool {
 
 // TouchOrganization initializes a sync for the provided organization id for relationships and memberships.
 func (s *service) TouchOrganization(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "TouchOrganization", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	logger := s.logger.With("organization.id", id.String())
 
 	org, err := s.metal.GetOrganizationDetails(ctx, id)
@@ -89,6 +95,10 @@ func (s *service) TouchOrganization(ctx context.Context, id gidx.PrefixedID) err
 
 // DeleteOrganization deletes the provided organization id.
 func (s *service) DeleteOrganization(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "DeleteOrganization", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	err := s.publisher.PublishChange(ctx, organizationEvent, events.ChangeMessage{
 		SubjectID: id,
 		EventType: string(events.DeleteChangeType),

internal/service/organizations_test.go

@@ -35,7 +35,7 @@ func TestTouchOrganizationEmpty(t *testing.T) {
 		},
 	}
 
-	userID := gidx.PrefixedID("idntusr-2s-9kVNPJBaInlHpRs3lAMsvU_kVkLaSlD4R_RhavDw")
+	userID := gidx.PrefixedID("idntusr-2aPXqbgfYSlld36XLfvNjEqUIIL9ekRZzgs2eBGOCBw")
 	user := &models.UserDetails{
 		ID: "usr1",
 	}
@@ -133,10 +133,10 @@ func TestTouchOrganizationCleanup(t *testing.T) {
 		},
 	}
 
-	oldUserID := gidx.PrefixedID("idntusr-2s-9kVNPJBaInlHpRs3lAMsvU_kVkLaSlD4R_RhavDw")
+	oldUserID := gidx.PrefixedID("idntusr-2aPXqbgfYSlld36XLfvNjEqUIIL9ekRZzgs2eBGOCBw")
 	deadProjectID := gidx.PrefixedID("metlprj-prj1")
 
-	userID := gidx.PrefixedID("idntusr-RnKvdujrwqm4o1dBDgfgaqeCpKFMaGeOtGnNbZky0Kg")
+	userID := gidx.PrefixedID("idntusr-Ewj_PJUue9eDIDoyCoWG48GtwlysqTj2Y4qWPiJPN1s")
 	user := &models.UserDetails{
 		ID: "usr2",
 	}
@@ -265,7 +265,7 @@ func TestTouchOrganizationNoChange(t *testing.T) {
 		},
 	}
 
-	userID := gidx.PrefixedID("idntusr-2s-9kVNPJBaInlHpRs3lAMsvU_kVkLaSlD4R_RhavDw")
+	userID := gidx.PrefixedID("idntusr-2aPXqbgfYSlld36XLfvNjEqUIIL9ekRZzgs2eBGOCBw")
 	user := &models.UserDetails{
 		ID: "usr1",
 	}

internal/service/process_memberships.go

@@ -5,6 +5,8 @@ import (
 	"strings"
 
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/exp/slices"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/permissions"
@@ -13,6 +15,15 @@ import (
 // processMemberships determines the changes between what is wanted and what is live and executes on the differences.
 // If skipDeletions is true, no deletes will be executed.
 func (s *service) processMemberships(ctx context.Context, relationships Relationships, skipDeletions bool) (int, int) {
+	ctx, span := tracer.Start(ctx, "processMemberships",
+		trace.WithAttributes(
+			attribute.String("resource.id", relationships.Resource.PrefixedID().String()),
+			attribute.Int("resource.memberships", len(relationships.Memberships)),
+		),
+	)
+
+	defer span.End()
+
 	if len(relationships.Memberships) == 0 {
 		return 0, 0
 	}
@@ -180,8 +191,6 @@ func (s *service) mapResourceWants(memberships []ResourceMemberships) (map[strin
 	roleActionsKey := make(map[string]string)
 
 	for role, actions := range s.roles {
-		slices.Sort(actions)
-
 		roleActionsKey[role] = strings.Join(actions, "|")
 	}
 

internal/service/process_relationships.go

@@ -5,6 +5,8 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/permissions"
 )
@@ -13,6 +15,15 @@ import (
 // Relationship creations use events.
 // Relationship deletions use the api, as delete events delete all related resources and not just the provided ones.
 func (s *service) processRelationships(ctx context.Context, eventType string, relationships Relationships) int {
+	ctx, span := tracer.Start(ctx, "processRelationships",
+		trace.WithAttributes(
+			attribute.String("resource.id", relationships.Resource.PrefixedID().String()),
+			attribute.Int("resource.subject_relationships", len(relationships.SubjectRelationships)),
+		),
+	)
+
+	defer span.End()
+
 	rlogger := s.logger.With("resource.id", relationships.Resource.PrefixedID())
 
 	wantParentRelationship, wantSubjectRelationships := s.mapRelationWants(relationships)

internal/service/projects.go

@@ -5,6 +5,8 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal/models"
 )
@@ -51,6 +53,10 @@ func (s *service) IsProjectID(id gidx.PrefixedID) bool {
 
 // TouchProject initializes a sync for the provided project id for relationships and memberships.
 func (s *service) TouchProject(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "TouchProject", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	logger := s.logger.With("project.id", id.String())
 
 	project, err := s.metal.GetProjectDetails(ctx, id)
@@ -82,6 +88,10 @@ func (s *service) TouchProject(ctx context.Context, id gidx.PrefixedID) error {
 
 // DeleteProject deletes the provided project id.
 func (s *service) DeleteProject(ctx context.Context, id gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "DeleteProject", trace.WithAttributes(attribute.String("resource.id", id.String())))
+
+	defer span.End()
+
 	err := s.publisher.PublishChange(ctx, projectEvent, events.ChangeMessage{
 		SubjectID: id,
 		EventType: string(events.DeleteChangeType),

internal/service/projects_test.go

@@ -24,7 +24,7 @@ func TestTouchProjectEmpty(t *testing.T) {
 		},
 	}
 
-	userID := gidx.PrefixedID("idntusr-2s-9kVNPJBaInlHpRs3lAMsvU_kVkLaSlD4R_RhavDw")
+	userID := gidx.PrefixedID("idntusr-2aPXqbgfYSlld36XLfvNjEqUIIL9ekRZzgs2eBGOCBw")
 	user := &models.UserDetails{
 		ID: "usr1",
 	}
@@ -110,10 +110,10 @@ func TestTouchProjectCleanup(t *testing.T) {
 		},
 	}
 
-	oldUserID := gidx.PrefixedID("idntusr-2s-9kVNPJBaInlHpRs3lAMsvU_kVkLaSlD4R_RhavDw")
+	oldUserID := gidx.PrefixedID("idntusr-2aPXqbgfYSlld36XLfvNjEqUIIL9ekRZzgs2eBGOCBw")
 	oldOrgID := gidx.PrefixedID("metlorg-org1")
 
-	userID := gidx.PrefixedID("idntusr-RnKvdujrwqm4o1dBDgfgaqeCpKFMaGeOtGnNbZky0Kg")
+	userID := gidx.PrefixedID("idntusr-Ewj_PJUue9eDIDoyCoWG48GtwlysqTj2Y4qWPiJPN1s")
 	user := &models.UserDetails{
 		ID: "usr2",
 	}
@@ -222,7 +222,7 @@ func TestTouchProjectNoChange(t *testing.T) {
 		},
 	}
 
-	userID := gidx.PrefixedID("idntusr-2s-9kVNPJBaInlHpRs3lAMsvU_kVkLaSlD4R_RhavDw")
+	userID := gidx.PrefixedID("idntusr-2aPXqbgfYSlld36XLfvNjEqUIIL9ekRZzgs2eBGOCBw")
 	user := &models.UserDetails{
 		ID: "usr1",
 	}

internal/service/service.go

@@ -5,6 +5,7 @@ import (
 
 	"go.infratographer.com/x/events"
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
 
 	"go.equinixmetal.net/infra9-metal-bridge/internal/metal"
@@ -22,6 +23,8 @@ const (
 	TypeUser ObjectType = "user"
 )
 
+var tracer = otel.Tracer("go.equinixmetal.net/infra9-metal-bridge/internal/service")
+
 // DefaultPrefixMap is the default id prefix to type relationship.
 var DefaultPrefixMap = map[string]ObjectType{
 	TypeOrganization.Prefix(): TypeOrganization,

internal/service/users.go

@@ -4,6 +4,8 @@ import (
 	"context"
 
 	"go.infratographer.com/x/gidx"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // IsUser checks the provided id has the metal user prefix.
@@ -29,8 +31,27 @@ func (s *service) IsAssignableResource(id gidx.PrefixedID) bool {
 	return false
 }
 
+func stringIDs(ids []gidx.PrefixedID) []string {
+	result := make([]string, len(ids))
+
+	for i, id := range ids {
+		result[i] = id.String()
+	}
+
+	return result
+}
+
 // Assignuser assigns the provided users to the given resource ids.
 func (s *service) AssignUser(ctx context.Context, userID gidx.PrefixedID, resourceIDs ...gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "AssignUser",
+		trace.WithAttributes(
+			attribute.String("user.id", userID.String()),
+			attribute.StringSlice("user.resource_assignments", stringIDs(resourceIDs)),
+		),
+	)
+
+	defer span.End()
+
 	var totalResources, rolesChanged, assignmentsChanged int
 
 	mlogger := s.logger.With("member.id", userID.String())
@@ -74,6 +95,15 @@ func (s *service) AssignUser(ctx context.Context, userID gidx.PrefixedID, resour
 
 // UnassignUser removes the assignment for the provided user id to the given resources.
 func (s *service) UnassignUser(ctx context.Context, userID gidx.PrefixedID, resourceIDs ...gidx.PrefixedID) error {
+	ctx, span := tracer.Start(ctx, "UnassignUser",
+		trace.WithAttributes(
+			attribute.String("user.id", userID.String()),
+			attribute.StringSlice("user.resource_assignments", stringIDs(resourceIDs)),
+		),
+	)
+
+	defer span.End()
+
 	for _, resourceID := range resourceIDs {
 		rlogger := s.logger.With("user.id", userID, "resource.id", resourceID)