Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 66 additions & 3 deletions platform/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,69 @@ type Tag struct {
Value string
}

// Config identifies the logical queue for metrics returned by a Factory.
type Config struct {
QueueName string
}

// Factory creates configured metric scopes from a shared Tally scope.
type Factory struct {
scope tally.Scope
}

// NewFactory returns a Factory backed by scope.
func NewFactory(scope tally.Scope) Factory {
return Factory{scope: scope}
}

// For returns a metric scope bound to the configured queue.
func (f Factory) For(config Config) Scope {
return Scope{
scope: f.scope,
configuredTags: []Tag{NewTag("queue", config.QueueName)},
}
}

// Base returns an emitter on the factory's underlying scope without adding a
// queue tag. Existing namespaces and inherited tags remain intact. It is
// intended for failures that happen before a message's queue can be decoded.
func (f Factory) Base() Scope {
return Scope{scope: f.scope}
}

// Scope emits named metrics on a factory's underlying Tally scope. Factory.For
// may bind configured tags that callers cannot override; Factory.Base returns
// the same type without adding tags.
type Scope struct {
scope tally.Scope
configuredTags []Tag
}

// NamedCounter increments the {name}.{counter} counter by value.
func (s Scope) NamedCounter(name string, counter string, value int64, tags ...Tag) {
tagged(s.scope, s.withConfiguredTags(tags)).SubScope(name).Counter(counter).Inc(value)
}

// NamedHistogram returns a tally.Histogram at {name}.{histogram} with the given
// bucket configuration.
func (s Scope) NamedHistogram(name string, histogram string, buckets tally.Buckets, tags ...Tag) tally.Histogram {
return tagged(s.scope, s.withConfiguredTags(tags)).SubScope(name).Histogram(histogram, buckets)
}

// NamedGauge sets the {name}.{gauge} gauge to value.
func (s Scope) NamedGauge(name string, gauge string, value float64, tags ...Tag) {
tagged(s.scope, s.withConfiguredTags(tags)).SubScope(name).Gauge(gauge).Update(value)
}

func (s Scope) withConfiguredTags(tags []Tag) []Tag {
if len(s.configuredTags) == 0 {
return tags
}
configuredTags := make([]Tag, 0, len(tags)+len(s.configuredTags))
configuredTags = append(configuredTags, tags...)
return append(configuredTags, s.configuredTags...)
}

// NewTag creates a Tag with the given key and value.
func NewTag(key, value string) Tag {
return Tag{Key: key, Value: value}
Expand Down Expand Up @@ -183,19 +246,19 @@ func (o Op) Complete(err error, tags ...Tag) {

// NamedCounter increments the {name}.{counter} counter by value.
func NamedCounter(scope tally.Scope, name string, counter string, value int64, tags ...Tag) {
tagged(scope, tags).SubScope(name).Counter(counter).Inc(value)
Scope{scope: scope}.NamedCounter(name, counter, value, tags...)
}

// NamedHistogram returns a tally.Histogram at {name}.{histogram} with the given
// bucket configuration. Store the returned histogram and call RecordDuration or
// RecordValue on each invocation.
func NamedHistogram(scope tally.Scope, name string, histogram string, buckets tally.Buckets, tags ...Tag) tally.Histogram {
return tagged(scope, tags).SubScope(name).Histogram(histogram, buckets)
return Scope{scope: scope}.NamedHistogram(name, histogram, buckets, tags...)
}

// NamedGauge sets the {name}.{gauge} gauge to value.
func NamedGauge(scope tally.Scope, name string, gauge string, value float64, tags ...Tag) {
tagged(scope, tags).SubScope(name).Gauge(gauge).Update(value)
Scope{scope: scope}.NamedGauge(name, gauge, value, tags...)
}

// tagsToMap converts a slice of Tag to a map for tally.
Expand Down
29 changes: 29 additions & 0 deletions platform/metrics/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,35 @@ func TestNamedGauge(t *testing.T) {
assert.Equal(t, float64(42), g.Value())
}

func TestFactory(t *testing.T) {
scope := tally.NewTestScope("", nil)
factory := NewFactory(scope)
queueScope := factory.For(Config{QueueName: "monorepo/main"})
base := factory.Base()

queueScope.NamedCounter("process", "attempts", 2, NewTag("result", "success"), NewTag("queue", "wrong"))
queueScope.NamedGauge("process", "in_flight", 3)
queueScope.NamedHistogram("process", "duration", StorageLatencyBuckets).RecordDuration(time.Second)
base.NamedCounter("process", "deserialize_errors", 1)
base.NamedGauge("process", "decode_in_flight", 1)
base.NamedHistogram("process", "decode_duration", StorageLatencyBuckets).RecordDuration(time.Second)

snapshot := scope.Snapshot()
counter, ok := snapshot.Counters()["process.attempts+queue=monorepo/main,result=success"]
assert.True(t, ok)
assert.EqualValues(t, 2, counter.Value())
_, ok = snapshot.Gauges()["process.in_flight+queue=monorepo/main"]
assert.True(t, ok)
_, ok = snapshot.Histograms()["process.duration+queue=monorepo/main"]
assert.True(t, ok)
_, ok = snapshot.Counters()["process.deserialize_errors+"]
assert.True(t, ok)
_, ok = snapshot.Gauges()["process.decode_in_flight+"]
assert.True(t, ok)
_, ok = snapshot.Histograms()["process.decode_duration+"]
assert.True(t, ok)
}

func TestLatencyBuckets_Sorted(t *testing.T) {
sets := map[string]tally.DurationBuckets{
"FastLatencyBuckets": FastLatencyBuckets,
Expand Down
39 changes: 20 additions & 19 deletions stovepipe/controller/build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ import (
// triggers a build for its already-decided scope, and publishes the resulting
// build id to buildsignal. Implements consumer.Controller.
type Controller struct {
logger *zap.SugaredLogger
metricsScope tally.Scope
stores storage.Factory
buildRunners buildrunner.Factory
registry consumer.TopicRegistry
topicKey consumer.TopicKey
consumerGroup string
logger *zap.SugaredLogger
metricsFactory metrics.Factory
stores storage.Factory
buildRunners buildrunner.Factory
registry consumer.TopicRegistry
topicKey consumer.TopicKey
consumerGroup string
}

// Verify Controller implements consumer.Controller interface at compile time.
Expand All @@ -66,13 +66,13 @@ func NewController(
consumerGroup string,
) *Controller {
return &Controller{
logger: logger.Named("build_controller"),
metricsScope: scope.SubScope("build_controller"),
stores: stores,
buildRunners: buildRunners,
registry: registry,
topicKey: topicKey,
consumerGroup: consumerGroup,
logger: logger.Named("build_controller"),
metricsFactory: metrics.NewFactory(scope.SubScope("build_controller")),
stores: stores,
buildRunners: buildRunners,
registry: registry,
topicKey: topicKey,
consumerGroup: consumerGroup,
}
}

Expand All @@ -84,28 +84,29 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er

br := &stovepipemq.BuildRequest{}
if err := stovepipemq.Unmarshal(msg.Payload, br); err != nil {
metrics.NamedCounter(c.metricsScope, _opName, "deserialize_errors", 1)
c.metricsFactory.Base().NamedCounter(_opName, "deserialize_errors", 1)
// Non-retryable: a malformed message will never succeed regardless of retries.
return fmt.Errorf("failed to deserialize build request: %w", err)
}
messageMetrics := c.metricsFactory.For(metrics.Config{QueueName: br.GetQueueName()})

store, err := c.stores.For(storage.Config{QueueName: br.GetQueueName()})
if err != nil {
metrics.NamedCounter(c.metricsScope, _opName, "storage_resolve_errors", 1)
messageMetrics.NamedCounter(_opName, "storage_resolve_errors", 1)
// Non-retryable: a missing or unresolvable queue is a malformed message.
return fmt.Errorf("failed to resolve storage for queue %q: %w", br.GetQueueName(), err)
}

request, err := c.loadRequest(ctx, store, br.Id)
if err != nil {
metrics.NamedCounter(c.metricsScope, _opName, "storage_errors", 1)
messageMetrics.NamedCounter(_opName, "storage_errors", 1)
return err
}

// The payload's queue must match the request's authoritative queue; a
// mismatch is a malformed message. Non-retryable — reject to the DLQ.
if br.GetQueueName() != "" && br.GetQueueName() != request.Queue {
metrics.NamedCounter(c.metricsScope, _opName, "queue_mismatch", 1)
messageMetrics.NamedCounter(_opName, "queue_mismatch", 1)
return fmt.Errorf("payload queue %q does not match queue %q of request %s", br.GetQueueName(), request.Queue, request.ID)
}

Expand All @@ -123,7 +124,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er

// process decided the scope; build never re-derives incremental-vs-full.
if request.BuildStrategy == entity.BuildStrategyUnknown {
metrics.NamedCounter(c.metricsScope, _opName, "strategy_not_visible", 1)
messageMetrics.NamedCounter(_opName, "strategy_not_visible", 1)
return errs.NewRetryableError(fmt.Errorf("request %s has no build strategy yet", request.ID))
}
baseURI := ""
Expand Down
20 changes: 18 additions & 2 deletions stovepipe/controller/build/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type buildMocks struct {
runnerFactory *buildrunnermock.MockFactory
runner *buildrunnermock.MockBuildRunner
publisher *mqmock.MockPublisher
metricsScope tally.TestScope
}

// staticStorageFactory resolves every queue to one fixed store aggregate.
Expand All @@ -63,12 +64,14 @@ func (f staticStorageFactory) For(storage.Config) (storage.Storage, error) { ret
func newController(t *testing.T, ctrl *gomock.Controller) (*Controller, buildMocks) {
t.Helper()

scope := tally.NewTestScope("test", nil)
m := buildMocks{
reqStore: storagemock.NewMockRequestStore(ctrl),
buildStore: storagemock.NewMockBuildStore(ctrl),
runnerFactory: buildrunnermock.NewMockFactory(ctrl),
runner: buildrunnermock.NewMockBuildRunner(ctrl),
publisher: mqmock.NewMockPublisher(ctrl),
metricsScope: scope,
}

store := storagemock.NewMockStorage(ctrl)
Expand All @@ -83,10 +86,23 @@ func newController(t *testing.T, ctrl *gomock.Controller) (*Controller, buildMoc
})
require.NoError(t, err)

c := NewController(zap.NewNop().Sugar(), tally.NewTestScope("test", nil), staticStorageFactory{store: store}, m.runnerFactory, registry, stovepipemq.TopicKeyBuild, "stovepipe-build")
c := NewController(zap.NewNop().Sugar(), scope, staticStorageFactory{store: store}, m.runnerFactory, registry, stovepipemq.TopicKeyBuild, "stovepipe-build")
return c, m
}

func TestProcessTagsMetricsWithQueue(t *testing.T) {
ctrl := gomock.NewController(t)
c, m := newController(t, ctrl)
m.reqStore.EXPECT().Get(gomock.Any(), testID).
Return(processingRequest(entity.BuildStrategyUnknown, ""), nil)
m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil)

require.Error(t, c.Process(context.Background(), delivery(t, ctrl, buildPayload(t, testID))))
counter, ok := m.metricsScope.Snapshot().Counters()["test.build_controller.build.strategy_not_visible+queue=monorepo/main"]
require.True(t, ok)
assert.EqualValues(t, 1, counter.Value())
}

func delivery(t *testing.T, ctrl *gomock.Controller, payload []byte) consumer.Delivery {
t.Helper()
d := consumermock.NewMockDelivery(ctrl)
Expand All @@ -97,7 +113,7 @@ func delivery(t *testing.T, ctrl *gomock.Controller, payload []byte) consumer.De

func buildPayload(t *testing.T, id string) []byte {
t.Helper()
b, err := stovepipemq.Marshal(&stovepipemq.BuildRequest{Id: id})
b, err := stovepipemq.Marshal(&stovepipemq.BuildRequest{Id: id, QueueName: testQueue})
require.NoError(t, err)
return b
}
Expand Down
Loading