Skip to content

Commit e3dbe23

Browse files
committed
feat(speculation): add outcome predictor implementation
## Summary ### Why? A scorer prices change content, but speculation also needs a separate contract for revising that price with evidence observed during a run. Keeping the concerns separate avoids adding path data that every content scorer would discard. ### What? Add the `predictor.Predictor` contract and an evidence implementation that converts the scorer probability to odds, applies factors for passed and failed all-succeed paths plus merging and cancelling states, and converts the result back to a probability. Include generated mocks and unit coverage for neutral factors, compounding evidence, path filtering, bounds, validation, and scorer failures. ## Test Plan - `bazel test //submitqueue/extension/speculation/predictor/...` - `make check-gazelle`
1 parent 3227132 commit e3dbe23

7 files changed

Lines changed: 644 additions & 0 deletions

File tree

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
load("@rules_go//go:def.bzl", "go_library")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["predictor.go"],
6+
importpath = "github.com/uber/submitqueue/submitqueue/extension/speculation/predictor",
7+
visibility = ["//visibility:public"],
8+
deps = ["//submitqueue/entity:go_default_library"],
9+
)
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
load("@rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["evidence.go"],
6+
importpath = "github.com/uber/submitqueue/submitqueue/extension/speculation/predictor/evidence",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//platform/metrics:go_default_library",
10+
"//submitqueue/entity:go_default_library",
11+
"//submitqueue/extension/speculation/predictor:go_default_library",
12+
"//submitqueue/extension/speculation/scorer:go_default_library",
13+
"@com_github_uber_go_tally//:go_default_library",
14+
],
15+
)
16+
17+
go_test(
18+
name = "go_default_test",
19+
srcs = ["evidence_test.go"],
20+
embed = [":go_default_library"],
21+
deps = [
22+
"//submitqueue/entity:go_default_library",
23+
"//submitqueue/extension/speculation/predictor:go_default_library",
24+
"//submitqueue/extension/speculation/scorer:go_default_library",
25+
"@com_github_stretchr_testify//assert:go_default_library",
26+
"@com_github_stretchr_testify//require:go_default_library",
27+
"@com_github_uber_go_tally//:go_default_library",
28+
],
29+
)
Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
// Copyright (c) 2025 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Package evidence revises a Scorer's price by multiplying its odds by one
16+
// factor per piece of evidence about the batch's progress.
17+
//
18+
// Odds rather than the probability itself, because a factor then means the same
19+
// thing wherever it applies and the result cannot leave [0, 1]. Written as logs
20+
// and summed, the same arithmetic is a logistic regression, which is what lets
21+
// hand-written factors later be replaced by fitted ones without changing the
22+
// form. See doc/rfc/submitqueue/outcome-predictor.md.
23+
package evidence
24+
25+
import (
26+
"fmt"
27+
"math"
28+
29+
"context"
30+
31+
"github.com/uber-go/tally"
32+
"github.com/uber/submitqueue/platform/metrics"
33+
"github.com/uber/submitqueue/submitqueue/entity"
34+
"github.com/uber/submitqueue/submitqueue/extension/speculation/predictor"
35+
"github.com/uber/submitqueue/submitqueue/extension/speculation/scorer"
36+
)
37+
38+
// Factors are the odds multipliers, one per piece of evidence. A factor of 1
39+
// leaves the price alone. Named fields rather than a keyed map, so an evidence
40+
// name that does not exist fails to compile instead of being ignored.
41+
type Factors struct {
42+
// PathPassed applies once when a build has passed on the batch's
43+
// all-succeed path.
44+
PathPassed float64
45+
// PathFailed applies once per failed all-succeed path, compounding.
46+
PathFailed float64
47+
// Merging applies while the batch is merging.
48+
Merging float64
49+
// Cancelling applies while the batch is cancelling.
50+
Cancelling float64
51+
}
52+
53+
// AllOnes is the neutral set: the prediction is the scorer's price.
54+
func AllOnes() Factors {
55+
return Factors{PathPassed: 1, PathFailed: 1, Merging: 1, Cancelling: 1}
56+
}
57+
58+
// epsilon bounds the price away from 0 and 1, which have no finite odds.
59+
// Without it a certain scorer could never be revised by any evidence — and
60+
// certainty about an unfinished batch is the scorer overstating what it sees.
61+
const epsilon = 1e-6
62+
63+
// evidence is a predictor.Predictor that revises a scorer's price.
64+
type evidence struct {
65+
// cfg is the per-queue identity this predictor was built for.
66+
cfg predictor.Config
67+
// base prices the batch's change; its price is what the factors revise.
68+
base scorer.Scorer
69+
// factors are the odds multipliers applied to that price.
70+
factors Factors
71+
// scope is the tally scope for emitting metrics.
72+
scope tally.Scope
73+
}
74+
75+
// New creates an evidence predictor bound to the queue named in cfg, revising
76+
// base's price by factors.
77+
//
78+
// It returns an error rather than panic on a nil base or a non-positive factor:
79+
// configuration rejects those today, but the fitted-factor file loader planned
80+
// in doc/rfc/submitqueue/outcome-predictor.md bypasses configuration entirely,
81+
// and on that path this check is the only guard.
82+
func New(cfg predictor.Config, base scorer.Scorer, factors Factors, scope tally.Scope) (predictor.Predictor, error) {
83+
if base == nil {
84+
return nil, fmt.Errorf("evidence.New: base must not be nil")
85+
}
86+
for name, factor := range map[string]float64{
87+
"PathPassed": factors.PathPassed,
88+
"PathFailed": factors.PathFailed,
89+
"Merging": factors.Merging,
90+
"Cancelling": factors.Cancelling,
91+
} {
92+
// Zero would pin the prediction to 0 and negative has no meaning as a
93+
// multiplier on odds.
94+
if !(factor > 0) {
95+
return nil, fmt.Errorf("evidence.New: factor %s must be positive, got %v", name, factor)
96+
}
97+
}
98+
return &evidence{cfg: cfg, base: base, factors: factors, scope: scope}, nil
99+
}
100+
101+
// Predict prices the batch's change through the base scorer, then multiplies
102+
// the odds of that price by one factor per piece of evidence.
103+
func (r *evidence) Predict(ctx context.Context, batch entity.Batch, paths entity.SpeculationPathSet) (ret predictor.Probability, retErr error) {
104+
op := metrics.Begin(r.scope, "predict", metrics.FastLatencyBuckets)
105+
defer func() { op.Complete(retErr) }()
106+
107+
price, err := r.base.Score(ctx, batch)
108+
if err != nil {
109+
return 0, err
110+
}
111+
// A price that is not a probability is a broken scorer, not a low opinion of
112+
// the batch. Saying so leaves the caller to fall back on its own default,
113+
// where clamping would hand back a number that looks deliberate.
114+
if !(price >= 0 && price <= 1) {
115+
return 0, fmt.Errorf("base scorer returned %v, which is not a probability", price)
116+
}
117+
118+
odds := oddsOf(math.Min(math.Max(price, epsilon), 1-epsilon))
119+
if hasPassedAllSucceedPath(paths) {
120+
odds *= r.factors.PathPassed
121+
}
122+
odds *= math.Pow(r.factors.PathFailed, float64(countFailed(paths)))
123+
switch batch.State {
124+
case entity.BatchStateMerging:
125+
odds *= r.factors.Merging
126+
case entity.BatchStateCancelling:
127+
odds *= r.factors.Cancelling
128+
}
129+
return probabilityOf(odds), nil
130+
}
131+
132+
// oddsOf converts a probability to odds. p is bounded away from 1, so this is
133+
// finite.
134+
func oddsOf(p float64) float64 {
135+
return p / (1 - p)
136+
}
137+
138+
// probabilityOf converts odds back to a probability. Overflowed odds read as
139+
// certainty rather than the NaN the division would produce.
140+
func probabilityOf(odds float64) predictor.Probability {
141+
if math.IsInf(odds, 1) {
142+
return 1
143+
}
144+
return predictor.Probability(odds / (1 + odds))
145+
}
146+
147+
// hasPassedAllSucceedPath reports a passed build on the batch's all-succeed
148+
// path. Only that path counts: one built without a dependency's changes says
149+
// nothing about a candidate that assumes the dependency lands.
150+
func hasPassedAllSucceedPath(paths entity.SpeculationPathSet) bool {
151+
for _, entry := range paths.Paths {
152+
if entry.Status != entity.SpeculationPathStatusPassed {
153+
continue
154+
}
155+
if assumesAllSucceed(entry.Path) {
156+
return true
157+
}
158+
}
159+
return false
160+
}
161+
162+
// assumesAllSucceed reports whether every dependency is assumed to succeed.
163+
func assumesAllSucceed(path entity.SpeculationPath) bool {
164+
for _, dep := range path.Dependencies {
165+
if dep.Assumption != entity.DependencyAssumptionSucceeds {
166+
return false
167+
}
168+
}
169+
return true
170+
}
171+
172+
// countFailed counts failed builds on the batch's all-succeed path; each one
173+
// compounds. Flip-subset failures are ignored: they were built under different
174+
// assumptions, the same filter PathPassed uses.
175+
func countFailed(paths entity.SpeculationPathSet) int {
176+
failed := 0
177+
for _, entry := range paths.Paths {
178+
if entry.Status == entity.SpeculationPathStatusFailed && assumesAllSucceed(entry.Path) {
179+
failed++
180+
}
181+
}
182+
return failed
183+
}

0 commit comments

Comments
 (0)