Skip to content

Commit e84cc72

Browse files
authored
feat(issue summary) Only show possible cause if confident and novel (#84349)
## Background `possible_cause_confidence` increases when there's less speculation (which correlates w/ slightly more accurate causes) `possible_cause_novelty` increases when there's more novelty / less redundancy wrt `whats_wrong` thresholds led to 50% of possible causes getting dropped on our autofix sentry issues ([notebook here](https://github.com/getsentry/data-analysis/blob/main/autofix/issue_summary/issue_summary_confidence.ipynb)) ## Backend changes corresponding backend change to sentry: #84346 corresponding backend change to seer: getsentry/seer#1788 without the sentry backend change (there are no `data.scores`), default to current behavior: always show possible cause ![before](https://github.com/user-attachments/assets/9f632a94-690b-4602-bdea-cbd2b95824b0) with the change, only show it if both scores are greater than the threshold ![after](https://github.com/user-attachments/assets/0b1e84af-cc0b-4d0a-8106-58cc615ea602) (note: this example is actually considered novel and confident-enough. I hardcoded the threshold to test the behavior)
1 parent 1f0d5ad commit e84cc72

File tree

3 files changed

+63
-10
lines changed

3 files changed

+63
-10
lines changed

static/app/components/group/groupSummary.spec.tsx

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,22 @@ describe('GroupSummary', function () {
2323
trace: 'Test trace',
2424
possibleCause: 'Test possible cause',
2525
headline: 'Test headline',
26+
scores: {
27+
possibleCauseConfidence: 0.9,
28+
possibleCauseNovelty: 0.8,
29+
},
30+
};
31+
32+
const mockSummaryDataWithLowScores = {
33+
groupId: '1',
34+
whatsWrong: 'Test whats wrong',
35+
trace: 'Test trace',
36+
possibleCause: 'Test possible cause',
37+
headline: 'Test headline',
38+
scores: {
39+
possibleCauseConfidence: 0.5,
40+
possibleCauseNovelty: 0.0,
41+
},
2642
};
2743

2844
beforeEach(() => {
@@ -62,6 +78,27 @@ describe('GroupSummary', function () {
6278
expect(screen.getByText('Test possible cause')).toBeInTheDocument();
6379
});
6480

81+
it('renders the summary without possible cause', async function () {
82+
MockApiClient.addMockResponse({
83+
url: `/organizations/${mockProject.organization.slug}/issues/${mockGroup.id}/summarize/`,
84+
method: 'POST',
85+
body: mockSummaryDataWithLowScores,
86+
});
87+
88+
render(<GroupSummary event={mockEvent} group={mockGroup} project={mockProject} />, {
89+
organization,
90+
});
91+
92+
await waitFor(() => {
93+
expect(screen.getByText("What's wrong")).toBeInTheDocument();
94+
});
95+
expect(screen.getByText('Test whats wrong')).toBeInTheDocument();
96+
expect(screen.getByText('In the trace')).toBeInTheDocument();
97+
expect(screen.getByText('Test trace')).toBeInTheDocument();
98+
expect(screen.queryByText('Possible cause')).not.toBeInTheDocument();
99+
expect(screen.queryByText('Test possible cause')).not.toBeInTheDocument();
100+
});
101+
65102
it('shows loading state', function () {
66103
MockApiClient.addMockResponse({
67104
url: `/organizations/${mockProject.organization.slug}/issues/${mockGroup.id}/summarize/`,
@@ -73,8 +110,8 @@ describe('GroupSummary', function () {
73110
organization,
74111
});
75112

76-
// Should show loading placeholders
77-
expect(screen.getAllByTestId('loading-placeholder')).toHaveLength(2);
113+
// Should show loading placeholders. Currently we load the whatsWrong section
114+
expect(screen.getAllByTestId('loading-placeholder')).toHaveLength(1);
78115
});
79116

80117
it('shows error state', async function () {

static/app/components/group/groupSummary.tsx

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,18 @@ import {useFeedbackForm} from 'sentry/utils/useFeedbackForm';
1616
import useOrganization from 'sentry/utils/useOrganization';
1717
import {useAiConfig} from 'sentry/views/issueDetails/streamline/hooks/useAiConfig';
1818

19+
const POSSIBLE_CAUSE_CONFIDENCE_THRESHOLD = 0.468;
20+
const POSSIBLE_CAUSE_NOVELTY_THRESHOLD = 0.419;
21+
1922
interface GroupSummaryData {
2023
groupId: string;
2124
headline: string;
2225
eventId?: string | null;
2326
possibleCause?: string | null;
27+
scores?: {
28+
possibleCauseConfidence: number;
29+
possibleCauseNovelty: number;
30+
} | null;
2431
trace?: string | null;
2532
whatsWrong?: string | null;
2633
}
@@ -152,6 +159,11 @@ export function GroupSummary({
152159
: []),
153160
];
154161

162+
const shouldShowPossibleCause =
163+
!data?.scores ||
164+
(data.scores.possibleCauseConfidence >= POSSIBLE_CAUSE_CONFIDENCE_THRESHOLD &&
165+
data.scores.possibleCauseNovelty >= POSSIBLE_CAUSE_NOVELTY_THRESHOLD);
166+
155167
const insightCards = [
156168
{
157169
id: 'whats_wrong',
@@ -167,13 +179,17 @@ export function GroupSummary({
167179
icon: <IconSpan size="sm" />,
168180
showWhenLoading: false,
169181
},
170-
{
171-
id: 'possible_cause',
172-
title: t('Possible cause'),
173-
insight: data?.possibleCause,
174-
icon: <IconFocus size="sm" />,
175-
showWhenLoading: true,
176-
},
182+
...(shouldShowPossibleCause
183+
? [
184+
{
185+
id: 'possible_cause',
186+
title: t('Possible cause'),
187+
insight: data?.possibleCause,
188+
icon: <IconFocus size="sm" />,
189+
showWhenLoading: false,
190+
},
191+
]
192+
: []),
177193
];
178194

179195
return (

static/app/views/issueDetails/streamline/sidebar/solutionsSection.spec.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ describe('SolutionsSection', () => {
116116
);
117117

118118
expect(screen.getByText('Solutions Hub')).toBeInTheDocument();
119-
expect(screen.getAllByTestId('loading-placeholder')).toHaveLength(3);
119+
expect(screen.getAllByTestId('loading-placeholder')).toHaveLength(2); // whatsWrong and Open Autofix
120120
});
121121

122122
it('renders summary when AI features are enabled and data is available', async () => {

0 commit comments

Comments
 (0)