telemetry(amazonq): Add changed IDE diagnostics after user acceptance

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt

@@ -21,6 +21,7 @@ import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.openapi.options.ShowSettingsUtil
 import com.intellij.psi.PsiDocumentManager
 import kotlinx.coroutines.cancelChildren
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.flow.catch
 import kotlinx.coroutines.flow.filter
 import kotlinx.coroutines.flow.first
@@ -36,6 +37,7 @@ import software.aws.toolkits.core.utils.getLogger
 import software.aws.toolkits.core.utils.info
 import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.core.coroutines.EDT
+import software.aws.toolkits.jetbrains.core.credentials.sono.isInternalUser
 import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitContext
 import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
 import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthNeededState
@@ -49,6 +51,8 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.settings.CodeWhisp
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererUserModificationTracker
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
 import software.aws.toolkits.jetbrains.services.cwc.InboundAppMessagesHandler
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
@@ -214,6 +218,7 @@ class ChatController private constructor(
             val caret: Caret = editor.caretModel.primaryCaret
             val offset: Int = caret.offset
 
+            val oldDiagnostics = getDocumentDiagnostics(editor.document, context.project)
             ApplicationManager.getApplication().runWriteAction {
                 WriteCommandAction.runWriteCommandAction(context.project) {
                     if (caret.hasSelection()) {
@@ -236,6 +241,12 @@ class ChatController private constructor(
                     )
                 }
             }
+            if (isInternalUser(getStartUrl(context.project))) {
+                // wait for the IDE itself to update its diagnostics for current file
+                delay(500)
+                val newDiagnostics = getDocumentDiagnostics(editor.document, context.project)
+                message.diagnosticsDifferences = getDiagnosticDifferences(oldDiagnostics, newDiagnostics)
+            }
         }
         telemetryHelper.recordInteractWithMessage(message)
 

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt

@@ -270,6 +270,8 @@ class TelemetryHelper(private val project: Project, private val sessionStorage:
                     acceptedCharacterCount(message.code.length)
                     acceptedLineCount(message.code.lines().size)
                     hasProjectLevelContext(getMessageHasProjectContext(message.messageId))
+                    addedIdeDiagnostics(message.diagnosticsDifferences?.added)
+                    removedIdeDiagnostics(message.diagnosticsDifferences?.removed)
                 }.build()
             }
 

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/messages/CwcMessage.kt

@@ -18,6 +18,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthFollowUpType
 import software.aws.toolkits.jetbrains.services.amazonq.messages.AmazonQMessage
 import software.aws.toolkits.jetbrains.services.amazonq.onboarding.OnboardingPageInteractionType
 import software.aws.toolkits.jetbrains.services.amazonq.util.HighlightCommand
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.FollowUpType
 import java.time.Instant
 
@@ -95,6 +96,7 @@ sealed interface IncomingCwcMessage : CwcMessage {
         val codeBlockIndex: Int?,
         val totalCodeBlocks: Int?,
         val codeBlockLanguage: String?,
+        var diagnosticsDifferences: DiagnosticDifferences?,
     ) : IncomingCwcMessage, TabId, MessageId
 
     data class TriggerTabIdReceived(

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt

@@ -44,6 +44,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitConte
 import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor
 import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererCustomization
 import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererModelConfigurator
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.ChatSession
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.CodeNamesImpl
@@ -475,6 +476,7 @@ class TelemetryHelperTest {
         val inserTionTargetType = "insertionTargetType"
         val eventId = "eventId"
         val code = "println()"
+        val diagnosticDifferences = DiagnosticDifferences(emptyList(), emptyList())
 
         sut.recordInteractWithMessage(
             IncomingCwcMessage.InsertCodeAtCursorPosition(
@@ -487,7 +489,8 @@ class TelemetryHelperTest {
                 eventId,
                 codeBlockIndex,
                 totalCodeBlocks,
-                lang
+                lang,
+                diagnosticDifferences
             )
         )
 
@@ -503,6 +506,8 @@ class TelemetryHelperTest {
                     acceptedLineCount(code.lines().size)
                     customizationArn(customizationArn)
                     hasProjectLevelContext(false)
+                    addedIdeDiagnostics(emptyList())
+                    removedIdeDiagnostics(emptyList())
                 }.build()
             )
         )

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt

@@ -37,6 +37,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.TargetCode
 import software.amazon.awssdk.services.codewhispererruntime.model.UserIntent
 import software.aws.toolkits.core.utils.debug
 import software.aws.toolkits.core.utils.getLogger
+import software.aws.toolkits.jetbrains.core.credentials.sono.isInternalUser
 import software.aws.toolkits.jetbrains.services.amazonq.codeWhispererUserContext
 import software.aws.toolkits.jetbrains.services.amazonq.profile.QRegionProfileManager
 import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererCustomization
@@ -47,6 +48,10 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.RequestCon
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.ResponseContext
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getTelemetryOptOutPreference
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.DiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDiagnosticDifferences
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
+import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
 import software.aws.toolkits.telemetry.CodewhispererCompletionType
 import software.aws.toolkits.telemetry.CodewhispererSuggestionState
 import java.time.Instant
@@ -340,7 +345,17 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         ) {
             e2eLatency = 0.0
         }
-
+        var diffDiagnostics = DiagnosticDifferences(
+            added = emptyList(),
+            removed = emptyList()
+        )
+        if (suggestionState == CodewhispererSuggestionState.Accept && isInternalUser(getStartUrl(project))) {
+            val oldDiagnostics = requestContext.diagnostics.orEmpty()
+            // wait for the IDE itself to update its diagnostics for current file
+            Thread.sleep(500)
+            val newDiagnostics = getDocumentDiagnostics(requestContext.editor.document, project)
+            diffDiagnostics = getDiagnosticDifferences(oldDiagnostics, newDiagnostics)
+        }
         return bearerClient().sendTelemetryEvent { requestBuilder ->
             requestBuilder.telemetryEvent { telemetryEventBuilder ->
                 telemetryEventBuilder.userTriggerDecisionEvent {
@@ -358,6 +373,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                     it.customizationArn(requestContext.customizationArn.nullize(nullizeSpaces = true))
                     it.numberOfRecommendations(numberOfRecommendations)
                     it.acceptedCharacterCount(acceptedCharCount)
+                    it.addedIdeDiagnostics(diffDiagnostics.added)
+                    it.removedIdeDiagnostics(diffDiagnostics.removed)
                 }
             }
             requestBuilder.optOutPreference(getTelemetryOptOutPreference())

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt

@@ -36,6 +36,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.Completion
 import software.amazon.awssdk.services.codewhispererruntime.model.FileContext
 import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsRequest
 import software.amazon.awssdk.services.codewhispererruntime.model.GenerateCompletionsResponse
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
 import software.amazon.awssdk.services.codewhispererruntime.model.ProgrammingLanguage
 import software.amazon.awssdk.services.codewhispererruntime.model.RecommendationsWithReferencesPreference
 import software.amazon.awssdk.services.codewhispererruntime.model.ResourceNotFoundException
@@ -87,6 +88,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhisperer
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.notifyErrorCodeWhispererUsageLimit
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.promptReAuth
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.FileContextProvider
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.getDocumentDiagnostics
 import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings
 import software.aws.toolkits.jetbrains.utils.isInjectedText
 import software.aws.toolkits.jetbrains.utils.isQExpired
@@ -691,6 +693,7 @@ class CodeWhispererService(private val cs: CoroutineScope) : Disposable {
         } catch (e: Exception) {
             LOG.warn { "Cannot get workspaceId from LSP'$e'" }
         }
+        val diagnostics = getDocumentDiagnostics(editor.document, project)
         return RequestContext(
             project,
             editor,
@@ -703,6 +706,7 @@ class CodeWhispererService(private val cs: CoroutineScope) : Disposable {
             customizationArn,
             profileArn,
             workspaceId,
+            diagnostics
         )
     }
 
@@ -895,6 +899,7 @@ data class RequestContext(
     val customizationArn: String?,
     val profileArn: String?,
     val workspaceId: String?,
+    val diagnostics: List<IdeDiagnostic>?,
 ) {
     // TODO: should make the entire getRequestContext() suspend function instead of making supplemental context only
     var supplementalContext: SupplementalContextInfo? = null

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererUtil.kt

@@ -3,12 +3,16 @@
 
 package software.aws.toolkits.jetbrains.services.codewhisperer.util
 
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.lookup.LookupManager
 import com.intellij.ide.BrowserUtil
+import com.intellij.lang.annotation.HighlightSeverity
 import com.intellij.notification.NotificationAction
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.runInEdt
+import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.Editor
+import com.intellij.openapi.editor.impl.DocumentMarkupModel
 import com.intellij.openapi.editor.impl.EditorImpl
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VfsUtil
@@ -22,7 +26,10 @@ import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.yield
 import software.amazon.awssdk.services.codewhispererruntime.model.Completion
+import software.amazon.awssdk.services.codewhispererruntime.model.IdeDiagnostic
 import software.amazon.awssdk.services.codewhispererruntime.model.OptOutPreference
+import software.amazon.awssdk.services.codewhispererruntime.model.Position
+import software.amazon.awssdk.services.codewhispererruntime.model.Range
 import software.aws.toolkits.core.utils.getLogger
 import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.core.credentials.AwsBearerTokenConnection
@@ -361,3 +368,88 @@ object CodeWhispererUtil {
 enum class CaretMovement {
     NO_CHANGE, MOVE_FORWARD, MOVE_BACKWARD
 }
+
+fun getDiagnosticsType(message: String): String {
+    val lowercaseMessage = message.lowercase()
+
+    val diagnosticPatterns = mapOf(
+        "TYPE_ERROR" to listOf("type", "cast"),
+        "SYNTAX_ERROR" to listOf("expected", "indent", "syntax"),
+        "REFERENCE_ERROR" to listOf("undefined", "not defined", "undeclared", "reference", "symbol"),
+        "BEST_PRACTICE" to listOf("deprecated", "unused", "uninitialized", "not initialized"),
+        "SECURITY" to listOf("security", "vulnerability")
+    )
+
+    return diagnosticPatterns
+        .entries
+        .firstOrNull { (_, keywords) ->
+            keywords.any { lowercaseMessage.contains(it) }
+        }
+        ?.key ?: "OTHER"
+}
+
+fun convertSeverity(severity: HighlightSeverity): String = when {
+    severity == HighlightSeverity.ERROR -> "ERROR"
+    severity == HighlightSeverity.WARNING ||
+        severity == HighlightSeverity.WEAK_WARNING -> "WARNING"
+    severity == HighlightSeverity.INFORMATION -> "INFORMATION"
+    severity.toString().contains("TEXT", ignoreCase = true) -> "HINT"
+    severity == HighlightSeverity.INFO -> "INFORMATION"
+    // For severities that might indicate performance issues
+    severity.toString().contains("PERFORMANCE", ignoreCase = true) -> "WARNING"
+    // For deprecation warnings
+    severity.toString().contains("DEPRECATED", ignoreCase = true) -> "WARNING"
+    // Default case
+    else -> "INFORMATION"
+}
+
+fun getDocumentDiagnostics(document: Document, project: Project): List<IdeDiagnostic> = runCatching {
+    DocumentMarkupModel.forDocument(document, project, true)
+        .allHighlighters
+        .mapNotNull { it.errorStripeTooltip as? HighlightInfo }
+        .filter { !it.description.isNullOrEmpty() }
+        .map { info ->
+            val startLine = document.getLineNumber(info.startOffset)
+            val endLine = document.getLineNumber(info.endOffset)
+
+            IdeDiagnostic.builder()
+                .ideDiagnosticType(getDiagnosticsType(info.description))
+                .severity(convertSeverity(info.severity))
+                .source(info.inspectionToolId)
+                .range(
+                    Range.builder()
+                        .start(
+                            Position.builder()
+                                .line(startLine)
+                                .character(document.getLineStartOffset(startLine))
+                                .build()
+                        )
+                        .end(
+                            Position.builder()
+                                .line(endLine)
+                                .character(document.getLineStartOffset(endLine))
+                                .build()
+                        )
+                        .build()
+                )
+                .build()
+        }
+}.getOrElse { e ->
+    getLogger<CodeWhispererUtil>().warn { "Failed to get document diagnostics ${e.message}" }
+    emptyList()
+}
+
+data class DiagnosticDifferences(
+    val added: List<IdeDiagnostic>,
+    val removed: List<IdeDiagnostic>,
+)
+
+fun serializeDiagnostics(diagnostic: IdeDiagnostic): String = "${diagnostic.source()}-${diagnostic.severity()}-${diagnostic.ideDiagnosticType()}"
+
+fun getDiagnosticDifferences(oldDiagnostic: List<IdeDiagnostic>, newDiagnostic: List<IdeDiagnostic>): DiagnosticDifferences {
+    val oldSet = oldDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
+    val newSet = newDiagnostic.map { i -> serializeDiagnostics(i) }.toSet()
+    val added = newDiagnostic.filter { i -> !oldSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
+    val removed = oldDiagnostic.filter { i -> !newSet.contains(serializeDiagnostics(i)) }.distinctBy { serializeDiagnostics(it) }
+    return DiagnosticDifferences(added, removed)
+}

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt

@@ -179,6 +179,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
             aString(),
             aString(),
             aString(),
+            emptyList()
         )
         val responseContext = ResponseContext("sessionId")
         val recommendationContext = RecommendationContext(

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererServiceTest.kt

@@ -215,6 +215,7 @@ class CodeWhispererServiceTest {
                 customizationArn = "fake-arn",
                 profileArn = "fake-arn",
                 workspaceId = null,
+                diagnostics = emptyList()
             )
         )