contribute a prompt benchmarking script

pkgs/sdk_triage_bot/bin/triage.dart

@@ -44,7 +44,7 @@ void main(List<String> arguments) async {
 
   var issue = results.rest.first;
   final dryRun = results.flag('dry-run');
-  final force = results.flag('force');
+  final forceTriage = results.flag('force');
 
   // Accept either an issue number or a url (i.e.,
   // https://github.com/dart-lang/sdk/issues/55816).
@@ -69,7 +69,7 @@ void main(List<String> arguments) async {
   await triage(
     int.parse(issue),
     dryRun: dryRun,
-    force: force,
+    forceTriage: forceTriage,
     githubService: githubService,
     geminiService: geminiService,
     logger: Logger(),

pkgs/sdk_triage_bot/lib/src/gemini.dart

@@ -6,22 +6,26 @@ import 'package:google_generative_ai/google_generative_ai.dart';
 import 'package:http/http.dart' as http;
 
 class GeminiService {
+  // gemini-1.5-pro-latest, gemini-1.5-flash-latest, gemini-1.0-pro-latest
+  static const String classificationModel = 'models/gemini-1.5-flash-latest';
+  static const String summarizationModel = 'models/gemini-1.5-flash-latest';
+
   final GenerativeModel _summarizeModel;
   final GenerativeModel _classifyModel;
 
   GeminiService({
     required String apiKey,
     required http.Client httpClient,
   })  : _summarizeModel = GenerativeModel(
-          model: 'models/gemini-1.5-flash-latest',
+          model: summarizationModel,
           apiKey: apiKey,
           generationConfig: GenerationConfig(temperature: 0.2),
           httpClient: httpClient,
         ),
         _classifyModel = GenerativeModel(
-          // TODO(devconcarew): substitute our tuned model
+          // TODO(devoncarew): substitute our tuned model
           // model: 'tunedModels/autotune-sdk-triage-tuned-prompt-1l96e2n',
-          model: 'models/gemini-1.5-flash-latest',
+          model: classificationModel,
           apiKey: apiKey,
           generationConfig: GenerationConfig(temperature: 0.2),
           httpClient: httpClient,
@@ -45,6 +49,6 @@ class GeminiService {
 
   Future<String> _query(GenerativeModel model, String prompt) async {
     final response = await model.generateContent([Content.text(prompt)]);
-    return response.text!.trim();
+    return (response.text ?? '').trim();
   }
 }

pkgs/sdk_triage_bot/lib/src/github.dart

@@ -43,10 +43,15 @@ class GithubService {
 
 Future<FetchIssuesResult> fetchIssues(
   String areaLabel, {
+  required bool includeClosed,
   String? cursor,
 }) async {
   final result = await _query(QueryOptions(
-    document: gql(_buildQueryString(areaLabel, cursor: cursor)),
+    document: gql(_buildQueryString(
+      areaLabel,
+      cursor: cursor,
+      includeClosed: includeClosed,
+    )),
     fetchPolicy: FetchPolicy.noCache,
     parserFn: (data) {
       final search = data['search'] as Map<String, dynamic>;
@@ -104,41 +109,46 @@ Future<QueryResult<T>> _query<T>(QueryOptions<T> options) {
   return _client.query<T>(options);
 }
 
-String _buildQueryString(String areaLabel, {String? cursor}) {
-  final cursorRef = cursor == null ? null : '"$cursor"';
+String _buildQueryString(
+  String areaLabel, {
+  required bool includeClosed,
+  String? cursor,
+}) {
+  final cursorTerm = cursor == null ? '' : 'after: "$cursor"';
+  final isOpen = includeClosed ? '' : 'is:open';
 
   return '''{
-  search(
-    query: "repo:dart-lang/sdk is:issue is:open label:$areaLabel"
-    type: ISSUE
-    first: 100,
-    after: $cursorRef
-  ) {
-    edges {
-      node {
-        ... on Issue {
-          title
-          number
-          state
-          bodyText
-          labels(first: 10) {
-            edges {
-              node {
-                name
+    search(
+      query: "repo:dart-lang/sdk is:issue $isOpen label:$areaLabel"
+      type: ISSUE
+      first: 100
+      $cursorTerm
+    ) {
+      edges {
+        node {
+          ... on Issue {
+            title
+            number
+            state
+            bodyText
+            labels(first: 10) {
+              edges {
+                node {
+                  name
+                }
               }
             }
           }
         }
       }
+      pageInfo {
+        endCursor
+        startCursor
+        hasNextPage
+        hasPreviousPage
+      }
     }
-    pageInfo {
-      endCursor
-      startCursor
-      hasNextPage
-      hasPreviousPage
-    }
-  }
-}''';
+  }''';
 }
 
 final GraphQLClient _client = _initGraphQLClient();
@@ -158,4 +168,17 @@ extension IssueExtension on Issue {
   ///
   /// Note that the original text for the issue is returned in the `body` field.
   bool get hasComments => commentsCount > 0;
+
+  /// Returns whether this issue has already been triaged.
+  ///
+  /// Generally, this means the the issue has had an `area-` label applied to
+  /// it, has had `needs-info` applied to it, or was closed.
+  bool get alreadyTriaged {
+    if (isClosed) return true;
+
+    return labels.any((label) {
+      final name = label.name;
+      return name == 'needs-info' || name.startsWith('area-');
+    });
+  }
 }

pkgs/sdk_triage_bot/lib/src/prompts.dart

@@ -28,10 +28,11 @@ area-infrastructure: Use area-infrastructure for SDK infrastructure issues, like
 area-intellij: Tracking issues for the Dart IntelliJ plugin.
 area-language: Dart language related items (some items might be better tracked at github.com/dart-lang/language).
 area-meta: Cross-cutting, high-level issues (for tracking many other implementation issues, ...).
+area-native-interop: Used for native interop related issues, including FFI.
 area-pkg: Used for miscellaneous pkg/ packages not associated with specific area- teams.
 area-sdk: Use area-sdk for general purpose SDK issues (packaging, distribution, …).
 area-test: Cross-cutting test issues (use area- labels for specific failures; not used for package:test).
-area-vm: Use area-vm for VM related issues, including code coverage, FFI, and the AOT and JIT backends.
+area-vm: Use area-vm for VM related issues, including code coverage, and the AOT and JIT backends.
 area-web: Use area-web for Dart web related issues, including the DDC and dart2js compilers and JS interop.
 
 Don't make up a new area.

pkgs/sdk_triage_bot/lib/triage.dart

@@ -17,7 +17,7 @@ final sdkSlug = RepositorySlug('dart-lang', 'sdk');
 Future<void> triage(
   int issueNumber, {
   bool dryRun = false,
-  bool force = false,
+  bool forceTriage = false,
   required GithubService githubService,
   required GeminiService geminiService,
   required Logger logger,
@@ -63,21 +63,22 @@ ${trimmedBody(comment.body ?? '')}
   }
 
   // decide if we should triage
-  final alreadyTriaged = labels.any((l) => l.startsWith('area-'));
-  if (alreadyTriaged && !force) {
-    logger.log('Exiting (issue is already triaged).');
-    return;
+  if (!forceTriage) {
+    if (issue.alreadyTriaged) {
+      logger.log('Exiting (issue is already triaged).');
+      return;
+    }
   }
 
   // ask for the summary
   var bodyTrimmed = trimmedBody(issue.body);
   String summary;
   try {
-    // Failures here can include things like gemini safety issues, ...
     summary = await geminiService.summarize(
       summarizeIssuePrompt(title: issue.title, body: bodyTrimmed),
     );
   } on GenerativeAIException catch (e) {
+    // Failures here can include things like gemini safety issues, ...
     stderr.writeln('gemini: $e');
     exit(1);
   }
@@ -88,21 +89,21 @@ ${trimmedBody(comment.body ?? '')}
   logger.log('');
 
   // ask for the 'area-' classification
-  List<String> classification;
+  List<String> newLabels;
   try {
-    // Failures here can include things like gemini safety issues, ...
-    classification = await geminiService.classify(
+    newLabels = await geminiService.classify(
       assignAreaPrompt(
           title: issue.title, body: bodyTrimmed, lastComment: lastComment),
     );
   } on GenerativeAIException catch (e) {
+    // Failures here can include things like gemini safety issues, ...
     stderr.writeln('gemini: $e');
     exit(1);
   }
 
   logger.log('## gemini classification');
   logger.log('');
-  logger.log(classification.toString());
+  logger.log(newLabels.toString());
   logger.log('');
 
   if (dryRun) {
@@ -113,7 +114,7 @@ ${trimmedBody(comment.body ?? '')}
   // perform changes
   logger.log('## github comment');
   logger.log('');
-  logger.log('labels: $classification');
+  logger.log('labels: $newLabels');
   logger.log('');
   logger.log(summary);
 
@@ -122,17 +123,16 @@ ${trimmedBody(comment.body ?? '')}
   // create github comment
   await githubService.createComment(sdkSlug, issueNumber, comment);
 
-  final allLabels = await githubService.getAllLabels(sdkSlug);
-  var newLabels = filterExistingLabels(allLabels, classification);
-  if (newLabels.any((l) => l.startsWith('area-'))) {
-    newLabels.add('triage-automation');
+  final allRepoLabels = (await githubService.getAllLabels(sdkSlug)).toSet();
+  final labelAdditions = newLabels.toSet().union(allRepoLabels).toList()
+    ..sort();
+  if (labelAdditions.isNotEmpty) {
+    labelAdditions.add('triage-automation');
   }
-  // remove any duplicates
-  newLabels = newLabels.toSet().toList();
 
   // apply github labels
   if (newLabels.isNotEmpty) {
-    await githubService.addLabelsToIssue(sdkSlug, issueNumber, newLabels);
+    await githubService.addLabelsToIssue(sdkSlug, issueNumber, labelAdditions);
   }
 
   logger.log('');

pkgs/sdk_triage_bot/test/triage_test.dart

@@ -63,7 +63,7 @@ void main() {
 
     await triage(
       mockIssueNumber,
-      force: true,
+      forceTriage: true,
       githubService: githubService,
       geminiService: geminiService,
       logger: TestLogger(),