diff --git a/dartdoc_options.yaml b/dartdoc_options.yaml index fc6bafd1bd734..204755d6a4a69 100644 --- a/dartdoc_options.yaml +++ b/dartdoc_options.yaml @@ -5,13 +5,13 @@ dartdoc: # The dev/bots/docs.sh script does this automatically. tools: snippet: - command: ["bin/cache/dart-sdk/bin/dart", "pub", "global", "run", "snippets", "--output-directory=doc/snippets", "--type=snippet"] + command: ["bin/cache/artifacts/snippets/snippets", "--output-directory=doc/snippets", "--type=snippet"] description: "Creates sample code documentation output from embedded documentation samples." sample: - command: ["bin/cache/dart-sdk/bin/dart", "pub", "global", "run", "snippets", "--output-directory=doc/snippets", "--type=sample"] + command: ["bin/cache/artifacts/snippets/snippets", "--output-directory=doc/snippets", "--type=sample"] description: "Creates full application sample code documentation output from embedded documentation samples." dartpad: - command: ["bin/cache/dart-sdk/bin/dart", "pub", "global", "run", "snippets", "--output-directory=doc/snippets", "--type=dartpad"] + command: ["bin/cache/artifacts/snippets/snippets", "--output-directory=doc/snippets", "--type=dartpad"] description: "Creates full application sample code documentation output from embedded documentation samples and displays it in an embedded DartPad." errors: ## Default errors of dartdoc: diff --git a/dev/bots/docs.sh b/dev/bots/docs.sh index e82ed0b1a4f4d..a91cc0086e50a 100755 --- a/dev/bots/docs.sh +++ b/dev/bots/docs.sh @@ -107,16 +107,25 @@ function parse_args() { fi } +function build_snippets_tool() ( + local snippets_dir="$FLUTTER_ROOT/dev/snippets" + local output_dir="$FLUTTER_BIN/cache/artifacts/snippets" + echo "Building snippets tool executable." + command cd "$snippets_dir" + mkdir -p "$output_dir" + dart pub get + dart compile exe -o "$output_dir/snippets" bin/snippets.dart +) + function generate_docs() { # Install and activate dartdoc. # When updating to a new dartdoc version, please also update # `dartdoc_options.yaml` to include newly introduced error and warning types. "$DART" pub global activate dartdoc 8.0.6 - # Install and activate the snippets tool, which resides in the - # assets-for-api-docs repo: - # https://github.com/flutter/assets-for-api-docs/tree/main/packages/snippets - "$DART" pub global activate snippets 0.4.3 + # Build and install the snippets tool, which resides in + # the dev/docs/snippets directory. + build_snippets_tool # This script generates a unified doc set, and creates # a custom index.html, placing everything into DOC_DIR. diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 84faee8de2095..84ed09e99d302 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -250,13 +250,14 @@ Future main(List args) async { 'web_long_running_tests': () => webLongRunningTestsRunner(flutterRoot), 'flutter_plugins': _runFlutterPackagesTests, 'skp_generator': skpGeneratorTestsRunner, - 'realm_checker': _runRealmCheckerTest, - 'customer_testing': _runCustomerTesting, - 'analyze': _runAnalyze, - 'fuchsia_precache': _runFuchsiaPrecache, - 'docs': _runDocs, - 'verify_binaries_codesigned': _runVerifyCodesigned, - kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc. + 'realm_checker': realmCheckerTestRunner, + 'customer_testing': customerTestingRunner, + 'analyze': analyzeRunner, + 'fuchsia_precache': fuchsiaPrecacheRunner, + 'snippets': _runSnippetsTests, + 'docs': docsRunner, + 'verify_binaries_codesigned': verifyCodesignedTestRunner, + kTestHarnessShardName: testHarnessTestsRunner, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc. }); } catch (error, stackTrace) { foundError([ @@ -506,6 +507,21 @@ Future _runToolTests() async { }); } +Future _runSnippetsTests() async { + final String snippetsPath = path.join(flutterRoot, 'dev', 'snippets'); + final List allTests = Directory(path.join(snippetsPath, 'test')) + .listSync(recursive: true).whereType() + .map((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath)) + .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList(); + + await runDartTest( + snippetsPath, + forceSingleCore: true, + testPaths: selectIndexOfTotalSubshard(allTests), + collectMetrics: true, + ); +} + Future runForbiddenFromReleaseTests() async { // Build a release APK to get the snapshot json. final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_forbidden_imports.'); diff --git a/dev/snippets/README.md b/dev/snippets/README.md new file mode 100644 index 0000000000000..a28e50db97230 --- /dev/null +++ b/dev/snippets/README.md @@ -0,0 +1,231 @@ +# Dartdoc Sample Generation + +The Flutter API documentation contains code blocks that help provide context or +a good starting point when learning to use any of Flutter's APIs. + +To generate these code blocks, Flutter uses dartdoc tools to turn documentation +in the source code into API documentation, as seen on [https://api.flutter.dev/] + +## Table of Contents + +- [Types of code blocks](#types-of-code-blocks) + - [Snippet tool](#snippet-tool) + - [Sample tool](#sample-tool) +- [Skeletons](#skeletons) +- [Test Doc Generation Workflow](#test-doc-generation-workflow) + +## Types of code blocks + +There are three kinds of code blocks. + +- A `snippet`, which is a more or less context-free code snippet that we + magically determine how to analyze. + +- A `dartpad` sample, which gets placed into a full-fledged application, and can + be executed inline in the documentation on the web page using + DartPad. + +- A `sample`, which gets placed into a full-fledged application, but isn't + placed into DartPad in the documentation because it doesn't make sense to do + so. + +Ideally, every sample is a DartPad sample, but some samples don't have any visual +representation and some just don't make sense that way (for example, sample +code for setting the system UI's notification area color on Android won't do +anything on the web). + +### Snippet Tool + +![Code snippet image](assets/code_snippet.png) + +The code `snippet` tool generates a block containing a description and example +code. Here is an example of the code `snippet` tool in use: + +```dart +/// {@tool snippet} +/// +/// If the avatar is to have an image, the image should be specified in the +/// [backgroundImage] property: +/// +/// ```dart +/// CircleAvatar( +/// backgroundImage: NetworkImage(userAvatarUrl), +/// ) +/// ``` +/// {@end-tool} +``` + +This will generate sample code that can be copied to the clipboard and added to +existing applications. + +This uses the skeleton for `snippet` snippets when generating the HTML to put +into the Dart docs. You can find this [template in the Flutter +repo](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/snippet.html). + +#### Analysis + +The +[`analyze_sample_code.dart`](https://github.com/flutter/flutter/blob/main/dev/bots/analyze_sample_code.dart) +script finds code inside the `@tool +snippet` sections and uses the Dart analyzer to check them. + +There are several kinds of sample code you can specify: + +- Constructor calls, typically showing what might exist in a build method. These + will be inserted into an assignment expression assigning to a variable of type + "dynamic" and followed by a semicolon, for analysis. + +- Class definitions. These start with "class", and are analyzed verbatim. + +- Other code. It gets included verbatim, though any line that says `// ...` is + considered to separate the block into multiple blocks to be processed + individually. + +The above means that it's tricky to include verbatim imperative code (e.g. a +call to a method) since it won't be valid to have such code at the top level. +Instead, wrap it in a function or even a whole class, or make it a valid +variable declaration. + +You can declare code that should be included in the analysis but not shown in +the API docs by adding a comment "// Examples can assume:" to the file (usually +at the top of the file, after the imports), following by one or more +commented-out lines of code. That code is included verbatim in the analysis. For +example: + +```dart +// Examples can assume: +// final BuildContext context; +// final String userAvatarUrl; +``` + +You can assume that the entire Flutter framework and most common +`dart:*` packages are imported and in scope; `dart:math` as `math` and +`dart:ui` as `ui`. + +### Sample Tool + +![Code sample image](assets/code_sample.png) + +The code `sample` and `dartpad` tools can expand sample code into full Flutter +applications. These sample applications can be directly copied and used to +demonstrate the API's functionality in a sample application, or used with the +`flutter create` command to create a local project with the sample code. The +`dartpad` samples are embedded into the API docs web page and are live +applications in the API documentation. + +```dart +/// {@tool sample --template=stateless_widget_material} +/// This example shows how to make a simple [FloatingActionButton] in a +/// [Scaffold], with a pink [backgroundColor] and a thumbs up [Icon]. +/// +/// ```dart +/// Widget build(BuildContext context) { +/// return Scaffold( +/// appBar: AppBar( +/// title: Text('Floating Action Button Sample'), +/// ), +/// body: Center( +/// child: Text('Press the button below!') +/// ), +/// floatingActionButton: FloatingActionButton( +/// onPressed: () { +/// // Add your onPressed code here! +/// }, +/// child: Icon(Icons.thumb_up), +/// backgroundColor: Colors.pink, +/// ), +/// ); +/// } +/// ``` +/// {@end-tool} +``` + +This uses the skeleton for [application](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/sample.html) +snippets in the Flutter repo. + +The `sample` and `dartpad` tools also allow for quick Flutter app generation +using the following command: + +```bash +flutter create --sample=[directory.File.sampleNumber] [name_of_project_directory] +``` + +This command is displayed as part of the sample in the API docs. + +#### Sample Analysis + +The [`../bots/analyze_sample_code.dart`](../bots/analyze_sample_code.dart) +script finds code inside the `@tool sample` sections and uses the Dart analyzer +to check the sample code. + +## Skeletons + +A skeleton (concerning this tool) is an HTML template into which the Dart +code blocks and descriptions are interpolated. + +There is currently one skeleton for +[application](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/sample.html) +samples, one for +[dartpad](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/dartpad-sample.html), +and one for +[snippet](https://github.com/flutter/flutter/blob/main/dev/snippets/config/skeletons/snippet.html) +code samples, but there could be more. + +Skeletons use mustache notation (e.g. `{{code}}`) to mark where components will +be interpolated into the template. It doesn't use the mustache +package since these are simple string substitutions, but it uses the same +syntax. + +The code block generation tools that process the source input and emit HTML for +output, which dartdoc places back into the documentation. Any options given to +the `{@tool ...}` directive are passed on verbatim to the tool. + +The `snippets` tool renders these examples through a combination of markdown +and HTML using the `{@inject-html}` dartdoc directive. + +## Test Doc Generation Workflow + +If you are making changes to an existing code block or are creating a new code +block, follow these steps to generate a local copy of the API docs and verify +that your code blocks are showing up correctly: + +1. Make an update to a code block or create a new code block. +2. From the root directory, run `./dev/bots/docs.sh`. This should start + generating a local copy of the API documentation. + Supplying the "--output" argument allows you to specify the output zip file + for the completed documentation. Defaults to `api_docs.zip`` in the current + directory. +3. Once complete, unzip the files to the desired location and open the `index.html` + within. + +Note that generating the sample output will not allow you to run your code in +DartPad, because DartPad pulls the code it runs from the appropriate docs server +(main or stable). + +Copy the generated code and paste it into a regular DartPad instance to test if +it runs in DartPad. To get the code that will be produced by your documentation +changes, run sample analysis locally (see the next section) and paste the output +into a DartPad at [https://dartpad.dartlang.org]. + +## Running sample analysis locally + +If all you want to do is analyze the sample code you have written locally, then +generating the entire docs output takes a long time. + +Instead, you can run the analysis locally with this command from the Flutter root: + +```bash +TMPDIR=/tmp bin/cache/dart-sdk/bin/dart dev/bots/analyze_sample_code.dart --temp=samples +``` + +This will analyze the samples, and leave the generated files in `/tmp/samples` + +You can find the sample you are working on in `/tmp/samples`. It is named using the +path to the file it is in, and the line of the file that the `{@tool ...}` directive +is on. + +For example, the file `sample.src.widgets.animated_list.52.dart` points to the sample +in `packages/flutter/src/widgets/animated_list.dart` at line 52. You can then take the +contents of that file, and paste it into [Dartpad](https://dartpad.dev) and see if it +works. If the sample relies on new features that have just landed, it may not work +until the features make it into the `dev` branch. diff --git a/dev/snippets/bin/snippets.dart b/dev/snippets/bin/snippets.dart new file mode 100644 index 0000000000000..8590cadf4ba23 --- /dev/null +++ b/dev/snippets/bin/snippets.dart @@ -0,0 +1,295 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' show ProcessResult, exitCode, stderr; + +import 'package:args/args.dart'; +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; +import 'package:snippets/snippets.dart'; + +const String _kElementOption = 'element'; +const String _kFormatOutputOption = 'format-output'; +const String _kHelpOption = 'help'; +const String _kInputOption = 'input'; +const String _kLibraryOption = 'library'; +const String _kOutputDirectoryOption = 'output-directory'; +const String _kOutputOption = 'output'; +const String _kPackageOption = 'package'; +const String _kSerialOption = 'serial'; +const String _kTemplateOption = 'template'; +const String _kTypeOption = 'type'; + +class GitStatusFailed implements Exception { + GitStatusFailed(this.gitResult); + + final ProcessResult gitResult; + + @override + String toString() { + return 'git status exited with a non-zero exit code: ' + '${gitResult.exitCode}:\n${gitResult.stderr}\n${gitResult.stdout}'; + } +} + +/// A singleton filesystem that can be set by tests to a memory filesystem. +FileSystem filesystem = const LocalFileSystem(); + +/// A singleton snippet generator that can be set by tests to a mock, so that +/// we can test the command line parsing. +SnippetGenerator snippetGenerator = SnippetGenerator(); + +/// A singleton platform that can be set by tests for use in testing command line +/// parsing. +Platform platform = const LocalPlatform(); + +/// A singleton process manager that can be set by tests for use in testing. +ProcessManager processManager = const LocalProcessManager(); + +/// Get the name of the channel these docs are from. +/// +/// First check env variable LUCI_BRANCH, then refer to the currently +/// checked out git branch. +String getChannelName({ + Platform platform = const LocalPlatform(), + ProcessManager processManager = const LocalProcessManager(), +}) { + final String? envReleaseChannel = platform.environment['LUCI_BRANCH']?.trim(); + if (['master', 'stable', 'main'].contains(envReleaseChannel)) { + // Backward compatibility: Still support running on "master", but pretend it is "main". + if (envReleaseChannel == 'master') { + return 'main'; + } + return envReleaseChannel!; + } + + final RegExp gitBranchRegexp = RegExp(r'^## (?.*)'); + final ProcessResult gitResult = processManager.runSync( + ['git', 'status', '-b', '--porcelain'], + // Use the FLUTTER_ROOT, if defined. + workingDirectory: platform.environment['FLUTTER_ROOT']?.trim() ?? + filesystem.currentDirectory.path, + // Adding extra debugging output to help debug why git status inexplicably fails + // (random non-zero error code) about 2% of the time. + environment: {'GIT_TRACE': '2', 'GIT_TRACE_SETUP': '2'}); + if (gitResult.exitCode != 0) { + throw GitStatusFailed(gitResult); + } + + final RegExpMatch? gitBranchMatch = gitBranchRegexp + .firstMatch((gitResult.stdout as String).trim().split('\n').first); + return gitBranchMatch == null + ? '' + : gitBranchMatch.namedGroup('branch')!.split('...').first; +} + +const List sampleTypes = [ + 'snippet', + 'sample', + 'dartpad', +]; + +// This is a hack to workaround the fact that git status inexplicably fails +// (with random non-zero error code) about 2% of the time. +String getChannelNameWithRetries({ + Platform platform = const LocalPlatform(), + ProcessManager processManager = const LocalProcessManager(), +}) { + int retryCount = 0; + + while (retryCount < 2) { + try { + return getChannelName(platform: platform, processManager: processManager); + } on GitStatusFailed catch (e) { + retryCount += 1; + stderr.write( + 'git status failed, retrying ($retryCount)\nError report:\n$e'); + } + } + + return getChannelName(platform: platform, processManager: processManager); +} + +/// Generates snippet dartdoc output for a given input, and creates any sample +/// applications needed by the snippet. +void main(List argList) { + final Map environment = platform.environment; + final ArgParser parser = ArgParser(); + + parser.addOption( + _kTypeOption, + defaultsTo: 'dartpad', + allowed: sampleTypes, + allowedHelp: { + 'dartpad': + 'Produce a code sample application complete with embedding the sample in an ' + 'application template for using in Dartpad.', + 'sample': + 'Produce a code sample application complete with embedding the sample in an ' + 'application template.', + 'snippet': + 'Produce a nicely formatted piece of sample code. Does not embed the ' + 'sample into an application template.', + }, + help: 'The type of snippet to produce.', + ); + // TODO(goderbauer): Remove template support, this is no longer used. + parser.addOption( + _kTemplateOption, + help: 'The name of the template to inject the code into.', + ); + parser.addOption( + _kOutputOption, + help: 'The output name for the generated sample application. Overrides ' + 'the naming generated by the --$_kPackageOption/--$_kLibraryOption/--$_kElementOption ' + 'arguments. Metadata will be written alongside in a .json file. ' + 'The basename of this argument is used as the ID. If this is a ' + 'relative path, will be placed under the --$_kOutputDirectoryOption location.', + ); + parser.addOption( + _kOutputDirectoryOption, + defaultsTo: '.', + help: 'The output path for the generated sample application.', + ); + parser.addOption( + _kInputOption, + defaultsTo: environment['INPUT'], + help: 'The input file containing the sample code to inject.', + ); + parser.addOption( + _kPackageOption, + defaultsTo: environment['PACKAGE_NAME'], + help: 'The name of the package that this sample belongs to.', + ); + parser.addOption( + _kLibraryOption, + defaultsTo: environment['LIBRARY_NAME'], + help: 'The name of the library that this sample belongs to.', + ); + parser.addOption( + _kElementOption, + defaultsTo: environment['ELEMENT_NAME'], + help: 'The name of the element that this sample belongs to.', + ); + parser.addOption( + _kSerialOption, + defaultsTo: environment['INVOCATION_INDEX'], + help: 'A unique serial number for this snippet tool invocation.', + ); + parser.addFlag( + _kFormatOutputOption, + defaultsTo: true, + help: 'Applies the Dart formatter to the published/extracted sample code.', + ); + parser.addFlag( + _kHelpOption, + negatable: false, + help: 'Prints help documentation for this command', + ); + + final ArgResults args = parser.parse(argList); + + if (args[_kHelpOption]! as bool) { + stderr.writeln(parser.usage); + exitCode = 0; + return; + } + + final String sampleType = args[_kTypeOption]! as String; + + if (args[_kInputOption] == null) { + stderr.writeln(parser.usage); + errorExit( + 'The --$_kInputOption option must be specified, either on the command ' + 'line, or in the INPUT environment variable.'); + return; + } + + final File input = filesystem.file(args['input']! as String); + if (!input.existsSync()) { + errorExit('The input file ${input.path} does not exist.'); + return; + } + + final bool formatOutput = args[_kFormatOutputOption]! as bool; + final String packageName = args[_kPackageOption] as String? ?? ''; + final String libraryName = args[_kLibraryOption] as String? ?? ''; + final String elementName = args[_kElementOption] as String? ?? ''; + final String serial = args[_kSerialOption] as String? ?? ''; + late String id; + File? output; + final Directory outputDirectory = + filesystem.directory(args[_kOutputDirectoryOption]! as String).absolute; + + if (args[_kOutputOption] != null) { + id = path.basenameWithoutExtension(args[_kOutputOption]! as String); + final File outputPath = filesystem.file(args[_kOutputOption]! as String); + if (outputPath.isAbsolute) { + output = outputPath; + } else { + output = + filesystem.file(path.join(outputDirectory.path, outputPath.path)); + } + } else { + final List idParts = []; + if (packageName.isNotEmpty && packageName != 'flutter') { + idParts.add(packageName.replaceAll(RegExp(r'\W'), '_').toLowerCase()); + } + if (libraryName.isNotEmpty) { + idParts.add(libraryName.replaceAll(RegExp(r'\W'), '_').toLowerCase()); + } + if (elementName.isNotEmpty) { + idParts.add(elementName); + } + if (serial.isNotEmpty) { + idParts.add(serial); + } + if (idParts.isEmpty) { + errorExit('Unable to determine ID. At least one of --$_kPackageOption, ' + '--$_kLibraryOption, --$_kElementOption, -$_kSerialOption, or the environment variables ' + 'PACKAGE_NAME, LIBRARY_NAME, ELEMENT_NAME, or INVOCATION_INDEX must be non-empty.'); + return; + } + id = idParts.join('.'); + output = outputDirectory.childFile('$id.dart'); + } + output.parent.createSync(recursive: true); + + final int? sourceLine = environment['SOURCE_LINE'] != null + ? int.tryParse(environment['SOURCE_LINE']!) + : null; + final String sourcePath = environment['SOURCE_PATH'] ?? 'unknown.dart'; + final SnippetDartdocParser sampleParser = SnippetDartdocParser(filesystem); + final SourceElement element = sampleParser.parseFromDartdocToolFile( + input, + startLine: sourceLine, + element: elementName, + sourceFile: filesystem.file(sourcePath), + type: sampleType, + ); + final Map metadata = { + 'channel': getChannelNameWithRetries( + platform: platform, processManager: processManager), + 'serial': serial, + 'id': id, + 'package': packageName, + 'library': libraryName, + 'element': elementName, + }; + + for (final CodeSample sample in element.samples) { + sample.metadata.addAll(metadata); + snippetGenerator.generateCode( + sample, + output: output, + formatOutput: formatOutput, + ); + print(snippetGenerator.generateHtml(sample)); + } + + exitCode = 0; +} diff --git a/dev/snippets/lib/snippets.dart b/dev/snippets/lib/snippets.dart new file mode 100644 index 0000000000000..116ae7970b4ec --- /dev/null +++ b/dev/snippets/lib/snippets.dart @@ -0,0 +1,11 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/analysis.dart'; +export 'src/configuration.dart'; +export 'src/data_types.dart'; +export 'src/import_sorter.dart'; +export 'src/snippet_generator.dart'; +export 'src/snippet_parser.dart'; +export 'src/util.dart'; diff --git a/dev/snippets/lib/src/analysis.dart b/dev/snippets/lib/src/analysis.dart new file mode 100644 index 0000000000000..73a871ae42784 --- /dev/null +++ b/dev/snippets/lib/src/analysis.dart @@ -0,0 +1,361 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:analyzer/dart/analysis/features.dart'; +import 'package:analyzer/dart/analysis/results.dart'; +import 'package:analyzer/dart/analysis/utilities.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/token.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:analyzer/file_system/file_system.dart' as afs; +import 'package:analyzer/file_system/physical_file_system.dart' as afs; +import 'package:analyzer/source/line_info.dart'; +import 'package:file/file.dart'; + +import 'data_types.dart'; +import 'util.dart'; + +/// Gets an iterable over all of the blocks of documentation comments in a file +/// using the analyzer. +/// +/// Each entry in the list is a list of source lines corresponding to the +/// documentation comment block. +Iterable> getFileDocumentationComments(File file) { + return getDocumentationComments(getFileElements(file)); +} + +/// Gets an iterable over all of the blocks of documentation comments from an +/// iterable over the [SourceElement]s involved. +Iterable> getDocumentationComments( + Iterable elements) { + return elements + .where((SourceElement element) => element.comment.isNotEmpty) + .map>((SourceElement element) => element.comment); +} + +/// Gets an iterable over the comment [SourceElement]s in a file. +Iterable getFileCommentElements(File file) { + return getCommentElements(getFileElements(file)); +} + +/// Filters the source `elements` to only return the comment elements. +Iterable getCommentElements(Iterable elements) { + return elements.where((SourceElement element) => element.comment.isNotEmpty); +} + +/// Reads the file content from a string, to avoid having to read the file more +/// than once if the caller already has the content in memory. +/// +/// The `file` argument is used to tag the lines with a filename that they came from. +Iterable getElementsFromString(String content, File file) { + final ParseStringResult parseResult = parseString( + featureSet: FeatureSet.fromEnableFlags2( + sdkLanguageVersion: FlutterInformation.instance.getDartSdkVersion(), + flags: [], + ), + content: content); + final _SourceVisitor visitor = + _SourceVisitor(file); + visitor.visitCompilationUnit(parseResult.unit); + visitor.assignLineNumbers(); + return visitor.elements; +} + +/// Gets an iterable over the [SourceElement]s in the given `file`. +/// +/// Takes an optional [ResourceProvider] to allow reading from a memory +/// filesystem. +Iterable getFileElements(File file, + {afs.ResourceProvider? resourceProvider}) { + resourceProvider ??= afs.PhysicalResourceProvider.INSTANCE; + final ParseStringResult parseResult = parseFile( + featureSet: FeatureSet.fromEnableFlags2( + sdkLanguageVersion: FlutterInformation.instance.getDartSdkVersion(), + flags: [], + ), + path: file.absolute.path, + resourceProvider: resourceProvider); + final _SourceVisitor visitor = + _SourceVisitor(file); + visitor.visitCompilationUnit(parseResult.unit); + visitor.assignLineNumbers(); + return visitor.elements; +} + +class _SourceVisitor extends RecursiveAstVisitor { + _SourceVisitor(this.file) : elements = {}; + + final Set elements; + String enclosingClass = ''; + + File file; + + void assignLineNumbers() { + final String contents = file.readAsStringSync(); + final LineInfo lineInfo = LineInfo.fromContent(contents); + + final Set removedElements = {}; + final Set replacedElements = {}; + for (final SourceElement element in elements) { + final List newLines = []; + for (final SourceLine line in element.comment) { + final CharacterLocation intervalLine = + lineInfo.getLocation(line.startChar); + newLines.add(line.copyWith(line: intervalLine.lineNumber)); + } + final int elementLine = lineInfo.getLocation(element.startPos).lineNumber; + replacedElements + .add(element.copyWith(comment: newLines, startLine: elementLine)); + removedElements.add(element); + } + elements.removeAll(removedElements); + elements.addAll(replacedElements); + } + + List _processComment(String element, Comment comment) { + final List result = []; + if (comment.tokens.isNotEmpty) { + for (final Token token in comment.tokens) { + result.add(SourceLine( + token.toString(), + element: element, + file: file, + startChar: token.charOffset, + endChar: token.charEnd, + )); + } + } + return result; + } + + @override + T? visitCompilationUnit(CompilationUnit node) { + elements.clear(); + return super.visitCompilationUnit(node); + } + + static bool isPublic(String name) { + return !name.startsWith('_'); + } + + static bool isInsideMethod(AstNode startNode) { + AstNode? node = startNode.parent; + while (node != null) { + if (node is MethodDeclaration) { + return true; + } + node = node.parent; + } + return false; + } + + @override + T? visitTopLevelVariableDeclaration(TopLevelVariableDeclaration node) { + for (final VariableDeclaration declaration in node.variables.variables) { + if (!isPublic(declaration.name.lexeme)) { + continue; + } + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment( + declaration.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.topLevelVariableType, + declaration.name.lexeme, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + ), + ); + } + return super.visitTopLevelVariableDeclaration(node); + } + + @override + T? visitGenericTypeAlias(GenericTypeAlias node) { + if (isPublic(node.name.lexeme)) { + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.typedefType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + ), + ); + } + return super.visitGenericTypeAlias(node); + } + + @override + T? visitFieldDeclaration(FieldDeclaration node) { + for (final VariableDeclaration declaration in node.fields.variables) { + if (!isPublic(declaration.name.lexeme) || !isPublic(enclosingClass)) { + continue; + } + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + assert(enclosingClass.isNotEmpty); + comment = _processComment('$enclosingClass.${declaration.name.lexeme}', + node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.fieldType, + declaration.name.lexeme, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + override: _isOverridden(node), + ), + ); + return super.visitFieldDeclaration(node); + } + return null; + } + + @override + T? visitConstructorDeclaration(ConstructorDeclaration node) { + final String fullName = + '$enclosingClass${node.name == null ? '' : '.${node.name}'}'; + if (isPublic(enclosingClass) && + (node.name == null || isPublic(node.name!.lexeme))) { + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment( + '$enclosingClass.$fullName', node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.constructorType, + fullName, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + ), + ); + } + return super.visitConstructorDeclaration(node); + } + + @override + T? visitFunctionDeclaration(FunctionDeclaration node) { + if (isPublic(node.name.lexeme)) { + List comment = []; + // Skip functions that are defined inside of methods. + if (!isInsideMethod(node)) { + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = + _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.functionType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + override: _isOverridden(node), + ), + ); + } + } + return super.visitFunctionDeclaration(node); + } + + @override + T? visitMethodDeclaration(MethodDeclaration node) { + if (isPublic(node.name.lexeme) && isPublic(enclosingClass)) { + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + assert(enclosingClass.isNotEmpty); + comment = _processComment( + '$enclosingClass.${node.name.lexeme}', node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.methodType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + className: enclosingClass, + comment: comment, + override: _isOverridden(node), + ), + ); + } + return super.visitMethodDeclaration(node); + } + + bool _isOverridden(AnnotatedNode node) { + return node.metadata.where((Annotation annotation) { + return annotation.name.name == 'override'; + }).isNotEmpty; + } + + @override + T? visitMixinDeclaration(MixinDeclaration node) { + enclosingClass = node.name.lexeme; + if (!node.name.lexeme.startsWith('_')) { + enclosingClass = node.name.lexeme; + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.classType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + ), + ); + } + final T? result = super.visitMixinDeclaration(node); + enclosingClass = ''; + return result; + } + + @override + T? visitClassDeclaration(ClassDeclaration node) { + enclosingClass = node.name.lexeme; + if (!node.name.lexeme.startsWith('_')) { + enclosingClass = node.name.lexeme; + List comment = []; + if (node.documentationComment != null && + node.documentationComment!.tokens.isNotEmpty) { + comment = _processComment(node.name.lexeme, node.documentationComment!); + } + elements.add( + SourceElement( + SourceElementType.classType, + node.name.lexeme, + node.beginToken.charOffset, + file: file, + comment: comment, + ), + ); + } + final T? result = super.visitClassDeclaration(node); + enclosingClass = ''; + return result; + } +} diff --git a/dev/snippets/lib/src/configuration.dart b/dev/snippets/lib/src/configuration.dart new file mode 100644 index 0000000000000..37176ce0dc609 --- /dev/null +++ b/dev/snippets/lib/src/configuration.dart @@ -0,0 +1,63 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:path/path.dart' as path; + +// Represents the locations of all of the data for snippets. +class SnippetConfiguration { + const SnippetConfiguration({ + required this.configDirectory, + required this.skeletonsDirectory, + required this.templatesDirectory, + this.filesystem = const LocalFileSystem(), + }); + + final FileSystem filesystem; + + /// This is the configuration directory for the snippets system, containing + /// the skeletons and templates. + final Directory configDirectory; + + /// The directory containing the HTML skeletons to be filled out with metadata + /// and returned to dartdoc for insertion in the output. + final Directory skeletonsDirectory; + + /// The directory containing the code templates that can be referenced by the + /// dartdoc. + final Directory templatesDirectory; + + /// Gets the skeleton file to use for the given [SampleType] and DartPad + /// preference. + File getHtmlSkeletonFile(String type) { + final String filename = + type == 'dartpad' ? 'dartpad-sample.html' : '$type.html'; + return filesystem.file(path.join(skeletonsDirectory.path, filename)); + } +} + +/// A class to compute the configuration of the snippets input and output +/// locations based in the current location of the snippets main.dart. +class FlutterRepoSnippetConfiguration extends SnippetConfiguration { + FlutterRepoSnippetConfiguration({required this.flutterRoot, super.filesystem}) + : super( + configDirectory: _underRoot(filesystem, flutterRoot, + const ['dev', 'snippets', 'config']), + skeletonsDirectory: _underRoot(filesystem, flutterRoot, + const ['dev', 'snippets', 'config', 'skeletons']), + templatesDirectory: _underRoot( + filesystem, + flutterRoot, + const ['dev', 'snippets', 'config', 'templates'], + ), + ); + + final Directory flutterRoot; + + static Directory _underRoot( + FileSystem fs, Directory flutterRoot, List dirs) => + fs.directory(path.canonicalize( + path.joinAll([flutterRoot.absolute.path, ...dirs]))); +} diff --git a/dev/snippets/lib/src/data_types.dart b/dev/snippets/lib/src/data_types.dart new file mode 100644 index 0000000000000..fd5bac98020b0 --- /dev/null +++ b/dev/snippets/lib/src/data_types.dart @@ -0,0 +1,567 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:args/args.dart'; +import 'package:file/file.dart'; + +import 'util.dart'; + +/// A class to represent a line of input code, with associated line number, file +/// and element name. +class SourceLine { + const SourceLine( + this.text, { + this.file, + this.element, + this.line = -1, + this.startChar = -1, + this.endChar = -1, + this.indent = 0, + }); + final File? file; + final String? element; + final int line; + final int startChar; + final int endChar; + final int indent; + final String text; + + String toStringWithColumn(int column) => + '$file:$line:${column + indent}: $text'; + + SourceLine copyWith({ + String? element, + String? text, + File? file, + int? line, + int? startChar, + int? endChar, + int? indent, + }) { + return SourceLine( + text ?? this.text, + element: element ?? this.element, + file: file ?? this.file, + line: line ?? this.line, + startChar: startChar ?? this.startChar, + endChar: endChar ?? this.endChar, + indent: indent ?? this.indent, + ); + } + + bool get hasFile => file != null; + + @override + String toString() => '$file:${line == -1 ? '??' : line}: $text'; +} + +/// A class containing the name and contents associated with a code block inside of a +/// code sample, for named injection into a template. +class TemplateInjection { + TemplateInjection(this.name, this.contents, {this.language = ''}); + final String name; + final List contents; + final String language; + Iterable get stringContents => + contents.map((SourceLine line) => line.text.trimRight()); + String get mergedContent => stringContents.join('\n'); +} + +/// A base class to represent a block of any kind of sample code, marked by +/// "{@tool (snippet|sample|dartdoc) ...}...{@end-tool}". +abstract class CodeSample { + CodeSample( + this.args, + this.input, { + required this.index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + _lineProto = lineProto, + sourceFile = null; + + CodeSample.fromFile( + this.args, + this.input, + this.sourceFile, { + required this.index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + _lineProto = lineProto; + + final File? sourceFile; + final List args; + final List input; + final SourceLine _lineProto; + String? _sourceFileContents; + String get sourceFileContents { + if (sourceFile != null && _sourceFileContents == null) { + // Strip lines until the first non-comment line. This gets rid of the + // copyright and comment directing the reader to the original source file. + final List stripped = []; + bool doneStrippingHeaders = false; + try { + for (final String line in sourceFile!.readAsLinesSync()) { + if (!doneStrippingHeaders && + RegExp(r'^\s*(\/\/.*)?$').hasMatch(line)) { + continue; + } + // Stop skipping lines after the first line that isn't stripped. + doneStrippingHeaders = true; + stripped.add(line); + } + } on FileSystemException catch (e) { + throw SnippetException( + 'Unable to read linked source file ${sourceFile!}: $e', + file: _lineProto.file?.absolute.path, + ); + } + // Remove any section markers + final RegExp sectionMarkerRegExp = RegExp( + r'(\/\/\*\*+\n)?\/\/\* [▼▲]+.*$(\n\/\/\*\*+)?\n\n?', + multiLine: true, + ); + _sourceFileContents = + stripped.join('\n').replaceAll(sectionMarkerRegExp, ''); + } + return _sourceFileContents ?? ''; + } + + Iterable get inputStrings => + input.map((SourceLine line) => line.text); + String get inputAsString => inputStrings.join('\n'); + + /// The index of this sample within the dartdoc comment it came from. + final int index; + String description = ''; + String get element => start.element ?? ''; + String output = ''; + Map metadata = {}; + List parts = []; + SourceLine get start => input.isEmpty ? _lineProto : input.first; + + String get template { + final ArgParser parser = ArgParser(); + parser.addOption('template', defaultsTo: ''); + final ArgResults parsedArgs = parser.parse(args); + return parsedArgs['template']! as String; + } + + @override + String toString() { + final StringBuffer buf = StringBuffer('${args.join(' ')}:\n'); + for (final SourceLine line in input) { + buf.writeln( + '${(line.line == -1 ? '??' : line.line).toString().padLeft(4)}: ${line.text} ', + ); + } + return buf.toString(); + } + + String get type; +} + +/// A class to represent a snippet of sample code, marked by "{@tool +/// snippet}...{@end-tool}". +/// +/// Snippets are code that is not meant to be run as a complete application, but +/// rather as a code usage example. +class SnippetSample extends CodeSample { + SnippetSample( + List input, { + required int index, + required SourceLine lineProto, + }) : assumptions = [], + super( + ['snippet'], + input, + index: index, + lineProto: lineProto, + ); + + factory SnippetSample.combine( + List sections, { + required int index, + required SourceLine lineProto, + }) { + final List code = + sections.expand((SnippetSample section) => section.input).toList(); + return SnippetSample(code, index: index, lineProto: lineProto); + } + + factory SnippetSample.fromStrings(SourceLine firstLine, List code, + {required int index}) { + final List codeLines = []; + int startPos = firstLine.startChar; + for (int i = 0; i < code.length; ++i) { + codeLines.add( + firstLine.copyWith( + text: code[i], + line: firstLine.line + i, + startChar: startPos, + ), + ); + startPos += code[i].length + 1; + } + return SnippetSample( + codeLines, + index: index, + lineProto: firstLine, + ); + } + + factory SnippetSample.surround( + String prefix, + List code, + String postfix, { + required int index, + }) { + return SnippetSample( + [ + if (prefix.isNotEmpty) SourceLine(prefix), + ...code, + if (postfix.isNotEmpty) SourceLine(postfix), + ], + index: index, + lineProto: code.first, + ); + } + + List assumptions; + + @override + String get template => ''; + + @override + SourceLine get start => + input.firstWhere((SourceLine line) => line.file != null); + + @override + String get type => 'snippet'; +} + +/// A class to represent a plain application sample in the dartdoc comments, +/// marked by `{@tool sample ...}...{@end-tool}`. +/// +/// Application samples are processed separately from [SnippetSample]s, because +/// they must be injected into templates in order to be analyzed. Each +/// [ApplicationSample] represents one `{@tool sample ...}...{@end-tool}` block +/// in the source file. +class ApplicationSample extends CodeSample { + ApplicationSample({ + List input = const [], + required List args, + required int index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + super(args, input, index: index, lineProto: lineProto); + + ApplicationSample.fromFile({ + List input = const [], + required List args, + required File sourceFile, + required int index, + required SourceLine lineProto, + }) : assert(args.isNotEmpty), + super.fromFile(args, input, sourceFile, + index: index, lineProto: lineProto); + + @override + String get type => 'sample'; +} + +/// A class to represent a Dartpad application sample in the dartdoc comments, +/// marked by `{@tool dartpad ...}...{@end-tool}`. +/// +/// Dartpad samples are processed separately from [SnippetSample]s, because they +/// must be injected into templates in order to be analyzed. Each +/// [DartpadSample] represents one `{@tool dartpad ...}...{@end-tool}` block in +/// the source file. +class DartpadSample extends ApplicationSample { + DartpadSample({ + super.input, + required super.args, + required super.index, + required super.lineProto, + }) : assert(args.isNotEmpty); + + DartpadSample.fromFile({ + super.input, + required super.args, + required super.sourceFile, + required super.index, + required super.lineProto, + }) : assert(args.isNotEmpty), + super.fromFile(); + + @override + String get type => 'dartpad'; +} + +/// The different types of Dart [SourceElement]s that can be found in a source file. +enum SourceElementType { + /// A class + classType, + + /// A field variable of a class. + fieldType, + + /// A constructor for a class. + constructorType, + + /// A method of a class. + methodType, + + /// A function typedef + typedefType, + + /// A top level (non-class) variable. + topLevelVariableType, + + /// A function, either top level, or embedded in another function. + functionType, + + /// An unknown type used for initialization. + unknownType, +} + +/// Converts the enun type [SourceElementType] to a human readable string. +String sourceElementTypeAsString(SourceElementType type) { + switch (type) { + case SourceElementType.classType: + return 'class'; + case SourceElementType.fieldType: + return 'field'; + case SourceElementType.methodType: + return 'method'; + case SourceElementType.constructorType: + return 'constructor'; + case SourceElementType.typedefType: + return 'typedef'; + case SourceElementType.topLevelVariableType: + return 'variable'; + case SourceElementType.functionType: + return 'function'; + case SourceElementType.unknownType: + return 'unknown'; + } +} + +/// A class that represents a Dart element in a source file. +/// +/// The element is one of the types in [SourceElementType]. +class SourceElement { + /// A factory constructor for SourceElements. + /// + /// This uses a factory so that the default for the `comment` and `samples` + /// lists can be modifiable lists. + factory SourceElement( + SourceElementType type, + String name, + int startPos, { + required File file, + String className = '', + List? comment, + int startLine = -1, + List? samples, + bool override = false, + }) { + comment ??= []; + samples ??= []; + final List commentLines = + comment.map((SourceLine line) => line.text).toList(); + final String commentString = commentLines.join('\n'); + return SourceElement._( + type, + name, + startPos, + file: file, + className: className, + comment: comment, + startLine: startLine, + samples: samples, + override: override, + commentString: commentString, + commentStringWithoutTools: _getCommentStringWithoutTools(commentString), + commentStringWithoutCode: _getCommentStringWithoutCode(commentString), + commentLines: commentLines, + ); + } + + const SourceElement._( + this.type, + this.name, + this.startPos, { + required this.file, + this.className = '', + this.comment = const [], + this.startLine = -1, + this.samples = const [], + this.override = false, + String commentString = '', + String commentStringWithoutTools = '', + String commentStringWithoutCode = '', + List commentLines = const [], + }) : _commentString = commentString, + _commentStringWithoutTools = commentStringWithoutTools, + _commentStringWithoutCode = commentStringWithoutCode, + _commentLines = commentLines; + + final String _commentString; + final String _commentStringWithoutTools; + final String _commentStringWithoutCode; + final List _commentLines; + + // Does not include the description of the sample code, just the text outside + // of any dartdoc tools. + static String _getCommentStringWithoutTools(String string) { + return string.replaceAll( + RegExp(r'(\{@tool ([^}]*)\}.*?\{@end-tool\}|/// ?)', dotAll: true), ''); + } + + // Includes the description text inside of an "@tool"-based sample, but not + // the code itself, or any dartdoc tags. + static String _getCommentStringWithoutCode(String string) { + return string.replaceAll( + RegExp(r'([`]{3}.*?[`]{3}|\{@\w+[^}]*\}|/// ?)', dotAll: true), ''); + } + + /// The type of the element + final SourceElementType type; + + /// The name of the element. + /// + /// For example, a method called "doSomething" that is part of the class + /// "MyClass" would have "doSomething" as its name. + final String name; + + /// The name of the class the element belongs to, if any. + /// + /// This is the empty string if it isn't part of a class. + /// + /// For example, a method called "doSomething" that is part of the class + /// "MyClass" would have "MyClass" as its `className`. + final String className; + + /// Whether or not this element has the "@override" annotation attached to it. + final bool override; + + /// The file that this [SourceElement] was parsed from. + final File file; + + /// The character position in the file that this [SourceElement] starts at. + final int startPos; + + /// The line in the file that the first position of [SourceElement] is on. + final int startLine; + + /// The list of [SourceLine]s that make up the documentation comment for this + /// [SourceElement]. + final List comment; + + /// The list of [CodeSample]s that are in the documentation comment for this + /// [SourceElement]. + /// + /// This field will be populated by calling [replaceSamples]. + final List samples; + + /// Get the comments as an iterable of lines. + Iterable get commentLines => _commentLines; + + /// Get the comments as a single string. + String get commentString => _commentString; + + /// Does not include the description of the sample code, just the text outside of any dartdoc tools. + String get commentStringWithoutTools => _commentStringWithoutTools; + + /// Includes the description text inside of an "@tool"-based sample, but not + /// the code itself, or any dartdoc tags. + String get commentStringWithoutCode => _commentStringWithoutCode; + + /// The number of samples in the dartdoc comment for this element. + int get sampleCount => samples.length; + + /// The number of [DartpadSample]s in the dartdoc comment for this element. + int get dartpadSampleCount => samples.whereType().length; + + /// The number of [ApplicationSample]s in the dartdoc comment for this element. + int get applicationSampleCount => samples.where((CodeSample sample) { + return sample is ApplicationSample && sample is! DartpadSample; + }).length; + + /// The number of [SnippetSample]s in the dartdoc comment for this element. + int get snippetCount => samples.whereType().length; + + /// Count of comment lines, not including lines of code in the comment. + int get lineCount => commentStringWithoutCode.split('\n').length; + + /// Count of comment words, not including words in any code in the comment. + int get wordCount { + return commentStringWithoutCode.split(RegExp(r'\s+')).length; + } + + /// Count of comment characters, not including any code samples in the + /// comment, after collapsing each run of whitespace to a single space. + int get charCount => + commentStringWithoutCode.replaceAll(RegExp(r'\s+'), ' ').length; + + /// Whether or not this element's documentation has a "See also:" section in it. + bool get hasSeeAlso => commentStringWithoutTools.contains('See also:'); + + int get referenceCount { + final RegExp regex = RegExp(r'\[[. \w]*\](?!\(.*\))'); + return regex.allMatches(commentStringWithoutCode).length; + } + + int get linkCount { + final RegExp regex = RegExp(r'\[[. \w]*\]\(.*\)'); + return regex.allMatches(commentStringWithoutCode).length; + } + + /// Returns the fully qualified name of this element. + /// + /// For example, a method called "doSomething" that is part of the class + /// "MyClass" would have "MyClass.doSomething" as its `elementName`. + String get elementName { + if (type == SourceElementType.constructorType) { + // Constructors already have the name of the class in them. + return name; + } + return className.isEmpty ? name : '$className.$name'; + } + + /// Returns the type of this element as a [String]. + String get typeAsString { + return '${override ? 'overridden ' : ''}${sourceElementTypeAsString(type)}'; + } + + void replaceSamples(Iterable samples) { + this.samples.clear(); + this.samples.addAll(samples); + } + + /// Copy the source element, with some attributes optionally replaced. + SourceElement copyWith({ + SourceElementType? type, + String? name, + int? startPos, + File? file, + String? className, + List? comment, + int? startLine, + List? samples, + bool? override, + }) { + return SourceElement( + type ?? this.type, + name ?? this.name, + startPos ?? this.startPos, + file: file ?? this.file, + className: className ?? this.className, + comment: comment ?? this.comment, + startLine: startLine ?? this.startLine, + samples: samples ?? this.samples, + override: override ?? this.override, + ); + } +} diff --git a/dev/snippets/lib/src/import_sorter.dart b/dev/snippets/lib/src/import_sorter.dart new file mode 100644 index 0000000000000..ef7fa29554179 --- /dev/null +++ b/dev/snippets/lib/src/import_sorter.dart @@ -0,0 +1,434 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:math'; + +import 'package:analyzer/dart/analysis/features.dart'; +import 'package:analyzer/dart/analysis/results.dart'; +import 'package:analyzer/dart/analysis/utilities.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/token.dart'; +import 'package:analyzer/error/error.dart'; +import 'package:analyzer/source/line_info.dart'; +import 'package:meta/meta.dart'; + +import 'util.dart'; + +/// Read the given source code, and return the new contents after sorting the +/// imports. +String sortImports(String contents) { + final ParseStringResult parseResult = parseString( + content: contents, + featureSet: FeatureSet.fromEnableFlags2( + sdkLanguageVersion: FlutterInformation.instance.getDartSdkVersion(), + flags: [], + ), + ); + final List errors = []; + final _ImportOrganizer organizer = + _ImportOrganizer(contents, parseResult.unit, errors); + final List<_SourceEdit> edits = organizer.organize(); + // Sort edits in reverse order + edits.sort((_SourceEdit a, _SourceEdit b) { + return b.offset.compareTo(a.offset); + }); + // Apply edits + for (final _SourceEdit edit in edits) { + contents = contents.replaceRange(edit.offset, edit.end, edit.replacement); + } + return contents; +} + +/// Organizer of imports (and other directives) in the [unit]. +// Adapted from the analysis_server package. +// This code is largely copied from: +// https://github.com/dart-lang/sdk/blob/c7405b9d86b4b47cf7610667491f1db72723b0dd/pkg/analysis_server/lib/src/services/correction/organize_imports.dart#L15 +// TODO(gspencergoog): If ImportOrganizer ever becomes part of the public API, +// this class should probably be replaced. +// https://github.com/flutter/flutter/issues/86197 +class _ImportOrganizer { + _ImportOrganizer(this.initialCode, this.unit, this.errors) + : code = initialCode { + endOfLine = getEOL(code); + hasUnresolvedIdentifierError = errors.any((AnalysisError error) { + return error.errorCode.isUnresolvedIdentifier; + }); + } + + final String initialCode; + + final CompilationUnit unit; + + final List errors; + + String code; + + String endOfLine = '\n'; + + bool hasUnresolvedIdentifierError = false; + + /// Returns the number of characters common to the end of [a] and [b]. + int findCommonSuffix(String a, String b) { + final int aLength = a.length; + final int bLength = b.length; + final int n = min(aLength, bLength); + for (int i = 1; i <= n; i++) { + if (a.codeUnitAt(aLength - i) != b.codeUnitAt(bLength - i)) { + return i - 1; + } + } + return n; + } + + /// Return the [_SourceEdit]s that organize imports in the [unit]. + List<_SourceEdit> organize() { + _organizeDirectives(); + // prepare edits + final List<_SourceEdit> edits = <_SourceEdit>[]; + if (code != initialCode) { + final int suffixLength = findCommonSuffix(initialCode, code); + final _SourceEdit edit = _SourceEdit(0, initialCode.length - suffixLength, + code.substring(0, code.length - suffixLength)); + edits.add(edit); + } + return edits; + } + + /// Organize all [Directive]s. + void _organizeDirectives() { + final LineInfo lineInfo = unit.lineInfo; + bool hasLibraryDirective = false; + final List<_DirectiveInfo> directives = <_DirectiveInfo>[]; + for (final Directive directive in unit.directives) { + if (directive is LibraryDirective) { + hasLibraryDirective = true; + } + if (directive is UriBasedDirective) { + final _DirectivePriority? priority = getDirectivePriority(directive); + if (priority != null) { + int offset = directive.offset; + int end = directive.end; + + final Token? leadingComment = + getLeadingComment(unit, directive, lineInfo); + final Token? trailingComment = + getTrailingComment(unit, directive, lineInfo, end); + + String? leadingCommentText; + if (leadingComment != null) { + leadingCommentText = + code.substring(leadingComment.offset, directive.offset); + offset = leadingComment.offset; + } + String? trailingCommentText; + if (trailingComment != null) { + trailingCommentText = + code.substring(directive.end, trailingComment.end); + end = trailingComment.end; + } + String? documentationText; + final Comment? documentationComment = directive.documentationComment; + if (documentationComment != null) { + documentationText = code.substring( + documentationComment.offset, documentationComment.end); + } + String? annotationText; + final Token? beginToken = directive.metadata.beginToken; + final Token? endToken = directive.metadata.endToken; + if (beginToken != null && endToken != null) { + annotationText = code.substring(beginToken.offset, endToken.end); + } + final String text = code.substring( + directive.firstTokenAfterCommentAndMetadata.offset, + directive.end); + final String uriContent = directive.uri.stringValue ?? ''; + directives.add( + _DirectiveInfo( + directive, + priority, + leadingCommentText, + documentationText, + annotationText, + uriContent, + trailingCommentText, + offset, + end, + text, + ), + ); + } + } + } + // nothing to do + if (directives.isEmpty) { + return; + } + final int firstDirectiveOffset = directives.first.offset; + final int lastDirectiveEnd = directives.last.end; + + // Without a library directive, the library comment is the comment of the + // first directive. + _DirectiveInfo? libraryDocumentationDirective; + if (!hasLibraryDirective && directives.isNotEmpty) { + libraryDocumentationDirective = directives.first; + } + + // sort + directives.sort(); + // append directives with grouping + String directivesCode; + { + final StringBuffer sb = StringBuffer(); + if (libraryDocumentationDirective != null && + libraryDocumentationDirective.documentationText != null) { + sb.write(libraryDocumentationDirective.documentationText); + sb.write(endOfLine); + } + _DirectivePriority currentPriority = directives.first.priority; + for (final _DirectiveInfo directiveInfo in directives) { + if (currentPriority != directiveInfo.priority) { + sb.write(endOfLine); + currentPriority = directiveInfo.priority; + } + if (directiveInfo.leadingCommentText != null) { + sb.write(directiveInfo.leadingCommentText); + } + if (directiveInfo != libraryDocumentationDirective && + directiveInfo.documentationText != null) { + sb.write(directiveInfo.documentationText); + sb.write(endOfLine); + } + if (directiveInfo.annotationText != null) { + sb.write(directiveInfo.annotationText); + sb.write(endOfLine); + } + sb.write(directiveInfo.text); + if (directiveInfo.trailingCommentText != null) { + sb.write(directiveInfo.trailingCommentText); + } + sb.write(endOfLine); + } + directivesCode = sb.toString(); + directivesCode = directivesCode.trimRight(); + } + // prepare code + final String beforeDirectives = code.substring(0, firstDirectiveOffset); + final String afterDirectives = code.substring(lastDirectiveEnd); + code = beforeDirectives + directivesCode + afterDirectives; + } + + static _DirectivePriority? getDirectivePriority(UriBasedDirective directive) { + final String uriContent = directive.uri.stringValue ?? ''; + if (directive is ImportDirective) { + if (uriContent.startsWith('dart:')) { + return _DirectivePriority.IMPORT_SDK; + } else if (uriContent.startsWith('package:')) { + return _DirectivePriority.IMPORT_PKG; + } else if (uriContent.contains('://')) { + return _DirectivePriority.IMPORT_OTHER; + } else { + return _DirectivePriority.IMPORT_REL; + } + } + if (directive is ExportDirective) { + if (uriContent.startsWith('dart:')) { + return _DirectivePriority.EXPORT_SDK; + } else if (uriContent.startsWith('package:')) { + return _DirectivePriority.EXPORT_PKG; + } else if (uriContent.contains('://')) { + return _DirectivePriority.EXPORT_OTHER; + } else { + return _DirectivePriority.EXPORT_REL; + } + } + if (directive is PartDirective) { + return _DirectivePriority.PART; + } + return null; + } + + /// Return the EOL to use for [code]. + static String getEOL(String code) { + if (code.contains('\r\n')) { + return '\r\n'; + } else { + return '\n'; + } + } + + /// Gets the first comment token considered to be the leading comment for this + /// directive. + /// + /// Leading comments for the first directive in a file are considered library + /// comments and not returned unless they contain blank lines, in which case + /// only the last part of the comment will be returned. + static Token? getLeadingComment( + CompilationUnit unit, UriBasedDirective directive, LineInfo lineInfo) { + if (directive.beginToken.precedingComments == null) { + return null; + } + + Token? firstComment = directive.beginToken.precedingComments; + Token? comment = firstComment; + Token? nextComment = comment?.next; + // Don't connect comments that have a blank line between them + while (comment != null && nextComment != null) { + final int currentLine = lineInfo.getLocation(comment.offset).lineNumber; + final int nextLine = lineInfo.getLocation(nextComment.offset).lineNumber; + if (nextLine - currentLine > 1) { + firstComment = nextComment; + } + comment = nextComment; + nextComment = comment.next; + } + + // Check if the comment is the first comment in the document + if (firstComment != unit.beginToken.precedingComments) { + final int previousDirectiveLine = + lineInfo.getLocation(directive.beginToken.previous!.end).lineNumber; + + // Skip over any comments on the same line as the previous directive + // as they will be attached to the end of it. + Token? comment = firstComment; + while (comment != null && + previousDirectiveLine == + lineInfo.getLocation(comment.offset).lineNumber) { + comment = comment.next; + } + return comment; + } + return null; + } + + /// Gets the last comment token considered to be the trailing comment for this + /// directive. + /// + /// To be considered a trailing comment, the comment must be on the same line + /// as the directive. + static Token? getTrailingComment(CompilationUnit unit, + UriBasedDirective directive, LineInfo lineInfo, int end) { + final int line = lineInfo.getLocation(end).lineNumber; + Token? comment = directive.endToken.next!.precedingComments; + while (comment != null) { + if (lineInfo.getLocation(comment.offset).lineNumber == line) { + return comment; + } + comment = comment.next; + } + return null; + } +} + +class _DirectiveInfo implements Comparable<_DirectiveInfo> { + _DirectiveInfo( + this.directive, + this.priority, + this.leadingCommentText, + this.documentationText, + this.annotationText, + this.uri, + this.trailingCommentText, + this.offset, + this.end, + this.text, + ); + + final UriBasedDirective directive; + final _DirectivePriority priority; + final String? leadingCommentText; + final String? documentationText; + final String? annotationText; + final String uri; + final String? trailingCommentText; + + /// The offset of the first token, usually the keyword but may include leading comments. + final int offset; + + /// The offset after the last token, including the end-of-line comment. + final int end; + + /// The text excluding comments, documentation and annotations. + final String text; + + @override + int compareTo(_DirectiveInfo other) { + if (priority == other.priority) { + return _compareUri(uri, other.uri); + } + return priority.ordinal - other.priority.ordinal; + } + + @override + String toString() => '(priority=$priority; text=$text)'; + + static int _compareUri(String a, String b) { + final List aList = _splitUri(a); + final List bList = _splitUri(b); + int result; + if ((result = aList[0].compareTo(bList[0])) != 0) { + return result; + } + if ((result = aList[1].compareTo(bList[1])) != 0) { + return result; + } + return 0; + } + + /// Split the given [uri] like `package:some.name/and/path.dart` into a list + /// like `[package:some.name, and/path.dart]`. + static List _splitUri(String uri) { + final int index = uri.indexOf('/'); + if (index == -1) { + return [uri, '']; + } + return [uri.substring(0, index), uri.substring(index + 1)]; + } +} + +enum _DirectivePriority { + IMPORT_SDK('IMPORT_SDK', 0), + IMPORT_PKG('IMPORT_PKG', 1), + IMPORT_OTHER('IMPORT_OTHER', 2), + IMPORT_REL('IMPORT_REL', 3), + EXPORT_SDK('EXPORT_SDK', 4), + EXPORT_PKG('EXPORT_PKG', 5), + EXPORT_OTHER('EXPORT_OTHER', 6), + EXPORT_REL('EXPORT_REL', 7), + PART('PART', 8); + + const _DirectivePriority(this.name, this.ordinal); + + final String name; + final int ordinal; + + @override + String toString() => name; +} + +/// SourceEdit +/// +/// { +/// "offset": int +/// "length": int +/// "replacement": String +/// "id": optional String +/// } +/// +/// Clients may not extend, implement or mix-in this class. +@immutable +class _SourceEdit { + const _SourceEdit(this.offset, this.length, this.replacement); + + /// The offset of the region to be modified. + final int offset; + + /// The length of the region to be modified. + final int length; + + /// The end of the region to be modified. + int get end => offset + length; + + /// The code that is to replace the specified region in the original code. + final String replacement; +} diff --git a/dev/snippets/lib/src/snippet_generator.dart b/dev/snippets/lib/src/snippet_generator.dart new file mode 100644 index 0000000000000..f83cf7ec1083a --- /dev/null +++ b/dev/snippets/lib/src/snippet_generator.dart @@ -0,0 +1,497 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:dart_style/dart_style.dart'; +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:path/path.dart' as path; + +import 'configuration.dart'; +import 'data_types.dart'; +import 'import_sorter.dart'; +import 'util.dart'; + +/// Generates the snippet HTML, as well as saving the output snippet main to +/// the output directory. +class SnippetGenerator { + SnippetGenerator( + {SnippetConfiguration? configuration, + FileSystem filesystem = const LocalFileSystem(), + Directory? flutterRoot}) + : flutterRoot = + flutterRoot ?? FlutterInformation.instance.getFlutterRoot(), + configuration = configuration ?? + FlutterRepoSnippetConfiguration( + filesystem: filesystem, + flutterRoot: flutterRoot ?? + FlutterInformation.instance.getFlutterRoot()); + + final Directory flutterRoot; + + /// The configuration used to determine where to get/save data for the + /// snippet. + final SnippetConfiguration configuration; + + static const JsonEncoder jsonEncoder = JsonEncoder.withIndent(' '); + + /// A Dart formatted used to format the snippet code and finished application + /// code. + static DartFormatter formatter = + DartFormatter(pageWidth: 80, fixes: StyleFix.all); + + /// Gets the path to the template file requested. + File? getTemplatePath(String templateName, {Directory? templatesDir}) { + final Directory templateDir = + templatesDir ?? configuration.templatesDirectory; + final File templateFile = configuration.filesystem + .file(path.join(templateDir.path, '$templateName.tmpl')); + return templateFile.existsSync() ? templateFile : null; + } + + /// Returns an iterable over the template files available in the templates + /// directory in the configuration. + Iterable getAvailableTemplates() sync* { + final Directory templatesDir = configuration.templatesDirectory; + for (final File file in templatesDir.listSync().whereType()) { + if (file.basename.endsWith('.tmpl')) { + yield file; + } + } + } + + /// Interpolates the [injections] into an HTML skeleton file. + /// + /// Similar to interpolateTemplate, but we are only looking for `code-` + /// components, and we care about the order of the injections. + /// + /// Takes into account the [type] and doesn't substitute in the id and the app + /// if not a [SnippetType.sample] snippet. + String interpolateSkeleton( + CodeSample sample, + String skeleton, + ) { + final List codeParts = []; + const HtmlEscape htmlEscape = HtmlEscape(); + String? language; + for (final TemplateInjection injection in sample.parts) { + if (!injection.name.startsWith('code')) { + continue; + } + codeParts.addAll(injection.stringContents); + if (injection.language.isNotEmpty) { + language = injection.language; + } + codeParts.addAll(['', '// ...', '']); + } + if (codeParts.length > 3) { + codeParts.removeRange(codeParts.length - 3, codeParts.length); + } + // Only insert a div for the description if there actually is some text there. + // This means that the {{description}} marker in the skeleton needs to + // be inside of an {@inject-html} block. + final String description = sample.description.trim().isNotEmpty + ? '
{@end-inject-html}${sample.description.trim()}{@inject-html}
' + : ''; + + // DartPad only supports stable or main as valid channels. Use main + // if not on stable so that local runs will work (although they will + // still take their sample code from the master docs server). + final String channel = + sample.metadata['channel'] == 'stable' ? 'stable' : 'main'; + + final Map substitutions = { + 'description': description, + 'code': htmlEscape.convert(codeParts.join('\n')), + 'language': language ?? 'dart', + 'serial': '', + 'id': sample.metadata['id']! as String, + 'channel': channel, + 'element': sample.metadata['element'] as String? ?? sample.element, + 'app': '', + }; + if (sample is ApplicationSample) { + substitutions + ..['serial'] = sample.metadata['serial']?.toString() ?? '0' + ..['app'] = htmlEscape.convert(sample.output); + } + return skeleton.replaceAllMapped( + RegExp('{{(${substitutions.keys.join('|')})}}'), (Match match) { + return substitutions[match[1]]!; + }); + } + + /// Consolidates all of the snippets and the assumptions into one snippet, in + /// order to create a compilable result. + Iterable consolidateSnippets(List samples, + {bool addMarkers = false}) { + if (samples.isEmpty) { + return []; + } + final Iterable snippets = samples.whereType(); + final List snippetLines = [ + ...snippets.first.assumptions, + ]; + for (final SnippetSample sample in snippets) { + parseInput(sample); + snippetLines.addAll(_processBlocks(sample)); + } + return snippetLines; + } + + /// A RegExp that matches a Dart constructor. + static final RegExp _constructorRegExp = + RegExp(r'(const\s+)?_*[A-Z][a-zA-Z0-9<>._]*\('); + + /// A serial number so that we can create unique expression names when we + /// generate them. + int _expressionId = 0; + + List _surround( + String prefix, Iterable body, String suffix) { + return [ + if (prefix.isNotEmpty) SourceLine(prefix), + ...body, + if (suffix.isNotEmpty) SourceLine(suffix), + ]; + } + + /// Process one block of sample code (the part inside of "```" markers). + /// Splits any sections denoted by "// ..." into separate blocks to be + /// processed separately. Uses a primitive heuristic to make sample blocks + /// into valid Dart code. + List _processBlocks(CodeSample sample) { + final List block = sample.parts + .expand((TemplateInjection injection) => injection.contents) + .toList(); + if (block.isEmpty) { + return []; + } + return _processBlock(block); + } + + List _processBlock(List block) { + final String firstLine = block.first.text; + if (firstLine.startsWith('new ') || + firstLine.startsWith(_constructorRegExp)) { + _expressionId += 1; + return _surround('dynamic expression$_expressionId = ', block, ';'); + } else if (firstLine.startsWith('await ')) { + _expressionId += 1; + return _surround( + 'Future expression$_expressionId() async { ', block, ' }'); + } else if (block.first.text.startsWith('class ') || + block.first.text.startsWith('enum ')) { + return block; + } else if ((block.first.text.startsWith('_') || + block.first.text.startsWith('final ')) && + block.first.text.contains(' = ')) { + _expressionId += 1; + return _surround( + 'void expression$_expressionId() { ', block.toList(), ' }'); + } else { + final List buffer = []; + int blocks = 0; + SourceLine? subLine; + final List subsections = []; + for (int index = 0; index < block.length; index += 1) { + // Each section of the dart code that is either split by a blank line, or with + // '// ...' is treated as a separate code block. + if (block[index].text.trim().isEmpty || block[index].text == '// ...') { + if (subLine == null) { + continue; + } + blocks += 1; + subsections.addAll(_processBlock(buffer)); + buffer.clear(); + assert(buffer.isEmpty); + subLine = null; + } else if (block[index].text.startsWith('// ')) { + if (buffer.length > 1) { + // don't include leading comments + // so that it doesn't start with "// " and get caught in this again + buffer.add(SourceLine('/${block[index].text}')); + } + } else { + subLine ??= block[index]; + buffer.add(block[index]); + } + } + if (blocks > 0) { + if (subLine != null) { + subsections.addAll(_processBlock(buffer)); + } + // Combine all of the subsections into one section, now that they've been processed. + return subsections; + } else { + return block; + } + } + } + + /// Parses the input for the various code and description segments, and + /// returns a set of template injections in the order found. + List parseInput(CodeSample sample) { + bool inCodeBlock = false; + final List description = []; + final List components = []; + String? language; + final RegExp codeStartEnd = + RegExp(r'^\s*```(?[-\w]+|[-\w]+ (?
[-\w]+))?\s*$'); + for (final SourceLine line in sample.input) { + final RegExpMatch? match = codeStartEnd.firstMatch(line.text); + if (match != null) { + // If we saw the start or end of a code block + inCodeBlock = !inCodeBlock; + if (match.namedGroup('language') != null) { + language = match[1]; + if (match.namedGroup('section') != null) { + components.add(TemplateInjection( + 'code-${match.namedGroup('section')}', [], + language: language!)); + } else { + components.add( + TemplateInjection('code', [], language: language!)); + } + } else { + language = null; + } + continue; + } + if (!inCodeBlock) { + description.add(line); + } else { + assert(language != null); + components.last.contents.add(line); + } + } + final List descriptionLines = []; + bool lastWasWhitespace = false; + for (final String line in description + .map((SourceLine line) => line.text.trimRight())) { + final bool onlyWhitespace = line.trim().isEmpty; + if (onlyWhitespace && descriptionLines.isEmpty) { + // Don't add whitespace lines until we see something without whitespace. + lastWasWhitespace = onlyWhitespace; + continue; + } + if (onlyWhitespace && lastWasWhitespace) { + // Don't add more than one whitespace line in a row. + continue; + } + descriptionLines.add(line); + lastWasWhitespace = onlyWhitespace; + } + sample.description = descriptionLines.join('\n').trimRight(); + sample.parts = [ + if (sample is SnippetSample) + TemplateInjection('#assumptions', sample.assumptions), + ...components, + ]; + return sample.parts; + } + + String _loadFileAsUtf8(File file) { + return file.readAsStringSync(); + } + + /// Generate the HTML using the skeleton file for the type of the given sample. + /// + /// Returns a string with the HTML needed to embed in a web page for showing a + /// sample on the web page. + String generateHtml(CodeSample sample) { + final String skeleton = + _loadFileAsUtf8(configuration.getHtmlSkeletonFile(sample.type)); + return interpolateSkeleton(sample, skeleton); + } + + // Sets the description string on the sample and in the sample metadata to a + // comment version of the description. + // Trims lines of extra whitespace, and strips leading and trailing blank + // lines. + String _getDescription(CodeSample sample) { + return sample.description.splitMapJoin( + '\n', + onMatch: (Match match) => match.group(0)!, + onNonMatch: (String nonmatch) => + nonmatch.trimRight().isEmpty ? '//' : '// ${nonmatch.trimRight()}', + ); + } + + /// The main routine for generating code samples from the source code doc comments. + /// + /// The `sample` is the block of sample code from a dartdoc comment. + /// + /// The optional `output` is the file to write the generated sample code to. + /// + /// If `addSectionMarkers` is true, then markers will be added before and + /// after each template section in the output. This is intended to facilitate + /// editing of the sample during the authoring process. + /// + /// If `includeAssumptions` is true, then the block in the "Examples can + /// assume:" block will also be included in the output. + /// + /// Returns a string containing the resulting code sample. + String generateCode( + CodeSample sample, { + File? output, + String? copyright, + String? description, + bool formatOutput = true, + bool addSectionMarkers = false, + bool includeAssumptions = false, + }) { + sample.metadata['copyright'] ??= copyright; + final List snippetData = parseInput(sample); + sample.description = description ?? sample.description; + sample.metadata['description'] = _getDescription(sample); + switch (sample.runtimeType) { + case DartpadSample _: + case ApplicationSample _: + String app; + if (sample.sourceFile == null) { + final String templateName = sample.template; + if (templateName.isEmpty) { + io.stderr + .writeln('Non-linked samples must have a --template argument.'); + io.exit(1); + } + final Directory templatesDir = configuration.templatesDirectory; + File? templateFile; + templateFile = + getTemplatePath(templateName, templatesDir: templatesDir); + if (templateFile == null) { + io.stderr.writeln( + 'The template $templateName was not found in the templates ' + 'directory ${templatesDir.path}'); + io.exit(1); + } + final String templateContents = _loadFileAsUtf8(templateFile); + final String templateRelativePath = + templateFile.absolute.path.contains(flutterRoot.absolute.path) + ? path.relative(templateFile.absolute.path, + from: flutterRoot.absolute.path) + : templateFile.absolute.path; + final String templateHeader = ''' +// Template: $templateRelativePath +// +// Comment lines marked with "▼▼▼" and "▲▲▲" are used for authoring +// of samples, and may be ignored if you are just exploring the sample. +'''; + app = interpolateTemplate( + snippetData, + addSectionMarkers + ? '$templateHeader\n$templateContents' + : templateContents, + sample.metadata, + addSectionMarkers: addSectionMarkers, + addCopyright: copyright != null, + ); + } else { + app = sample.sourceFileContents; + } + sample.output = app; + if (formatOutput) { + final DartFormatter formatter = + DartFormatter(pageWidth: 80, fixes: StyleFix.all); + try { + sample.output = formatter.format(sample.output); + } on FormatterException catch (exception) { + io.stderr + .write('Code to format:\n${_addLineNumbers(sample.output)}\n'); + errorExit('Unable to format sample code: $exception'); + } + sample.output = sortImports(sample.output); + } + if (output != null) { + output.writeAsStringSync(sample.output); + + final File metadataFile = configuration.filesystem.file(path.join( + path.dirname(output.path), + '${path.basenameWithoutExtension(output.path)}.json')); + sample.metadata['file'] = path.basename(output.path); + final Map metadata = sample.metadata; + if (metadata.containsKey('description')) { + metadata['description'] = (metadata['description']! as String) + .replaceAll(RegExp(r'^// ?', multiLine: true), ''); + } + metadataFile.writeAsStringSync(jsonEncoder.convert(metadata)); + } + case SnippetSample _: + if (sample is SnippetSample) { + String app; + if (sample.sourceFile == null) { + String templateContents; + if (includeAssumptions) { + templateContents = + '${headers.map((SourceLine line) => line.text).join('\n')}\n{{#assumptions}}\n{{description}}\n{{code}}'; + } else { + templateContents = '{{description}}\n{{code}}'; + } + app = interpolateTemplate( + snippetData, + templateContents, + sample.metadata, + addSectionMarkers: addSectionMarkers, + addCopyright: copyright != null, + ); + } else { + app = sample.inputAsString; + } + sample.output = app; + } + } + return sample.output; + } + + String _addLineNumbers(String code) { + final StringBuffer buffer = StringBuffer(); + int count = 0; + for (final String line in code.split('\n')) { + count++; + buffer.writeln('${count.toString().padLeft(5)}: $line'); + } + return buffer.toString(); + } + + /// Computes the headers needed for each snippet file. + /// + /// Not used for "sample" and "dartpad" samples, which use their own template. + List get headers { + return _headers ??= [ + '// generated code', + '// ignore_for_file: unused_import', + '// ignore_for_file: unused_element', + '// ignore_for_file: unused_local_variable', + "import 'dart:async';", + "import 'dart:convert';", + "import 'dart:math' as math;", + "import 'dart:typed_data';", + "import 'dart:ui' as ui;", + "import 'package:flutter_test/flutter_test.dart';", + for (final File file in _listDartFiles(FlutterInformation.instance + .getFlutterRoot() + .childDirectory('packages') + .childDirectory('flutter') + .childDirectory('lib'))) ...[ + '', + '// ${file.path}', + "import 'package:flutter/${path.basename(file.path)}';", + ], + ].map((String code) => SourceLine(code)).toList(); + } + + List? _headers; + + static List _listDartFiles(Directory directory, + {bool recursive = false}) { + return directory + .listSync(recursive: recursive, followLinks: false) + .whereType() + .where((File file) => path.extension(file.path) == '.dart') + .toList(); + } +} diff --git a/dev/snippets/lib/src/snippet_parser.dart b/dev/snippets/lib/src/snippet_parser.dart new file mode 100644 index 0000000000000..9af60b483453e --- /dev/null +++ b/dev/snippets/lib/src/snippet_parser.dart @@ -0,0 +1,426 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/file.dart'; +import 'package:path/path.dart' as path; + +import 'data_types.dart'; +import 'util.dart'; + +/// Parses [CodeSample]s from the source file given to one of the parsing routines. +/// +/// - [parseFromDartdocToolFile] parses the output of the dartdoc `@tool` +/// directive, which contains the dartdoc comment lines (with comment markers +/// stripped) contained between the tool markers. +/// +/// - [parseAndAddAssumptions] parses the assumptions in the "Examples can +/// assume:" block at the top of the file and adds them to the code samples +/// contained in the given [SourceElement] iterable. +class SnippetDartdocParser { + SnippetDartdocParser(this.filesystem); + + final FileSystem filesystem; + + /// The prefix of each comment line + static const String _dartDocPrefix = '///'; + + /// The prefix of each comment line with a space appended. + static const String _dartDocPrefixWithSpace = '$_dartDocPrefix '; + + /// A RegExp that matches the beginning of a dartdoc snippet or sample. + static final RegExp _dartDocSampleBeginRegex = + RegExp(r'\{@tool (?sample|snippet|dartpad)(?:| (?[^}]*))\}'); + + /// A RegExp that matches the end of a dartdoc snippet or sample. + static final RegExp _dartDocSampleEndRegex = RegExp(r'\{@end-tool\}'); + + /// A RegExp that matches the start of a code block within dartdoc. + static final RegExp _codeBlockStartRegex = RegExp(r'///\s+```dart.*$'); + + /// A RegExp that matches the end of a code block within dartdoc. + static final RegExp _codeBlockEndRegex = RegExp(r'///\s+```\s*$'); + + /// A RegExp that matches a linked sample pointer. + static final RegExp _filePointerRegex = + RegExp(r'\*\* See code in (?[^\]]+) \*\*'); + + /// Parses the assumptions in the "Examples can assume:" block at the top of + /// the `assumptionsFile` and adds them to the code samples contained in the + /// given `elements` iterable. + void parseAndAddAssumptions( + Iterable elements, + File assumptionsFile, { + bool silent = true, + }) { + final List assumptions = parseAssumptions(assumptionsFile); + for (final CodeSample sample in elements + .expand((SourceElement element) => element.samples)) { + if (sample is SnippetSample) { + sample.assumptions = assumptions; + } + sample.metadata.addAll({ + 'id': '${sample.element}.${sample.index}', + 'element': sample.element, + 'sourcePath': assumptionsFile.path, + 'sourceLine': sample.start.line, + }); + } + } + + /// Parses a file containing the output of the dartdoc `@tool` directive, + /// which contains the dartdoc comment lines (with comment markers stripped) + /// between the tool markers. + /// + /// This is meant to be run as part of a dartdoc tool that handles snippets. + SourceElement parseFromDartdocToolFile( + File input, { + int? startLine, + String? element, + required File sourceFile, + String type = '', + bool silent = true, + }) { + final List lines = []; + int lineNumber = startLine ?? 0; + final List inputStrings = [ + // The parser wants to read the arguments from the input, so we create a new + // tool line to match the given arguments, so that we can use the same parser for + // editing and docs generation. + '/// {@tool $type}', + // Snippet input comes in with the comment markers stripped, so we add them + // back to make it conform to the source format, so we can use the same + // parser for editing samples as we do for processing docs. + ...input + .readAsLinesSync() + .map((String line) => '/// $line'.trimRight()), + '/// {@end-tool}', + ]; + for (final String line in inputStrings) { + lines.add( + SourceLine(line, + element: element ?? '', line: lineNumber, file: sourceFile), + ); + lineNumber++; + } + // No need to get assumptions: dartdoc won't give that to us. + final SourceElement newElement = SourceElement( + SourceElementType.unknownType, element!, -1, + file: input, comment: lines); + parseFromComments([newElement], silent: silent); + for (final CodeSample sample in newElement.samples) { + sample.metadata.addAll({ + 'id': '${sample.element}.${sample.index}', + 'element': sample.element, + 'sourcePath': sourceFile.path, + 'sourceLine': sample.start.line, + }); + } + return newElement; + } + + /// This parses the assumptions in the "Examples can assume:" block from the + /// given `file`. + List parseAssumptions(File file) { + // Whether or not we're in the file-wide preamble section ("Examples can assume"). + bool inPreamble = false; + final List preamble = []; + int lineNumber = 0; + int charPosition = 0; + for (final String line in file.readAsLinesSync()) { + if (inPreamble && line.trim().isEmpty) { + // Reached the end of the preamble. + break; + } + if (!line.startsWith('// ')) { + lineNumber++; + charPosition += line.length + 1; + continue; + } + if (line == '// Examples can assume:') { + inPreamble = true; + lineNumber++; + charPosition += line.length + 1; + continue; + } + if (inPreamble) { + preamble.add(SourceLine( + line.substring(3), + startChar: charPosition, + endChar: charPosition + line.length + 1, + element: '#assumptions', + file: file, + line: lineNumber, + )); + } + lineNumber++; + charPosition += line.length + 1; + } + return preamble; + } + + /// This parses the code snippets from the documentation comments in the given + /// `elements`, and sets the resulting samples as the `samples` member of + /// each element in the supplied iterable. + void parseFromComments( + Iterable elements, { + bool silent = true, + }) { + int dartpadCount = 0; + int sampleCount = 0; + int snippetCount = 0; + + for (final SourceElement element in elements) { + if (element.comment.isEmpty) { + continue; + } + parseComment(element); + for (final CodeSample sample in element.samples) { + switch (sample.runtimeType) { + case DartpadSample _: + dartpadCount++; + case ApplicationSample _: + sampleCount++; + case SnippetSample _: + snippetCount++; + } + } + } + + if (!silent) { + print('Found:\n' + ' $snippetCount snippet code blocks,\n' + ' $sampleCount non-dartpad sample code sections, and\n' + ' $dartpadCount dartpad sections.\n'); + } + } + + /// This parses the documentation comment on a single [SourceElement] and + /// assigns the resulting samples to the `samples` member of the given + /// `element`. + void parseComment(SourceElement element) { + // Whether or not we're in a snippet code sample. + bool inSnippet = false; + // Whether or not we're in a '```dart' segment. + bool inDart = false; + bool foundSourceLink = false; + bool foundDartSection = false; + File? linkedFile; + List block = []; + List snippetArgs = []; + final List samples = []; + final Directory flutterRoot = FlutterInformation.instance.getFlutterRoot(); + + int index = 0; + for (final SourceLine line in element.comment) { + final String trimmedLine = line.text.trim(); + if (inSnippet) { + if (!trimmedLine.startsWith(_dartDocPrefix)) { + throw SnippetException('Snippet section unterminated.', + file: line.file?.path, line: line.line); + } + if (_dartDocSampleEndRegex.hasMatch(trimmedLine)) { + switch (snippetArgs.first) { + case 'snippet': + samples.add( + SnippetSample( + block, + index: index++, + lineProto: line, + ), + ); + case 'sample': + if (linkedFile != null) { + samples.add( + ApplicationSample.fromFile( + input: block, + args: snippetArgs, + sourceFile: linkedFile, + index: index++, + lineProto: line, + ), + ); + break; + } + samples.add( + ApplicationSample( + input: block, + args: snippetArgs, + index: index++, + lineProto: line, + ), + ); + case 'dartpad': + if (linkedFile != null) { + samples.add( + DartpadSample.fromFile( + input: block, + args: snippetArgs, + sourceFile: linkedFile, + index: index++, + lineProto: line, + ), + ); + break; + } + samples.add( + DartpadSample( + input: block, + args: snippetArgs, + index: index++, + lineProto: line, + ), + ); + default: + throw SnippetException( + 'Unknown snippet type ${snippetArgs.first}'); + } + snippetArgs = []; + block = []; + inSnippet = false; + foundSourceLink = false; + foundDartSection = false; + linkedFile = null; + } else if (_filePointerRegex.hasMatch(trimmedLine)) { + foundSourceLink = true; + if (foundDartSection) { + throw SnippetException( + 'Snippet contains a source link and a dart section. Cannot contain both.', + file: line.file?.path, + line: line.line, + ); + } + if (linkedFile != null) { + throw SnippetException( + 'Found more than one linked sample. Only one linked file per sample is allowed.', + file: line.file?.path, + line: line.line, + ); + } + final RegExpMatch match = _filePointerRegex.firstMatch(trimmedLine)!; + linkedFile = filesystem.file( + path.join(flutterRoot.absolute.path, match.namedGroup('file'))); + } else { + block.add(line.copyWith( + text: line.text.replaceFirst(RegExp(r'\s*/// ?'), ''))); + } + } else { + if (_dartDocSampleEndRegex.hasMatch(trimmedLine)) { + if (inDart) { + throw SnippetException( + "Dart section didn't terminate before end of sample", + file: line.file?.path, + line: line.line); + } + } + if (inDart) { + if (_codeBlockEndRegex.hasMatch(trimmedLine)) { + inDart = false; + block = []; + } else if (trimmedLine == _dartDocPrefix) { + block.add(line.copyWith(text: '')); + } else { + final int index = line.text.indexOf(_dartDocPrefixWithSpace); + if (index < 0) { + throw SnippetException( + 'Dart section inexplicably did not contain "$_dartDocPrefixWithSpace" prefix.', + file: line.file?.path, + line: line.line, + ); + } + block.add(line.copyWith(text: line.text.substring(index + 4))); + } + } else if (_codeBlockStartRegex.hasMatch(trimmedLine)) { + if (foundSourceLink) { + throw SnippetException( + 'Snippet contains a source link and a dart section. Cannot contain both.', + file: line.file?.path, + line: line.line, + ); + } + assert(block.isEmpty); + inDart = true; + foundDartSection = true; + } + } + if (!inSnippet && !inDart) { + final RegExpMatch? sampleMatch = + _dartDocSampleBeginRegex.firstMatch(trimmedLine); + if (sampleMatch != null) { + inSnippet = sampleMatch.namedGroup('type') == 'snippet' || + sampleMatch.namedGroup('type') == 'sample' || + sampleMatch.namedGroup('type') == 'dartpad'; + if (inSnippet) { + if (sampleMatch.namedGroup('args') != null) { + // There are arguments to the snippet tool to keep track of. + snippetArgs = [ + sampleMatch.namedGroup('type')!, + ..._splitUpQuotedArgs(sampleMatch.namedGroup('args')!) + ]; + } else { + snippetArgs = [ + sampleMatch.namedGroup('type')!, + ]; + } + } + } + } + } + for (final CodeSample sample in samples) { + sample.metadata.addAll({ + 'id': '${sample.element}.${sample.index}', + 'element': sample.element, + 'sourcePath': sample.start.file?.path ?? '', + 'sourceLine': sample.start.line, + }); + } + element.replaceSamples(samples); + } + + // Helper to process arguments given as a (possibly quoted) string. + // + // First, this will split the given [argsAsString] into separate arguments, + // taking any quoting (either ' or " are accepted) into account, including + // handling backslash-escaped quotes. + // + // Then, it will prepend "--" to any args that start with an identifier + // followed by an equals sign, allowing the argument parser to treat any + // "foo=bar" argument as "--foo=bar" (which is a dartdoc-ism). + Iterable _splitUpQuotedArgs(String argsAsString) { + // This function is used because the arg parser package doesn't handle + // quoted args. + + // Regexp to take care of splitting arguments, and handling the quotes + // around arguments, if any. + // + // Match group 1 (option) is the "foo=" (or "--foo=") part of the option, if any. + // Match group 2 (quote) contains the quote character used (which is discarded). + // Match group 3 (value) is a quoted arg, if any, without the quotes. + // Match group 4 (unquoted) is the unquoted arg, if any. + final RegExp argMatcher = RegExp( + r'(?