From 61d518fcf76091235bec7bdc81d778d6355bb423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20So=C3=B3s?= Date: Thu, 21 Mar 2024 18:12:00 +0100 Subject: [PATCH] Admin action + some tests for PackageVersion.isModerated. (#7560) --- app/lib/admin/actions/actions.dart | 3 +- .../actions/moderate_package_versions.dart | 113 +++++++++ app/lib/package/backend.dart | 3 + app/lib/package/models.dart | 12 + app/lib/shared/exceptions.dart | 3 + .../moderate_package_version_test.dart | 235 ++++++++++++++++++ 6 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 app/lib/admin/actions/moderate_package_versions.dart create mode 100644 app/test/package/moderate_package_version_test.dart diff --git a/app/lib/admin/actions/actions.dart b/app/lib/admin/actions/actions.dart index 2db66242e..d2758df47 100644 --- a/app/lib/admin/actions/actions.dart +++ b/app/lib/admin/actions/actions.dart @@ -3,11 +3,11 @@ // BSD-style license that can be found in the LICENSE file. import '../../shared/exceptions.dart'; - import 'create_publisher.dart'; import 'delete_publisher.dart'; import 'merge_moderated_package_into_existing.dart'; import 'moderate_package.dart'; +import 'moderate_package_versions.dart'; import 'moderate_user.dart'; import 'publisher_block.dart'; import 'publisher_members_list.dart'; @@ -73,6 +73,7 @@ final class AdminAction { deletePublisher, mergeModeratedPackageIntoExisting, moderatePackage, + moderatePackageVersion, moderateUser, publisherBlock, publisherMembersList, diff --git a/app/lib/admin/actions/moderate_package_versions.dart b/app/lib/admin/actions/moderate_package_versions.dart new file mode 100644 index 000000000..c97d3ed6a --- /dev/null +++ b/app/lib/admin/actions/moderate_package_versions.dart @@ -0,0 +1,113 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:_pub_shared/utils/dart_sdk_version.dart'; +import 'package:clock/clock.dart'; + +import '../../package/backend.dart'; +import '../../package/models.dart'; +import '../../shared/datastore.dart'; +import '../../shared/versions.dart'; +import '../../task/backend.dart'; +import '../../tool/maintenance/update_public_bucket.dart'; + +import 'actions.dart'; + +final moderatePackageVersion = AdminAction( + name: 'moderate-package-version', + summary: + 'Set the moderated flag on a package version (making it not visible).', + description: ''' +Set the moderated flag on a package version (updating the flag and the timestamp). +''', + options: { + 'package': 'The package name to be moderated', + 'version': 'The version to be moderated', + 'state': + 'Set moderated state true / false. Returns current state if omitted.', + }, + invoke: (options) async { + final package = options['package']; + InvalidInputException.check( + package != null && package.isNotEmpty, + 'package must be given', + ); + final version = options['version']; + InvalidInputException.check( + version != null && version.isNotEmpty, + 'version must be given', + ); + + final state = options['state']; + bool? valueToSet; + switch (state) { + case 'true': + valueToSet = true; + break; + case 'false': + valueToSet = false; + break; + } + + final p = await packageBackend.lookupPackage(package!); + if (p == null) { + throw NotFoundException.resource(package); + } + final pv = await packageBackend.lookupPackageVersion(package, version!); + if (pv == null) { + throw NotFoundException.resource('$package $version'); + } + + PackageVersion? pv2; + if (valueToSet != null) { + final currentDartSdk = + await getDartSdkVersion(lastKnownStable: toolStableDartSdkVersion); + pv2 = await withRetryTransaction(dbService, (tx) async { + final v = await tx.lookupValue(pv.key); + v.updateIsModerated(isModerated: valueToSet!); + tx.insert(v); + + // Update references to latest versions. + final pkg = await tx.lookupValue(p.key); + if (pkg.mayAffectLatestVersions(v.semanticVersion)) { + final versions = + await tx.query(pkg.key).run().toList(); + pkg.updateLatestVersionReferences( + versions, + dartSdkVersion: currentDartSdk.semanticVersion, + replaced: v, + ); + } + pkg.updated = clock.now().toUtc(); + tx.insert(pkg); + + return v; + }); + + // retract or re-populate public archive files + await updatePublicArchiveBucket( + package: package, + ageCheckThreshold: Duration.zero, + deleteIfOlder: Duration.zero, + ); + + await taskBackend.trackPackage(package); + await purgePackageCache(package); + } + + return { + 'package': p.name, + 'version': pv.version, + 'before': { + 'isModerated': pv.isModerated, + 'moderatedAt': pv.moderatedAt?.toIso8601String(), + }, + if (pv2 != null) + 'after': { + 'isModerated': pv2.isModerated, + 'moderatedAt': pv2.moderatedAt?.toIso8601String(), + }, + }; + }, +); diff --git a/app/lib/package/backend.dart b/app/lib/package/backend.dart index 40dcdc8e9..b1c2f51f4 100644 --- a/app/lib/package/backend.dart +++ b/app/lib/package/backend.dart @@ -459,6 +459,9 @@ class PackageBackend { if (pv == null) { throw NotFoundException.resource(version); } + if (pv.isModerated) { + throw ModeratedException.packageVersion(package, version); + } if (options.isRetracted != null && options.isRetracted != pv.isRetracted) { diff --git a/app/lib/package/models.dart b/app/lib/package/models.dart index 27f148d1e..727497c9f 100644 --- a/app/lib/package/models.dart +++ b/app/lib/package/models.dart @@ -289,7 +289,12 @@ class Package extends db.ExpandoModel { }) { final versions = allVersions .map((v) => v.version == replaced?.version ? replaced! : v) + .where((v) => !v.isModerated) .toList(); + if (versions.isEmpty) { + throw NotAcceptableException('No visible versions left.'); + } + final oldStableVersion = latestSemanticVersion; final oldPrereleaseVersion = latestPrereleaseSemanticVersion; final oldPreviewVersion = latestPreviewSemanticVersion; @@ -666,6 +671,13 @@ class PackageVersion extends db.ExpandoModel { bool get canUndoRetracted => isRetracted && retracted!.isAfter(clock.now().toUtc().subtract(const Duration(days: 7))); + + void updateIsModerated({ + required bool isModerated, + }) { + this.isModerated = isModerated; + moderatedAt = isModerated ? clock.now().toUtc() : null; + } } /// A derived entity that holds derived/cleaned content of [PackageVersion]. diff --git a/app/lib/shared/exceptions.dart b/app/lib/shared/exceptions.dart index 3b10074e6..b112efc5e 100644 --- a/app/lib/shared/exceptions.dart +++ b/app/lib/shared/exceptions.dart @@ -577,6 +577,9 @@ class RemovedPackageException extends NotFoundException { class ModeratedException extends NotFoundException { ModeratedException.package(String package) : super('Package "$package" has been moderated.'); + + ModeratedException.packageVersion(String package, String version) + : super('PackageVersion "$package" "$version" has been moderated.'); } /// Thrown when API endpoint is not implemented. diff --git a/app/test/package/moderate_package_version_test.dart b/app/test/package/moderate_package_version_test.dart new file mode 100644 index 000000000..0bd9a6d6f --- /dev/null +++ b/app/test/package/moderate_package_version_test.dart @@ -0,0 +1,235 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:_pub_shared/data/admin_api.dart'; +import 'package:_pub_shared/data/package_api.dart'; +import 'package:clock/clock.dart'; +import 'package:http/http.dart' as http; +import 'package:pub_dev/fake/backend/fake_auth_provider.dart'; +import 'package:pub_dev/package/backend.dart'; +import 'package:pub_dev/search/backend.dart'; +import 'package:pub_dev/shared/configuration.dart'; +import 'package:pub_dev/tool/maintenance/update_public_bucket.dart'; +import 'package:test/test.dart'; + +import '../shared/handlers_test_utils.dart'; +import '../shared/test_models.dart'; +import '../shared/test_services.dart'; +import 'backend_test_utils.dart'; + +void main() { + group('Moderate package version', () { + Future _moderate( + String package, + String version, { + bool? state, + }) async { + final api = createPubApiClient(authToken: siteAdminToken); + return await api.adminInvokeAction( + 'moderate-package-version', + AdminInvokeActionArguments(arguments: { + 'package': package, + 'version': version, + if (state != null) 'state': state.toString(), + }), + ); + } + + testWithProfile('update state', fn: () async { + final r1 = await _moderate('oxygen', '1.0.0'); + expect(r1.output, { + 'package': 'oxygen', + 'version': '1.0.0', + 'before': {'isModerated': false, 'moderatedAt': null}, + }); + + final r2 = await _moderate('oxygen', '1.0.0', state: true); + expect(r2.output, { + 'package': 'oxygen', + 'version': '1.0.0', + 'before': {'isModerated': false, 'moderatedAt': null}, + 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + }); + final p1 = await packageBackend.lookupPackage('oxygen'); + expect(p1!.isModerated, isFalse); + final pv2 = await packageBackend.lookupPackageVersion('oxygen', '1.0.0'); + expect(pv2!.isModerated, isTrue); + + // cannot redact this version + await expectApiException( + (await createFakeAuthPubApiClient(email: adminAtPubDevEmail)) + .setVersionOptions( + 'oxygen', '1.0.0', VersionOptions(isRetracted: true)), + code: 'NotFound', + status: 404, + message: 'PackageVersion \"oxygen\" \"1.0.0\" has been moderated.', + ); + + // can redact other version + final optionsUpdates = + await (await createFakeAuthPubApiClient(email: adminAtPubDevEmail)) + .setVersionOptions( + 'oxygen', '1.2.0', VersionOptions(isRetracted: true)); + expect(optionsUpdates.isRetracted, true); + final p2 = await packageBackend.lookupPackage('oxygen'); + expect(p2!.latestVersion, '2.0.0-dev'); + }); + + testWithProfile('clear moderation flag', fn: () async { + final r1 = await _moderate('oxygen', '1.0.0', state: true); + expect(r1.output, { + 'package': 'oxygen', + 'version': '1.0.0', + 'before': {'isModerated': false, 'moderatedAt': null}, + 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + }); + final p1 = await packageBackend.lookupPackage('oxygen'); + expect(p1!.isModerated, isFalse); + final pv1 = await packageBackend.lookupPackageVersion('oxygen', '1.0.0'); + expect(pv1!.isModerated, isTrue); + + // clear flag + final r2 = await _moderate('oxygen', '1.0.0', state: false); + expect(r2.output, { + 'package': 'oxygen', + 'version': '1.0.0', + 'before': {'isModerated': true, 'moderatedAt': isNotEmpty}, + 'after': {'isModerated': false, 'moderatedAt': null}, + }); + final p2 = await packageBackend.lookupPackage('oxygen'); + expect(p2!.isModerated, isFalse); + final pv2 = await packageBackend.lookupPackageVersion('oxygen', '1.0.0'); + expect(pv2!.isModerated, isFalse); + + // can redact this version + final optionsUpdates = + await (await createFakeAuthPubApiClient(email: adminAtPubDevEmail)) + .setVersionOptions( + 'oxygen', '1.0.0', VersionOptions(isRetracted: true)); + expect(optionsUpdates.isRetracted, true); + }); + + testWithProfile('cannot moderated last visible version', fn: () async { + await _moderate('oxygen', '1.2.0', state: true); + final p1 = await packageBackend.lookupPackage('oxygen'); + expect(p1!.latestVersion, '1.0.0'); + expect(p1.latestPrereleaseVersion, '2.0.0-dev'); + expect(p1.latestPreviewVersion, '1.0.0'); + + await _moderate('oxygen', '2.0.0-dev', state: true); + final p2 = await packageBackend.lookupPackage('oxygen'); + expect(p2!.latestVersion, '1.0.0'); + expect(p2.latestPrereleaseVersion, '1.0.0'); + expect(p2.latestPreviewVersion, '1.0.0'); + + await expectApiException( + _moderate('oxygen', '1.0.0', state: true), + code: 'NotAcceptable', + status: 406, + message: 'No visible versions left.', + ); + }); + + testWithProfile('can publish new version', fn: () async { + await _moderate('oxygen', '1.0.0', state: true); + + final pubspecContent = generatePubspecYaml('oxygen', '3.0.0'); + final bytes = await packageArchiveBytes(pubspecContent: pubspecContent); + final message = await createPubApiClient(authToken: adminClientToken) + .uploadPackageBytes(bytes); + expect(message.success.message, contains('Successfully uploaded')); + }); + + testWithProfile('can not re-publish moderated version', fn: () async { + await _moderate('oxygen', '1.0.0', state: true); + + final pubspecContent = generatePubspecYaml('oxygen', '1.0.0'); + final bytes = await packageArchiveBytes(pubspecContent: pubspecContent); + await expectApiException( + createPubApiClient(authToken: adminClientToken) + .uploadPackageBytes(bytes), + code: 'PackageRejected', + status: 400, + message: 'Version 1.0.0 of package oxygen already exists.', + ); + }); + + testWithProfile('archive file is removed from public bucket', fn: () async { + Future expectStatusCode(int statusCode, + {String version = '1.0.0'}) async { + final publicUri = Uri.parse('${activeConfiguration.storageBaseUrl}' + '/${activeConfiguration.publicPackagesBucketName}' + '/packages/oxygen-$version.tar.gz'); + final rs1 = await http.get(publicUri); + expect(rs1.statusCode, statusCode); + return rs1.bodyBytes; + } + + final bytes = await expectStatusCode(200); + + await _moderate('oxygen', '1.0.0', state: true); + await expectStatusCode(404); + await expectStatusCode(200, version: '1.2.0'); + + // another check after background tasks are running + await updatePublicArchiveBucket(); + await expectStatusCode(404); + await expectStatusCode(200, version: '1.2.0'); + + await _moderate('oxygen', '1.0.0', state: false); + await expectStatusCode(200); + await updatePublicArchiveBucket(); + final restoredBytes = await expectStatusCode(200); + expect(restoredBytes, bytes); + }); + + testWithProfile('search is updated with new version', fn: () async { + await searchBackend.doCreateAndUpdateSnapshot( + FakeGlobalLockClaim(clock.now().add(Duration(seconds: 3))), + concurrency: 2, + sleepDuration: Duration(milliseconds: 300), + ); + final docs = await searchBackend.fetchSnapshotDocuments(); + expect(docs!.firstWhere((d) => d.package == 'oxygen').version, '1.2.0'); + + await _moderate('oxygen', '1.2.0', state: true); + final minimumIndex = + await searchBackend.loadMinimumPackageIndex().toList(); + expect(minimumIndex.firstWhere((d) => d.package == 'oxygen').version, + '1.0.0'); + + await searchBackend.doCreateAndUpdateSnapshot( + FakeGlobalLockClaim(clock.now().add(Duration(seconds: 3))), + concurrency: 2, + sleepDuration: Duration(milliseconds: 300), + ); + final docs2 = await searchBackend.fetchSnapshotDocuments(); + expect(docs2!.firstWhere((d) => d.package == 'oxygen').version, '1.0.0'); + + await _moderate('oxygen', '1.2.0', state: false); + + final minimumIndex2 = + await searchBackend.loadMinimumPackageIndex().toList(); + expect(minimumIndex2.firstWhere((d) => d.package == 'oxygen').version, + '1.2.0'); + + await searchBackend.doCreateAndUpdateSnapshot( + FakeGlobalLockClaim(clock.now().add(Duration(seconds: 3))), + concurrency: 2, + sleepDuration: Duration(milliseconds: 300), + ); + final docs3 = await searchBackend.fetchSnapshotDocuments(); + expect(docs3!.firstWhere((d) => d.package == 'oxygen').version, '1.2.0'); + }); + + // TODO(https://github.com/dart-lang/pub-dev/issues/7535): + // moderated version is not visible in API (other version is) + // moderated version pages are not visible (other version is) + // moderated versions tab does not display version + // moderated version is not selected for analysis + // moderated analysis is cleared and new analysis is scheduled + }); +}