isModerated flags + backfill in Package, PackageVersion, Publisher an…

…d User (#7538)

CHANGELOG.md

@@ -6,6 +6,7 @@ AppEngine version, listed here to ease deployment and troubleshooting.
  * Upgraded stable Flutter analysis SDK to `3.19.2`.
  * Upgraded dartdoc to `8.0.6`.
  * Note: preview analysis SDKs are downloaded dynamically.
+ * Note: started to backfill `isModerated` flags in `Package`, `PackageVersion`, `Publisher` and `User`.
  * Resyncing all `SecurityAdvisory` entities to get `pub_display_url` in the
   `database_specific` field backfilled.
 

app/lib/account/backend.dart

@@ -411,14 +411,12 @@ class AccountBackend {
       }
 
       // Create new user with oauth2 user_id mapping
-      final user = User()
+      final user = User.init()
         ..parentKey = emptyKey
         ..id = createUuid()
         ..oauthUserId = auth.oauthUserId
         ..email = auth.email
-        ..created = clock.now().toUtc()
-        ..isBlocked = false
-        ..isDeleted = false;
+        ..created = clock.now().toUtc();
 
       tx.insert(user);
       tx.insert(

app/lib/account/models.dart

@@ -53,6 +53,22 @@ class User extends db.ExpandoModel<String> {
   /// to perform any action.
   @db.BoolProperty(required: true)
   bool isBlocked = false;
+
+  /// `true` if user was moderated (pending moderation or deletion).
+  /// TODO: set this to true after backfill and all versions use it
+  @db.BoolProperty(required: false)
+  bool? isModerated;
+
+  /// The timestamp when the user was moderated.
+  @db.DateTimeProperty()
+  DateTime? moderatedAt;
+
+  User();
+  User.init() {
+    isBlocked = false;
+    isDeleted = false;
+    isModerated = false;
+  }
 }
 
 /// Maps Oauth user_id to User.id

app/lib/package/backend.dart

@@ -1754,7 +1754,7 @@ Future<_UploadEntities> _createUploadEntities(
   final packageKey = db.emptyKey.append(Package, id: pubspec.name);
   final versionString = canonicalizeVersion(pubspec.nonCanonicalVersion);
 
-  final version = PackageVersion()
+  final version = PackageVersion.init()
     ..id = versionString
     ..parentKey = packageKey
     ..version = versionString
@@ -1763,8 +1763,7 @@ Future<_UploadEntities> _createUploadEntities(
     ..pubspec = pubspec
     ..libraries = archive.libraries
     ..uploader = agent.agentId
-    ..sha256 = sha256Hash
-    ..isRetracted = false;
+    ..sha256 = sha256Hash;
 
   final derived = derivePackageVersionEntities(
     archive: archive,

app/lib/package/models.dart

@@ -137,6 +137,15 @@ class Package extends db.ExpandoModel<String> {
   @db.DateTimeProperty()
   DateTime? blocked;
 
+  /// `true` if package was moderated (pending moderation or deletion).
+  /// TODO: set this to true after backfill and all versions use it
+  @db.BoolProperty(required: false)
+  bool? isModerated;
+
+  /// The timestamp when the package was moderated.
+  @db.DateTimeProperty()
+  DateTime? moderatedAt;
+
   /// Tags that are assigned to this package.
   ///
   /// The permissions required to assign a tag typically depends on the tag.
@@ -187,6 +196,7 @@ class Package extends db.ExpandoModel<String> {
       ..isDiscontinued = false
       ..isUnlisted = false
       ..isBlocked = false
+      ..isModerated = false
       ..assignedTags = []
       ..deletedVersions = [];
   }
@@ -579,6 +589,22 @@ class PackageVersion extends db.ExpandoModel<String> {
   @db.DateTimeProperty()
   DateTime? retracted;
 
+  /// `true` if package version was moderated (pending moderation or deletion).
+  /// TODO: set this to true after backfill and all versions use it
+  @db.BoolProperty(required: false)
+  bool? isModerated;
+
+  /// The timestamp when the package version was moderated.
+  @db.DateTimeProperty()
+  DateTime? moderatedAt;
+
+  PackageVersion();
+
+  PackageVersion.init() {
+    isModerated = false;
+    isRetracted = false;
+  }
+
   // Convenience Fields:
 
   late final semanticVersion = Version.parse(version!);

app/lib/publisher/models.dart

@@ -64,6 +64,15 @@ class Publisher extends db.ExpandoModel<String> {
   @db.BoolProperty(required: true)
   bool isBlocked = false;
 
+  /// `true` if publisher was moderated (pending moderation or deletion).
+  /// TODO: set this to true after backfill and all versions use it
+  @db.BoolProperty(required: false)
+  bool? isModerated;
+
+  /// The timestamp when the publisher was moderated.
+  @db.DateTimeProperty()
+  DateTime? moderatedAt;
+
   Publisher();
 
   Publisher.init({
@@ -80,6 +89,7 @@ class Publisher extends db.ExpandoModel<String> {
     websiteUrl = defaultPublisherWebsite(publisherId);
     isAbandoned = false;
     isBlocked = false;
+    isModerated = false;
   }
 
   /// Clears most properties on the entity and sets the [isBlocked] flag.

app/lib/tool/backfill/backfill_new_fields.dart

@@ -3,7 +3,11 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:logging/logging.dart';
+import 'package:pub_dev/account/models.dart';
+import 'package:pub_dev/package/models.dart';
+import 'package:pub_dev/publisher/models.dart';
 import 'package:pub_dev/service/security_advisories/sync_security_advisories.dart';
+import 'package:pub_dev/shared/datastore.dart';
 
 final _logger = Logger('backfill_new_fields');
 
@@ -16,4 +20,39 @@ Future<void> backfillNewFields() async {
   _logger.info('Resyncing all security advisories...');
   // This will backfill the `pub_display_url` in the `database_specific` field on the `SecurityAdvisory` entity.
   await syncSecurityAdvisories(resync: true);
+
+  _logger.info('Backfilling isModerated fileds...');
+  await for (final e in dbService.query<Package>().run()) {
+    if (e.isModerated != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final o = await tx.lookupValue<Package>(e.key);
+      o.isModerated = false;
+      tx.insert(o);
+    });
+  }
+  await for (final e in dbService.query<PackageVersion>().run()) {
+    if (e.isModerated != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final o = await tx.lookupValue<PackageVersion>(e.key);
+      o.isModerated = false;
+      tx.insert(o);
+    });
+  }
+  await for (final e in dbService.query<User>().run()) {
+    if (e.isModerated != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final o = await tx.lookupValue<User>(e.key);
+      o.isModerated = false;
+      tx.insert(o);
+    });
+  }
+  await for (final e in dbService.query<Publisher>().run()) {
+    if (e.isModerated != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final o = await tx.lookupValue<Publisher>(e.key);
+      o.isModerated = false;
+      tx.insert(o);
+    });
+  }
+  _logger.info('Backfill completed.');
 }

app/test/audit/backend_test.dart

@@ -44,7 +44,7 @@ void main() {
         package: 'pkg',
         version: '1.0.0',
         uploader: AuthenticatedUser(
-          User()
+          User.init()
             ..id = 'user-id'
             ..email = 'user@pub.dev',
           audience: 'fake-client-audience',

app/test/package/models_test.dart

@@ -329,7 +329,7 @@ class _PublishSequence {
 
   void publish(String version, {int sdk = 0}) {
     final minSdk = sdk > 0 ? _futureSdk : (sdk < 0 ? _pastSdk : _currentSdk);
-    final pv = PackageVersion()
+    final pv = PackageVersion.init()
       ..parentKey = _p.key
       ..id = version
       ..version = version