Merge branch 'master' into 8.4

.github/workflows/benchmark.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-20.04
     name: Benchmark TypeScript Types
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
         with:
           fetch-depth: 0
       - name: Setup node

.github/workflows/codeql.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/documentation.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Lint Markdown files
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-20.04
     name: Test Generating Docs
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       - run: git fetch --depth=1 --tags # download all tags for documentation
 
       - name: Setup node

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Lint JS-Files
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -58,7 +58,7 @@ jobs:
       MONGOMS_PREFER_GLOBAL_PATH: 1
       FORCE_COLOR: true
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -94,7 +94,7 @@ jobs:
       MONGOMS_PREFER_GLOBAL_PATH: 1
       FORCE_COLOR: true
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
         with:
@@ -122,7 +122,7 @@ jobs:
     env:
       FORCE_COLOR: true
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
         with:
@@ -139,6 +139,6 @@ jobs:
       contents: read
     steps:
       - name: Check out repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       - name: Dependency review
         uses: actions/dependency-review-action@v4

.github/workflows/tidelift-alignment.yml

@@ -15,7 +15,7 @@ jobs:
     if: github.repository == 'Automattic/mongoose'
     steps:
       - name: Checkout
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
         with:

.github/workflows/tsd.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Lint TS-Files
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -38,7 +38,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Test Typescript Types
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       - name: Setup node
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2

CHANGELOG.md

@@ -1,3 +1,13 @@
+8.3.3 / 2024-04-29
+==================
+ * perf(document): add fast path for applying non-nested virtuals to JSON #14543
+ * fix: make hydrate() recursively hydrate virtual populate docs if hydratedPopulatedDocs is set #14533 #14503
+ * fix: improve timestamps option handling in bulkWrite #14546 #14536 [sderrow](https://github.com/sderrow)
+ * fix(model): make recompileSchema() overwrite existing document array discriminators #14527
+ * types(schema): correctly infer Array<Schema.Types.*> #14534 #14367
+ * types(query+populate): apply populate overrides to doc toObject() result #14525 #14441
+ * types: add null to select override return type for findOne #14545 [sderrow](https://github.com/sderrow)
+
 8.3.2 / 2024-04-16
 ==================
  * fix(populate): avoid match function filtering out null values in populate result #14518 #14494

docs/connections.md

@@ -457,30 +457,10 @@ module.exports = userSchema;
 // module.exports = mongoose.model('User', userSchema);
 ```
 
-If you use the export schema pattern, you still need to create models
-somewhere. There are two common patterns. First is to export a connection
-and register the models on the connection in the file:
-
-```javascript
-// connections/fast.js
-const mongoose = require('mongoose');
-
-const conn = mongoose.createConnection(process.env.MONGODB_URI);
-conn.model('User', require('../schemas/user'));
-
-module.exports = conn;
-
-// connections/slow.js
-const mongoose = require('mongoose');
-
-const conn = mongoose.createConnection(process.env.MONGODB_URI);
-conn.model('User', require('../schemas/user'));
-conn.model('PageView', require('../schemas/pageView'));
-
-module.exports = conn;
-```
-
-Another alternative is to register connections with a dependency injector
+If you use the export schema pattern, you still need to create models somewhere.
+There are two common patterns.
+The first is to create a function that instantiates a new connection and registers all models on that connection.
+With this pattern, you may also register connections with a dependency injector
 or another [inversion of control (IOC) pattern](https://thecodebarbarian.com/using-ramda-as-a-dependency-injector).
 
 ```javascript
@@ -496,6 +476,23 @@ module.exports = function connectionFactory() {
 };
 ```
 
+Exporting a function that creates a new connection is the most flexible pattern.
+However, that pattern can make it tricky to get access to your connection from your route handlers or wherever your business logic is.
+An alternative pattern is to export a connection and register the models on the connection in the file's top-level scope as follows.
+
+```javascript
+// connections/index.js
+const mongoose = require('mongoose');
+
+const conn = mongoose.createConnection(process.env.MONGODB_URI);
+conn.model('User', require('../schemas/user'));
+
+module.exports = conn;
+```
+
+You can create separate files for each connection, like `connections/web.js` and `connections/mobile.js` if you want to create separate connections for your web API backend and your mobile API backend.
+Your business logic can then `require()` or `import` the connection it needs.
+
 <h2 id="connection_pools"><a href="#connection_pools">Connection Pools</a></h2>
 
 Each `connection`, whether created with `mongoose.connect` or

docs/typescript/schemas.md

@@ -1,89 +1,98 @@
 # Schemas in TypeScript
 
 Mongoose [schemas](../guide.html) are how you tell Mongoose what your documents look like.
-Mongoose schemas are separate from TypeScript interfaces, so you need to either define both a *document interface* and a *schema*; or rely on Mongoose to automatically infer the type from the schema definition.
+Mongoose schemas are separate from TypeScript interfaces, so you need to either define both a *raw document interface* and a *schema*; or rely on Mongoose to automatically infer the type from the schema definition.
 
-## Separate document interface definition
+## Automatic type inference
+
+Mongoose can automatically infer the document type from your schema definition as follows.
+We recommend relying on automatic type inference when defining schemas and models.
 
 ```typescript
 import { Schema } from 'mongoose';
-
-// Document interface
-interface User {
-  name: string;
-  email: string;
-  avatar?: string;
-}
-
 // Schema
-const schema = new Schema<User>({
+const schema = new Schema({
   name: { type: String, required: true },
   email: { type: String, required: true },
   avatar: String
 });
+
+// `UserModel` will have `name: string`, etc.
+const UserModel = mongoose.model('User', schema);
+
+const doc = new UserModel({ name: 'test', email: 'test' });
+doc.name; // string
+doc.email; // string
+doc.avatar; // string | undefined | null
 ```
 
-By default, Mongoose does **not** check if your document interface lines up with your schema.
-For example, the above code won't throw an error if `email` is optional in the document interface, but `required` in `schema`.
+There are a few caveats for using automatic type inference:
 
-## Automatic type inference
+1. You need to set `strictNullChecks: true` or `strict: true` in your `tsconfig.json`. Or, if you're setting flags at the command line, `--strictNullChecks` or `--strict`. There are [known issues](https://github.com/Automattic/mongoose/issues/12420) with automatic type inference with strict mode disabled.
+2. You need to define your schema in the `new Schema()` call. Don't assign your schema definition to a temporary variable. Doing something like `const schemaDefinition = { name: String }; const schema = new Schema(schemaDefinition);` will not work.
+3. Mongoose adds `createdAt` and `updatedAt` to your schema if you specify the `timestamps` option in your schema, *except* if you also specify `methods`, `virtuals`, or `statics`. There is a [known issue](https://github.com/Automattic/mongoose/issues/12807) with type inference with timestamps and methods/virtuals/statics options. If you use methods, virtuals, and statics, you're responsible for adding `createdAt` and `updatedAt` to your schema definition.
+
+If automatic type inference doesn't work for you, you can always fall back to document interface definitions.
 
-Mongoose can also automatically infer the document type from your schema definition as follows.
+## Separate document interface definition
+
+If automatic type inference doesn't work for you, you can define a separate raw document interface as follows.
 
 ```typescript
-import { Schema, InferSchemaType } from 'mongoose';
+import { Schema } from 'mongoose';
 
-// Document interface
-// No need to define TS interface any more.
-// interface User {
-//   name: string;
-//   email: string;
-//   avatar?: string;
-// }
+// Raw document interface. Contains the data type as it will be stored
+// in MongoDB. So you can ObjectId, Buffer, and other custom primitive data types.
+// But no Mongoose document arrays or subdocuments.
+interface User {
+  name: string;
+  email: string;
+  avatar?: string;
+}
 
 // Schema
-const schema = new Schema({
+const schema = new Schema<User>({
   name: { type: String, required: true },
   email: { type: String, required: true },
   avatar: String
 });
-
-type User = InferSchemaType<typeof schema>;
-// InferSchemaType will determine the type as follows:
-// type User = {
-//   name: string;
-//   email: string;
-//   avatar?: string;
-// }
-
-// `UserModel` will have `name: string`, etc.
-const UserModel = mongoose.model('User', schema);
 ```
 
-There are a few caveats for using automatic type inference:
-
-1. You need to set `strictNullChecks: true` or `strict: true` in your `tsconfig.json`. Or, if you're setting flags at the command line, `--strictNullChecks` or `--strict`. There are [known issues](https://github.com/Automattic/mongoose/issues/12420) with automatic type inference with strict mode disabled.
-2. You need to define your schema in the `new Schema()` call. Don't assign your schema definition to a temporary variable. Doing something like `const schemaDefinition = { name: String }; const schema = new Schema(schemaDefinition);` will not work.
-3. Mongoose adds `createdAt` and `updatedAt` to your schema if you specify the `timestamps` option in your schema, *except* if you also specify `methods`, `virtuals`, or `statics`. There is a [known issue](https://github.com/Automattic/mongoose/issues/12807) with type inference with timestamps and methods/virtuals/statics options. If you use methods, virtuals, and statics, you're responsible for adding `createdAt` and `updatedAt` to your schema definition.
-
-If automatic type inference doesn't work for you, you can always fall back to document interface definitions.
+By default, Mongoose does **not** check if your raw document interface lines up with your schema.
+For example, the above code won't throw an error if `email` is optional in the document interface, but `required` in `schema`.
 
 ## Generic parameters
 
-The Mongoose `Schema` class in TypeScript has 4 [generic parameters](https://www.typescriptlang.org/docs/handbook/2/generics.html):
+The Mongoose `Schema` class in TypeScript has 9 [generic parameters](https://www.typescriptlang.org/docs/handbook/2/generics.html):
 
-* `DocType` - An interface describing how the data is saved in MongoDB
-* `M` - The Mongoose model type. Can be omitted if there are no query helpers or instance methods to be defined.
+* `RawDocType` - An interface describing how the data is saved in MongoDB
+* `TModelType` - The Mongoose model type. Can be omitted if there are no query helpers or instance methods to be defined.
   * default: `Model<DocType, any, any>`
 * `TInstanceMethods` - An interface containing the methods for the schema.
   * default: `{}`
 * `TQueryHelpers` - An interface containing query helpers defined on the schema. Defaults to `{}`.
+* `TVirtuals` - An interface containing virtuals defined on the schema. Defaults to `{}`
+* `TStaticMethods` - An interface containing methods on a model. Defaults to `{}`
+* `TSchemaOptions` - The type passed as the 2nd option to `Schema()` constructor. Defaults to `DefaultSchemaOptions`.
+* `DocType` - The inferred document type from the schema.
+* `THydratedDocumentType` - The hydrated document type. This is the default return type for `await Model.findOne()`, `Model.hydrate()`, etc.
 
 <details>
   <summary>View TypeScript definition</summary>
 
   ```typescript
-  class Schema<DocType = any, M = Model<DocType, any, any>, TInstanceMethods = {}, TQueryHelpers = {}> extends events.EventEmitter {
+  export class Schema<
+    RawDocType = any,
+    TModelType = Model<RawDocType, any, any, any>,
+    TInstanceMethods = {},
+    TQueryHelpers = {},
+    TVirtuals = {},
+    TStaticMethods = {},
+    TSchemaOptions = DefaultSchemaOptions,
+    DocType = ...,
+    THydratedDocumentType = HydratedDocument<FlatRecord<DocType>, TVirtuals & TInstanceMethods>
+  >
+    extends events.EventEmitter {
     // ...
   }
   ```
@@ -154,33 +163,47 @@ This is because Mongoose has numerous features that add paths to your schema tha
 
 ## Arrays
 
-When you define an array in a document interface, we recommend using Mongoose's `Types.Array` type for primitive arrays or `Types.DocumentArray` for arrays of documents.
+When you define an array in a document interface, we recommend using vanilla JavaScript arrays, **not** Mongoose's `Types.Array` type or `Types.DocumentArray` type.
+Instead, use the `THydratedDocumentType` generic to define that the hydrated document type has paths of type `Types.Array` and `Types.DocumentArray`.
 
 ```typescript
-import { Schema, Model, Types } from 'mongoose';
+import mongoose from 'mongoose'
+const { Schema } = mongoose;
 
-interface BlogPost {
-  _id: Types.ObjectId;
-  title: string;
+interface IOrder {
+  tags: Array<{ name: string }>
 }
 
-interface User {
-  tags: Types.Array<string>;
-  blogPosts: Types.DocumentArray<BlogPost>;
-}
-
-const schema = new Schema<User, Model<User>>({
-  tags: [String],
-  blogPosts: [{ title: String }]
+// Define a HydratedDocumentType that describes what type Mongoose should use
+// for fully hydrated docs returned from `findOne()`, etc.
+type OrderHydratedDocument = mongoose.HydratedDocument<
+  IOrder,
+  { tags: mongoose.Types.DocumentArray<{ name: string }> }
+>;
+type OrderModelType = mongoose.Model<
+  IOrder,
+  {},
+  {},
+  {},
+  OrderHydratedDocument
+>;
+
+const orderSchema = new mongoose.Schema<IOrder, OrderModelType>({
+  tags: [{ name: { type: String, required: true } }]
 });
-```
+const OrderModel = mongoose.model<IOrder, OrderModelType>('Order', orderSchema);
 
-Using `Types.DocumentArray` is helpful when dealing with defaults.
-For example, `BlogPost` has an `_id` property that Mongoose will set by default.
-If you use `Types.DocumentArray` in the above case, you'll be able to `push()` a subdocument without an `_id`.
+// Demonstrating return types from OrderModel
+const doc = new OrderModel({ tags: [{ name: 'test' }] });
 
-```typescript
-const user = new User({ blogPosts: [] });
+doc.tags; // mongoose.Types.DocumentArray<{ name: string }>
+doc.toObject().tags; // Array<{ name: string }>
+
+async function run() {
+  const docFromDb = await OrderModel.findOne().orFail();
+  docFromDb.tags; // mongoose.Types.DocumentArray<{ name: string }>
 
-user.blogPosts.push({ title: 'test' }); // Would not work if you did `blogPosts: BlogPost[]`
+  const leanDoc = await OrderModel.findOne().orFail().lean();
+  leanDoc.tags; // Array<{ name: string }>
+};
 ```