diff --git a/src/controller/subtitle-stream-controller.ts b/src/controller/subtitle-stream-controller.ts
index 2008f90efb8..ac23616e8f6 100644
--- a/src/controller/subtitle-stream-controller.ts
+++ b/src/controller/subtitle-stream-controller.ts
@@ -274,8 +274,8 @@ export class SubtitleStreamController
       this.warn(`Subtitle tracks were reset while loading level ${trackId}`);
       return;
     }
-    const track: Level = levels[currentTrackId];
-    if (trackId >= levels.length || trackId !== currentTrackId || !track) {
+    const track: Level = levels[trackId];
+    if (trackId >= levels.length || !track) {
       return;
     }
     this.log(
@@ -320,6 +320,10 @@ export class SubtitleStreamController
     track.details = newDetails;
     this.levelLastLoaded = track;
 
+    if (trackId !== currentTrackId) {
+      return;
+    }
+
     if (!this.startFragRequested && (this.mainDetails || !newDetails.live)) {
       this.setStartPosition(this.mainDetails || newDetails, sliding);
     }
diff --git a/tests/unit/controller/subtitle-track-controller.ts b/tests/unit/controller/subtitle-track-controller.ts
index 4dafd12832a..573ddf2dfe4 100644
--- a/tests/unit/controller/subtitle-track-controller.ts
+++ b/tests/unit/controller/subtitle-track-controller.ts
@@ -506,6 +506,33 @@ describe('SubtitleTrackController', function () {
       expect(playlistLoadedSpy).to.have.been.calledOnce;
     });
 
+    it('retains loaded details on track if active track synchronously set to something else', function () {
+      const playlistLoadedSpy = sandbox.spy(
+        subtitleTrackController,
+        'playlistLoaded',
+      );
+      subtitleTrackController.startLoad();
+      (subtitleTrackController as any).trackId = 1;
+      (subtitleTrackController as any).currentTrack = subtitleTracks[1];
+
+      const mockLoadedEvent = {
+        id: 1,
+        groupId: 'default-text-group',
+        details: { foo: 'bar' } as any,
+        stats: new LoadStats(),
+        networkDetails: {},
+        deliveryDirectives: null,
+      };
+
+      hls.subtitleTrack = -1;
+      hls.trigger(Events.SUBTITLE_TRACK_LOADED, mockLoadedEvent);
+
+      expect(subtitleTracks[1].details).not.to.be.undefined;
+      expect((subtitleTrackController as any).timer).to.equal(-1);
+      // We will still emit playlist loaded since we did load and store the details
+      expect(playlistLoadedSpy).to.have.been.called;
+    });
+
     it('does not set the reload timer if loading has not started', function () {
       const details = new LevelDetails('');
       subtitleTrackController.stopLoad();