fix: four regressions in main (headers, archives, fragmented MP4) and two subtitle bugs

chrome/background/NetRequestRuleManager.mjs

@@ -77,10 +77,14 @@ export class RuleManager {
         requestHeaders: requestHeaderCommands,
       },
       condition: {
-        // Escape urlFilter's special characters (\ * ^ |) so a URL that
-        // happens to contain one is matched literally instead of being
-        // reinterpreted as a wildcard/anchor by declarativeNetRequest.
-        urlFilter: '||' + url.replace('https://', '').replace('http://', '').replace(/[\\*^|]/g, '\\$&'),
+        // The URL itself, unescaped: urlFilter has no escape character. A `\`
+        // is a literal character to match (Firefox's CompiledUrlFilter in
+        // ExtensionDNR.sys.mjs; Chrome documents only * | || ^), so escaping
+        // a `*` made the rule demand a backslash no URL has, and a stream URL
+        // like an Akamai `acl=/*` token got no Referer/Origin. Unescaped, a
+        // `*`, `^` or `|` from the URL can only widen the match, never exclude
+        // the URL itself, and the rule is limited to one tab and 5 seconds.
+        urlFilter: '||' + url.replace('https://', '').replace('http://', ''),
         tabIds: [tabId],
       },
     };

chrome/player/network/DownloadManager.mjs

@@ -89,6 +89,18 @@ export class DownloadManager {
     return this.storage.get(key);
   }
 
+  /**
+   * Drops a finished stored copy, so the next getFile fetches the file again. A manifest a
+   * player loads a second time is a live one being refreshed, and its stored copy is the
+   * old window. A copy still downloading is left alone.
+   * @param {Object} details - The request (url, range and response type).
+   */
+  forgetCompletedFile(details) {
+    if (this.getEntry(details)?.status === DownloadStatus.DOWNLOAD_COMPLETE) {
+      this.removeFile(details);
+    }
+  }
+
   removeFile(details) {
     const key = this.getIdentifier(details);
     const storedEntry = this.storage.get(key);

chrome/player/players/dash/DashLoader.mjs

@@ -135,6 +135,15 @@ export function DASHLoaderFactory(player) {
       };
 
       const downloadManager = player.getClient().downloadManager;
+      if (request.type === 'MPD') {
+        // A player's first load of the manifest may be answered from the store, and the copy
+        // has to stay there for "dump buffer" archives, a player opened from one, and the
+        // seek preview. dash.js loads a live manifest again to learn of new segments: then
+        // the stored copy is the old one, and the stream would stop where it ends.
+        const key = downloadManager.getIdentifier(context);
+        if (player.loadedManifests.has(key)) downloadManager.forgetCompletedFile(context);
+        player.loadedManifests.add(key);
+      }
       const loader = downloadManager.getFile({
         ...context,
         preProcessor: async (entry, request) => {
@@ -152,9 +161,6 @@ export function DASHLoaderFactory(player) {
       }, {
         onSuccess: async (entry, xhr) => {
           const data = await entry.getDataFromBlob();
-          // dash.js loads a live manifest again to learn of new segments; from the store,
-          // every reload would be this one, and the stream would stop where it ends.
-          if (request.type === 'MPD') downloadManager.removeFile(entry);
           httpRequest.customData.onSuccess(data, entry.responseURL);
         },
         onProgress: (stats, context, data, xhr)=> {

chrome/player/players/dash/DashPlayer.mjs

@@ -22,6 +22,9 @@ export default class DashPlayer extends EventEmitter {
 
     this.fragmentRequester = new DashFragmentRequester(this);
     this.activeRequests = [];
+    // Download-manager keys of the manifests this player has loaded; a second load of one
+    // is a live refresh (DashLoader).
+    this.loadedManifests = new Set();
   }
 
   async setup() {

chrome/player/players/hls/HLSLoader.mjs

@@ -168,20 +168,29 @@ export function HLSLoaderFactory(player) {
       // A playlist, not a key or a fragment. hls.js loads one again to see what is new.
       const isPlaylist = this.context.frag === undefined;
       const downloadManager = player.getClient().downloadManager;
-      this.loader = downloadManager.getFile({
+      const details = {
         ...this.context,
         config: this.config,
         headers: {
           ...this.context.headers,
           ...player.source.headers,
         },
-      }, {
+      };
+      if (isPlaylist) {
+        // A player's first load of a playlist may be answered from the store, and the copy
+        // has to stay there: "dump buffer" writes the store into the archive, and a player
+        // opened from an archive, or the seek preview beside the main player, finds it
+        // there. hls.js loads a playlist again only to see what is new (a live stream):
+        // then the stored copy is the old window, and answering with it would stop the
+        // stream where that window ends.
+        const key = downloadManager.getIdentifier(details);
+        if (player.loadedManifests.has(key)) downloadManager.forgetCompletedFile(details);
+        player.loadedManifests.add(key);
+      }
+      this.loader = downloadManager.getFile(details, {
         onSuccess: async (entry, xhr) => {
           this.copyStats(entry.stats);
           const data = await entry.getDataFromBlob();
-          // Or the next load of a live playlist would be answered with this one from the
-          // store, and the stream would stop where this window ends.
-          if (isPlaylist) downloadManager.removeFile(entry);
 
           if (this.callbacks) {
             this.callbacks.onSuccess({

chrome/player/players/hls/HLSPlayer.mjs

@@ -34,6 +34,9 @@ export default class HLSPlayer extends EventEmitter {
     this.defaultQuality = client.options.defaultQuality || 'Auto';
     this.source = null;
     this.activeRequests = [];
+    // Download-manager keys of the playlists this player has loaded; a second load of
+    // one is a live refresh (HLSLoader).
+    this.loadedManifests = new Set();
     this.fragmentRequester = new HLSFragmentRequester(this);
     this.video = document.createElement(this.isAudioOnly ? 'audio' : 'video');
     if (!Hls.isSupported()) {

chrome/player/players/mp4/MP4Player.mjs

@@ -350,13 +350,31 @@ export default class MP4Player extends EventEmitter {
       return;
     }
 
+    // A fragmented file's sample list grows with each moof parsed, so "every sample
+    // appended" also holds between two fragments, whenever a range ends there: ending the
+    // stream then cuts the duration to what is buffered, and the video stops early. Its
+    // samples are all known only once the range with the file's last byte is in.
+    if (this.metaData.isFragmented && !this.hasLastRange()) {
+      return;
+    }
+
     try {
       this.mediaSource.endOfStream();
     } catch (e) {
       console.warn('Could not end the MediaSource stream', e);
     }
   }
 
+  /**
+   * Whether the range holding the file's last byte has reached mp4box since the last
+   * reset: resetHLS() empties currentFragments and seeks. False while the file's length is
+   * not known.
+   * @return {boolean}
+   */
+  hasLastRange() {
+    return this.fileLength > 0 && this.currentFragments.some((frag) => frag.rangeEnd >= this.fileLength);
+  }
+
   initializeFragments() {
     const max = Math.ceil(this.fileLength / FRAGMENT_SIZE);
     const levelID = this.getCurrentVideoLevelID();
@@ -490,6 +508,19 @@ export default class MP4Player extends EventEmitter {
               this.loader = null;
             } else return;
 
+            // The file's length has to be known before mp4box parses the data: parsing the
+            // moov runs onMetadataParsed, which otherwise works the length out from the
+            // samples known so far. For a fragmented file those are only the fragments in
+            // this range, so the ranges after it were never made, and the file was
+            // taken to end there.
+            if (!this.fileLength) {
+              const total = parseInt(entry.responseHeaders['content-range']?.split('/')[1]);
+              if (total > 0) {
+                this.fileLength = total;
+                this.initializeFragments();
+              }
+            }
+
             this.mp4box.appendBuffer(data);
             this.currentFragments.push(frag);
             frag.addReference(ReferenceTypes.MP4PLAYER, true);

chrome/player/ui/SaveManager.mjs

@@ -7,7 +7,6 @@ import {streamSaver} from '../modules/StreamSaver.mjs';
 import {AlertPolyfill} from '../utils/AlertPolyfill.mjs';
 import {EnvUtils} from '../utils/EnvUtils.mjs';
 import {FastStreamArchiveUtils} from '../utils/FastStreamArchiveUtils.mjs';
-import {RequestUtils} from '../utils/RequestUtils.mjs';
 import {StringUtils} from '../utils/StringUtils.mjs';
 import {URLUtils} from '../utils/URLUtils.mjs';
 import {Utils} from '../utils/Utils.mjs';
@@ -429,17 +428,8 @@ export class SaveManager {
         newSource = new VideoSource(file, {}, mode);
         newSource.identifier = file.name + 'size' + file.size;
       } else if (ext === 'fsa') {
-        // Read once, then drop the URL - an .fsa archive is a whole video, so
-        // holding this would pin the entire file for the tab's lifetime.
-        const archiveURL = window.URL.createObjectURL(file);
-        let buffer;
         try {
-          buffer = await RequestUtils.httpGetLarge(archiveURL);
-        } finally {
-          window.URL.revokeObjectURL(archiveURL);
-        }
-        try {
-          const {source, entries, currentLevel, currentAudioLevel} = await FastStreamArchiveUtils.parseFSA(buffer, (progress)=>{
+          const {source, entries, currentLevel, currentAudioLevel} = await FastStreamArchiveUtils.parseFSAFile(file, (progress)=>{
             this.setStatusMessage('save-video', Localize.getMessage('player_archive_loading', [Math.floor(progress * 100)]), 'info');
           }, this.client.downloadManager);
 

chrome/player/ui/subtitles/SubtitlesManager.mjs

@@ -670,7 +670,12 @@ export class SubtitlesManager extends EventEmitter {
         if (!cue.dom) {
           cue.dom = WebVTT.convertCueToDOMTree(window, cue.text);
         }
-        toAdd.push(cue.dom);
+        // A cue with no text (a SubRip cue with nothing under its timestamp) has no tree,
+        // and appending null throws: timeUpdated() then stopped here for as long as the
+        // cue lasted, before the other tracks and the skip segments.
+        if (cue.dom) {
+          toAdd.push(cue.dom);
+        }
         cueIndex++;
       }
 

chrome/player/utils/FastStreamArchiveUtils.mjs

@@ -1,11 +1,16 @@
 import {DownloadStatus} from '../enums/DownloadStatus.mjs';
 import {DownloadEntry} from '../network/DownloadEntry.mjs';
+import {RequestUtils} from './RequestUtils.mjs';
 import {Utils} from './Utils.mjs';
 
 /**
  * Utility functions for working with FastStream archive files and streams.
  */
 export class FastStreamArchiveUtils {
+  // Bytes per read of a local archive file (parseFSAFile). A test lowers it, so that a
+  // small archive spans many chunks the way a large one does.
+  static fileChunkSize = 1e9 / 4;
+
   /**
    * Writes FastStream archive data to a stream.
    * @param {WritableStream} filestream - The writable stream.
@@ -95,6 +100,27 @@ export class FastStreamArchiveUtils {
     writer.close();
   }
 
+  /**
+   * Reads an archive from a local file, such as one dropped on the player.
+   *
+   * httpGetLarge fetches the file in chunks as parseFSA reads it, so the object URL has to
+   * live until parseFSA is done. Revoked any earlier, every chunk past the first two failed,
+   * and any archive over 500 MB could not be opened.
+   * @param {File|Blob} file - The .fsa file.
+   * @param {Function} [progressCallback] - Called with the share of entries read.
+   * @param {Object} [downloadManager] - Receives the entries' data.
+   * @return {Promise<Object>} What parseFSA returns.
+   */
+  static async parseFSAFile(file, progressCallback, downloadManager) {
+    const url = URL.createObjectURL(file);
+    try {
+      const buffer = await RequestUtils.httpGetLarge(url, this.fileChunkSize);
+      return await this.parseFSA(buffer, progressCallback, downloadManager);
+    } finally {
+      URL.revokeObjectURL(url);
+    }
+  }
+
   static async parseFSA(largeBuffer, progressCallback, downloadManager) {
     const entries = [];
 

chrome/player/utils/RequestUtils.mjs

@@ -185,13 +185,14 @@ export class RequestUtils {
 
   /**
    * Downloads a large file in fragments using range requests and returns a LargeBuffer.
+   * The fragments are fetched as the buffer is read, not up front, so `source` has to stay
+   * reachable until the last byte has been read.
    * @param {string} source - The URL of the large file.
+   * @param {number} [fragSize] - Bytes per range request.
    * @return {Promise<LargeBuffer>} Resolves with a LargeBuffer containing the file data.
    * @throws {Error} If the request fails or headers are missing.
    */
-  static async httpGetLarge(source) {
-    const fragSize = 1e9 / 4;
-
+  static async httpGetLarge(source, fragSize = 1e9 / 4) {
     const headersXHR = await this.request({
       url: source,
       responseType: 'arraybuffer',

chrome/player/utils/SubtitleUtils.mjs

@@ -59,10 +59,11 @@ export class SubtitleUtils {
     let srt = data.replace(/\r+/g, '');
     // trim white space start and end
     srt = srt.replace(/^\s+|\s+$/g, '');
-    // get cues: a cue ends at a blank line, however many follow it, and a line of
-    // spaces or tabs is as blank as an empty one; a cue also starts at its own timestamp
-    // when no blank line comes before it
-    const cuelist = srt.split(/\n(?:[ \t]*\n)+/).flatMap((block) => this.splitAtCueStarts(block));
+    // get cues: a cue ends at a blank line, however many follow it, and a cue also starts
+    // at its own timestamp when no blank line comes before it. So a line of spaces ends
+    // the cue before a timestamp, and inside a cue it is text, as ffmpeg and VLC read it:
+    // splitting at it too lost the text below it.
+    const cuelist = srt.split(/\n\n+/).flatMap((block) => this.splitAtCueStarts(block));
     let result = '';
     if (cuelist.length > 0) {
       result += 'WEBVTT\n\n';

tests/e2e/ext-specs/header-rules.e2e.mjs

@@ -0,0 +1,78 @@
+// A stream's Referer reaches the server, whatever characters its URL contains.
+//
+// Sites that check Referer/Origin get them through SET_HEADERS: the background adds a
+// declarativeNetRequest session rule for the request URL, and the rule sets the headers.
+// The rule's urlFilter has no escape character, so when the URL's `*`, `^` and `|` were
+// "escaped" with a backslash, the rule demanded a backslash no URL has. A URL with a `*` -
+// an Akamai `acl=/*` token, say - then went out with no Referer at all.
+//
+// Driven on the installed extension, so the rule runs in Firefox's real rule engine: an
+// extension page asks for the header rule, fetches the URL, and the server reports what
+// Referer it saw. The plain URL is the control: it shows the mechanism works at all here.
+
+import http from 'node:http';
+
+import {expect} from '@wdio/globals';
+
+import {inExtensionPage} from '../extension-page.mjs';
+
+const SITE_PORT = 41984;
+const SITE = `http://127.0.0.1:${SITE_PORT}`;
+const REFERER = 'https://video-site.test/watch/1';
+
+let siteServer;
+const seen = new Map();
+
+describe('Header rules for a stream URL', function() {
+  before(async function() {
+    siteServer = http.createServer((req, res) => {
+      const id = new URL(req.url, SITE).searchParams.get('id');
+      if (id) seen.set(id, req.headers.referer || null);
+      res.writeHead(200, {
+        'Content-Type': 'text/plain',
+        'Access-Control-Allow-Origin': '*',
+      });
+      res.end('ok');
+    });
+    await new Promise((resolve, reject) => {
+      siteServer.on('error', reject);
+      siteServer.listen(SITE_PORT, '127.0.0.1', resolve);
+    });
+  });
+
+  after(async function() {
+    if (siteServer) await new Promise((resolve) => siteServer.close(resolve));
+  });
+
+  /**
+   * Asks the background for a Referer rule on the URL, then fetches it from the page.
+   * @param {string} url - The URL to fetch.
+   * @return {Promise<Object>} What the page saw.
+   */
+  function fetchWithReferer(url) {
+    return inExtensionPage((args, done) => {
+      chrome.runtime.sendMessage({
+        type: 'SET_HEADERS',
+        url: args.url,
+        commands: [{operation: 'set', header: 'referer', value: args.referer}],
+      }).then(() => fetch(args.url, {cache: 'no-store'}))
+          .then((response) => done({status: response.status, url: response.url}))
+          .catch((e) => done({error: String(e)}));
+    }, {url, referer: REFERER});
+  }
+
+  const cases = [
+    ['a plain URL (the control)', 'plain', '/stream/master.m3u8?id=plain'],
+    ['a URL with a *', 'star', '/stream/master.m3u8?acl=/*~hmac=5f&id=star'],
+    ['a URL with a ^ and a |', 'caret', '/stream/master.m3u8?sig=a^b|c&id=caret'],
+  ];
+
+  for (const [name, id, path] of cases) {
+    it(`sends the Referer for ${name}`, async function() {
+      const page = await fetchWithReferer(SITE + path);
+      console.log('      page:', JSON.stringify(page), 'server saw:', seen.get(id));
+      expect(page.status).toBe(200);
+      expect(seen.get(id)).toBe(REFERER);
+    });
+  }
+});