feat(detection): play the stream the page's own video played

chrome/background/background.mjs

@@ -1,5 +1,6 @@
 import {PlayerModes} from '../player/enums/PlayerModes.mjs';
 import {StreamLength} from '../player/utils/StreamLength.mjs';
+import {StreamPick} from '../player/utils/StreamPick.mjs';
 import {StringUtils} from '../player/utils/StringUtils.mjs';
 import {URLUtils} from '../player/utils/URLUtils.mjs';
 import {Utils} from '../player/utils/Utils.mjs';
@@ -101,6 +102,10 @@ const Lengths = new StreamLengths({
 });
 // The longest a player waits for the lengths still being read.
 const SourceLengthWaitMs = 2500;
+// The longest a player waits for the page to say what its video played (getPlayedVideo).
+// The page answers within milliseconds unless its own scripts keep it busy; then the
+// longest decides, as before the question.
+const PlayedVideoWaitMs = 1000;
 // Where this background's own requests come from.
 const OwnOrigin = chrome.runtime.getURL('');
 
@@ -625,7 +630,7 @@ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
     }
     return;
   } else if (msg.type === MessageTypes.MPV_USER_PLAY) {
-    onUserPlay(sender, typeof msg.src === 'string' ? msg.src : '');
+    onUserPlay(sender, typeof msg.src === 'string' ? msg.src : '', msg.video || null);
     return;
   } else if (msg.type === MessageTypes.SHORTCUT_CANCELLED) {
     if (sender.tab && typeof msg.key === 'string' && typeof msg.code === 'string') {
@@ -1396,9 +1401,18 @@ async function sendSources(frame) {
   const continuationOptions = frame.tab.continuationOptions;
   frame.tab.continuationOptions = null;
 
-  // The player plays the longest of them: a little time for the lengths still being read.
+  // The player plays the one the page's video played, or the longest of them: a little
+  // time for the lengths still being read. The video is asked about after that, when the
+  // page's player has most likely set its length; then a wait for the sources detected
+  // meanwhile, if any (one still unread from before is not waited for twice).
+  let video = null;
   if (collectSources(frame).sources.length > 1) {
     await Lengths.settle(() => collectSources(frame).sources, SourceLengthWaitMs);
+    if (frame.parent) {
+      const known = new Set(collectSources(frame).sources.map((source) => source.url));
+      video = await getPlayedVideo(frame);
+      await Lengths.settle(() => collectSources(frame).sources.filter((source) => !known.has(source.url)), SourceLengthWaitMs);
+    }
   }
 
   const {subtitles, sources} = collectSources(frame, true);
@@ -1411,6 +1425,7 @@ async function sendSources(frame) {
     type: MessageTypes.SOURCES,
     subtitles: subtitles,
     sources: sources,
+    video,
     autoSetSource: true,
     continuationOptions: continuationOptions,
   }, {
@@ -1437,6 +1452,32 @@ async function scrapeCaptionsTags(frame) {
   });
 }
 
+/**
+ * What the page's video that a player replaced played (content.js replacedVideo), asked
+ * of the frame the player's iframe is in.
+ * @param {Object} player - The player's frame.
+ * @return {Promise<?{src: string, duration: ?number}>} Its file's URL and its length, or
+ *   null when the player replaced no video (it fills the page), or the page did not answer
+ *   within PlayedVideoWaitMs.
+ */
+async function getPlayedVideo(player) {
+  return new Promise((resolve) => {
+    // Content scripts share the page's event loop: a page that never lets go would hold
+    // the player without sources.
+    const timer = setTimeout(() => resolve(null), PlayedVideoWaitMs);
+    chrome.tabs.sendMessage(player.tab.tabId, {
+      type: MessageTypes.GET_PLAYED_VIDEO,
+      frameId: player.frameId,
+    }, {
+      frameId: player.parent.frameId,
+    }, (video) => {
+      BackgroundUtils.checkMessageError('get_played_video');
+      clearTimeout(timer);
+      resolve(video || null);
+    });
+  });
+}
+
 async function getVideoSize(frame) {
   return new Promise((resolve, reject) => {
     chrome.tabs.sendMessage(frame.tab.tabId, {
@@ -1657,8 +1698,9 @@ function pauseTabMedia(tabId) {
  *
  * @param {Object} sender - The message sender: its tab and frameId.
  * @param {string} src - The video element's currentSrc.
+ * @param {?Object} video - What it plays (content.js playedVideo).
  */
-async function onUserPlay(sender, src) {
+async function onUserPlay(sender, src, video) {
   await ensureOptions();
 
   if (!Options.mpvMode || !sender.tab || typeof sender.frameId !== 'number') {
@@ -1670,7 +1712,7 @@ async function onUserPlay(sender, src) {
     return;
   }
 
-  const source = await findPlayedSource(tab, sender.frameId, src);
+  const source = await findPlayedSource(tab, sender.frameId, src, video);
   // The tab may have left MPV while the lengths were read.
   if (!tab.isOn || !tab.isMpv || !tab.mpvOnPlay) {
     return;
@@ -1686,18 +1728,20 @@ async function onUserPlay(sender, src) {
 /**
  * The detected source a started video plays. Its own URL first: a progressive
  * file, often preloaded long before the click, which no later request would
- * detect again. Otherwise the newest of the longest sources of the frame it
- * plays in, since an MSE player's src is a blob: and its manifest a request of
- * that frame - and one that ran an ad first has the ad's among them. Their
+ * detect again. Otherwise the stream as long as the video (StreamPick), or the
+ * newest of the longest sources of the frame it plays in, since an MSE player's
+ * src is a blob: and its manifest a request of that frame - and one that ran an
+ * ad first has the ad's among them. Their
  * lengths may not be read yet: the page can have asked for them before this
  * tab's MPV started, when nothing read them.
  *
  * @param {Object} tab - TabHolder the video is in.
  * @param {number} frameId - Frame the video is in.
  * @param {string} src - The video element's currentSrc.
+ * @param {?Object} video - What it plays (content.js playedVideo).
  * @return {Promise<Object|null>} The source, or null when none is detected yet.
  */
-async function findPlayedSource(tab, frameId, src) {
+async function findPlayedSource(tab, frameId, src, video) {
   if (/^https?:\/\//i.test(src)) {
     for (const frame of tab.getFrames()) {
       const match = getSourceFromURL(frame, src);
@@ -1715,20 +1759,22 @@ async function findPlayedSource(tab, frameId, src) {
   if (frame.getSources().length > 1) {
     await Lengths.settle(() => frame.getSources(), SourceLengthWaitMs);
   }
-  return newestOfLongest(frame.getSources());
+  return newestOfLongest(frame.getSources(), video);
 }
 
 /**
- * The newest of the longest sources: an ad runs for seconds, the video for minutes
- * (StreamLength.longest), and of streams that tie, the one the page asked for last is
- * the one it plays now.
+ * The newest of the streams a video plays (StreamPick.played), or else of the longest
+ * sources: an ad runs for seconds, the video for minutes (StreamLength.longest). Of
+ * streams that tie, the one the page asked for last is the one it plays now.
  *
  * @param {Array<Object>} sources - Detected sources.
+ * @param {?Object} [video] - What the page's video plays (content.js playedVideo).
  * @return {Object|null} One of them, or null when there are none.
  */
-function newestOfLongest(sources) {
+function newestOfLongest(sources, video = null) {
   const lengths = Lengths.lengthsOf(sources);
-  const longest = StreamLength.longest(sources.map((source, i) => ({source, duration: lengths[i]})));
+  const measured = sources.map((source, i) => ({source, url: source.url, duration: lengths[i]}));
+  const longest = StreamPick.played(measured, video) || StreamLength.longest(measured);
   let newest = null;
   for (const {source} of longest) {
     if (!newest || source.time > newest.time) {

chrome/content.js

@@ -5,6 +5,7 @@
     PONG_TAB: 'PONG_TAB',
     OPEN_PLAYER: 'OPEN_PLAYER',
     GET_VIDEO_SIZE: 'GET_VIDEO_SIZE',
+    GET_PLAYED_VIDEO: 'GET_PLAYED_VIDEO',
     FRAME_ADDED: 'FRAME_ADDED',
     REMOVE_PLAYERS: 'REMOVE_PLAYERS',
     IS_FULL: 'IS_FULL',
@@ -111,6 +112,11 @@
         sendResponse(video ? video.size : 0);
       });
       return true;
+    } else if (request.type === MessageTypes.GET_PLAYED_VIDEO) {
+      // The asking player's, once it linked up with its iframe; before that, the latest
+      // replaced (one player opens per frame at a time, background.mjs openPlayer).
+      const linked = iframeMap.get(request.frameId);
+      sendResponse(replacedVideo(linked ? linked.replacedData : replacedPlayerQueue[replacedPlayerQueue.length - 1]));
     } else if (request.type === MessageTypes.SCRAPE_CAPTIONS) {
       return handleCaptionsScrape(request, sender, sendResponse);
     } else if (request.type === MessageTypes.TOGGLE_MINIPLAYER) {
@@ -319,6 +325,13 @@
           softReplace,
           fillScreen: playerFillsScreen,
         };
+        // The video itself, for the stream it played (GET_PLAYED_VIDEO). A site's own query
+        // may have found its player's box: the one video in it, and none when there are
+        // more, an ad's or a preview's among them as likely as not.
+        const inBox = video.highest.tagName === 'VIDEO' ? [video.highest] :
+          querySelectorAllIncludingShadows('video', video.highest);
+        pobj.video = video.video || (inBox.length === 1 ? inBox[0] : null);
+        pobj.played = playedVideo(pobj.video);
         replacedPlayerQueue.push(pobj);
 
         updateReplacedPlayer(video.highest, iframe, softReplace);
@@ -1577,6 +1590,41 @@
     }
   });
 
+  /**
+   * What a video plays, for the player to play the same stream (StreamPick.played).
+   * @param {HTMLVideoElement|null|undefined} video - The video.
+   * @return {?{src: string, duration: ?number, playing: string}} Its file's URL (not a
+   *   blob: URL, which a detected source never has), its length in seconds (Infinity when
+   *   live), and its currentSrc as it is, blob: or not; or null for no video.
+   */
+  function playedVideo(video) {
+    if (!video) {
+      return null;
+    }
+    const src = video.currentSrc || '';
+    return {
+      src: /^https?:\/\//i.test(src) ? src : '',
+      duration: video.duration > 0 ? video.duration : null,
+      playing: src,
+    };
+  }
+
+  /**
+   * What the video a player replaced plays: as it is now, while it plays what it played
+   * then (a page's player sets its length a moment after it asked for the manifest), or
+   * else as it was then.
+   * @param {Object|undefined} player - The replaced player (replacedPlayerQueue).
+   * @return {?{src: string, duration: ?number}} See playedVideo.
+   */
+  function replacedVideo(player) {
+    if (!player || !player.played) {
+      return null;
+    }
+    const now = playedVideo(player.video);
+    const {src, duration} = now && now.playing === player.played.playing ? now : player.played;
+    return {src, duration};
+  }
+
   // MPV started by its shortcut hands over only a video the user starts, not
   // whatever the page loads or autoplays on its own (previews, background
   // clips, a preloaded player). A 'play' while the page is still handling a
@@ -1590,6 +1638,7 @@
       chrome.runtime.sendMessage({
         type: MessageTypes.MPV_USER_PLAY,
         src: video.currentSrc || '',
+        video: playedVideo(video),
       }, () => {
         void chrome.runtime.lastError;
       });

chrome/player/enums/MessageTypes.mjs

@@ -13,6 +13,7 @@ export const MessageTypes = {
   OPEN_PLAYER: 'OPEN_PLAYER',
   SET_HEADERS: 'SET_HEADERS',
   GET_VIDEO_SIZE: 'GET_VIDEO_SIZE',
+  GET_PLAYED_VIDEO: 'GET_PLAYED_VIDEO',
   FRAME_ADDED: 'FRAME_ADDED',
   REMOVE_PLAYERS: 'REMOVE_PLAYERS',
   IS_FULL: 'IS_FULL',

chrome/player/main.mjs

@@ -8,6 +8,7 @@ import {SubtitleTrack} from './SubtitleTrack.mjs';
 import {EnvUtils} from './utils/EnvUtils.mjs';
 import {RequestUtils} from './utils/RequestUtils.mjs';
 import {StreamLength} from './utils/StreamLength.mjs';
+import {StreamPick} from './utils/StreamPick.mjs';
 import {URLUtils} from './utils/URLUtils.mjs';
 import {Utils} from './utils/Utils.mjs';
 import {VideoSource} from './VideoSource.mjs';
@@ -92,9 +93,10 @@ async function recieveSources(request, sendResponse) {
     return;
   }
 
-  // Of the longest: an ad or an intro runs for seconds, the video for minutes or hours.
-  // The rest stay in the list, to choose by hand.
-  const candidates = StreamLength.longest(sources);
+  // The streams the page's video played, when it tells them; else the longest: an ad or an
+  // intro runs for seconds, the video for minutes or hours. The rest stay in the list, to
+  // choose by hand.
+  const candidates = StreamPick.played(sources, request.video) || StreamLength.longest(sources);
 
   // Sources are ordered by time, so we can just choose the first one and it will be the oldest.
   // But we also want to minimize depth

chrome/player/utils/StreamPick.mjs

@@ -0,0 +1,96 @@
+import {StreamLength} from './StreamLength.mjs';
+
+// A video's length and a stream's match within this many seconds, or this share of the
+// length when that is more. Both come from the same manifest or movie header: an MSE player
+// (hls.js, Shaka, dash.js) sets its video's length from the manifest's total, and the media
+// it appends may run a little past it. Another episode is seconds shorter or longer.
+const MATCH_S = 3;
+const MATCH_SHARE = 0.0025;
+
+// When a stream this many times as long as the page's video plays, or any longer one while
+// the video runs less than SHORT_S, the video is as likely as not an ad, a trailer or a
+// preview (they run a few minutes at most), and the longest decides, as it did before.
+const DWARF_RATIO = 5;
+const SHORT_S = 180;
+
+/**
+ * Which of a page's streams its video plays: the one FastStream replaced, or the one the
+ * user started. The player plays that stream, not the page's longest, when it can tell: a
+ * page may load the next episode, or another stream as long, beside the video the user
+ * watches. Never over a far longer stream, nor a short video over any longer one: then
+ * the longest plays, as it did before.
+ */
+export class StreamPick {
+  /**
+   * The streams the page's video plays.
+   * @template {{url: string, duration?: number|null}} T
+   * @param {T[]} sources - Detected sources, each with its length in seconds, when known.
+   * @param {?{src?: string, duration?: number|null}} video - What the video plays: its URL,
+   *   for a file it plays itself (not a blob:), and its length in seconds (content.js).
+   * @return {T[]|null} Those of the sources, or null when the video tells none of them:
+   *   then the longest play, as StreamLength.longest tells them.
+   */
+  static played(sources, video) {
+    if (!video) {
+      return null;
+    }
+
+    const src = typeof video.src === 'string' && /^https?:\/\//i.test(video.src) ? video.src : '';
+    let length = StreamPick.lengthOf(video.duration);
+    let played = src ? sources.filter((source) => source.url === src) : [];
+    if (played.length > 0) {
+      // Its file: the length read from it, when there is one.
+      length = StreamPick.lengthOf(played[0].duration) ?? length;
+    } else if (length !== null) {
+      played = sources.filter((source) => StreamPick.sameLength(source.duration, length));
+    }
+    if (played.length === 0) {
+      return null;
+    }
+
+    if (length === null) {
+      // How long it runs is not known: it decides only among the longest.
+      const longest = StreamLength.longest(sources);
+      played = played.filter((source) => longest.includes(source));
+      return played.length > 0 ? played : null;
+    }
+
+    // Nothing runs longer than a live stream.
+    if (length === Infinity) {
+      return played;
+    }
+    const best = Math.max(...sources.map((source) => StreamLength.rankLength(source.duration)));
+    if (best >= length * DWARF_RATIO) {
+      return null;
+    }
+    if (length < SHORT_S && best > length && !StreamPick.sameLength(best, length)) {
+      return null;
+    }
+    return played;
+  }
+
+  /**
+   * Whether a stream's length is a video's.
+   * @param {number|null|undefined} duration - The stream's length in seconds, when known.
+   * @param {number} length - The video's (a number over 0, or Infinity when live).
+   * @return {boolean} Whether they match.
+   */
+  static sameLength(duration, length) {
+    if (StreamPick.lengthOf(duration) === null) {
+      return false;
+    }
+    if (duration === Infinity || length === Infinity) {
+      return duration === length;
+    }
+    return Math.abs(duration - length) <= Math.max(MATCH_S, length * MATCH_SHARE);
+  }
+
+  /**
+   * A length, when it is one.
+   * @param {*} value - Seconds, maybe.
+   * @return {number|null} The seconds (a number over 0, or Infinity), or null.
+   */
+  static lengthOf(value) {
+    return typeof value === 'number' && value > 0 ? value : null;
+  }
+}