refactor: bake ffmpeg into npm instead of CDN (#3799)

2020-09-08 15:10:36 -07:00 · 2020-09-08 15:10:36 -07:00 · 143adc1634
parent 1d4601b479
commit 143adc1634
11 changed files with 245 additions and 36 deletions
--- a/bin/README.md
+++ b/bin/README.md
@ -0,0 +1,221 @@
+# Binary Files
+
+<!-- GEN:toc-top-level -->
+- [Building `PrintDeps.exe`](#building-printdepsexe)
+- [Building `ffmpeg-mac`](#building-ffmpeg-mac)
+- [Building `ffmpeg-win64.exe`](#building-ffmpeg-win64exe)
+- [Building `ffmpeg-win32.exe`](#building-ffmpeg-win32exe)
+<!-- GEN:stop -->
+
+## Building `PrintDeps.exe`
+
+See instructions at [`//browser_patches/tools/PrintDepsWindows/README.md`](../browser_patches/tools/PrintDepsWindows/README.md)
+
+## Building `ffmpeg-mac`
+
+> FFMPEG: [`n4.3.1`](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.3.1)
+> libvpx: [`v1.9.0`](https://github.com/webmproject/libvpx/releases/tag/v1.9.0)
+
+I mostly followed steps at https://trac.ffmpeg.org/wiki/CompilationGuide/macOS
+
+1. Clone repo & checkout release tag (we're building release v4.3.1)
+
+```sh
+~$ git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg
+~$ cd ffmpeg
+~/ffmpeg$ git checkout n4.3.1
+```
+
+2. Install brew dependencies
+
+```sh
+~/ffmpeg$ brew install automake fdk-aac git lame libass libtool libvorbis libvpx \
+opus sdl shtool texi2html theora wget x264 x265 xvid nasm
+```
+
+3. Prepare output folders
+
+```sh
+~/ffmpeg$ mkdir -p output/bin
+```
+
+4. Configure with vpx 
+
+```sh
+~/ffmpeg$ ./configure --prefix=$PWD/output \
+                      --pkg-config-flags="--static" \
+                      --bindir=$PWD/output/bin \
+                      --disable-everything \
+                      --enable-ffmpeg \
+                      --enable-protocol=pipe \
+                      --enable-protocol=file \
+                      --enable-parser=mjpeg \
+                      --enable-decoder=mjpeg \
+                      --enable-demuxer=image2pipe \
+                      --enable-filter=pad \
+                      --enable-filter=crop \
+                      --enable-filter=scale \
+                      --enable-muxer=webm \
+                      --enable-encoder=libvpx_vp8 \
+                      --enable-libvpx \
+                      --enable-static
+```
+
+5. Make & install to the `output/bin`
+
+```sh
+~/ffmpeg$ make && make install
+```
+
+## Building `ffmpeg-win64.exe`
+
+> FFMPEG: [`n4.3.1`](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.3.1)
+> libvpx: [`d1a7897`](https://github.com/webmproject/libvpx/commit/d1a78971ebcfd728c9c73b0cfbee69f470d4dc72)
+
+1. Install `MSYS2` from `https://www.msys2.org` and install to `c:\msys64`
+
+2. Launch `c:\msys64\mingw64` to launch a shell with a proper environment.
+
+3. `MSYS2` uses `pacman` to install dependencies. Run the following commands to update & install packages:
+
+```sh
+$ pacman -Syu
+$ pacman -Su
+$ pacman -S make pkgconf diffutils yasm
+$ pacman -S mingw-w64-x86_64-nasm mingw-w64-x86_64-gcc mingw-w64-x86_64-SDL2
+```
+
+notes:
+- `yasm` is needed for `libvpx`
+- the rest is a general compilation toolchain
+
+4. Clone `libvpx` of proper version, compile manually and install to a well-known location (prefix):
+
+```sh
+$ cd /c/
+$ git clone https://chromium.googlesource.com/webm/libvpx
+$ cd libvpx
+$ git checkout d1a78971ebcfd728c9c73b0cfbee69f470d4dc72
+$ ./configure --prefix=/eee64 --target=x86_64-win64-gcc --enable-static --disable-shared --disable-docs --disable-tools --disable-unit-tests --disable-examples
+$ make && make install
+```
+
+Note: `libvpx` has a useful readme: https://chromium.googlesource.com/webm/libvpx/+/master/README
+
+5. Once `libvpx` is compiled, compile `ffmpeg` using the just-compiled `libvpx`:
+
+```sh
+$ cd /c/
+$ git clone git://source.ffmpeg.org/ffmpeg.git
+$ cd ffmpeg
+$ git checkout n4.3.1
+$ ./configure --extra-cflags="-I/eee64/include" \
+              --extra-ldflags="-L/eee64/lib -static" \
+              --prefix=/eee64 \
+              --pkg-config-flags="--static" \
+              --bindir=$PWD/output/bin \
+              --disable-everything \
+              --enable-ffmpeg \
+              --enable-protocol=pipe \
+              --enable-protocol=file \
+              --enable-parser=mjpeg \
+              --enable-decoder=mjpeg \
+              --enable-demuxer=image2pipe \
+              --enable-filter=pad \
+              --enable-filter=crop \
+              --enable-filter=scale \
+              --enable-muxer=webm \
+              --enable-libvpx \
+              --enable-static \
+              --enable-encoder=libvpx_vp8 \
+              --disable-pthreads \
+              --disable-zlib \
+              --disable-iconv \
+              --disable-bzlib \
+              --disable-w32threads
+$ make && make install
+```
+
+note: the following resources helped me to deal with some dynamic dependencies in the resulting `ffmpeg.exe`:
+- https://stackoverflow.com/questions/13768515/how-to-do-static-linking-of-libwinpthread-1-dll-in-mingw
+
+## Building `ffmpeg-win32.exe`
+
+> FFMPEG: [`n4.3.1`](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.3.1)
+> libvpx: [`d1a7897`](https://github.com/webmproject/libvpx/commit/d1a78971ebcfd728c9c73b0cfbee69f470d4dc72)
+
+> NOTE: these steps assume that `ffmpeg-win64.exe` was just built on the machine.
+
+1. Launch `c:\msys64\mingw32` to launch a shell with a proper environment. Not sure if this is required or everything could be done from `mingw64` - but it worked.
+
+2. Update libraries for mingw32
+
+```sh
+$ pacman -Syu
+$ pacman -Su
+```
+
+3. Uninstall the `x86_64` compilers that we installed to build win64 version:
+
+```sh
+$ pacman -R mingw-w64-x86_64-nasm mingw-w64-x86_64-gcc mingw-w64-x86_64-SDL2
+```
+
+4. Install the i686 compilers instead of their x86_64 counterparts:
+
+```sh
+$ pacman -S mingw-w64-i686-nasm mingw-w64-i686-gcc mingw-w64-i686-SDL2
+```
+
+5. Remove all old source folders - we'll re-clone everything later for simplicity
+
+```sh
+$ rm -rf /c/ffmpeg && rm -rf /c/libvpx
+```
+
+6. Clone & compile libvpx. Notice a change: `--target=x86-win32-gcc`
+
+```sh
+$ cd /c/
+$ git clone https://chromium.googlesource.com/webm/libvpx
+$ cd libvpx
+$ ./configure --prefix=/eee32 --target=x86-win32-gcc --enable-static --disable-shared --disable-docs --disable-tools --disable-unit-tests --disable-examples
+$ make && make install
+```
+
+7. Clone & compile ffmpeg
+
+```sh
+$ cd /c/
+$ git clone git://source.ffmpeg.org/ffmpeg.git
+$ cd ffmpeg
+$ git checkout n4.3.1
+$ ./configure --extra-cflags="-I/eee32/include" \
+              --extra-ldflags="-L/eee32/lib -static" \
+              --prefix=/eee32 \
+              --pkg-config-flags="--static" \
+              --bindir=$PWD/output/bin \
+              --disable-everything \
+              --enable-ffmpeg \
+              --enable-protocol=pipe \
+              --enable-protocol=file \
+              --enable-parser=mjpeg \
+              --enable-decoder=mjpeg \
+              --enable-demuxer=image2pipe \
+              --enable-filter=pad \
+              --enable-filter=crop \
+              --enable-filter=scale \
+              --enable-muxer=webm \
+              --enable-libvpx \
+              --enable-static \
+              --enable-encoder=libvpx_vp8 \
+              --disable-pthreads \
+              --disable-zlib \
+              --disable-iconv \
+              --disable-bzlib \
+              --disable-w32threads
+$ make && make install
+```
+
+note: using `-j` for make somehow breaks compilation - so I'd suggest compiling everything in one thread.
+
--- a/bin/ffmpeg-mac
+++ b/bin/ffmpeg-mac
--- a/bin/ffmpeg-win32.exe
+++ b/bin/ffmpeg-win32.exe
--- a/bin/ffmpeg-win64.exe
+++ b/bin/ffmpeg-win64.exe
--- a/src/server/chromium/chromium.ts
+++ b/src/server/chromium/chromium.ts
@ -24,7 +24,6 @@ import { rewriteErrorMessage } from '../../utils/stackTrace';
 import { BrowserTypeBase } from '../browserType';
 import { ConnectionTransport, ProtocolRequest } from '../transport';
 import type { BrowserDescriptor } from '../../utils/browserPaths';
-import { browserDirectory, browsersPath, ffmpegPath} from '../../utils/browserPaths';
 import { CRDevTools } from './crDevTools';
 import { BrowserOptions } from '../browser';
 import * as types from '../types';
@ -33,7 +32,6 @@ import { isDebugMode, getFromENV } from '../../utils/utils';
 export class Chromium extends BrowserTypeBase {
  private _devtools: CRDevTools | undefined;
  private _debugPort: number | undefined;
-  private _ffmpegPath: string;

  constructor(packagePath: string, browser: BrowserDescriptor) {
    const debugPortStr = getFromENV('PLAYWRIGHT_CHROMIUM_DEBUG_PORT');
@ -45,8 +43,6 @@ export class Chromium extends BrowserTypeBase {

    super(packagePath, browser, debugPort ? { webSocketRegex: /^DevTools listening on (ws:\/\/.*)$/, stream: 'stderr' } : null);
    this._debugPort = debugPort;
-    const browserDir = browserDirectory(browsersPath(packagePath), browser);
-    this._ffmpegPath = ffmpegPath(browserDir, browser);
    if (isDebugMode())
      this._devtools = this._createDevTools();
  }
@ -61,7 +57,7 @@ export class Chromium extends BrowserTypeBase {
      devtools = this._createDevTools();
      await (options as any).__testHookForDevTools(devtools);
    }
-    return CRBrowser.connect(transport, options, this._ffmpegPath, devtools);
+    return CRBrowser.connect(transport, options, devtools);
  }

  _rewriteStartupError(error: Error): Error {
--- a/src/server/chromium/crBrowser.ts
+++ b/src/server/chromium/crBrowser.ts
@ -44,11 +44,10 @@ export class CRBrowser extends Browser {
  private _tracingRecording = false;
  private _tracingPath: string | null = '';
  private _tracingClient: CRSession | undefined;
-  private _ffmpegPath: string;

-  static async connect(transport: ConnectionTransport, options: BrowserOptions, ffmpegPath: string, devtools?: CRDevTools): Promise<CRBrowser> {
+  static async connect(transport: ConnectionTransport, options: BrowserOptions, devtools?: CRDevTools): Promise<CRBrowser> {
    const connection = new CRConnection(transport);
-    const browser = new CRBrowser(connection, options, ffmpegPath);
+    const browser = new CRBrowser(connection, options);
    browser._devtools = devtools;
    const session = connection.rootSession;
    const version = await session.send('Browser.getVersion');
@ -89,10 +88,9 @@ export class CRBrowser extends Browser {
    return browser;
  }

-  constructor(connection: CRConnection, options: BrowserOptions, ffmpegPath: string) {
+  constructor(connection: CRConnection, options: BrowserOptions) {
    super(options);
    this._connection = connection;
-    this._ffmpegPath = ffmpegPath;
    this._session = this._connection.rootSession;
    this._connection.on(ConnectionEvents.Disconnected, () => this._didClose());
    this._session.on('Target.attachedToTarget', this._onAttachedToTarget.bind(this));
@ -148,7 +146,7 @@ export class CRBrowser extends Browser {
    assert(!this._serviceWorkers.has(targetInfo.targetId), 'Duplicate target ' + targetInfo.targetId);

    if (targetInfo.type === 'background_page') {
-      const backgroundPage = new CRPage(session, targetInfo.targetId, context, null, false, this._ffmpegPath);
+      const backgroundPage = new CRPage(session, targetInfo.targetId, context, null, false);
      this._backgroundPages.set(targetInfo.targetId, backgroundPage);
      backgroundPage.pageOrError().then(() => {
        context!.emit(CRBrowserContext.CREvents.BackgroundPage, backgroundPage._page);
@ -158,7 +156,7 @@ export class CRBrowser extends Browser {

    if (targetInfo.type === 'page') {
      const opener = targetInfo.openerId ? this._crPages.get(targetInfo.openerId) || null : null;
-      const crPage = new CRPage(session, targetInfo.targetId, context, opener, true, this._ffmpegPath);
+      const crPage = new CRPage(session, targetInfo.targetId, context, opener, true);
      this._crPages.set(targetInfo.targetId, crPage);
      crPage.pageOrError().then(pageOrError => {
        const page = crPage._page;
--- a/src/server/chromium/crPage.ts
+++ b/src/server/chromium/crPage.ts
@ -56,7 +56,6 @@ export class CRPage implements PageDelegate {
  readonly _browserContext: CRBrowserContext;
  private readonly _pagePromise: Promise<Page | Error>;
  _initializedPage: Page | null = null;
-  readonly _ffmpegPath: string;

  // Holds window features for the next popup being opened via window.open,
  // until the popup target arrives. This could be racy if two oopifs
@ -65,10 +64,9 @@ export class CRPage implements PageDelegate {
  // of new popup targets.
  readonly _nextWindowOpenPopupFeatures: string[][] = [];

-  constructor(client: CRSession, targetId: string, browserContext: CRBrowserContext, opener: CRPage | null, hasUIWindow: boolean, ffmpegPath: string) {
+  constructor(client: CRSession, targetId: string, browserContext: CRBrowserContext, opener: CRPage | null, hasUIWindow: boolean) {
    this._targetId = targetId;
    this._opener = opener;
-    this._ffmpegPath = ffmpegPath;
    this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac);
    this.rawMouse = new RawMouseImpl(client);
    this._pdf = new CRPDF(client);
@ -752,7 +750,7 @@ class FrameSession {
  async _startScreencast(screencastId: string, options: types.PageScreencastOptions): Promise<void> {
    if (this._screencastState !== 'stopped')
      throw new Error('Already started');
-    const videoRecorder = await VideoRecorder.launch(this._crPage._ffmpegPath, options);
+    const videoRecorder = await VideoRecorder.launch(options);
    this._screencastState = 'starting';
    try {
      await this._client.send('Page.startScreencast', {
--- a/src/server/chromium/videoRecorder.ts
+++ b/src/server/chromium/videoRecorder.ts
@ -18,6 +18,8 @@ import { launchProcess } from '../processLauncher';
 import { ChildProcess } from 'child_process';
 import { Progress, runAbortableTask } from '../progress';
 import * as types from '../types';
+import * as path from 'path';
+import * as os from 'os';
 import { assert } from '../../utils/utils';

 const fps = 25;
@ -30,22 +32,20 @@ export class VideoRecorder {
  private _lastFrameBuffer: Buffer | null = null;
  private _lastWriteTimestamp: number = 0;
  private readonly _progress: Progress;
-  private readonly _ffmpegPath: string;

-  static async launch(ffmpegPath: string, options: types.PageScreencastOptions): Promise<VideoRecorder> {
+  static async launch(options: types.PageScreencastOptions): Promise<VideoRecorder> {
    if (!options.outputFile.endsWith('.webm'))
      throw new Error('File must have .webm extension');

    return await runAbortableTask(async progress => {
-      const recorder = new VideoRecorder(ffmpegPath, progress);
+      const recorder = new VideoRecorder(progress);
      await recorder._launch(options);
      return recorder;
    }, 0, 'browser');
  }

-  private constructor(ffmpegPath: string, progress: Progress) {
+  private constructor(progress: Progress) {
    this._progress = progress;
-    this._ffmpegPath = ffmpegPath;
    this._lastWritePromise = Promise.resolve();
  }

@ -56,8 +56,15 @@ export class VideoRecorder {
    const args = `-loglevel error -f image2pipe -c:v mjpeg -i - -y -an -r ${fps} -c:v vp8 -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' ');
    args.push(options.outputFile);
    const progress = this._progress;
+
+    let ffmpegPath = 'ffmpeg';
+    const binPath = path.join(__dirname, '../../../bin/');
+    if (os.platform() === 'win32')
+      ffmpegPath = path.join(binPath, os.arch() === 'x64' ? 'ffmpeg-win64.exe' : 'ffmpeg-win32.exe');
+    else if (os.platform() === 'darwin')
+      ffmpegPath = path.join(binPath, 'ffmpeg-mac');
    const { launchedProcess, gracefullyClose } = await launchProcess({
-      executablePath: this._ffmpegPath,
+      executablePath: ffmpegPath,
      args,
      pipeStdin: true,
      progress,
--- a/src/server/electron/electron.ts
+++ b/src/server/electron/electron.ts
@ -184,7 +184,7 @@ export class Electron  {
        close: gracefullyClose,
        kill
      };
-      const browser = await CRBrowser.connect(chromeTransport, { name: 'electron', headful: true, persistent: { noDefaultViewport: true }, browserProcess }, '');
+      const browser = await CRBrowser.connect(chromeTransport, { name: 'electron', headful: true, persistent: { noDefaultViewport: true }, browserProcess });
      app = new ElectronApplication(browser, nodeConnection);
      await app._init();
      return app;
--- a/src/utils/browserPaths.ts
+++ b/src/utils/browserPaths.ts
@ -72,18 +72,6 @@ export function windowsExeAndDllDirectories(browserPath: string, browser: Browse
  return [];
 }

-export function ffmpegPath(browserPath: string, browser: BrowserDescriptor): string {
-  if (browser.name !== 'chromium')
-    return '';
-  if (os.platform() === 'linux')
-    return 'ffmpeg';
-  if (os.platform() === 'win32')
-    return path.join(browserPath, 'chrome-win', 'ffmpeg.exe');
-  if (os.platform() === 'darwin')
-    return path.join(browserPath, 'chrome-mac', 'ffmpeg');
-  return '';
-}
-
 export function executablePath(browserPath: string, browser: BrowserDescriptor): string | undefined {
  let tokens: string[] | undefined;
  if (browser.name === 'chromium') {
--- a/utils/doclint/cli.js
+++ b/utils/doclint/cli.js
@ -44,10 +44,11 @@ async function run() {
  // Documentation checks.
  {
    const readme = await Source.readFile(path.join(PROJECT_DIR, 'README.md'));
+    const binReadme = await Source.readFile(path.join(PROJECT_DIR, 'bin', 'README.md'));
    const contributing = await Source.readFile(path.join(PROJECT_DIR, 'CONTRIBUTING.md'));
    const api = await Source.readFile(path.join(PROJECT_DIR, 'docs', 'api.md'));
    const docs = await Source.readdir(path.join(PROJECT_DIR, 'docs'), '.md');
-    const mdSources = [readme, api, contributing, ...docs];
+    const mdSources = [readme, binReadme, api, contributing, ...docs];

    const preprocessor = require('./preprocessor');
    const browserVersions = await getBrowserVersions();