async picking

Apps/Sandcastle/gallery/3D Tiles Feature Picking.html

@@ -37,6 +37,7 @@
           terrain: Cesium.Terrain.fromWorldTerrain(),
         });
 
+        const async = true;
         viewer.scene.globe.depthTestAgainstTerrain = true;
 
         // Set the initial camera view to look at Manhattan
@@ -157,13 +158,20 @@
           );
 
           // Silhouette a feature blue on hover.
-          viewer.screenSpaceEventHandler.setInputAction(function onMouseMove(movement) {
+          viewer.screenSpaceEventHandler.setInputAction(async function onMouseMove(
+            movement,
+          ) {
+            // Pick a new feature
+            let pickedFeature;
+            if (async) {
+              pickedFeature = await viewer.scene.pickAsync(movement.endPosition);
+            } else {
+              pickedFeature = viewer.scene.pick(movement.endPosition);
+            }
+
             // If a feature was previously highlighted, undo the highlight
             silhouetteBlue.selected = [];
 
-            // Pick a new feature
-            const pickedFeature = viewer.scene.pick(movement.endPosition);
-
             updateNameOverlay(pickedFeature, movement.endPosition);
 
             if (!Cesium.defined(pickedFeature)) {
@@ -177,12 +185,20 @@
           }, Cesium.ScreenSpaceEventType.MOUSE_MOVE);
 
           // Silhouette a feature on selection and show metadata in the InfoBox.
-          viewer.screenSpaceEventHandler.setInputAction(function onLeftClick(movement) {
+          viewer.screenSpaceEventHandler.setInputAction(async function onLeftClick(
+            movement,
+          ) {
+            // Pick a new feature
+            let pickedFeature;
+            if (async) {
+              pickedFeature = await viewer.scene.pickAsync(movement.position);
+            } else {
+              pickedFeature = viewer.scene.pick(movement.position);
+            }
+
             // If a feature was previously selected, undo the highlight
             silhouetteGreen.selected = [];
 
-            // Pick a new feature
-            const pickedFeature = viewer.scene.pick(movement.position);
             if (!Cesium.defined(pickedFeature)) {
               clickHandler(movement);
               return;
@@ -216,14 +232,23 @@
           };
 
           // Color a feature yellow on hover.
-          viewer.screenSpaceEventHandler.setInputAction(function onMouseMove(movement) {
+          viewer.screenSpaceEventHandler.setInputAction(async function onMouseMove(
+            movement,
+          ) {
+            // Pick a new feature
+            let pickedFeature;
+            if (async) {
+              pickedFeature = await viewer.scene.pickAsync(movement.endPosition);
+            } else {
+              pickedFeature = viewer.scene.pick(movement.endPosition);
+            }
+
             // If a feature was previously highlighted, undo the highlight
             if (Cesium.defined(highlighted.feature)) {
               highlighted.feature.color = highlighted.originalColor;
               highlighted.feature = undefined;
             }
-            // Pick a new feature
-            const pickedFeature = viewer.scene.pick(movement.endPosition);
+
             updateNameOverlay(pickedFeature, movement.endPosition);
 
             if (!Cesium.defined(pickedFeature)) {
@@ -239,14 +264,23 @@
           }, Cesium.ScreenSpaceEventType.MOUSE_MOVE);
 
           // Color a feature on selection and show metadata in the InfoBox.
-          viewer.screenSpaceEventHandler.setInputAction(function onLeftClick(movement) {
+          viewer.screenSpaceEventHandler.setInputAction(async function onLeftClick(
+            movement,
+          ) {
+            // Pick a new feature
+            let pickedFeature;
+            if (async) {
+              pickedFeature = await viewer.scene.pickAsync(movement.position);
+            } else {
+              pickedFeature = viewer.scene.pick(movement.position);
+            }
+
             // If a feature was previously selected, undo the highlight
             if (Cesium.defined(selected.feature)) {
               selected.feature.color = selected.originalColor;
               selected.feature = undefined;
             }
-            // Pick a new feature
-            const pickedFeature = viewer.scene.pick(movement.position);
+
             if (!Cesium.defined(pickedFeature)) {
               clickHandler(movement);
               return;

Specs/pick.js

@@ -66,7 +66,7 @@ function pick(frameState, primitives, x, y) {
 
   frameState.passes = oldPasses;
 
-  const p = pickFramebuffer.end(rectangle);
+  const p = pickFramebuffer.end(rectangle, frameState);
   pickFramebuffer.destroy();
 
   return p;

packages/engine/Source/Renderer/Buffer.js

@@ -72,6 +72,24 @@ function Buffer(options) {
   this.vertexArrayDestroyable = true;
 }
 
+Buffer.createPixelBuffer = function (options) {
+  //>>includeStart('debug', pragmas.debug);
+  Check.defined("options.context", options.context);
+  //>>includeEnd('debug');
+
+  if (!options.context._webgl2) {
+    throw new DeveloperError("A WebGL 2 context is required.");
+  }
+
+  return new Buffer({
+    context: options.context,
+    bufferTarget: WebGLConstants.PIXEL_PACK_BUFFER,
+    typedArray: options.typedArray,
+    sizeInBytes: options.sizeInBytes,
+    usage: options.usage,
+  });
+};
+
 /**
  * Creates a vertex buffer, which contains untyped vertex data in GPU-controlled memory.
  * <br /><br />
@@ -242,6 +260,18 @@ Buffer.prototype._getBuffer = function () {
   return this._buffer;
 };
 
+Buffer.prototype._bind = function () {
+  const gl = this._gl;
+  const target = this._bufferTarget;
+  gl.bindBuffer(target, this._buffer);
+};
+
+Buffer.prototype._unBind = function () {
+  const gl = this._gl;
+  const target = this._bufferTarget;
+  gl.bindBuffer(target, null);
+};
+
 Buffer.prototype.copyFromArrayView = function (arrayView, offsetInBytes) {
   offsetInBytes = offsetInBytes ?? 0;
 

packages/engine/Source/Renderer/BufferUsage.js

@@ -7,12 +7,14 @@ const BufferUsage = {
   STREAM_DRAW: WebGLConstants.STREAM_DRAW,
   STATIC_DRAW: WebGLConstants.STATIC_DRAW,
   DYNAMIC_DRAW: WebGLConstants.DYNAMIC_DRAW,
+  DYNAMIC_READ: WebGLConstants.DYNAMIC_READ,
 
   validate: function (bufferUsage) {
     return (
       bufferUsage === BufferUsage.STREAM_DRAW ||
       bufferUsage === BufferUsage.STATIC_DRAW ||
-      bufferUsage === BufferUsage.DYNAMIC_DRAW
+      bufferUsage === BufferUsage.DYNAMIC_DRAW ||
+      bufferUsage === BufferUsage.DYNAMIC_READ
     );
   },
 };

packages/engine/Source/Renderer/Context.js

@@ -1,3 +1,4 @@
+import Buffer from "./Buffer.js";
 import Check from "../Core/Check.js";
 import Color from "../Core/Color.js";
 import ComponentDatatype from "../Core/ComponentDatatype.js";
@@ -1449,6 +1450,7 @@ Context.prototype.endFrame = function () {
  * @param {number} [readState.width=this.drawingBufferWidth] The width of the rectangle to read from.
  * @param {number} [readState.height=this.drawingBufferHeight] The height of the rectangle to read from.
  * @param {Framebuffer} [readState.framebuffer] The framebuffer to read from. If undefined, the read will be from the default framebuffer.
+ * @param {Framebuffer} [readState.pbo] If true pixel data is read to PBO instead of TypedArray.
  * @returns {Uint8Array|Uint16Array|Float32Array|Uint32Array} The pixels in the specified rectangle.
  */
 Context.prototype.readPixels = function (readState) {
@@ -1460,6 +1462,11 @@ Context.prototype.readPixels = function (readState) {
   const width = readState.width ?? this.drawingBufferWidth;
   const height = readState.height ?? this.drawingBufferHeight;
   const framebuffer = readState.framebuffer;
+  const pbo = readState.pbo ?? false;
+
+  if (pbo && !this._webgl2) {
+    throw new DeveloperError("A WebGL 2 context is required.");
+  }
 
   //>>includeStart('debug', pragmas.debug);
   Check.typeOf.number.greaterThan("readState.width", width, 0);
@@ -1467,28 +1474,58 @@ Context.prototype.readPixels = function (readState) {
   //>>includeEnd('debug');
 
   let pixelDatatype = PixelDatatype.UNSIGNED_BYTE;
+  let pixelFormat = PixelFormat.RGBA;
   if (defined(framebuffer) && framebuffer.numberOfColorAttachments > 0) {
     pixelDatatype = framebuffer.getColorTexture(0).pixelDatatype;
+    pixelFormat = framebuffer.getColorTexture(0).pixelFormat;
   }
 
-  const pixels = PixelFormat.createTypedArray(
-    PixelFormat.RGBA,
-    pixelDatatype,
-    width,
-    height,
-  );
+  let pixels;
+  if (pbo) {
+    pixels = Buffer.createPixelBuffer({
+      context: this,
+      sizeInBytes: PixelFormat.textureSizeInBytes(
+        pixelFormat,
+        pixelDatatype,
+        width,
+        height,
+      ),
+      usage: BufferUsage.DYNAMIC_READ,
+    });
+  } else {
+    pixels = PixelFormat.createTypedArray(
+      pixelFormat,
+      pixelDatatype,
+      width,
+      height,
+    );
+  }
 
   bindFramebuffer(this, framebuffer);
 
-  gl.readPixels(
-    x,
-    y,
-    width,
-    height,
-    PixelFormat.RGBA,
-    PixelDatatype.toWebGLConstant(pixelDatatype, this),
-    pixels,
-  );
+  if (pbo) {
+    pixels._bind();
+    gl.readPixels(
+      x,
+      y,
+      width,
+      height,
+      pixelFormat,
+      PixelDatatype.toWebGLConstant(pixelDatatype, this),
+      0,
+    );
+    pixels._unBind();
+  } else {
+    gl.readPixels(
+      x,
+      y,
+      width,
+      height,
+      PixelFormat.RGBA,
+      PixelDatatype.toWebGLConstant(pixelDatatype, this),
+      pixels,
+    );
+  }
 
   return pixels;
 };

packages/engine/Source/Renderer/Sync.js

@@ -0,0 +1,46 @@
+import Check from "../Core/Check";
+import destroyObject from "../Core/destroyObject";
+import DeveloperError from "../Core/DeveloperError";
+import Frozen from "../Core/Frozen";
+import WebGLConstants from "../Core/WebGLConstants";
+
+/**
+ * @private
+ */
+function Sync(options) {
+  options = options ?? Frozen.EMPTY_OBJECT;
+
+  //>>includeStart('debug', pragmas.debug);
+  Check.defined("options.context", options.context);
+  //>>includeEnd('debug');
+
+  if (!options.context._webgl2) {
+    throw new DeveloperError("A WebGL 2 context is required.");
+  }
+
+  const context = options.context;
+  const gl = context._gl;
+
+  const sync = gl.fenceSync(WebGLConstants.SYNC_GPU_COMMANDS_COMPLETE, 0);
+
+  this._gl = gl;
+  this._sync = sync;
+}
+Sync.create = function (options) {
+  return new Sync(options);
+};
+Sync.prototype.getStatus = function () {
+  const status = this._gl.getSyncParameter(
+    this._sync,
+    WebGLConstants.SYNC_STATUS,
+  );
+  return status;
+};
+Sync.prototype.isDestroyed = function () {
+  return false;
+};
+Sync.prototype.destroy = function () {
+  this._gl.deleteSync(this._sync);
+  return destroyObject(this);
+};
+export default Sync;

packages/engine/Source/Scene/PickFramebuffer.js

@@ -4,6 +4,10 @@ import defined from "../Core/defined.js";
 import destroyObject from "../Core/destroyObject.js";
 import FramebufferManager from "../Renderer/FramebufferManager.js";
 import PassState from "../Renderer/PassState.js";
+import PixelDatatype from "../Renderer/PixelDatatype.js";
+import PixelFormat from "../Core/PixelFormat.js";
+import WebGLConstants from "../Core/WebGLConstants.js";
+import Sync from "../Renderer/Sync.js";
 
 /**
  * @private
@@ -50,25 +54,7 @@ PickFramebuffer.prototype.begin = function (screenSpaceRectangle, viewport) {
 
 const colorScratchForPickFramebuffer = new Color();
 
-/**
- * Return the picked object rendered within a given rectangle.
- *
- * @param {BoundingRectangle} screenSpaceRectangle
- * @returns {object|undefined} The object rendered in the middle of the rectangle, or undefined if nothing was rendered.
- */
-PickFramebuffer.prototype.end = function (screenSpaceRectangle) {
-  const width = screenSpaceRectangle.width ?? 1.0;
-  const height = screenSpaceRectangle.height ?? 1.0;
-
-  const context = this._context;
-  const pixels = context.readPixels({
-    x: screenSpaceRectangle.x,
-    y: screenSpaceRectangle.y,
-    width: width,
-    height: height,
-    framebuffer: this._fb.framebuffer,
-  });
-
+function colorScratchForObject(context, pixels, width, height) {
   const max = Math.max(width, height);
   const length = max * max;
   const halfWidth = Math.floor(width * 0.5);
@@ -125,6 +111,129 @@ PickFramebuffer.prototype.end = function (screenSpaceRectangle) {
   }
 
   return undefined;
+}
+
+let i = 0;
+PickFramebuffer.prototype.endAsync = async function (
+  screenSpaceRectangle,
+  frameState,
+) {
+  const width = screenSpaceRectangle.width ?? 1.0;
+  const height = screenSpaceRectangle.height ?? 1.0;
+
+  const context = this._context;
+  const framebuffer = this._fb.framebuffer;
+
+  let pixelDatatype = PixelDatatype.UNSIGNED_BYTE;
+  let pixelFormat = PixelFormat.RGBA;
+
+  if (defined(framebuffer) && framebuffer.numberOfColorAttachments > 0) {
+    pixelDatatype = framebuffer.getColorTexture(0).pixelDatatype;
+    pixelFormat = framebuffer.getColorTexture(0).pixelFormat;
+  }
+
+  const pbo = context.readPixels({
+    x: screenSpaceRectangle.x,
+    y: screenSpaceRectangle.y,
+    width: width,
+    height: height,
+    framebuffer: framebuffer,
+    pbo: true,
+  });
+
+  const sync = Sync.create({
+    context: context,
+  });
+
+  i++;
+
+  const pickState = {
+    id: i, // TODO: remove
+    context: context,
+    frameState: frameState,
+    frameNumber: frameState.frameNumber,
+    sync: sync,
+    pbo: pbo,
+    pixelFormat: pixelFormat,
+    pixelDatatype: pixelDatatype,
+    width: width,
+    height: width,
+  };
+
+  return new Promise((resolve, reject) => {
+    //console.log("[async] Pick", `#${pickState.id}`, pickState.frameNumber, screenSpaceRectangle.x, screenSpaceRectangle.y);
+    frameState.afterRender.push(
+      createAsyncPick(pickState, (context, signaled) => {
+        const pbo = pickState.pbo;
+        //const frameDelta = frameState.frameNumber - pickState.frameNumber; // how many frames passed since inital request
+        const pixels = PixelFormat.createTypedArray(
+          pickState.pixelFormat,
+          pickState.pixelDatatype,
+          pickState.width,
+          pickState.height,
+        );
+        pbo.getBufferData(pixels);
+        pbo.destroy();
+        const obj = colorScratchForObject(
+          context,
+          pixels,
+          pickState.width,
+          pickState.height,
+        );
+        //console.log("[async] Return", `#${pickState.id}`, frameDelta, signaled, obj);
+        if (signaled) {
+          resolve(obj);
+        } else {
+          reject("Picking Request Timeout");
+        }
+      }),
+    );
+  });
+};
+
+// TODO: comment
+function createAsyncPick(pickState, onSignalCallback) {
+  return () => {
+    const context = pickState.context;
+    const sync = pickState.sync;
+    const frameState = pickState.frameState;
+    const ttl = pickState.ttl ?? 10;
+    const syncStatus = sync.getStatus();
+    const signaled = syncStatus === WebGLConstants.SIGNALED;
+    const frameDelta = frameState.frameNumber - pickState.frameNumber; // how many frames passed since inital request
+    if (signaled || frameDelta > ttl) {
+      //console.log("signal", `#${pickState.id}`, pickState.frameNumber, frameState.frameNumber, frameDelta);
+      sync.destroy();
+      onSignalCallback(context, signaled);
+    } else {
+      //console.log("no-signal", `#${pickState.id}`, pickState.frameNumber, frameState.frameNumber, frameDelta);
+      frameState.afterRender.push(createAsyncPick(pickState, onSignalCallback));
+    }
+  };
+}
+
+/**
+ * Return the picked object rendered within a given rectangle.
+ *
+ * @param {BoundingRectangle} screenSpaceRectangle
+ * @returns {object|undefined} The object rendered in the middle of the rectangle, or undefined if nothing was rendered.
+ */
+PickFramebuffer.prototype.end = function (screenSpaceRectangle, frameState) {
+  const width = screenSpaceRectangle.width ?? 1.0;
+  const height = screenSpaceRectangle.height ?? 1.0;
+
+  //console.log("[sync] Pick# ", i, frameState.frameNumber);
+
+  const context = this._context;
+  const pixels = context.readPixels({
+    x: screenSpaceRectangle.x,
+    y: screenSpaceRectangle.y,
+    width: width,
+    height: height,
+    framebuffer: this._fb.framebuffer,
+  });
+
+  return colorScratchForObject(context, pixels, width, height);
 };
 
 /**

packages/engine/Source/Scene/Picking.js

@@ -272,15 +272,23 @@ function computePickingDrawingBufferRectangle(
  * @param {Cartesian2} windowPosition Window coordinates to perform picking on.
  * @param {number} [width=3] Width of the pick rectangle.
  * @param {number} [height=3] Height of the pick rectangle.
- * @returns {object | undefined} Object containing the picked primitive.
+ * @param {boolean} [async=false] Use async non GPU blocking picking.
+ * @returns {object | Promise<object | undefined> | undefined} Object containing the picked primitive.
  */
-Picking.prototype.pick = function (scene, windowPosition, width, height) {
+Picking.prototype.pick = function (
+  scene,
+  windowPosition,
+  width,
+  height,
+  async,
+) {
   //>>includeStart('debug', pragmas.debug);
   Check.defined("windowPosition", windowPosition);
   //>>includeEnd('debug');
 
   const { context, frameState, defaultView } = scene;
   const { viewport, pickFramebuffer } = defaultView;
+  async = async ?? false;
 
   scene.view = defaultView;
 
@@ -328,7 +336,12 @@ Picking.prototype.pick = function (scene, windowPosition, width, height) {
   scene.updateAndExecuteCommands(passState, scratchColorZero);
   scene.resolveFramebuffers(passState);
 
-  const object = pickFramebuffer.end(drawingBufferRectangle);
+  let object;
+  if (async) {
+    object = pickFramebuffer.endAsync(drawingBufferRectangle, frameState); // Promise
+  } else {
+    object = pickFramebuffer.end(drawingBufferRectangle, frameState); // Object
+  }
   context.endFrame();
   return object;
 };
@@ -1016,7 +1029,7 @@ function getRayIntersection(
   scene.resolveFramebuffers(passState);
 
   let position;
-  const object = view.pickFramebuffer.end(drawingBufferRectangle);
+  const object = view.pickFramebuffer.end(drawingBufferRectangle, frameState);
 
   if (scene.context.depthTexture) {
     const { frustumCommandsList } = view;

packages/engine/Source/Scene/Scene.js

@@ -3821,14 +3821,15 @@ function callAfterRenderFunctions(scene) {
   // Functions are queued up during primitive update and executed here in case
   // the function modifies scene state that should remain constant over the frame.
   const functions = scene._frameState.afterRender;
-  for (let i = 0; i < functions.length; ++i) {
-    const shouldRequestRender = functions[i]();
+  const functionsCpy = functions.slice(); // Snapshot before iterate allows callbacks to add functions for next frame
+  functions.length = 0;
+
+  for (let i = 0; i < functionsCpy.length; ++i) {
+    const shouldRequestRender = functionsCpy[i]();
     if (shouldRequestRender) {
       scene.requestRender();
     }
   }
-
-  functions.length = 0;
 }
 
 function getGlobeHeight(scene) {
@@ -4405,6 +4406,10 @@ Scene.prototype.pick = function (windowPosition, width, height) {
   return this._picking.pick(this, windowPosition, width, height);
 };
 
+Scene.prototype.pickAsync = async function (windowPosition, width, height) {
+  return this._picking.pick(this, windowPosition, width, height, true); // TODO: merge apis?
+};
+
 /**
  * Returns a {@link VoxelCell} for the voxel sample rendered at a particular window coordinate,
  * or <code>undefined</code> if no voxel is rendered at that position.