Skip to content
This repository has been archived by the owner on Feb 1, 2022. It is now read-only.

Commit

Permalink
WebGL section initial content. Updated the examples.
Browse files Browse the repository at this point in the history 
Renoves the splitting depth to red and green component.
  • Loading branch information
@astojilj
astojilj committed Mar 3, 2017
1 parent 9079d6a commit b66a489
Show file tree
Hide file tree
Showing 2 changed files with 305 additions and 77 deletions.
235 changes: 187 additions & 48 deletions index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,9 @@
.mjx-chtml {display: inline-block; line-height: 0; text-indent: 0; text-align: left; text-transform: none; font-style: normal; font-weight: normal; font-size: 100%; font-size-adjust: none; letter-spacing: normal; word-wrap: normal; word-spacing: normal; white-space: nowrap; float: none; direction: ltr; max-width: none; max-height: none; min-width: 0; min-height: 0; border: 0; margin: 0; padding: 1px 0}
.MJXc-display {display: block; text-align: center; margin: 1em 0; padding: 0}
.mjx-chtml[tabindex]:focus, body :focus .mjx-chtml[tabindex] {display: inline-table}
.mjx-full-width {text-align: center; display: table-cell!important; width: 10000em}
.mjx-math {display: inline-block; border-collapse: separate; border-spacing: 0}
.mjx-math * {display: inline-block; text-align: left}
.mjx-math * {display: inline-block; -webkit-box-sizing: content-box!important; -moz-box-sizing: content-box!important; box-sizing: content-box!important; text-align: left}
.mjx-numerator {display: block; text-align: center}
.mjx-denominator {display: block; text-align: center}
.MJXc-stacked {height: 0; position: relative}
Expand All @@ -115,11 +116,12 @@
.mjx-mtr {display: table-row}
.mjx-mlabeledtr {display: table-row}
.mjx-mtd {display: table-cell; text-align: center}
.mjx-label {display: block}
.mjx-label {display: table-row}
.mjx-box {display: inline-block}
.mjx-block {display: block}
.mjx-span {display: inline}
.mjx-char {display: block; white-space: pre}
.mjx-itable {display: inline-table}
.mjx-itable {display: inline-table; width: auto}
.mjx-row {display: table-row}
.mjx-cell {display: table-cell}
.mjx-table {display: table; width: 100%}
Expand All @@ -130,6 +132,8 @@
.MJXc-space2 {margin-left: .222em}
.MJXc-space3 {margin-left: .278em}
.mjx-ex-box-test {position: absolute; width: 1px; height: 60ex}
.mjx-line-box-test {display: table!important}
.mjx-line-box-test span {display: table-cell!important; width: 10000em!important; min-width: 0; max-width: none; padding: 0; border: 0; margin: 0}
.MJXc-TeX-unknown-R {font-family: monospace; font-style: normal; font-weight: normal}
.MJXc-TeX-unknown-I {font-family: monospace; font-style: italic; font-weight: normal}
.MJXc-TeX-unknown-B {font-family: monospace; font-style: normal; font-weight: bold}
Expand Down Expand Up @@ -470,8 +474,8 @@ <h2>
</p>
<p>
<span id="MathJax-Element-1-Frame" class="mjx-chtml"><span id=
"MJXc-Node-1" class="mjx-math" role="math"><span id="MJXc-Node-2"
class="mjx-mrow"><span id="MJXc-Node-3" class="mjx-mstyle"><span id=
"MJXc-Node-1" class="mjx-math"><span id="MJXc-Node-2" class=
"mjx-mrow"><span id="MJXc-Node-3" class="mjx-mstyle"><span id=
"MJXc-Node-4" class="mjx-mrow"><span id="MJXc-Node-5" class=
"mjx-msub"><span class="mjx-base" style=
"margin-right: -0.003em;"><span id="MJXc-Node-6" class=
Expand All @@ -495,7 +499,7 @@ <h2>
class="mjx-mo" style=
"padding-left: 0.267em; padding-right: 0.267em;"><span class=
"mjx-char MJXc-TeX-main-R" style=
"margin-top: 0.004em; padding-bottom: 0.298em;">−</span></span><span id="MJXc-Node-14"
"padding-top: 0.298em; padding-bottom: 0.446em;">−</span></span><span id="MJXc-Node-14"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.225em; padding-bottom: 0.298em;">n</span></span><span id="MJXc-Node-15"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
Expand All @@ -515,7 +519,7 @@ <h2>
class="mjx-mo" style=
"padding-left: 0.267em; padding-right: 0.267em;"><span class=
"mjx-char MJXc-TeX-main-R" style=
"margin-top: 0.004em; padding-bottom: 0.298em;">−</span></span><span id="MJXc-Node-23"
"padding-top: 0.298em; padding-bottom: 0.446em;">−</span></span><span id="MJXc-Node-23"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.225em; padding-bottom: 0.298em;">n</span></span><span id="MJXc-Node-24"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
Expand All @@ -530,8 +534,8 @@ <h2>
</p>
<p>
<span id="MathJax-Element-2-Frame" class="mjx-chtml"><span id=
"MJXc-Node-27" class="mjx-math" role="math"><span id="MJXc-Node-28"
class="mjx-mrow"><span id="MJXc-Node-29" class="mjx-mstyle"><span id=
"MJXc-Node-27" class="mjx-math"><span id="MJXc-Node-28" class=
"mjx-mrow"><span id="MJXc-Node-29" class="mjx-mstyle"><span id=
"MJXc-Node-30" class="mjx-mrow"><span id="MJXc-Node-31" class=
"mjx-msub"><span class="mjx-base" style=
"margin-right: -0.003em;"><span id="MJXc-Node-32" class=
Expand Down Expand Up @@ -589,10 +593,10 @@ <h2>
<var>d<sub>16bit</sub></var> to [0, 1] range:
<p>
<span id="MathJax-Element-3-Frame" class="mjx-chtml"><span id=
"MJXc-Node-49" class="mjx-math" role="math"><span id=
"MJXc-Node-50" class="mjx-mrow"><span id="MJXc-Node-51" class=
"mjx-mstyle"><span id="MJXc-Node-52" class="mjx-mrow"><span id=
"MJXc-Node-53" class="mjx-msub"><span class="mjx-base" style=
"MJXc-Node-49" class="mjx-math"><span id="MJXc-Node-50" class=
"mjx-mrow"><span id="MJXc-Node-51" class="mjx-mstyle"><span id=
"MJXc-Node-52" class="mjx-mrow"><span id="MJXc-Node-53" class=
"mjx-msub"><span class="mjx-base" style=
"margin-right: -0.003em;"><span id="MJXc-Node-54" class=
"mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.446em; padding-bottom: 0.298em; padding-right: 0.003em;">
Expand Down Expand Up @@ -635,11 +639,10 @@ <h2>
d</var>:
<p>
<span id="MathJax-Element-4-Frame" class="mjx-chtml"><span id=
"MJXc-Node-67" class="mjx-math" role="math"><span id=
"MJXc-Node-68" class="mjx-mrow"><span id="MJXc-Node-69" class=
"mjx-mstyle"><span id="MJXc-Node-70" class="mjx-mrow"><span id=
"MJXc-Node-71" class="mjx-mi"><span class=
"mjx-char MJXc-TeX-math-I" style=
"MJXc-Node-67" class="mjx-math"><span id="MJXc-Node-68" class=
"mjx-mrow"><span id="MJXc-Node-69" class="mjx-mstyle"><span id=
"MJXc-Node-70" class="mjx-mrow"><span id="MJXc-Node-71" class=
"mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.446em; padding-bottom: 0.298em; padding-right: 0.003em;">
d</span></span><span id="MJXc-Node-72" class="mjx-mo" style=
"padding-left: 0.333em; padding-right: 0.333em;"><span class=
Expand Down Expand Up @@ -673,7 +676,7 @@ <h2>
class="mjx-mo" style=
"padding-left: 0.267em; padding-right: 0.267em;"><span class=
"mjx-char MJXc-TeX-main-R" style=
"margin-top: 0.004em; padding-bottom: 0.298em;">−</span></span><span id="MJXc-Node-86"
"padding-top: 0.298em; padding-bottom: 0.446em;">−</span></span><span id="MJXc-Node-86"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.225em; padding-bottom: 0.298em;">n</span></span><span id="MJXc-Node-87"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
Expand Down Expand Up @@ -1193,17 +1196,105 @@ <h3>
<div class="note">
This section is currently work in progress, and subject to change.
</div>
<p>
There are several use-cases which are a good fit to be, at least
partially, implemented on the GPU, such as motion recognition,
pattern recognition, background removal, as well as 3D point cloud.
</p>
<p>
This section explains which APIs can be used for some of these
mentioned use-cases; the concrete usage examples are provided in
the <a href=
"https://www.w3.org/wiki/Media_Capture_Depth_Stream_Extension#Examples">
Examples</a> section.
</p>
<h3>
Upload video frame to WebGL texture
</h3>
<p>
A <a>video</a> element whose source is a <a>MediaStream</a> object
containing a <a>depth stream track</a> may be uploaded to a WebGL
texture of format <code>RGB</code> and type
<code>UNSIGNED_BYTE</code>. [[WEBGL]]
texture of format <code>RGBA</code> or <code>RED</code> and type
<code>FLOAT</code>. See the specification [[WEBGL]] and the
<a>upload to float texture</a> example code.
</p>
<p>
For each pixel of this WebGL texture, the R component represents
the lower 8 bit value of 16 bit depth value, the G component
represents the upper 8 bit value of 16 bit depth value and the
value in B component is not defined.
normalized 16-bit value following the formula: <span id=
"MathJax-Element-5-Frame" class="mjx-chtml"><span id="MJXc-Node-97"
class="mjx-math"><span id="MJXc-Node-98" class="mjx-mrow"><span id=
"MJXc-Node-99" class="mjx-mstyle"><span id="MJXc-Node-100" class=
"mjx-mrow"><span id="MJXc-Node-101" class="mjx-msub"><span class=
"mjx-base" style="margin-right: -0.003em;"><span id="MJXc-Node-102"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.446em; padding-bottom: 0.298em; padding-right: 0.003em;">
d</span></span></span><span class="mjx-sub" style=
"font-size: 70.7%; vertical-align: -0.23em; padding-right: 0.071em;"><span id="MJXc-Node-103"
class="mjx-mrow" style=""><span id="MJXc-Node-104" class=
"mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.519em; padding-bottom: 0.519em; padding-right: 0.06em;">f</span></span><span id="MJXc-Node-105"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.446em; padding-bottom: 0.298em;">l</span></span><span id="MJXc-Node-106"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.225em; padding-bottom: 0.298em;">o</span></span><span id="MJXc-Node-107"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.225em; padding-bottom: 0.298em;">a</span></span><span id="MJXc-Node-108"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.372em; padding-bottom: 0.298em;">t</span></span></span></span></span><span id="MJXc-Node-109"
class="mjx-mo" style=
"padding-left: 0.333em; padding-right: 0.333em;"><span class=
"mjx-char MJXc-TeX-main-R" style=
"padding-top: 0.077em; padding-bottom: 0.298em;">=</span></span><span id="MJXc-Node-110"
class="mjx-mfrac"><span class="mjx-box MJXc-stacked" style=
"width: 3.478em; padding: 0px 0.12em;"><span class="mjx-numerator"
style="width: 3.478em; top: -1.447em;"><span id="MJXc-Node-111"
class="mjx-msub"><span class="mjx-base" style=
"margin-right: -0.003em;"><span id="MJXc-Node-112" class=
"mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.446em; padding-bottom: 0.298em; padding-right: 0.003em;">d</span></span></span><span class="mjx-sub"
style=
"font-size: 70.7%; vertical-align: -0.219em; padding-right: 0.071em;"><span id="MJXc-Node-113"
class="mjx-mrow" style=""><span id="MJXc-Node-114" class=
"mjx-mn"><span class="mjx-char MJXc-TeX-main-R" style=
"padding-top: 0.372em; padding-bottom: 0.372em;">16</span></span><span id="MJXc-Node-115"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.446em; padding-bottom: 0.298em;">b</span></span><span id="MJXc-Node-116"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.446em; padding-bottom: 0.298em;">i</span></span><span id="MJXc-Node-117"
class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style=
"padding-top: 0.372em; padding-bottom: 0.298em;">t</span></span></span></span></span></span><span class="mjx-denominator"
style="width: 3.478em; bottom: -0.778em;"><span id="MJXc-Node-118"
class="mjx-mn"><span class="mjx-char MJXc-TeX-main-R" style=
"padding-top: 0.372em; padding-bottom: 0.372em;">65535.0</span></span></span><span style="border-bottom: 1.3px solid; top: -0.296em; width: 3.478em;"
class="mjx-line"></span></span><span style=
"height: 2.225em; vertical-align: -0.778em;" class=
"mjx-vsize"></span></span></span></span></span></span></span>
</p>
<h3>
Read the data from a WebGL texture
</h3>
<p>
Here we list some of the possible approaches.
</p>
<ul>
<li>Synchronous readPixels requires the least amount of code and it
is available om WebGL1. See the <a>readPixels from float</a>
example for further details.
</li>
<li>Asynchronous readPixels using pixel buffer objects to avoid
blocking the readPixels call.
</li>
<li>Transform feedback with GetBufferSubData(Async) provides
synchronous and asynchronous read access to depth and color texture
data processed in the vertex shader.
</li>
</ul>
<p></p>
<p class="note">
Performance of synchronous <a>readPixels from float</a> example in
the current implementation suffice for some of the use cases. The
reason is that there is no rendering to the float texture bound to
named framebuffer.
</p>
</section>
</section>
Expand Down Expand Up @@ -1239,12 +1330,11 @@ <h3>
);
</pre>
<h3>
WebGL Fragment Shader based post-processing
WebGL2: <dfn>upload to float texture</dfn>
</h3>
<pre class="example">
// This code sets up a video element from a depth stream, uploads it to a WebGL
// texture, and samples that texture in the fragment shader, reconstructing the
// 16-bit depth values from the red and green channels.
// This code sets up a video element from a depth stream, uploads it to a WebGL2
// float texture containing the full precision data as the 16-bit depth map.
navigator.mediaDevices.getUserMedia({
video: {videoKind: {exact: "depth"}}
}).then(function (stream) {
Expand All @@ -1256,31 +1346,80 @@ <h3>
// handle gUM error here
});

// ... later, in the rendering loop ...
var gl = canvas.getContext("webgl2");
// Activate the extension to use single component R32F texture format.
gl.getExtension('EXT_color_buffer_float');

// Later, in the rendering loop ...
gl.bindTexture(gl.TEXTURE_2D, depthTexture);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.RGB,
gl.RGB,
gl.UNSIGNED_BYTE,
depthVideo
);
gl.R32F,
gl.RED,
gl.FLOAT,
depthVideo);

&lt;script id="fragment-shader" type="x-shader/x-fragment"&gt;
varying vec2 v_texCoord;
// u_tex points to the texture unit containing the depth texture.
uniform sampler2D u_tex;
uniform float far;
uniform float near;
void main() {
vec4 floatColor = texture2D(u_tex, v_texCoord);
float dn = floatColor.r;
float depth = 0.;
depth = far * near / ( far - dn * ( far - near));
// ...
}
&lt;/script&gt;
</pre>
<h3>
WebGL2: <dfn>readPixels from float</dfn> texture
</h3>
<p>
This example extends <a>upload to float texture</a> example.
</p>
<pre class="example">
// This code creates the texture to which we will upload the depth video frame.
// Then, it sets up a named framebuffer, attach the texture as color attachment
// and, after uploading the depth video to the texture, reads the texture
// content to Float32Array.

// Initialize texture and framebuffer for reading back the texture...
var depthTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, depthTexture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);

var framebuffer = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
gl.framebufferTexture2D(
gl.FRAMEBUFFER,
gl.COLOR_ATTACHMENT0,
gl.TEXTURE_2D,
depthTexture,
0);

// Later, in the rendering loop ...
gl.bindTexture(gl.TEXTURE_2D, depthTexture);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.R32F,
gl.RED,
gl.FLOAT,
depthVideo);

// It is recommended to allocate this buffer once.
const buffer = new Float32Array(depthVideo.videoWidth * depthVideo.videoHeight);

gl.readPixels(
0,
0,
depthVideo.videoWidth,
depthVideo.videoHeight,
gl.RED,
gl.FLOAT,
buffer);

</pre>
<div class="note">
<p>
Use
<code>gl.getParameter(gl.IMPLEMENTATION_COLOR_READ_FORMAT);</code> to
check whether readPixels to gl.RED or gl.RGBA float is supported.
</p>
</div>
</section>
<section class="informative">
<h2>
Expand Down
Loading

0 comments on commit b66a489

Please sign in to comment.