/* * Copyright 2016 Google Inc. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ var CardboardUI = require('./cardboard-ui.js'); var Util = require('./util.js'); var WGLUPreserveGLState = require('./deps/wglu-preserve-state.js'); var distortionVS = [ 'attribute vec2 position;', 'attribute vec3 texCoord;', 'varying vec2 vTexCoord;', 'uniform vec4 viewportOffsetScale[2];', 'void main() {', ' vec4 viewport = viewportOffsetScale[int(texCoord.z)];', ' vTexCoord = (texCoord.xy * viewport.zw) + viewport.xy;', ' gl_Position = vec4( position, 1.0, 1.0 );', '}', ].join('\n'); var distortionFS = [ 'precision mediump float;', 'uniform sampler2D diffuse;', 'varying vec2 vTexCoord;', 'void main() {', ' gl_FragColor = texture2D(diffuse, vTexCoord);', '}', ].join('\n'); /** * A mesh-based distorter. */ function CardboardDistorter(gl) { this.gl = gl; this.ctxAttribs = gl.getContextAttributes(); this.meshWidth = 20; this.meshHeight = 20; this.bufferScale = WebVRConfig.BUFFER_SCALE; this.bufferWidth = gl.drawingBufferWidth; this.bufferHeight = gl.drawingBufferHeight; // Patching support this.realBindFramebuffer = gl.bindFramebuffer; this.realEnable = gl.enable; this.realDisable = gl.disable; this.realColorMask = gl.colorMask; this.realClearColor = gl.clearColor; this.realViewport = gl.viewport; if (!Util.isIOS()) { this.realCanvasWidth = Object.getOwnPropertyDescriptor(gl.canvas.__proto__, 'width'); this.realCanvasHeight = Object.getOwnPropertyDescriptor(gl.canvas.__proto__, 'height'); } this.isPatched = false; // State tracking this.lastBoundFramebuffer = null; this.cullFace = false; this.depthTest = false; this.blend = false; this.scissorTest = false; this.stencilTest = false; this.viewport = [0, 0, 0, 0]; this.colorMask = [true, true, true, true]; this.clearColor = [0, 0, 0, 0]; this.attribs = { position: 0, texCoord: 1 }; this.program = Util.linkProgram(gl, distortionVS, distortionFS, this.attribs); this.uniforms = Util.getProgramUniforms(gl, this.program); this.viewportOffsetScale = new Float32Array(8); this.setTextureBounds(); this.vertexBuffer = gl.createBuffer(); this.indexBuffer = gl.createBuffer(); this.indexCount = 0; this.renderTarget = gl.createTexture(); this.framebuffer = gl.createFramebuffer(); this.depthStencilBuffer = null; this.depthBuffer = null; this.stencilBuffer = null; if (this.ctxAttribs.depth && this.ctxAttribs.stencil) { this.depthStencilBuffer = gl.createRenderbuffer(); } else if (this.ctxAttribs.depth) { this.depthBuffer = gl.createRenderbuffer(); } else if (this.ctxAttribs.stencil) { this.stencilBuffer = gl.createRenderbuffer(); } this.patch(); this.onResize(); if (!WebVRConfig.CARDBOARD_UI_DISABLED) { this.cardboardUI = new CardboardUI(gl); } }; /** * Tears down all the resources created by the distorter and removes any * patches. */ CardboardDistorter.prototype.destroy = function() { var gl = this.gl; this.unpatch(); gl.deleteProgram(this.program); gl.deleteBuffer(this.vertexBuffer); gl.deleteBuffer(this.indexBuffer); gl.deleteTexture(this.renderTarget); gl.deleteFramebuffer(this.framebuffer); if (this.depthStencilBuffer) { gl.deleteRenderbuffer(this.depthStencilBuffer); } if (this.depthBuffer) { gl.deleteRenderbuffer(this.depthBuffer); } if (this.stencilBuffer) { gl.deleteRenderbuffer(this.stencilBuffer); } if (this.cardboardUI) { this.cardboardUI.destroy(); } }; /** * Resizes the backbuffer to match the canvas width and height. */ CardboardDistorter.prototype.onResize = function() { var gl = this.gl; var self = this; var glState = [ gl.RENDERBUFFER_BINDING, gl.TEXTURE_BINDING_2D, gl.TEXTURE0 ]; WGLUPreserveGLState(gl, glState, function(gl) { // Bind real backbuffer and clear it once. We don't need to clear it again // after that because we're overwriting the same area every frame. self.realBindFramebuffer.call(gl, gl.FRAMEBUFFER, null); // Put things in a good state if (self.scissorTest) { self.realDisable.call(gl, gl.SCISSOR_TEST); } self.realColorMask.call(gl, true, true, true, true); self.realViewport.call(gl, 0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); self.realClearColor.call(gl, 0, 0, 0, 1); gl.clear(gl.COLOR_BUFFER_BIT); // Now bind and resize the fake backbuffer self.realBindFramebuffer.call(gl, gl.FRAMEBUFFER, self.framebuffer); gl.bindTexture(gl.TEXTURE_2D, self.renderTarget); gl.texImage2D(gl.TEXTURE_2D, 0, self.ctxAttribs.alpha ? gl.RGBA : gl.RGB, self.bufferWidth, self.bufferHeight, 0, self.ctxAttribs.alpha ? gl.RGBA : gl.RGB, gl.UNSIGNED_BYTE, null); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, self.renderTarget, 0); if (self.ctxAttribs.depth && self.ctxAttribs.stencil) { gl.bindRenderbuffer(gl.RENDERBUFFER, self.depthStencilBuffer); gl.renderbufferStorage(gl.RENDERBUFFER, gl.DEPTH_STENCIL, self.bufferWidth, self.bufferHeight); gl.framebufferRenderbuffer(gl.FRAMEBUFFER, gl.DEPTH_STENCIL_ATTACHMENT, gl.RENDERBUFFER, self.depthStencilBuffer); } else if (self.ctxAttribs.depth) { gl.bindRenderbuffer(gl.RENDERBUFFER, self.depthBuffer); gl.renderbufferStorage(gl.RENDERBUFFER, gl.DEPTH_COMPONENT16, self.bufferWidth, self.bufferHeight); gl.framebufferRenderbuffer(gl.FRAMEBUFFER, gl.DEPTH_ATTACHMENT, gl.RENDERBUFFER, self.depthBuffer); } else if (self.ctxAttribs.stencil) { gl.bindRenderbuffer(gl.RENDERBUFFER, self.stencilBuffer); gl.renderbufferStorage(gl.RENDERBUFFER, gl.STENCIL_INDEX8, self.bufferWidth, self.bufferHeight); gl.framebufferRenderbuffer(gl.FRAMEBUFFER, gl.STENCIL_ATTACHMENT, gl.RENDERBUFFER, self.stencilBuffer); } if (!gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE) { console.error('Framebuffer incomplete!'); } self.realBindFramebuffer.call(gl, gl.FRAMEBUFFER, self.lastBoundFramebuffer); if (self.scissorTest) { self.realEnable.call(gl, gl.SCISSOR_TEST); } self.realColorMask.apply(gl, self.colorMask); self.realViewport.apply(gl, self.viewport); self.realClearColor.apply(gl, self.clearColor); }); if (this.cardboardUI) { this.cardboardUI.onResize(); } }; CardboardDistorter.prototype.patch = function() { if (this.isPatched) { return; } var self = this; var canvas = this.gl.canvas; var gl = this.gl; if (!Util.isIOS()) { canvas.width = Util.getScreenWidth() * this.bufferScale; canvas.height = Util.getScreenHeight() * this.bufferScale; Object.defineProperty(canvas, 'width', { configurable: true, enumerable: true, get: function() { return self.bufferWidth; }, set: function(value) { self.bufferWidth = value; self.realCanvasWidth.set.call(canvas, value); self.onResize(); } }); Object.defineProperty(canvas, 'height', { configurable: true, enumerable: true, get: function() { return self.bufferHeight; }, set: function(value) { self.bufferHeight = value; self.realCanvasHeight.set.call(canvas, value); self.onResize(); } }); } this.lastBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING); if (this.lastBoundFramebuffer == null) { this.lastBoundFramebuffer = this.framebuffer; this.gl.bindFramebuffer(gl.FRAMEBUFFER, this.framebuffer); } this.gl.bindFramebuffer = function(target, framebuffer) { self.lastBoundFramebuffer = framebuffer ? framebuffer : self.framebuffer; // Silently make calls to bind the default framebuffer bind ours instead. self.realBindFramebuffer.call(gl, target, self.lastBoundFramebuffer); }; this.cullFace = gl.getParameter(gl.CULL_FACE); this.depthTest = gl.getParameter(gl.DEPTH_TEST); this.blend = gl.getParameter(gl.BLEND); this.scissorTest = gl.getParameter(gl.SCISSOR_TEST); this.stencilTest = gl.getParameter(gl.STENCIL_TEST); gl.enable = function(pname) { switch (pname) { case gl.CULL_FACE: self.cullFace = true; break; case gl.DEPTH_TEST: self.depthTest = true; break; case gl.BLEND: self.blend = true; break; case gl.SCISSOR_TEST: self.scissorTest = true; break; case gl.STENCIL_TEST: self.stencilTest = true; break; } self.realEnable.call(gl, pname); }; gl.disable = function(pname) { switch (pname) { case gl.CULL_FACE: self.cullFace = false; break; case gl.DEPTH_TEST: self.depthTest = false; break; case gl.BLEND: self.blend = false; break; case gl.SCISSOR_TEST: self.scissorTest = false; break; case gl.STENCIL_TEST: self.stencilTest = false; break; } self.realDisable.call(gl, pname); }; this.colorMask = gl.getParameter(gl.COLOR_WRITEMASK); gl.colorMask = function(r, g, b, a) { self.colorMask[0] = r; self.colorMask[1] = g; self.colorMask[2] = b; self.colorMask[3] = a; self.realColorMask.call(gl, r, g, b, a); }; this.clearColor = gl.getParameter(gl.COLOR_CLEAR_VALUE); gl.clearColor = function(r, g, b, a) { self.clearColor[0] = r; self.clearColor[1] = g; self.clearColor[2] = b; self.clearColor[3] = a; self.realClearColor.call(gl, r, g, b, a); }; this.viewport = gl.getParameter(gl.VIEWPORT); gl.viewport = function(x, y, w, h) { self.viewport[0] = x; self.viewport[1] = y; self.viewport[2] = w; self.viewport[3] = h; self.realViewport.call(gl, x, y, w, h); }; this.isPatched = true; Util.safariCssSizeWorkaround(canvas); }; CardboardDistorter.prototype.unpatch = function() { if (!this.isPatched) { return; } var gl = this.gl; var canvas = this.gl.canvas; if (!Util.isIOS()) { Object.defineProperty(canvas, 'width', this.realCanvasWidth); Object.defineProperty(canvas, 'height', this.realCanvasHeight); } canvas.width = this.bufferWidth; canvas.height = this.bufferHeight; gl.bindFramebuffer = this.realBindFramebuffer; gl.enable = this.realEnable; gl.disable = this.realDisable; gl.colorMask = this.realColorMask; gl.clearColor = this.realClearColor; gl.viewport = this.realViewport; // Check to see if our fake backbuffer is bound and bind the real backbuffer // if that's the case. if (this.lastBoundFramebuffer == this.framebuffer) { gl.bindFramebuffer(gl.FRAMEBUFFER, null); } this.isPatched = false; setTimeout(function() { Util.safariCssSizeWorkaround(canvas); }, 1); }; CardboardDistorter.prototype.setTextureBounds = function(leftBounds, rightBounds) { if (!leftBounds) { leftBounds = [0, 0, 0.5, 1]; } if (!rightBounds) { rightBounds = [0.5, 0, 0.5, 1]; } // Left eye this.viewportOffsetScale[0] = leftBounds[0]; // X this.viewportOffsetScale[1] = leftBounds[1]; // Y this.viewportOffsetScale[2] = leftBounds[2]; // Width this.viewportOffsetScale[3] = leftBounds[3]; // Height // Right eye this.viewportOffsetScale[4] = rightBounds[0]; // X this.viewportOffsetScale[5] = rightBounds[1]; // Y this.viewportOffsetScale[6] = rightBounds[2]; // Width this.viewportOffsetScale[7] = rightBounds[3]; // Height }; /** * Performs distortion pass on the injected backbuffer, rendering it to the real * backbuffer. */ CardboardDistorter.prototype.submitFrame = function() { var gl = this.gl; var self = this; var glState = []; if (!WebVRConfig.DIRTY_SUBMIT_FRAME_BINDINGS) { glState.push( gl.CURRENT_PROGRAM, gl.ARRAY_BUFFER_BINDING, gl.ELEMENT_ARRAY_BUFFER_BINDING, gl.TEXTURE_BINDING_2D, gl.TEXTURE0 ); } WGLUPreserveGLState(gl, glState, function(gl) { // Bind the real default framebuffer self.realBindFramebuffer.call(gl, gl.FRAMEBUFFER, null); // Make sure the GL state is in a good place if (self.cullFace) { self.realDisable.call(gl, gl.CULL_FACE); } if (self.depthTest) { self.realDisable.call(gl, gl.DEPTH_TEST); } if (self.blend) { self.realDisable.call(gl, gl.BLEND); } if (self.scissorTest) { self.realDisable.call(gl, gl.SCISSOR_TEST); } if (self.stencilTest) { self.realDisable.call(gl, gl.STENCIL_TEST); } self.realColorMask.call(gl, true, true, true, true); self.realViewport.call(gl, 0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); // If the backbuffer has an alpha channel clear every frame so the page // doesn't show through. if (self.ctxAttribs.alpha || Util.isIOS()) { self.realClearColor.call(gl, 0, 0, 0, 1); gl.clear(gl.COLOR_BUFFER_BIT); } // Bind distortion program and mesh gl.useProgram(self.program); gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, self.indexBuffer); gl.bindBuffer(gl.ARRAY_BUFFER, self.vertexBuffer); gl.enableVertexAttribArray(self.attribs.position); gl.enableVertexAttribArray(self.attribs.texCoord); gl.vertexAttribPointer(self.attribs.position, 2, gl.FLOAT, false, 20, 0); gl.vertexAttribPointer(self.attribs.texCoord, 3, gl.FLOAT, false, 20, 8); gl.activeTexture(gl.TEXTURE0); gl.uniform1i(self.uniforms.diffuse, 0); gl.bindTexture(gl.TEXTURE_2D, self.renderTarget); gl.uniform4fv(self.uniforms.viewportOffsetScale, self.viewportOffsetScale); // Draws both eyes gl.drawElements(gl.TRIANGLES, self.indexCount, gl.UNSIGNED_SHORT, 0); if (self.cardboardUI) { self.cardboardUI.renderNoState(); } // Bind the fake default framebuffer again self.realBindFramebuffer.call(self.gl, gl.FRAMEBUFFER, self.framebuffer); // If preserveDrawingBuffer == false clear the framebuffer if (!self.ctxAttribs.preserveDrawingBuffer) { self.realClearColor.call(gl, 0, 0, 0, 0); gl.clear(gl.COLOR_BUFFER_BIT); } if (!WebVRConfig.DIRTY_SUBMIT_FRAME_BINDINGS) { self.realBindFramebuffer.call(gl, gl.FRAMEBUFFER, self.lastBoundFramebuffer); } // Restore state if (self.cullFace) { self.realEnable.call(gl, gl.CULL_FACE); } if (self.depthTest) { self.realEnable.call(gl, gl.DEPTH_TEST); } if (self.blend) { self.realEnable.call(gl, gl.BLEND); } if (self.scissorTest) { self.realEnable.call(gl, gl.SCISSOR_TEST); } if (self.stencilTest) { self.realEnable.call(gl, gl.STENCIL_TEST); } self.realColorMask.apply(gl, self.colorMask); self.realViewport.apply(gl, self.viewport); if (self.ctxAttribs.alpha || !self.ctxAttribs.preserveDrawingBuffer) { self.realClearColor.apply(gl, self.clearColor); } }); // Workaround for the fact that Safari doesn't allow us to patch the canvas // width and height correctly. After each submit frame check to see what the // real backbuffer size has been set to and resize the fake backbuffer size // to match. if (Util.isIOS()) { var canvas = gl.canvas; if (canvas.width != self.bufferWidth || canvas.height != self.bufferHeight) { self.bufferWidth = canvas.width; self.bufferHeight = canvas.height; self.onResize(); } } }; /** * Call when the deviceInfo has changed. At this point we need * to re-calculate the distortion mesh. */ CardboardDistorter.prototype.updateDeviceInfo = function(deviceInfo) { var gl = this.gl; var self = this; var glState = [gl.ARRAY_BUFFER_BINDING, gl.ELEMENT_ARRAY_BUFFER_BINDING]; WGLUPreserveGLState(gl, glState, function(gl) { var vertices = self.computeMeshVertices_(self.meshWidth, self.meshHeight, deviceInfo); gl.bindBuffer(gl.ARRAY_BUFFER, self.vertexBuffer); gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW); // Indices don't change based on device parameters, so only compute once. if (!self.indexCount) { var indices = self.computeMeshIndices_(self.meshWidth, self.meshHeight); gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, self.indexBuffer); gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, indices, gl.STATIC_DRAW); self.indexCount = indices.length; } }); }; /** * Build the distortion mesh vertices. * Based on code from the Unity cardboard plugin. */ CardboardDistorter.prototype.computeMeshVertices_ = function(width, height, deviceInfo) { var vertices = new Float32Array(2 * width * height * 5); var lensFrustum = deviceInfo.getLeftEyeVisibleTanAngles(); var noLensFrustum = deviceInfo.getLeftEyeNoLensTanAngles(); var viewport = deviceInfo.getLeftEyeVisibleScreenRect(noLensFrustum); var vidx = 0; var iidx = 0; for (var e = 0; e < 2; e++) { for (var j = 0; j < height; j++) { for (var i = 0; i < width; i++, vidx++) { var u = i / (width - 1); var v = j / (height - 1); // Grid points regularly spaced in StreoScreen, and barrel distorted in // the mesh. var s = u; var t = v; var x = Util.lerp(lensFrustum[0], lensFrustum[2], u); var y = Util.lerp(lensFrustum[3], lensFrustum[1], v); var d = Math.sqrt(x * x + y * y); var r = deviceInfo.distortion.distortInverse(d); var p = x * r / d; var q = y * r / d; u = (p - noLensFrustum[0]) / (noLensFrustum[2] - noLensFrustum[0]); v = (q - noLensFrustum[3]) / (noLensFrustum[1] - noLensFrustum[3]); // Convert u,v to mesh screen coordinates. var aspect = deviceInfo.device.widthMeters / deviceInfo.device.heightMeters; // FIXME: The original Unity plugin multiplied U by the aspect ratio // and didn't multiply either value by 2, but that seems to get it // really close to correct looking for me. I hate this kind of "Don't // know why it works" code though, and wold love a more logical // explanation of what needs to happen here. u = (viewport.x + u * viewport.width - 0.5) * 2.0; //* aspect; v = (viewport.y + v * viewport.height - 0.5) * 2.0; vertices[(vidx * 5) + 0] = u; // position.x vertices[(vidx * 5) + 1] = v; // position.y vertices[(vidx * 5) + 2] = s; // texCoord.x vertices[(vidx * 5) + 3] = t; // texCoord.y vertices[(vidx * 5) + 4] = e; // texCoord.z (viewport index) } } var w = lensFrustum[2] - lensFrustum[0]; lensFrustum[0] = -(w + lensFrustum[0]); lensFrustum[2] = w - lensFrustum[2]; w = noLensFrustum[2] - noLensFrustum[0]; noLensFrustum[0] = -(w + noLensFrustum[0]); noLensFrustum[2] = w - noLensFrustum[2]; viewport.x = 1 - (viewport.x + viewport.width); } return vertices; } /** * Build the distortion mesh indices. * Based on code from the Unity cardboard plugin. */ CardboardDistorter.prototype.computeMeshIndices_ = function(width, height) { var indices = new Uint16Array(2 * (width - 1) * (height - 1) * 6); var halfwidth = width / 2; var halfheight = height / 2; var vidx = 0; var iidx = 0; for (var e = 0; e < 2; e++) { for (var j = 0; j < height; j++) { for (var i = 0; i < width; i++, vidx++) { if (i == 0 || j == 0) continue; // Build a quad. Lower right and upper left quadrants have quads with // the triangle diagonal flipped to get the vignette to interpolate // correctly. if ((i <= halfwidth) == (j <= halfheight)) { // Quad diagonal lower left to upper right. indices[iidx++] = vidx; indices[iidx++] = vidx - width - 1; indices[iidx++] = vidx - width; indices[iidx++] = vidx - width - 1; indices[iidx++] = vidx; indices[iidx++] = vidx - 1; } else { // Quad diagonal upper left to lower right. indices[iidx++] = vidx - 1; indices[iidx++] = vidx - width; indices[iidx++] = vidx; indices[iidx++] = vidx - width; indices[iidx++] = vidx - 1; indices[iidx++] = vidx - width - 1; } } } } return indices; }; CardboardDistorter.prototype.getOwnPropertyDescriptor_ = function(proto, attrName) { var descriptor = Object.getOwnPropertyDescriptor(proto, attrName); // In some cases (ahem... Safari), the descriptor returns undefined get and // set fields. In this case, we need to create a synthetic property // descriptor. This works around some of the issues in // https://github.com/borismus/webvr-polyfill/issues/46 if (descriptor.get === undefined || descriptor.set === undefined) { descriptor.configurable = true; descriptor.enumerable = true; descriptor.get = function() { return this.getAttribute(attrName); }; descriptor.set = function(val) { this.setAttribute(attrName, val); }; } return descriptor; }; module.exports = CardboardDistorter;