From 4165d81e0c9be1cb7b574c7cd7800ed7a26d3410 Mon Sep 17 00:00:00 2001 From: lgtst Date: Tue, 21 Feb 2023 18:06:16 +0000 Subject: [PATCH 1/2] Fix for showing videos on iphones. ServerManager downloads videos and saves them as PIXI.Texture. MovieStim prepared to work with texture resource. --- src/core/ServerManager.js | 64 +++++++++++++++++++++++++++---- src/visual/MovieStim.js | 79 ++++++++++++++++++++++++++++----------- 2 files changed, 114 insertions(+), 29 deletions(-) diff --git a/src/core/ServerManager.js b/src/core/ServerManager.js index 1b59a994..e92a53ce 100644 --- a/src/core/ServerManager.js +++ b/src/core/ServerManager.js @@ -16,6 +16,7 @@ import { PsychObject } from "../util/PsychObject.js"; import * as util from "../util/Util.js"; import { Scheduler } from "../util/Scheduler.js"; import { PsychoJS } from "./PsychoJS.js"; +import * as PIXI from "pixi.js-legacy"; /** *

This manager handles all communications between the experiment running in the participant's browser and the @@ -1182,6 +1183,27 @@ export class ServerManager extends PsychObject }); } + /** + * Check if all the resources were loaded and if so set the READY status and emit the DOWNLOAD_COMPLETED event. + * + * @protected + * @returns boolean - if downloading is done or not. + */ + _checkIfDownloadingIsDone (resourcesTotal) + { + if (this._nbLoadedResources === resourcesTotal) + { + this.setStatus(ServerManager.Status.READY); + this.emit(ServerManager.Event.RESOURCE, { + message: ServerManager.Event.DOWNLOAD_COMPLETED, + }); + + return true; + } + + return false; + } + /** * Download the specified resources. * @@ -1214,8 +1236,9 @@ export class ServerManager extends PsychObject const surveyModelResources = []; for (const name of resources) { - const nameParts = name.toLowerCase().split("."); - const extension = (nameParts.length > 1) ? nameParts.pop() : undefined; + const pathStatusData = this._resources.get(name); + const pathParts = pathStatusData.path.toLowerCase().split("."); + const extension = (pathParts.length > 1) ? pathParts.pop() : undefined; // warn the user if the resource does not have any extension: if (typeof extension === "undefined") @@ -1223,7 +1246,6 @@ export class ServerManager extends PsychObject this.psychoJS.logger.warn(`"${name}" does not appear to have an extension, which may negatively impact its loading. We highly recommend you add an extension.`); } - const pathStatusData = this._resources.get(name); if (typeof pathStatusData === "undefined") { throw Object.assign(response, { error: name + " has not been previously registered" }); @@ -1233,9 +1255,6 @@ export class ServerManager extends PsychObject throw Object.assign(response, { error: name + " is already downloaded or is currently already downloading" }); } - const pathParts = pathStatusData.path.toLowerCase().split("."); - const pathExtension = (pathParts.length > 1) ? pathParts.pop() : undefined; - // preload.js with forced binary: if (["csv", "odp", "xls", "xlsx", "json"].indexOf(extension) > -1) { @@ -1265,7 +1284,7 @@ export class ServerManager extends PsychObject } // font files: - else if (["ttf", "otf", "woff", "woff2"].indexOf(pathExtension) > -1) + else if (["ttf", "otf", "woff", "woff2"].indexOf(extension) > -1) { fontResources.push(name); } @@ -1276,6 +1295,37 @@ export class ServerManager extends PsychObject surveyModelResources.push(name); } + // Videos (compatible with PIXI): + else if (["mp4", "m4v", "webm", "ogv", "h264", "avi", "mov"].indexOf(extension) > -1) + { + pathStatusData.data = PIXI.Texture.from( + pathStatusData.path, + { + resourceOptions: { autoPlay: false } + } + ); + + pathStatusData.data.baseTexture.resource.source.addEventListener( + "loadeddata", + () => + { + pathStatusData.status = ServerManager.ResourceStatus.DOWNLOADED; + this.emit(ServerManager.Event.RESOURCE, { + message: ServerManager.Event.RESOURCE_DOWNLOADED, + resource: name, + }); + + this._nbLoadedResources++; + this._checkIfDownloadingIsDone(resources.size); + }); + + pathStatusData.status = ServerManager.ResourceStatus.DOWNLOADING; + this.emit(ServerManager.Event.RESOURCE, { + message: ServerManager.Event.DOWNLOADING_RESOURCE, + resource: name, + }); + } + // all other extensions handled by preload.js (download type decided by preload.js): else { diff --git a/src/visual/MovieStim.js b/src/visual/MovieStim.js index 8b4f0394..2ebd8320 100644 --- a/src/visual/MovieStim.js +++ b/src/visual/MovieStim.js @@ -57,6 +57,8 @@ export class MovieStim extends VisualStim this.psychoJS.logger.debug("create a new MovieStim with name: ", name); + this._pixiTextureResource = undefined; + // movie and movie control: this._addAttribute( "movie", @@ -149,6 +151,9 @@ export class MovieStim extends VisualStim try { + let htmlVideo = undefined; + this._pixiTextureResource = undefined; + // movie is undefined: that's fine but we raise a warning in case this is // a symptom of an actual problem if (typeof movie === "undefined") @@ -160,19 +165,24 @@ export class MovieStim extends VisualStim else { - // if movie is a string, then it should be the name of a resource, which we get: + let videoResource; + + // If movie is a string, then it should be the name of a resource, which we get: if (typeof movie === "string") { - movie = this.psychoJS.serverManager.getResource(movie); + videoResource = this.psychoJS.serverManager.getResource(movie); } - - // if movie is an instance of camera, get a video element from it: + // If movie is a HTMLVideoElement, pass it as is: + else if (movie instanceof HTMLVideoElement) + { + videoResource = movie; + } + // If movie is an instance of camera, get a video element from it: else if (movie instanceof Camera) { // old behaviour: feeding a Camera to MovieStim plays the live stream: - const video = movie.getVideo(); - // TODO remove previous one if there is one - movie = video; + videoResource = movie.getVideo(); + // TODO remove previous movie one if there is one /* // new behaviour: feeding a Camera to MovieStim replays the video previously recorded by the Camera: @@ -181,27 +191,38 @@ export class MovieStim extends VisualStim */ } - // check that movie is now an HTMLVideoElement - if (!(movie instanceof HTMLVideoElement)) + if (videoResource instanceof HTMLVideoElement) { - throw `${movie.toString()} is not a video`; + htmlVideo = videoResource; + htmlVideo.playsInline = true; + this._pixiTextureResource = PIXI.Texture.from(htmlVideo, { resourceOptions: { autoPlay: false } }); + } + else if (videoResource instanceof PIXI.Texture) + { + htmlVideo = videoResource.baseTexture.resource.source; + this._pixiTextureResource = videoResource; + } + else + { + throw `${videoResource.toString()} is not a HTMLVideoElement nor PIXI.Texture!`; } - this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${movie.src}, size= ${movie.videoWidth}x${movie.videoHeight}, duration= ${movie.duration}s`); + this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${htmlVideo.src}, size= ${htmlVideo.videoWidth}x${htmlVideo.videoHeight}, duration= ${htmlVideo.duration}s`); // ensure we have only one onended listener per HTMLVideoElement, since we can have several // MovieStim with the same underlying HTMLVideoElement // https://stackoverflow.com/questions/11455515 - if (!movie.onended) + // TODO: make it actually work! + if (!htmlVideo.onended) { - movie.onended = () => + htmlVideo.onended = () => { this.status = PsychoJS.Status.FINISHED; }; } } - this._setAttribute("movie", movie, log); + this._setAttribute("movie", htmlVideo, log); this._needUpdate = true; this._needPixiUpdate = true; } @@ -369,13 +390,24 @@ export class MovieStim extends VisualStim return; } + // No PIXI.Texture, also return immediately. + if (this._pixiTextureResource === undefined) + { + return; + } + // create a PixiJS video sprite: - this._texture = PIXI.Texture.from(this._movie, { resourceOptions: { autoPlay: this.autoPlay } }); - this._pixi = new PIXI.Sprite(this._texture); + this._pixiTextureResource.baseTexture.resource.autoPlay = this._autoPlay; + this._pixi = new PIXI.Sprite(this._pixiTextureResource); - // since _texture.width may not be immedialy available but the rest of the code needs its value + if (this._autoPlay) + { + this._pixiTextureResource.baseTexture.resource.source.play(); + } + + // since _pixiTextureResource.width may not be immedialy available but the rest of the code needs its value // we arrange for repeated calls to _updateIfNeeded until we have a width: - if (this._texture.width === 0) + if (this._pixiTextureResource.width === 0) { this._needUpdate = true; this._needPixiUpdate = true; @@ -396,11 +428,14 @@ export class MovieStim extends VisualStim // set the scale: const displaySize = this._getDisplaySize(); const size_px = util.to_px(displaySize, this.units, this.win); - const scaleX = size_px[0] / this._texture.width; - const scaleY = size_px[1] / this._texture.height; + const scaleX = size_px[0] / this._pixiTextureResource.width; + const scaleY = size_px[1] / this._pixiTextureResource.height; this._pixi.scale.x = this.flipHoriz ? -scaleX : scaleX; this._pixi.scale.y = this.flipVert ? scaleY : -scaleY; + this._pixi.width = size_px[0]; + this._pixi.height = size_px[1]; + // set the position, rotation, and anchor (movie centered on pos): this._pixi.position = to_pixiPoint(this.pos, this.units, this.win); this._pixi.rotation = -this.ori * Math.PI / 180; @@ -424,9 +459,9 @@ export class MovieStim extends VisualStim if (typeof displaySize === "undefined") { // use the size of the texture, if we have access to it: - if (typeof this._texture !== "undefined" && this._texture.width > 0) + if (typeof this._pixiTextureResource !== "undefined" && this._pixiTextureResource.width > 0) { - const textureSize = [this._texture.width, this._texture.height]; + const textureSize = [this._pixiTextureResource.width, this._pixiTextureResource.height]; displaySize = util.to_unit(textureSize, "pix", this.win, this.units); } } From f42d806f768a04ea9f78ed94e6c4a72a5b9c1423 Mon Sep 17 00:00:00 2001 From: lightest Date: Wed, 3 Apr 2024 23:48:29 +0100 Subject: [PATCH 2/2] texture instantiation moved back to update method. --- src/visual/MovieStim.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/visual/MovieStim.js b/src/visual/MovieStim.js index b9c2ff40..a9ee51d2 100644 --- a/src/visual/MovieStim.js +++ b/src/visual/MovieStim.js @@ -92,7 +92,6 @@ export class MovieStim extends VisualStim this._youtubePlayer = undefined; this._ytPlayerIsReady = false; - // movie and movie control: this._addAttribute( "movie", movie, @@ -242,28 +241,29 @@ export class MovieStim extends VisualStim { htmlVideo = videoResource; htmlVideo.playsInline = true; - this._pixiTextureResource = PIXI.Texture.from(htmlVideo, { resourceOptions: { autoPlay: false } }); - // Not using PIXI.Texture.from() on purpose, as it caches both PIXI.Texture and PIXI.BaseTexture. - // As a result of that we can have multiple MovieStim instances using same PIXI.BaseTexture, - // thus changing texture related properties like interpolation, or calling _pixi.destroy(true) - // will affect all MovieStims which happen to share that BaseTexture. - this._pixiTextureResource = new PIXI.Texture(new PIXI.BaseTexture( - this._movie, - { - resourceOptions: { autoPlay: this.autoPlay } - } - )); } else if (videoResource instanceof PIXI.Texture) { htmlVideo = videoResource.baseTexture.resource.source; - this._pixiTextureResource = videoResource; } else { throw `${videoResource.toString()} is not a HTMLVideoElement nor PIXI.Texture!`; } + // Not using PIXI.Texture.from() on purpose, as it caches both PIXI.Texture and PIXI.BaseTexture. + // As a result of that we can have multiple MovieStim instances using same PIXI.BaseTexture, + // thus changing texture related properties like interpolation, or calling _pixi.destroy(true) + // will affect all MovieStims which happen to share that BaseTexture. + this._pixiTextureResource = new PIXI.Texture( + new PIXI.BaseTexture( + htmlVideo, + { + resourceOptions: { autoPlay: this.autoPlay } + } + ) + ); + this.psychoJS.logger.debug(`set the movie of MovieStim: ${this._name} as: src= ${htmlVideo.src}, size= ${htmlVideo.videoWidth}x${htmlVideo.videoHeight}, duration= ${htmlVideo.duration}s`); // ensure we have only one onended listener per HTMLVideoElement, since we can have several