From 4019c2e63930fbcc400e84a69d25577ef41e9f50 Mon Sep 17 00:00:00 2001 From: DavidMockler Date: Thu, 8 Aug 2024 12:33:54 +0100 Subject: [PATCH] created synthesis generation/storing pipeline on creation Mongodb does not support cascade deletion so some work needs to be done to delete the audio files as well when a story is deleted --- ...t.js => UNIMPLEMENTED averageWordCount.js} | 0 ...e.js => UNIMPLEMENTED getStoriesByDate.js} | 0 ...tats.ts => UNIMPLEMENTED getStoryStats.ts} | 0 ...IMPLEMENTED updateStoryAndCheckGrammar.js} | 0 .../endpoint/drStory/countGrammarErrors.js | 68 ------ api/src/endpoint/drStory/feedbackAudio.js | 63 ------ api/src/endpoint/drStory/storeSynthAudio.ts | 59 +++++ api/src/endpoint/drStory/viewFeedback.js | 28 --- api/src/endpoint/drStory/withId.js | 26 --- api/src/models/drSentenceAudio.ts | 29 +++ api/src/routes/drStory.route.ts | 22 +- .../src/app/core/services/dr-story.service.ts | 73 ++++++- .../dr-story-builder.component.html | 205 ++++++++---------- .../dr-story-builder.component.scss | 108 ++++++++- .../dr-story-builder.component.ts | 168 +++++++------- .../dr-story-viewer.component.scss | 2 + .../digital-reader.component.ts | 37 +++- 17 files changed, 481 insertions(+), 407 deletions(-) rename api/src/endpoint/drStory/{averageWordCount.js => UNIMPLEMENTED averageWordCount.js} (100%) rename api/src/endpoint/drStory/{getStoriesByDate.js => UNIMPLEMENTED getStoriesByDate.js} (100%) rename api/src/endpoint/drStory/{getStoryStats.ts => UNIMPLEMENTED getStoryStats.ts} (100%) rename api/src/endpoint/drStory/{updateStoryAndCheckGrammar.js => UNIMPLEMENTED updateStoryAndCheckGrammar.js} (100%) delete mode 100644 api/src/endpoint/drStory/countGrammarErrors.js delete mode 100644 api/src/endpoint/drStory/feedbackAudio.js create mode 100644 api/src/endpoint/drStory/storeSynthAudio.ts delete mode 100644 api/src/endpoint/drStory/viewFeedback.js delete mode 100644 api/src/endpoint/drStory/withId.js create mode 100644 api/src/models/drSentenceAudio.ts diff --git a/api/src/endpoint/drStory/averageWordCount.js b/api/src/endpoint/drStory/UNIMPLEMENTED averageWordCount.js similarity index 100% rename from api/src/endpoint/drStory/averageWordCount.js rename to api/src/endpoint/drStory/UNIMPLEMENTED averageWordCount.js diff --git a/api/src/endpoint/drStory/getStoriesByDate.js b/api/src/endpoint/drStory/UNIMPLEMENTED getStoriesByDate.js similarity index 100% rename from api/src/endpoint/drStory/getStoriesByDate.js rename to api/src/endpoint/drStory/UNIMPLEMENTED getStoriesByDate.js diff --git a/api/src/endpoint/drStory/getStoryStats.ts b/api/src/endpoint/drStory/UNIMPLEMENTED getStoryStats.ts similarity index 100% rename from api/src/endpoint/drStory/getStoryStats.ts rename to api/src/endpoint/drStory/UNIMPLEMENTED getStoryStats.ts diff --git a/api/src/endpoint/drStory/updateStoryAndCheckGrammar.js b/api/src/endpoint/drStory/UNIMPLEMENTED updateStoryAndCheckGrammar.js similarity index 100% rename from api/src/endpoint/drStory/updateStoryAndCheckGrammar.js rename to api/src/endpoint/drStory/UNIMPLEMENTED updateStoryAndCheckGrammar.js diff --git a/api/src/endpoint/drStory/countGrammarErrors.js b/api/src/endpoint/drStory/countGrammarErrors.js deleted file mode 100644 index a72b08ec2..000000000 --- a/api/src/endpoint/drStory/countGrammarErrors.js +++ /dev/null @@ -1,68 +0,0 @@ -// @ts-nocheck -const StoryGrammarErrors = require('../../models/storygrammarerrors'); -const {API404Error} = require('../../utils/APIError'); -const mongoose = require('mongoose'); - -/** - * Returns a dictionary of errors and their counts for a given student/story id - * - * @param {Object} req story id - * @param {Object} res - * @return {Promise} error dictionary - */ -async function getGrammarErrors(req, res) { - if (! mongoose.Types.ObjectId.isValid(req.params.id)) { - return res.status(400).json({ - invalidObjectId: req.params.id, - }); - } - // get grammar error objects by student id - const grammarErrors = await StoryGrammarErrors.find({'owner': req.params.id}); - if (grammarErrors.length > 0) { - const errorSet = []; - - // create an array of error objects: {error, sentence, timestamp} - for (const errorObject of grammarErrors) { - for (const entry of errorObject.sentences) { - if (entry.errors.length > 0) { - for (const error of entry.errors) { - // add to array: {error, sentence, timestamp} - const errorEntry = { - error: error.type, - sentence: entry.sentence, - date: new Date(+errorObject.timestamp) - .toISOString() - .slice(0, 10), - }; - errorSet.push(errorEntry); - } - } - } - } - - // filter out errors in array that appear in the same sentence on the same day - const unique = errorSet.filter( - (o, i) => i === errorSet.findIndex((oo) => o.error === oo.error && - o.sentence === oo.sentence && o.date === oo.date), - ); - - const errorDict = {}; - - // create a dictionary of errors and counts - for (const entry of unique) { - if (entry.error in errorDict) { - errorDict[entry.error] += 1; - } else { - errorDict[entry.error] = 1; - } - } - - return res.status(200).json(errorDict); - } else { - return res.status(200).json({}); - } - - throw new API404Error('Could not find a story with id ' + req.params.id); -} - -module.exports = getGrammarErrors; diff --git a/api/src/endpoint/drStory/feedbackAudio.js b/api/src/endpoint/drStory/feedbackAudio.js deleted file mode 100644 index 26d3b9c94..000000000 --- a/api/src/endpoint/drStory/feedbackAudio.js +++ /dev/null @@ -1,63 +0,0 @@ -const mongodb = require('mongodb'); -const mongoose = require('mongoose'); -const Story = require('../../models/story'); - -/** - * Get feedback audio from a given story - * @param {Object} req params: Story ID - * @param {Object} res - * @return {Promise} Audio stream - */ -module.exports = async (req, res) => { - if (! mongoose.Types.ObjectId.isValid(req.params.id)) { - return res.status(400).json({ - invalidObjectId: req.params.id, - }); - } - Story.findById(req.params.id, (err, story) => { - if (err) { - console.log(err); - res.status(404).json(err); - } else if (story) { - if (story.feedback.audioId) { - let audioId; - // get the audio id from the audio id set to the story - try { - audioId = new mongodb.ObjectID(story.feedback.audioId); - } catch (err) { - return res.status(400).json({ - message: 'Invalid trackID in URL parameter. ' + - 'Must be a single String of 12 bytes ' + - 'or a string of 24 hex characters'}); - } - - res.set('content-type', 'audio/mp3'); - res.set('accept-ranges', 'bytes'); - // get collection name for audio files - const bucket = new mongodb.GridFSBucket(mongoose.connection.db, { - bucketName: 'audioFeedback', - }); - // create a new stream of file data using the bucket name - const downloadStream = bucket.openDownloadStream(audioId); - // write stream data to response if data is found - downloadStream.on('data', (chunk) => { - res.write(chunk); - }); - - downloadStream.on('error', () => { - res.sendStatus(404); - }); - // close the stream after data sent to response - downloadStream.on('end', () => { - res.end(); - }); - } else { - // res.status(404).json({ - // message: "No audio feedback has been associated with this story"}); - res.json(null); - } - } else { - res.status(404).json({message: 'Story does not exist'}); - } - }); -}; diff --git a/api/src/endpoint/drStory/storeSynthAudio.ts b/api/src/endpoint/drStory/storeSynthAudio.ts new file mode 100644 index 000000000..908ce2d28 --- /dev/null +++ b/api/src/endpoint/drStory/storeSynthAudio.ts @@ -0,0 +1,59 @@ +import { API500Error } from "../../utils/APIError"; + +const DigitalReaderSentenceAudio = require('../../models/drSentenceAudio'); +//const User = require('../../models/user'); +const mongoose = require('mongoose'); +const {API404Error} = require('../../utils/APIError'); + +const handler = async (req, res) => { + + function yes() { + res.status(200).json({id: audio._id}); + } + function no(status=404, msg='not found') { + res.status(status).json(msg); + } + + console.log(req.body); + + //if (!req.body.audioPromise) return no(501, 'no audio promise provided'); + if (!req.body.audioUrl) return no(501, 'no audio url provided'); + if (!req.body.audioTiming) return no(501, 'no audio timing provided'); + if (!req.body.drStoryId) return no(501, 'no story id provided'); + if (req.body.sentId === undefined) return no(501, 'no sentence id provided'); + if (!req.body.voice) return no(501, 'no voice code parameter provided'); + + // check that the current user owns the provided story (or is an admin) (?) + + let audio; + + //const audioPromise:Promise = req.body.audioPromise; + //console.log(audioPromise); + + //audioPromise.then( + //async (response:any) => { + //console.log(response); + //const audioUrl = response.audioContent; + //const timing = response.timing; + //console.log(audioUrl); + //if (audioUrl) { + audio = await DigitalReaderSentenceAudio.create({ + drStoryId: req.body.storyId, + sentenceId: req.body.sentId, + voice: req.body.voice, + audioUrl: req.body.audioUrl, + timing: req.body.audioTiming, + }); + if (!audio) { + throw new API500Error('Unable to save audio file to DB. It may be too large'); + } + yes(); + //} + //} + //); + + return no(); + +}; + +export = handler; \ No newline at end of file diff --git a/api/src/endpoint/drStory/viewFeedback.js b/api/src/endpoint/drStory/viewFeedback.js deleted file mode 100644 index 89d14afff..000000000 --- a/api/src/endpoint/drStory/viewFeedback.js +++ /dev/null @@ -1,28 +0,0 @@ -const Story = require('../../models/story'); -const {API404Error} = require('../../utils/APIError'); -const mongoose = require('mongoose'); - -/** - * Set feedback status of story to 'viewed' - * @param {Object} req params: Story ID - * @param {Object} res - * @return {Promise} Success or Error Message - */ -module.exports = async (req, res) => { - if (! mongoose.Types.ObjectId.isValid(req.params.id)) { - return res.status(400).json({ - invalidObjectId: req.params.id, - }); - } - const story = await Story.findById(req.params.id); - if (story) { - story.feedback.seenByStudent = true; - story.save(); - return res.status(200).json({ - message: 'Feedback viewed successfully', - }); - } - - throw new API404Error( - 'Could not find a story with id ' + req.params.id); -}; diff --git a/api/src/endpoint/drStory/withId.js b/api/src/endpoint/drStory/withId.js deleted file mode 100644 index 33cc9c8db..000000000 --- a/api/src/endpoint/drStory/withId.js +++ /dev/null @@ -1,26 +0,0 @@ -const Story = require('../../models/story'); -const mongoose = require('mongoose'); - -/** - * Set feedback status of story to 'viewed' - * @param {Object} req user: User information; params: story ID - * @param {Object} res - * @param {Object} next - * @return {Promise} Story object - */ -module.exports = async (req, res, next) => { - function yes() { - res.json(story); - } - function no(status=404, msg='not found') { - res.status(status).json(msg); - } - if (!req.user) return no(400, 'need to know user'); - if (!req.user._id) return no(400, 'need to know user\'s id'); - const story = await Story.findById(new mongoose.mongo.ObjectId(req.params.id)); - if (!story) return no(); - // owner is of type ObjectId, and req.user._id is of type string, so the second half - // of this statement was never read => added the toString() - if (story.owner.toString() === req.user._id) return yes(); - return no(); -}; diff --git a/api/src/models/drSentenceAudio.ts b/api/src/models/drSentenceAudio.ts new file mode 100644 index 000000000..73ac11628 --- /dev/null +++ b/api/src/models/drSentenceAudio.ts @@ -0,0 +1,29 @@ +const mongoose = require('mongoose'); +const Schema = mongoose.Schema; + +const DigitalReaderSentenceAudio = new Schema( + { + drStoryId: { + type: mongoose.Types.ObjectId, + index: true, + }, + sentenceId: { + type: Number, + }, + voice: { + type: String, + }, + timing: { + type: Array, + }, + audioUrl: { + type: String, + }, + }, + { + collection: 'drSentenceAudio', + timestamps: true + }, +); + +export = mongoose.model('DigitalReaderSentenceAudio', DigitalReaderSentenceAudio); diff --git a/api/src/routes/drStory.route.ts b/api/src/routes/drStory.route.ts index e48a1e610..1cb3e88b6 100644 --- a/api/src/routes/drStory.route.ts +++ b/api/src/routes/drStory.route.ts @@ -38,6 +38,7 @@ let storyRoutes; // POST const create = require("../endpoint/drStory/create"); + const storeSynthAudio = require("../endpoint/drStory/storeSynthAudio"); /*const viewFeedback = require("../endpoint/story/viewFeedback"); const updateStoryAndCheckGrammar = require("../endpoint/story/updateStoryAndCheckGrammar"); const averageWordCount = require("../endpoint/story/averageWordCount"); @@ -60,6 +61,7 @@ let storyRoutes; post: { "/create": create, "/getMatchingWords": getMatchingWords, + "/storeSynthAudio": storeSynthAudio, //"/viewFeedback/:id": viewFeedback, //"/updateStoryAndCheckGrammar": updateStoryAndCheckGrammar, /*"/averageWordCount/:studentId": averageWordCount, @@ -74,7 +76,7 @@ let storyRoutes; * @param {Object} req params: Date classroom created * @return {Object} List of stories */ -storyRoutes +/*storyRoutes .route("/getStoriesForClassroom/:owner/:date") .get(function (req, res) { const conditions = { owner: req.params.owner }; @@ -91,7 +93,7 @@ storyRoutes res.json(stories); } }); - }); + });*/ /** * Get total number of stories for a given user (owner) with optional classroom creation date filter @@ -99,7 +101,7 @@ storyRoutes * @param {Object} req params: Date classroom created * @return {Object} List of stories */ -storyRoutes.route("/getNumberOfStories/:owner/:date").get(function (req, res) { +/*storyRoutes.route("/getNumberOfStories/:owner/:date").get(function (req, res) { const conditions = { owner: req.params.owner }; if (req.params.date != "empty") { conditions["date"] = { @@ -114,14 +116,14 @@ storyRoutes.route("/getNumberOfStories/:owner/:date").get(function (req, res) { res.json(count); } }); -}); +});*/ /** * Get a story by ID * @param {Object} req params: Story ID * @return {Object} Story object */ -storyRoutes.route("/viewStory/:id").get(function (req, res) { +/*storyRoutes.route("/viewStory/:id").get(function (req, res) { Story.find({ _id: req.params.id }, (err, story) => { if (err) { console.log(err); @@ -130,7 +132,7 @@ storyRoutes.route("/viewStory/:id").get(function (req, res) { res.json(story); } }); -}); +});*/ /** * Update story information @@ -138,7 +140,7 @@ storyRoutes.route("/viewStory/:id").get(function (req, res) { * @param {Object} req body: new story data * @return {Object} Success or error message */ -storyRoutes.route("/update/:id").post((req, res) => { +/*storyRoutes.route("/update/:id").post((req, res) => { Story.findById(req.params.id, function (err, story) { if (err) { console.log(err); @@ -163,7 +165,7 @@ storyRoutes.route("/update/:id").post((req, res) => { (err) => res.status(400).json(err) ); }); -}); +});*/ /** * Update story title @@ -171,7 +173,7 @@ storyRoutes.route("/update/:id").post((req, res) => { * @param {Object} req body: new story title * @return {Object} Success or error message */ -storyRoutes.route("/updateTitle/:id").post((req, res) => { +/*storyRoutes.route("/updateTitle/:id").post((req, res) => { Story.findById(req.params.id, function (err, story) { if (err) { console.log(err); @@ -189,7 +191,7 @@ storyRoutes.route("/updateTitle/:id").post((req, res) => { (err) => res.status(400).json(err) ); }); -}); +});*/ /** * Delete a story by ID diff --git a/ngapp/src/app/core/services/dr-story.service.ts b/ngapp/src/app/core/services/dr-story.service.ts index ab37d451b..3629c0304 100644 --- a/ngapp/src/app/core/services/dr-story.service.ts +++ b/ngapp/src/app/core/services/dr-story.service.ts @@ -4,7 +4,7 @@ import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { HttpHeaders } from '@angular/common/http'; import { Router } from '@angular/router'; import { AuthenticationService } from 'app/core/services/authentication.service'; -import { Observable } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { EngagementService } from 'app/core/services/engagement.service'; import { EventType } from 'app/core/models/event'; import { TranslationService } from 'app/core/services/translation.service'; @@ -67,6 +67,10 @@ export class DigitalReaderStoryService { return segmentedSentences } + parseSegId(id:string, _class:string) { + return parseInt(id.replace(_class, '')); + } + reformatExtractedSentences(sentences:Array) { const reformattedSentences:Array = [] for (let sentenceText of sentences) { @@ -161,12 +165,8 @@ export class DigitalReaderStoryService { title: title, collections: collections, thumbnail: thumbnail, - //text: text, story: story, public: isPublic, - //author: author, - //createdWithPrompts: createdWithPrompts, - //activeRecording: null }; console.log('here!') console.log(drStoryObj.thumbnail); @@ -174,6 +174,69 @@ export class DigitalReaderStoryService { return this.http.post<{id: string}>(this.baseUrl + 'drStory/create', drStoryObj); } + storeSynthAudio(drStoryId: string, sentId: number, /*audioPromise:Promise*/audioObservable:Observable, voiceCode:string) { + + let idObj:Observable<{id:string}> = of(); + + audioObservable.subscribe( + response => { + console.log(response) + + const drSentenceAudioObj = { + drStoryId: drStoryId, + sentId: sentId, + voice: voiceCode, + audioUrl: response.audioUrl, + audioTiming: response.timing + } + + console.log(drSentenceAudioObj) + + idObj = this.http.post<{id: string}>(this.baseUrl + 'drStory/storeSynthAudio', drSentenceAudioObj) + + console.log(idObj) + + idObj.subscribe( + (data) => {console.log(data)} + ) + } + ) + + /*audioObservable.subscribe( + response => { + const drSentenceAudioObj = { + drStoryId: drStoryId, + sentId: sentId, + voice: voiceCode, + audioUrl: response.audioUrl, + timing: response.timing + }; + idObj = this.http.post<{id: string}>(this.baseUrl + 'drStory/storeSynthAudio', drSentenceAudioObj) + } + )*/ + + /*console.log(audioPromise); + audioPromise.then( + (response) => { + console.log('gets to here!') + const drSentenceAudioObj = { + drStoryId: drStoryId, + sentId: sentId, + voice: voiceCode, + audioUrl: response.audioUrl, + timing: response.timing + }; + idObj = this.http.post<{id: string}>(this.baseUrl + 'drStory/storeSynthAudio', drSentenceAudioObj) + } + )*/ + + return idObj; + } + + getSynthAudio() { + + } + getDRStoriesByOwner(owner: string) : Observable { return this.http.get(this.baseUrl + 'drStory/owner/' + owner); } diff --git a/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.html b/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.html index 512155024..4f5a69b2e 100644 --- a/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.html +++ b/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.html @@ -1,129 +1,100 @@ -
- - - - - +
-
- - - - - + + +
+ +
+ - - - + - - - -
-

+ + + + + + + + + +
+
+ + + -
-
+
+
+
-
- - -
--> + - \ No newline at end of file + \ No newline at end of file diff --git a/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.scss b/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.scss index 1e9ab1bed..174dd858d 100644 --- a/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.scss +++ b/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.scss @@ -4,6 +4,8 @@ animation-duration: 1s; animation-name: fadeIn; text-align: left; + + min-width: 320px; } .storyContainer { @@ -65,25 +67,55 @@ .Toolbar { position: sticky; top: 0; + + padding: 10px 5px; background-color: var(--scealai-green); max-width: 60%; - margin: auto; + display: flex; flex-direction: row; - justify-content: center; - margin: 0 auto; + vertical-align: middle; + margin: auto; + //align-items: center; + //justify-content: center; + + border-radius: 4px; + + min-width: 320px; +} + +.Toolbar > div { + //something here about the containing div + height: 40px; + //width:50%; } .Toolbar > * { + //margin-right: 5px; + //margin-left: 5px; + padding: 0; + + display: inline-flex; + align-items: center; + justify-content: center; +} + +.Toolbar > .playbackControls > button { border: none; background-color: white; color: var(--scealai-green); vertical-align: middle; - width: 70px; - height: 40px; + + // the logic is wrong somewhere here + min-width: 20px; + //max-width: 40px; + width: 15%; + // + + height: inherit; margin-right: 5px; margin-left: 5px; @@ -98,11 +130,67 @@ border-radius: 5px; } +.Toolbar > .voiceSelection { + // These two attributes go together + position: absolute; + right: 5px; + // + + //width: 200px; + width: 50%; + //height: 40px; +} + +.Toolbar > .playbackControls { + // These two attributes go together + //position: absolute; + //right: 5px; + // + width: 50%; + + //width: 50%; + //height: 40px; +} + +.Toolbar > .voiceSelection > button { + background-color: white; + color: var(--scealai-green); + + height: inherit; + vertical-align: middle; + + font-size: 20px; + + width: 70%; + //min-width: 200px; + //min-width: 100px; + + /*overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap;*/ + + /*overflow-x: hidden; + overflow-y: none; + text-overflow: ellipsis; + white-space: nowrap;*/ +} + +.Toolbar > .voiceSelection > button > span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.Toolbar > .voiceSelection > button > mat-icon { + transform: scale(1.5); + min-width: 18px; +} + .play { font-size: 40px; } -button.playing { +.Toolbar > .playbackControls > button.playing, .Toolbar > .playbackControls > button.paused { background-color: var(--scealai-cream); } @@ -111,16 +199,16 @@ button.playing { } .voiceSelection { - display: flex; - flex-direction: row; + //display: flex; + //flex-direction: row; //right: 0; } -.voiceSelection mat-form-field { +/*.voiceSelection mat-form-field { //margin: auto auto; max-width: 150px; //max-width: min-content; -} +}*/ /*html { scroll-behavior: smooth; diff --git a/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.ts b/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.ts index 7cdb2dea9..119971165 100644 --- a/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.ts +++ b/ngapp/src/app/dr-story-viewer/dr-story-builder/dr-story-builder.component.ts @@ -30,6 +30,7 @@ import { MatSelectModule } from "@angular/material/select"; import { MatMenuModule } from "@angular/material/menu"; import { MatIconModule } from "@angular/material/icon"; import { MatButtonModule } from "@angular/material/button"; +import { MatSidenavModule } from "@angular/material/sidenav"; const dialectToVoiceIndex = new Map([ ["Connacht f", 0], @@ -48,7 +49,8 @@ const dialectToVoiceIndex = new Map([ MatSelectModule, MatMenuModule, MatIconModule, - MatButtonModule + MatButtonModule, + MatSidenavModule ], selector: "app-dr-story-builder", templateUrl: "./dr-story-builder.component.html", @@ -75,16 +77,18 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { public timings:any[] = [] public voiceSpeed = 1; //public speakerBaseSpeed = 6; // testing for Áine - public speakerAlignmentConstant = .03; // testing for Áine + public speakerAlignmentConstant = .03; // testing resync of audio // below boolean is needed to meaningfully distinguish audio.ended and audio.paused public audioPaused:Boolean = false; public audioPlaying:Boolean = false; + public playContinuously:Boolean = false; //public voiceDialect:string | null = 'Connacht'; //public voiceGender:string | null = 'f'; - public voiceIndex:number = 0; // defaults to Sibéal nemo + public voiceIndex:number = 0; + public speaker = voices[this.voiceIndex]; // defaults to Sibéal nemo constructor( @@ -112,7 +116,7 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { const firstSentSpans = this.content?.querySelectorAll('.sentence') for (let i=0;i<3;i++) { const sent = firstSentSpans.item(i) - this.synthRequest(sent?.textContent, voices[this.voiceIndex]).then( (data) => { + this.synthRequest(sent?.textContent, this.speaker).then( (data) => { console.log(data) this.listOfAudios[this.voiceIndex][i] = data // only for testing console.log(this.listOfAudios) @@ -133,6 +137,7 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { const voiceIndex:number|undefined = dialectToVoiceIndex.get(speaker); if (voiceIndex!==undefined) { this.voiceIndex = voiceIndex; + this.speaker = voices[this.voiceIndex]; } if (this.audioPlaying) { this.playFromCurrentWord(); @@ -208,6 +213,7 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { const wordInd = this.getWordPositionIndex(word, childWordSpans) // for testing as ASR seems to split only on spaces. + // **actual logic needed to be at the span Element level - see: disconnectedFromPreviousWord() //const sentenceText:string = this.recreateSentenceFromWord(childWordSpans.item(wordInd)) //const numTimings = sentenceText.split(' ').length //console.log(numTimings) @@ -385,8 +391,7 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { setTimeout( () => { // to keep a trail of word highlights for a short time prevWord.classList.remove('currentWord') console.log(prevWord?.textContent, 'highlight removed') - } - , delay) + }, delay) } else { this.currentWord.classList.remove('currentWord') } @@ -406,107 +411,115 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { // this.audioPaused is repeated in an attempt to mitigate any issues regarding asynchronisation if (!this.audioPaused) { this.updateCurrentWord(nextWord, 200) - setTimeout( - () => this.seekNextWord() // added in the case of a very fast speaker (e.g Áine) - , 60)//30) + setTimeout(() => this.seekNextWord(), 60) // added in the case of a very fast speaker (e.g Áine) } } } } - parseSegId(id:string, _class:string) { + /*parseSegId(id:string, _class:string) { return parseInt(id.replace(_class, '')); - } + }*/ async getCurrentAudioObject() { - const sentId = this.parseSegId(this.currentSentence.getAttribute('id'), 'sentence') + const sentId = this.drStoryService.parseSegId(this.currentSentence.getAttribute('id'), 'sentence') let audioObj = this.listOfAudios[this.voiceIndex][sentId]; // if the audio has not yet been created, synthesise it and add it to the list. if (!audioObj) { - audioObj = await this.synthRequest(this.currentSentence?.textContent, voices[this.voiceIndex]); + audioObj = await this.synthRequest(this.currentSentence?.textContent, this.speaker); this.listOfAudios[this.voiceIndex][sentId] = audioObj; } return audioObj; } - async playFromCurrentWord(audioSrc:string, continuous=false) { + async playFromCurrentWord() { if (this.audio) this.pause(); // to avoid multiple audio instances at the same time + // clear/destroy current audio obj (?) + this.audioPaused = false; - this.audioPlaying = true; - - //const timing = this.timings.shift() - const timing = this.timings[0] - if (timing) { + const sentAudioObj = await this.getCurrentAudioObject(); + + if (sentAudioObj) { - //const buffer = .05; - //const start = Math.max(timing.start-buffer, 0); - const start = timing.start + this.timings = this.getWordTimings(this.currentWord, this.currentSentence, sentAudioObj) - const audio = document.createElement('audio') - this.audio = audio; - this.audio.src = audioSrc + //const timing = this.timings.shift() + const timing = this.timings[0] - this.audio.ontimeupdate = () => { - this.seekNextWord() - } + if (timing) { + + //const buffer = .05; + //const start = Math.max(timing.start-buffer, 0); + if (this.audioPaused) return; + + const start = timing.start + + const audio = document.createElement('audio') + this.audio = audio; + this.audio.src = sentAudioObj.audioUrl - console.log(this.timings) - - //this.seekNextWord() - - this.audio.onended = async (event) => { - - setTimeout( async () => { - - // TODO: delete the current audio element + listener (if needed (?)) - - if (continuous) { - let nextSent = this.checkForNextSiblingSeg(this.currentSentence, 'sentence'); - console.log(nextSent) - // if this section does not contain anymore sentences - if (!nextSent) - nextSent = this.checkForNextSentence(this.currentSentence); - if (nextSent) { - this.currentSentence = nextSent; - const firstWord = nextSent.querySelector('.word') // TODO : maybe factor out to function - this.updateCurrentWord(firstWord); - - if (this.currentWord) { - // TODO : factor out into function - //const sentAudioObj = this.listOfAudios[parseInt(this.currentWord.getAttribute('sentid'))] - const sentAudioObj = await this.getCurrentAudioObject(); - - - /*console.log(this.audio) - console.log(this.audio.paused) - console.log(this.audio.ended)*/ - //if (sentAudioObj && (this.audio && (!this.audio.paused || this.audio.ended))) { - if (sentAudioObj && !this.audioPaused) { - this.timings = this.getWordTimings(this.currentWord, this.currentSentence, sentAudioObj) - //this.setCurrentWord() - await this.playFromCurrentWord(sentAudioObj.audioUrl, true) + this.audio.ontimeupdate = () => { + this.seekNextWord() + } + + console.log(this.timings) + + //this.seekNextWord() + + this.audio.onended = async (event) => { + + this.audioPlaying = false; + setTimeout( async () => { + + // TODO: delete the current audio element + listener (if needed (?)) + + if (this.playContinuously) { + let nextSent = this.checkForNextSiblingSeg(this.currentSentence, 'sentence'); + console.log(nextSent) + // if this section does not contain anymore sentences + if (!nextSent) + nextSent = this.checkForNextSentence(this.currentSentence); + if (nextSent) { + this.currentSentence = nextSent; + const firstWord = nextSent.querySelector('.word') // TODO : maybe factor out to function + this.updateCurrentWord(firstWord); + + if (this.currentWord) { + // TODO : factor out into function + + //const sentAudioObj = await this.getCurrentAudioObject(); + + //if (!this.audioPaused) { + if (!this.audioPaused && !this.audioPlaying) { + //this.timings = this.getWordTimings(this.currentWord, this.currentSentence, sentAudioObj) + //this.setCurrentWord() + await this.playFromCurrentWord() + } } } + } else { + this.updateCurrentWord(null) } - } else { - this.updateCurrentWord(null) - } - }, 400) // keep final word highlighted for a bit longer - } + }, 400) // keep final word highlighted for a bit longer + } - this.audio.currentTime = start - this.audio.play(); + this.audio.currentTime = start + this.audio.play(); + this.audioPlaying = true; + } } } async playWord() { + this.playContinuously = false; // may not be necessary + if (this.currentWord) { - const audioObj = await this.synthRequest(this.currentWord.textContent, voices[this.voiceIndex]); + const audioObj = await this.synthRequest(this.currentWord.textContent, this.speaker); const audio = document.createElement('audio') audio.src = audioObj.audioUrl; @@ -557,8 +570,8 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { // needs complete refactoring - the outer if statement can probably be done away with // different functions can be used for playing a sentence etc. async playStory() { - console.log(this.currentWord) - console.log(this.currentSentence) + + this.playContinuously = true; if (!this.currentWord) { // below 2 lines really need to be refactored - should create a reference to the story root node // this.content is not enough as it does not reference the actual rendered elements @@ -566,13 +579,14 @@ export class DigitalReaderStoryBuilderComponent implements OnInit { this.currentWord = this.currentSentence.querySelector('.word') // only for testing // } - const sentAudioObj = await this.getCurrentAudioObject(); - console.log(sentAudioObj) + //const sentAudioObj = await this.getCurrentAudioObject(); + //console.log(sentAudioObj) - this.timings = this.getWordTimings(this.currentWord, this.currentSentence, sentAudioObj) + //this.timings = this.getWordTimings(this.currentWord, this.currentSentence, sentAudioObj) console.log(this.timings) this.setCurrentWord() - await this.playFromCurrentWord(sentAudioObj.audioUrl, true) + //await this.playFromCurrentWord(sentAudioObj.audioUrl, true) + await this.playFromCurrentWord() } /*async playNextSent() { diff --git a/ngapp/src/app/dr-story-viewer/dr-story-viewer/dr-story-viewer.component.scss b/ngapp/src/app/dr-story-viewer/dr-story-viewer/dr-story-viewer.component.scss index a6dd0e56e..8098078cb 100644 --- a/ngapp/src/app/dr-story-viewer/dr-story-viewer/dr-story-viewer.component.scss +++ b/ngapp/src/app/dr-story-viewer/dr-story-viewer/dr-story-viewer.component.scss @@ -1,8 +1,10 @@ + .container { max-width: 70%; padding-bottom: 20px; animation-duration: 1s; animation-name: fadeIn; + min-width: 320px; } .playback { diff --git a/ngapp/src/app/nav-bar/digital-reader/digital-reader.component.ts b/ngapp/src/app/nav-bar/digital-reader/digital-reader.component.ts index d219bbee2..f04de3875 100644 --- a/ngapp/src/app/nav-bar/digital-reader/digital-reader.component.ts +++ b/ngapp/src/app/nav-bar/digital-reader/digital-reader.component.ts @@ -2,7 +2,7 @@ import { Component, OnInit, Output, EventEmitter } from '@angular/core'; import { TranslationService } from 'app/core/services/translation.service'; import { AuthenticationService } from 'app/core/services/authentication.service'; -import { firstValueFrom } from "rxjs"; +import { firstValueFrom, from } from "rxjs"; import { User } from "app/core/models/user"; import { UserService } from "app/core/services/user.service"; //import { createClient } from "@supabase/supabase-js" @@ -23,6 +23,7 @@ import { constructJSON } from '@phonlab-tcd/html2json'; import { objectUtil } from 'zod'; import config from '../anScealaiStoryCollectionsConf' +import { SynthesisService, Voice, voices } from 'app/core/services/synthesis.service'; @Component({ selector: 'app-digital-reader', @@ -52,7 +53,8 @@ export class DigitalReaderComponent implements OnInit { public userService: UserService, public drStoryService: DigitalReaderStoryService, public http: HttpClient, - private dialog: MatDialog) { + private dialog: MatDialog, + private synth: SynthesisService,) { this.dialectOptions = [this.ts.l.connacht, this.ts.l.munster, this.ts.l.ulster] //console.log(adminStoryCollectionOpts) @@ -105,6 +107,34 @@ export class DigitalReaderComponent implements OnInit { } + async synthesiseStory(htmlBody:Document, storyId:string) { + + //const parser = new DOMParser; + + //const html = parser.parseFromString(htmlString, 'text/html'); + //const html = this.convertedHTMLDoc?.body; + + const firstSentSpans = htmlBody?.querySelectorAll('.sentence') + for (let i=0;i = firstValueFrom(this.synth.synthesiseText(sentText, voice, false, undefined, 1)); + + const audioObservable = from(audioPromise); + + //this.drStoryService.storeSynthAudio(storyId, sentId, audioPromise, voice.code) + this.drStoryService.storeSynthAudio(storyId, sentId, audioObservable, voice.code) + /*.subscribe({ + next(response) {console.log(response.id)} + });*/ + } + } + } + async createNewStory() { this.storyState = '' @@ -196,9 +226,10 @@ export class DigitalReaderComponent implements OnInit { //.saveDRStory(res.title, dialects, story, res.public) .saveDRStory(res.title, collections, thumbnail, story, res.public) .subscribe({ - next: () => { + next: (response) => { console.log('a response was received') this.storyState = 'processed' + this.synthesiseStory(this.convertedHTMLDoc, response.id); }, error: () => { alert("Not able to create a new story");