From b6dc683477d3458fd4270e4be0450b6ba4f8f439 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 12:39:35 +0100 Subject: [PATCH 1/8] NF: Allow Microphone to record forever by deleting old samples when the buffer fills up --- .../components/microphone/__init__.py | 26 +++++--- psychopy/hardware/microphone.py | 65 ++++++++++++------- psychopy/sound/microphone.py | 33 ++++++++++ 3 files changed, 92 insertions(+), 32 deletions(-) diff --git a/psychopy/experiment/components/microphone/__init__.py b/psychopy/experiment/components/microphone/__init__.py index d864ab0928..87d239f402 100644 --- a/psychopy/experiment/components/microphone/__init__.py +++ b/psychopy/experiment/components/microphone/__init__.py @@ -56,6 +56,7 @@ def __init__(self, exp, parentName, name='mic', channels='auto', device=None, sampleRate='DVD Audio (48kHz)', maxSize=24000, outputType='default', speakTimes=False, trimSilent=False, + policyWhenFull='warn', transcribe=False, transcribeBackend="none", transcribeLang="en-US", transcribeWords="", transcribeWhisperModel="base", @@ -158,7 +159,21 @@ def getDeviceNames(): hint=msg, label=_translate("Output file type") ) - + self.params['policyWhenFull'] = Param( + policyWhenFull, valType="str", inputType="choice", categ="Data", + updates="set every repeat", + allowedVals=["warn", "roll", "error"], + allowedLabels=[ + _translate("Discard incoming data"), + _translate("Clear oldest data"), + _translate("Raise error"), + ], + label=_translate("Full buffer policy"), + hint=_translate( + "What to do when we reach the max amount of audio data which can be safely stored " + "in memory?" + ) + ) msg = _translate( "Tick this to save times when the participant starts and stops speaking") self.params['speakTimes'] = Param( @@ -418,15 +433,6 @@ def writeFrameCode(self, buff): inits = getInitVals(self.params) inits['routine'] = self.parentName - # If stop time is blank, substitute max stop - if self.params['stopVal'] in ('', None, -1, 'None'): - self.params['stopVal'].val = at.audioMaxDuration( - bufferSize=float(self.params['maxSize'].val) * 1000, - freq=float(sampleRates[self.params['sampleRate'].val]) - ) - # Show alert - alert(4125, strFields={'name': self.params['name'].val, 'stopVal': self.params['stopVal'].val}) - # Start the recording indented = self.writeStartTestCode(buff) if indented: diff --git a/psychopy/hardware/microphone.py b/psychopy/hardware/microphone.py index 3077bfc8b4..c6b5715575 100644 --- a/psychopy/hardware/microphone.py +++ b/psychopy/hardware/microphone.py @@ -120,7 +120,7 @@ def __init__(self, channels=None, streamBufferSecs=2.0, maxRecordingSize=24000, - policyWhenFull='warn', + policyWhenFull='roll', audioLatencyMode=None, audioRunMode=0): @@ -307,6 +307,28 @@ def __init__(self, # list to store listeners in self.listeners = [] + + @property + def policyWhenFull(self): + """ + Until a file is saved, the audio data from a Microphone needs to be stored in RAM. To avoid + a memory leak, we limit the amount which can be stored by a single Microphone object. The + `policyWhenFull` parameter tells the Microphone what to do when it's reached that limit. + + Parameters + ---------- + value : str + One of: + - "ignore": When full, just don't record any new samples + - "warn"/"warning": Same as ignore, but will log a warning + - "error": When full, will raise an error + - "roll"/"rolling": When full, clears the start of the buffer to make room for new samples + """ + return self._recording._policyWhenFull + + @policyWhenFull.setter + def policyWhenFull(self, value): + self._recording._policyWhenFull = value def findBestDevice(self, index, sampleRateHz, channels): """ @@ -940,19 +962,6 @@ def dispatchMessages(self, clear=True): self.getCurrentVolume(), device=self, ) - # clear recording if requested (helps with continuous running) - if clear and self.isRecBufferFull: - # work out how many samples is 0.1s - toSave = min( - int(0.2 * self._sampleRateHz), - int(self.maxRecordingSize / 2) - ) - # get last 0.1s so we still have enough for volume measurement - savedSamples = self._recording._samples[-toSave:, :] - # clear samples - self._recording.clear() - # reassign saved samples - self._recording.write(savedSamples) # dispatch to listeners for listener in self.listeners: listener.receiveMessage(message) @@ -1004,10 +1013,6 @@ def __init__(self, sampleRateHz=SAMPLE_RATE_48kHz, channels=2, self._spaceRemaining = None # set in `_allocRecBuffer` self._totalSamples = None # set in `_allocRecBuffer` - # check if the value is valid - if policyWhenFull not in ['ignore', 'warn', 'error']: - raise ValueError("Invalid value for `policyWhenFull`.") - self._policyWhenFull = policyWhenFull self._warnedRecBufferFull = False self._loops = 0 @@ -1158,9 +1163,8 @@ def write(self, samples): """ nSamples = len(samples) if self.isFull: - if self._policyWhenFull == 'ignore': - return nSamples # samples lost - elif self._policyWhenFull == 'warn': + if self._policyWhenFull in ('warn', 'warning'): + # if policy is warn, we log a warning then proceed as if ignored if not self._warnedRecBufferFull: logging.warning( f"Audio recording buffer filled! This means that no " @@ -1171,10 +1175,27 @@ def write(self, samples): self._warnedRecBufferFull = True return nSamples elif self._policyWhenFull == 'error': + # if policy is error, we fully error raise AudioRecordingBufferFullError( "Cannot write samples, recording buffer is full.") + elif self._policyWhenFull == ('rolling', 'roll'): + # if policy is rolling, we clear the first half of the buffer + toSave = int(self._totalSamples / 2) + # get last 0.1s so we still have enough for volume measurement + savedSamples = self._recording._samples[-toSave:, :] + # log + logging.debug( + f"Microphone buffer reached, as policy when full is 'roll'/'rolling' all but " + f"the most recent {toSave} samples will be cleared to make room for new " + f"samples." + ) + # clear samples + self._recording.clear() + # reassign saved samples + self._recording.write(savedSamples) else: - return nSamples # whatever + # if policy is to ignore, we simply don't write new samples + return nSamples if not nSamples: # no samples came out of the stream, just return return diff --git a/psychopy/sound/microphone.py b/psychopy/sound/microphone.py index 0701a04e3a..70abeac3d3 100644 --- a/psychopy/sound/microphone.py +++ b/psychopy/sound/microphone.py @@ -13,6 +13,7 @@ from pathlib import Path from psychopy.constants import NOT_STARTED from psychopy.hardware import DeviceManager +from psychopy.tools.attributetools import logAttrib class Microphone: @@ -56,6 +57,8 @@ def __init__( audioLatencyMode=audioLatencyMode, audioRunMode=audioRunMode ) + # set policy when full (in case device already existed) + self.device.policyWhenFull = policyWhenFull # setup clips and transcripts dicts self.clips = {} self.lastClip = None @@ -66,6 +69,36 @@ def __init__( def __del__(self): self.saveClips() + + @property + def policyWhenFull(self): + """ + Until a file is saved, the audio data from a Microphone needs to be stored in RAM. To avoid + a memory leak, we limit the amount which can be stored by a single Microphone object. The + `policyWhenFull` parameter tells the Microphone what to do when it's reached that limit. + + Parameters + ---------- + value : str + One of: + - "ignore": When full, just don't record any new samples + - "warn": Same as ignore, but will log a warning + - "error": When full, will raise an error + - "rolling": When full, clears the start of the buffer to make room for new samples + """ + return self.device.policyWhenFull + + @policyWhenFull.setter + def policyWhenFull(self, value): + return self.device.policyWhenFull + + def setPolicyWhenFull(self, value): + self.policyWhenFull = value + # log + logAttrib( + obj=self, log=True, attrib="policyWhenFull", value=value + ) + setPolicyWhenFull.__doc__ = policyWhenFull.__doc__ @property def recording(self): From 0e7db88656790cf0cb22b05fe50186c8278c3125 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 13:00:42 +0100 Subject: [PATCH 2/8] FF: Only clear as many samples as needed, so once full the buffer remains at capacity --- psychopy/hardware/microphone.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/psychopy/hardware/microphone.py b/psychopy/hardware/microphone.py index c6b5715575..bfd535635c 100644 --- a/psychopy/hardware/microphone.py +++ b/psychopy/hardware/microphone.py @@ -1180,15 +1180,17 @@ def write(self, samples): "Cannot write samples, recording buffer is full.") elif self._policyWhenFull == ('rolling', 'roll'): # if policy is rolling, we clear the first half of the buffer - toSave = int(self._totalSamples / 2) + toSave = self._totalSamples - len(samples) # get last 0.1s so we still have enough for volume measurement savedSamples = self._recording._samples[-toSave:, :] # log - logging.debug( - f"Microphone buffer reached, as policy when full is 'roll'/'rolling' all but " - f"the most recent {toSave} samples will be cleared to make room for new " - f"samples." - ) + if not self._warnedRecBufferFull: + logging.warning( + f"Microphone buffer reached, as policy when full is 'roll'/'rolling' the " + f"oldest samples will be cleared to make room for new samples." + ) + logging.flush() + self._warnedRecBufferFull = True # clear samples self._recording.clear() # reassign saved samples From d9510595e5d4b23abd68c4ad3b60daf79ec18927 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 13:49:18 +0100 Subject: [PATCH 3/8] ENH: Set max recording size each repeat so it isn't ignored when device is initialised elsewhere --- .../components/microphone/__init__.py | 1 + psychopy/hardware/microphone.py | 21 +++++++++++++++ psychopy/sound/microphone.py | 27 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/psychopy/experiment/components/microphone/__init__.py b/psychopy/experiment/components/microphone/__init__.py index 87d239f402..1f5fff1062 100644 --- a/psychopy/experiment/components/microphone/__init__.py +++ b/psychopy/experiment/components/microphone/__init__.py @@ -143,6 +143,7 @@ def getDeviceNames(): ) self.params['maxSize'] = Param( maxSize, valType='num', inputType="single", categ='Device', + updates="set every repeat", label=_translate("Max recording size (kb)"), hint=_translate( "To avoid excessively large output files, what is the biggest file size you are " diff --git a/psychopy/hardware/microphone.py b/psychopy/hardware/microphone.py index bfd535635c..04af90fba9 100644 --- a/psychopy/hardware/microphone.py +++ b/psychopy/hardware/microphone.py @@ -308,6 +308,27 @@ def __init__(self, # list to store listeners in self.listeners = [] + @property + def maxRecordingSize(self): + """ + Until a file is saved, the audio data from a Microphone needs to be stored in RAM. To avoid + a memory leak, we limit the amount which can be stored by a single Microphone object. The + `maxRecordingSize` parameter defines what this limit is. + + Parameters + ---------- + value : int + How much data (in kb) to allow, default is 24mb (so 24,000kb) + """ + return self._recording.maxRecordingSize + + @maxRecordingSize.setter + def maxRecordingSize(self, value): + # set size + self._recording.maxRecordingSize = value + # re-allocate + self._recording._allocRecBuffer() + @property def policyWhenFull(self): """ diff --git a/psychopy/sound/microphone.py b/psychopy/sound/microphone.py index 70abeac3d3..49bc611bbd 100644 --- a/psychopy/sound/microphone.py +++ b/psychopy/sound/microphone.py @@ -70,6 +70,33 @@ def __init__( def __del__(self): self.saveClips() + @property + def maxRecordingSize(self): + """ + Until a file is saved, the audio data from a Microphone needs to be stored in RAM. To avoid + a memory leak, we limit the amount which can be stored by a single Microphone object. The + `maxRecordingSize` parameter defines what this limit is. + + Parameters + ---------- + value : int + How much data (in kb) to allow, default is 24mb (so 24,000kb) + """ + return self.device.maxRecordingSize + + @maxRecordingSize.setter + def maxRecordingSize(self, value): + # set size + self.device.maxRecordingSize = value + + def setMaxRecordingSize(self, value): + self.maxRecordingSize = value + # log + logAttrib( + obj=self, log=True, attrib="maxRecordingSize", value=value + ) + setMaxRecordingSize.__doc__ == maxRecordingSize.__doc__ + @property def policyWhenFull(self): """ From 6b71f37af63b911adf961b2ad8278f3eec90b215 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 13:53:08 +0100 Subject: [PATCH 4/8] FF: Alias setMaxRecordingSize to fit Builder's param name --- psychopy/sound/microphone.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psychopy/sound/microphone.py b/psychopy/sound/microphone.py index 49bc611bbd..a50f8fc36b 100644 --- a/psychopy/sound/microphone.py +++ b/psychopy/sound/microphone.py @@ -96,6 +96,9 @@ def setMaxRecordingSize(self, value): obj=self, log=True, attrib="maxRecordingSize", value=value ) setMaxRecordingSize.__doc__ == maxRecordingSize.__doc__ + + # the Builder param has a different name + setMaxSize = setMaxRecordingSize @property def policyWhenFull(self): From f9df83d884494b4a812c298b6d2da7602e29e6f6 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 17:14:28 +0100 Subject: [PATCH 5/8] NF: Add `open` and `reopen` methods to MicrophoneDevice --- psychopy/hardware/microphone.py | 101 ++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 39 deletions(-) diff --git a/psychopy/hardware/microphone.py b/psychopy/hardware/microphone.py index 04af90fba9..555da778a6 100644 --- a/psychopy/hardware/microphone.py +++ b/psychopy/hardware/microphone.py @@ -246,48 +246,14 @@ def __init__(self, # PTB specific stuff self._mode = 2 # open a stream in capture mode - # Handle for the recording stream, should only be opened once per - # session - logging.debug('Opening audio stream for device #{}'.format( - self._device.deviceIndex)) - if self._device.deviceIndex not in MicrophoneDevice._streams: - MicrophoneDevice._streams[self._device.deviceIndex] = audio.Stream( - device_id=self._device.deviceIndex, - latency_class=self._audioLatencyMode, - mode=self._mode, - freq=self._device.defaultSampleRate, - channels=self._device.inputChannels) - logging.debug('Stream opened') - else: - logging.debug( - "Stream already created for device at index {}, using created stream.".format( - self._device.deviceIndex - ) - ) - - # store reference to stream in this instance - self._stream = MicrophoneDevice._streams[self._device.deviceIndex] - + # get audio run mode assert isinstance(audioRunMode, (float, int)) and \ (audioRunMode == 0 or audioRunMode == 1) self._audioRunMode = int(audioRunMode) - self._stream.run_mode = self._audioRunMode - logging.debug('Set run mode to `{}`'.format( - self._audioRunMode)) - - # set latency bias - self._stream.latency_bias = 0.0 - - logging.debug('Set stream latency bias to {} ms'.format( - self._stream.latency_bias)) - - # pre-allocate recording buffer, called once - self._stream.get_audio_data(self._streamBufferSecs) - - logging.debug( - 'Allocated stream buffer to hold {} seconds of data'.format( - self._streamBufferSecs)) + # open stream + self._stream = None + self.open() # status flag for Builder self._statusFlag = NOT_STARTED @@ -817,8 +783,51 @@ def pause(self, blockUntilStopped=True, stopTime=None): """ return self.stop(blockUntilStopped=blockUntilStopped, stopTime=stopTime) + def open(self): + """ + Open the audio stream. + """ + # do nothing if stream is already open + if self._stream is not None and not self._stream._closed: + return + + # search for open streams and if there is one, use it + if self._device.deviceIndex in MicrophoneDevice._streams: + logging.debug( + f"Assigning audio stream for device #{self._device.deviceIndex} to a new " + f"MicrophoneDevice object." + ) + self._stream = MicrophoneDevice._streams[self._device.deviceIndex] + return + + # if no open streams, make one + logging.debug( + f"Opening new audio stream for device #{self._device.deviceIndex}." + ) + self._stream = MicrophoneDevice._streams[self._device.deviceIndex] = audio.Stream( + device_id=self._device.deviceIndex, + latency_class=self._audioLatencyMode, + mode=self._mode, + freq=self._device.defaultSampleRate, + channels=self._device.inputChannels + ) + # set run mode + self._stream.run_mode = self._audioRunMode + logging.debug('Set run mode to `{}`'.format( + self._audioRunMode)) + # set latency bias + self._stream.latency_bias = 0.0 + logging.debug('Set stream latency bias to {} ms'.format( + self._stream.latency_bias)) + # pre-allocate recording buffer, called once + self._stream.get_audio_data(self._streamBufferSecs) + logging.debug( + 'Allocated stream buffer to hold {} seconds of data'.format( + self._streamBufferSecs)) + def close(self): - """Close the stream. + """ + Close the audio stream. """ self.clearListeners() if self._stream._closed: @@ -827,6 +836,20 @@ def close(self): MicrophoneDevice._streams.pop(self._device.deviceIndex) self._stream.close() logging.debug('Stream closed') + + def reopen(self): + """ + Calls self.close() then self.open() to reopen the stream. + """ + # start timer + start = time.time() + # close then open + self.close() + self.open() + # log time it took + logging.info( + f"Reopened microphone #{self.index}, took {time.time() - start:.3f}s" + ) def poll(self): """Poll audio samples. From faa0c63fb64bc914975d1e118351dba1ab799a03 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 17:19:49 +0100 Subject: [PATCH 6/8] ENH: Detect when Microphone appears to have gone to sleep --- psychopy/hardware/microphone.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/psychopy/hardware/microphone.py b/psychopy/hardware/microphone.py index 555da778a6..82d56081ae 100644 --- a/psychopy/hardware/microphone.py +++ b/psychopy/hardware/microphone.py @@ -265,7 +265,7 @@ def __init__(self, maxRecordingSize=maxRecordingSize, policyWhenFull=policyWhenFull ) - + self._possiblyAsleep = False self._isStarted = False # internal state logging.debug('Audio capture device #{} ready'.format( @@ -883,6 +883,24 @@ def poll(self): # figure out what to do with this other information audioData, absRecPosition, overflow, cStartTime = \ self._stream.get_audio_data() + + audioData = [] + + if len(audioData): + # if we got samples, the device is awake, so stop figuring out if it's asleep + self._possiblyAsleep = False + elif self._possiblyAsleep is False: + # if it was awake and now we've got no samples, store the time + self._possiblyAsleep = time.time() + elif self._possiblyAsleep + 1 < time.time(): + # if we've not had any evidence of it being awake for 1s, reopen + logging.error( + f"Microphone device appears to have gone to sleep, reopening to wake it up." + ) + # mark as stopped so we don't recursively poll forever when stopping + self._isStarted = False + # reopen + self.reopen() if overflow: logging.warning( From d0c4840d3cb1986cb5626481dd8a4a1908254b28 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 17:21:51 +0100 Subject: [PATCH 7/8] FF: Remove test override --- psychopy/hardware/microphone.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/psychopy/hardware/microphone.py b/psychopy/hardware/microphone.py index 82d56081ae..c065360cf7 100644 --- a/psychopy/hardware/microphone.py +++ b/psychopy/hardware/microphone.py @@ -884,8 +884,6 @@ def poll(self): audioData, absRecPosition, overflow, cStartTime = \ self._stream.get_audio_data() - audioData = [] - if len(audioData): # if we got samples, the device is awake, so stop figuring out if it's asleep self._possiblyAsleep = False From 8984c15c79b55461a54b5ae14b2b970c491cc676 Mon Sep 17 00:00:00 2001 From: Todd OST Date: Tue, 10 Sep 2024 17:24:43 +0100 Subject: [PATCH 8/8] ENH: Start mic after reopening --- psychopy/hardware/microphone.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psychopy/hardware/microphone.py b/psychopy/hardware/microphone.py index c065360cf7..6ebf5e8216 100644 --- a/psychopy/hardware/microphone.py +++ b/psychopy/hardware/microphone.py @@ -899,6 +899,8 @@ def poll(self): self._isStarted = False # reopen self.reopen() + # start again + self.start() if overflow: logging.warning(