Assessing the performance of the energy-detection sampler in enhancing speech rates

[lucasgautheron]

The energy-based sampler samples portions of the recordings above some energy level. It can be used to sample from the portions of the recordings that are not pure silence. It is also a model-independent way of targeting speech segments.

In order to assess the performance of the energy-based sampler, we ran it on the VanDam daylong dataset using energy thresholds ranging from 0% (= no selection) to 90% (= discard the 90% lowest energy windows) for a window-length of 30 seconds. We then plotted the average amount of OCH and CHI vocalization time per window passing the energy cut; the fraction of those windows that have at least some OCH/CHI speech; and the ratio of OCH/CHI voc. time as estimated by the VTC, the LENA, and the CHAT transcripts.

Note: in the CHAT annotations, every portion of audio is labelled with a speaker, even when the audio is silent. Therefore, speech time is completely unreliable; the presence of speech in some window is more reliable, and it is even more so as the signal energy increases.

#!/usr/bin/env python3

from ChildProject.projects import ChildProject
from ChildProject.annotations import AnnotationManager
from ChildProject.pipelines.samplers import EnergyDetectionSampler

import numpy as np
import pandas as pd

project = ChildProject('.')
project.read()

am = AnnotationManager(project)
am.read()

window_length = 30*1000
thresholds = np.arange(0, 100, 10)/100
speakers = ['OCH', 'CHI', 'MAL', 'FEM']
baseline = ['vtc', 'its', 'cha']

sampler = EnergyDetectionSampler(
    project,
    window_length,
    window_length,
    10000,
    0,
    0,
    threads = 1,
    profile = 'standard',
    low_freq = 50,
    high_freq = 3000
)

recording = project.recordings.to_dict(orient = 'records')[0]
windows = sampler.get_recording_windows(recording).rename(columns = {
    'segment_onset': 'range_onset',
    'segment_offset': 'range_offset'
}).assign(set = 'energy', time_seek = 0).sort_values(['recording_filename', 'energy'], ascending = False)
windows.to_csv('windows_cuts.csv', index = False)

windows = pd.read_csv('windows_cuts.csv')

results = []
for threshold in thresholds:
    _windows = windows.copy()
    n_windows = int((1-threshold)*len(windows))
    _windows = _windows.groupby('recording_filename').head(n_windows)        

    annotations = pd.concat([
        am.annotations[am.annotations['set'].isin(baseline)],
        _windows
    ])

    intersection = am.intersection(annotations, baseline + ['energy'])
    duration = _windows['range_offset'].sum()-_windows['range_onset'].sum()

    segments = am.get_collapsed_segments(intersection[intersection['set'].isin(baseline)])
    segments = segments[segments['speaker_type'].isin(speakers)]
    segments['duration'] = segments['segment_offset'] - segments['segment_onset']

    classes = segments.groupby(['set', 'position', 'speaker_type'])\
        .agg(
            duration = ('duration', 'sum')
        )
    
    classes = classes.groupby(['set', 'speaker_type'])\
        .agg(
            n = ('duration', 'count'),
            duration = ('duration', 'sum')
        )
    
    classes = classes.assign(
        threshold = threshold,
        total_duration = duration,
        total_n = n_windows
    )

    results.append(classes)

results = pd.concat(results)
results.to_csv('results_cuts.csv')

results = pd.read_csv('results_cuts.csv')
results['avg_duration'] = results['duration']/results['total_n']/1000
results['frac_n'] = results['n']/results['total_n']

results = results.pivot_table(index = ['set', 'speaker_type', 'threshold'])

sets = {
    'vtc': 'VTC',
    'its': 'LENA',
    'cha': 'CHAT'
}

from matplotlib import pyplot as plt 

for speaker in ['OCH', 'CHI']:
    for set in sets:
        plt.plot(results.loc[set, speaker].index.values, results.loc[set, speaker]['avg_duration'], label = sets[set])
    
    plt.xlabel('Energy threshold')
    plt.ylabel(f'Avg total {speaker} time per 30s window (s)')
    plt.legend()
    plt.savefig(f'{speaker}_time.png')
    plt.clf()

for speaker in ['OCH', 'CHI']:
    for set in sets:
        plt.plot(results.loc[set, speaker].index.values, results.loc[set, speaker]['frac_n'], label = sets[set])

    plt.xlabel('Energy threshold')
    plt.ylabel(f'Fraction of 30s windows with at least some {speaker} speech')
    plt.legend()
    plt.savefig(f'{speaker}_n.png')
    plt.clf()

for set in sets:
    plt.plot(results.loc[set, 'OCH'].index.values, results.loc[set, 'OCH']['avg_duration']/results.loc[set, 'CHI']['avg_duration'], label = sets[set])

plt.xlabel('Energy threshold')
plt.ylabel(f'OCH/CHI time ratio')
plt.legend()
plt.savefig(f'OCH_CHI_ratio.png')