diff --git a/human_experiments/datasette_interface/bin/copy_screenshots_to_server b/human_experiments/datasette_interface/bin/copy_screenshots_to_server deleted file mode 100755 index 60ebb7969..000000000 --- a/human_experiments/datasette_interface/bin/copy_screenshots_to_server +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -# Script to copy screenshots to the ivilab webserver. This is meant to be run -# on an IVILab machine on which /data is mounted. - -time rsync \ - -adrPm \ # archive mode, show progress, prune empty directories - --no-inc-recursive \ - --copy-links \ # Copy files and not just symlinks - --info=progress2 \ - --include="*/" \ - --include="*/*/Screenshots/*.png" \ - --include="*/*/screenshots/*.png" \ # Include lowercased variant - --exclude="*" \ - /data/tomcat/raw/LangLab/experiments/study_3_pilot/group ivilab:/var/www/data/tomcat/ diff --git a/human_experiments/datasette_interface/copy_screenshots_to_server.sh b/human_experiments/datasette_interface/bin/copy_screenshots_to_server.sh similarity index 100% rename from human_experiments/datasette_interface/copy_screenshots_to_server.sh rename to human_experiments/datasette_interface/bin/copy_screenshots_to_server.sh diff --git a/human_experiments/datasette_interface/bin/launch_dev b/human_experiments/datasette_interface/bin/launch_dev index 9c6e806e9..36fdd730b 100755 --- a/human_experiments/datasette_interface/bin/launch_dev +++ b/human_experiments/datasette_interface/bin/launch_dev @@ -3,7 +3,7 @@ set -euo pipefail datasette \ - /space/${USER}/tomcat/tomcat.db \ + /space/paulosoares/tomcat/tomcat.db \ --metadata metadata.yml \ --reload \ --template-dir=templates \ diff --git a/human_experiments/datasette_interface/bin/update_inspect_file b/human_experiments/datasette_interface/bin/update_inspect_file index 84958d0c8..54ac658ac 100755 --- a/human_experiments/datasette_interface/bin/update_inspect_file +++ b/human_experiments/datasette_interface/bin/update_inspect_file @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Update inspect-data.json -time datasette inspect /space/$USER/tomcat/tomcat.db > inspect-data.json +time datasette inspect /space/paulosoares/tomcat/tomcat.db > inspect-data.json diff --git a/human_experiments/datasette_interface/docker-compose.prod.yml b/human_experiments/datasette_interface/docker-compose.prod.yml index e7a29995b..f0e2898f0 100644 --- a/human_experiments/datasette_interface/docker-compose.prod.yml +++ b/human_experiments/datasette_interface/docker-compose.prod.yml @@ -17,7 +17,7 @@ services: --metadata /repo/metadata.yml --template-dir /repo/templates --plugins-dir /repo/plugins - --inspect-file /repo/inspect_file.json + --inspect-file /repo/inspect-data.json --static assets:/repo/static --setting sql_time_limit_ms 10000 --setting facet_time_limit_ms 10000 diff --git a/human_experiments/datasette_interface/inspect-data.json b/human_experiments/datasette_interface/inspect-data.json new file mode 100644 index 000000000..6a3c05e73 --- /dev/null +++ b/human_experiments/datasette_interface/inspect-data.json @@ -0,0 +1,78 @@ +{ + "tomcat": { + "hash": "e658dd324b33f88e17a02a77b45a7e16f0807b719ddeba440b552968efb54eae", + "size": 460870664192, + "file": "/space/paulosoares/tomcat/tomcat.db", + "tables": { + "task": { + "count": 9 + }, + "modality": { + "count": 3 + }, + "group_session": { + "count": 40 + }, + "participant": { + "count": 114 + }, + "station": { + "count": 4 + }, + "rest_state_task": { + "count": 40 + }, + "ping_pong_competitive_task_observation": { + "count": 647242 + }, + "eeg_device": { + "count": 120 + }, + "affective_task_event": { + "count": 14586 + }, + "eeg_raw": { + "count": 320700934 + }, + "ping_pong_cooperative_task_observation": { + "count": 323806 + }, + "finger_tapping_task_observation": { + "count": 99875 + }, + "data_validity": { + "count": 3060 + }, + "gaze_raw": { + "count": 129554477 + }, + "minecraft_mission": { + "count": 100 + }, + "minecraft_testbed_message": { + "count": 6370941 + }, + "screen_capture": { + "count": 4100470 + }, + "audio_vocalics": { + "count": 33049497 + }, + "eeg_sync": { + "count": 124351398 + }, + "fnirs_raw": { + "count": 6849295 + }, + "fnirs_sync": { + "count": 125359661 + }, + "gsr_sync": { + "count": 124351398 + }, + "ekg_sync": { + "count": 97446792 + } + } + } +} \ No newline at end of file diff --git a/human_experiments/datasette_interface/static/db_diagram.png b/human_experiments/datasette_interface/static/db_diagram.png index f64df8266..f3819fa58 100644 Binary files a/human_experiments/datasette_interface/static/db_diagram.png and b/human_experiments/datasette_interface/static/db_diagram.png differ diff --git a/human_experiments/datasette_interface/templates/base.html b/human_experiments/datasette_interface/templates/base.html index ce81536bc..dbc81902f 100644 --- a/human_experiments/datasette_interface/templates/base.html +++ b/human_experiments/datasette_interface/templates/base.html @@ -50,6 +50,7 @@ diff --git a/human_experiments/datasette_interface/templates/index.html b/human_experiments/datasette_interface/templates/index.html index 52db25b37..f795c7af0 100644 --- a/human_experiments/datasette_interface/templates/index.html +++ b/human_experiments/datasette_interface/templates/index.html @@ -57,9 +57,13 @@

{{ render_markdown(""" ## Funding Acknowledgment -The creation of this dataset was funded by the Army Research Office and was +- The creation of this dataset was funded by the Army Research Office and was accomplished under Grant Number W911NF-20-1-0002. The grant was awarded through -the Defense Advanced Research Projects Agency (DARPA). We would also like to -acknowledge intramural seed funding from the University of Arizona's SensorLab. +the Defense Advanced Research Projects Agency (DARPA). +- We would also like to acknowledge intramural seed funding from the University +of Arizona's SensorLab. +- Continued support (documentation updates, replying to questions from dataset +users, etc.) is supported by Army Research Office (ARO) Award Number +W911NF-24-2-0034. """) }} {% endblock %} diff --git a/human_experiments/datasette_interface/templates/pages/updates.html b/human_experiments/datasette_interface/templates/pages/updates.html index c2fcb55fc..429e26202 100644 --- a/human_experiments/datasette_interface/templates/pages/updates.html +++ b/human_experiments/datasette_interface/templates/pages/updates.html @@ -12,7 +12,10 @@ ## List of updates -- [2023-07-19](/updates/2023-07-19): This update adds the following: +- [2024-02-22](/updates/2024-02-22): Added new tables with vocalic data, +synchronized data, fixed a couple of data issues. +- [2023-07-19](/updates/2023-07-19): This update adds the following (see +[here](/updates/2023-07-19) for details): - [Data Products document](http://ivilab.cs.arizona.edu/data/tomcat/data_products.pdf), which contains descriptions of both the raw data (this is primarily meant for internal documentation purposes) and the diff --git a/human_experiments/datasette_interface/templates/pages/updates/2024-02-22.html b/human_experiments/datasette_interface/templates/pages/updates/2024-02-22.html new file mode 100644 index 000000000..c0dafaac3 --- /dev/null +++ b/human_experiments/datasette_interface/templates/pages/updates/2024-02-22.html @@ -0,0 +1,90 @@ +{% extends "base.html" %} + +{% block body_class %}index{% endblock %} + +{% block content %} +{{ render_markdown( +""" +# 2024-02-22 update + +## 1. New data + +### 1.1 Synchronized physio data tables + +Added new tables (`fnirs_sync`, `gsr_sync`, `eeg_sync`, and `ekg_sync`) +to store preprocessed signals synchronized to a main clock for four modalities +(fNIRS, GSR, EEG, and EKG) across experiments, with filtering for artifact +removal. For each experiment, a main clock was defined with start time at 5 +seconds before the beginning of the first task (rest state), end time at 5 +seconds after the end of last task (Minecraft), and frequency at 200 Hz. + +The changes are introduced in this PR: +https://github.com/ml4ai/tomcat/pull/558. + +### 1.2 Vocalic features table + +Added a table called `audio_vocalics` that contains vocalic features extracted +using [OpenSMILE](https://github.com/audeering/opensmile/releases/tag/v3.0.0). + +### 1.3 Updated `fnirs_raw` table + +Expanded fNIRS data by adding raw signals for 760 nm and 850 nm wavelengths to +the `fnirs_raw` table, in case users wish to transform those into HbO and HbR +themselves. + +### 1.4 Added advisor information + +Added a column to the `group_session` table containing information about which +advisor was present during the trials. + +## 2. Timestamp corrections + +### 2.1 Timestamp corrections for exp_2023_01_31_14 + +**Summary**: Corrected Minecraft start/end timestamps for exp_2023_01_31_14 +which had inconsistent dates in the future. + +**Explanation**: `exp_2023_01_31_14` had dates for trial start/stop and mission +stop (same as trial stop due to the absence of a mission stop message in the +corresponding `.metadata` file) in the past due to inconsistency. We noticed +that the code was not taking the most recent trial dates into consideration in +case of multiple trial start/stop. We fixed the data processing code and +updated the dates manually. The wrong dates were causing the main clock not to +have any ticks because rest state (with correct timestamp) had timestamp after +the Minecraft task which is used to compute the final timestamp of the main +clock when synchronizing signals. + +### 2.2 Timestamps for rest state task in exp_2022_12_05_12. + +**Summary**: Added start and end timestamps for the rest state task in +`exp_2022_12_05_12`. + +**Explanation**: If there are no start and end timestamps saved in the raw data +file for the rest state task (prior to 2023-04, the timestamps for the rest +state task were being saved in a CSV file), we use the file modification date +as an approximation to the start timestamp when it started and add 300 seconds +to the start timestamp to obtain the end timestamp. The file is created when +the task start so the offset should be minimal. So far, only experiment +`exp_2022_12_05_12` has this issue. We use the start time of the rest state +when defining the main clock, so having that information is important for us +not to lose any data. + +## 3. Fixed swapped signals for EKG and GSR for some experiments. + +Addressed an issue where EKG and GSR signals were swapped in specific +experiments: + +- `exp_2022_09_30_10` +- `exp_2022_10_04_09` for the `lion` station +- `exp_2022_10_04_09` for the `tiger` station) by updating the raw EEG data +processing processing to swap the signals for these specific instances. + +## Other updates + +- Updated funding acknowledgment section on the landing page to include new +grant from ARO. +- Updated entity-relationship diagram. + +""") +}} +{% endblock %}