From 444d8cbccd74d359ca89e1f7df6ece50cd912cd1 Mon Sep 17 00:00:00 2001 From: mcshicks Date: Wed, 19 May 2021 15:05:31 -0700 Subject: [PATCH 01/14] Mseed (#31) * Initial version for mseed and multiple architectures on single branch * fixed ffmpeg command * made files safe names * Hardcod URL for now, made env vars to int * removed from project * Fixed debug message * make data directory if it doesn't exist * initial combo version * initial version * run actual commands, add logspout * fix typos, use shutil.move instead of os.rename for docker * Fixed m3u8 path * fixed to encode in realtime, wait for file * change to ts files * use a bunch of ts files * force new dummy.ts on start * Fix Dockerfile case sensitive name * Added logging * make streaming privleged to use nice * fix audio sampling rate at 64000 * check before deleting/moving files, fixed audio rate, speed up start * change to mono * updates * Split docker-compose.yml into rpi and x86 version * Fix file permsissions * chagne segment to minutes, debug statements * make node run for 24 hours * Fixed rpi baseline build * added support for virtual streaming mode * moved to node folder * made different compose files for build and pull * Added info about logical soundcard names * checking .env adding hls loopback * add image tag for rpi * Removed empty build section To be updated * Fix typo in readme sentence. * Fix minor typos in README paragraph 1 Co-authored-by: steve Co-authored-by: Scott Veirs --- Dockerfile | 85 ----- README.md | 17 +- base/DockerCommon | 34 ++ base/amd64/Dockerfile | 14 + base/amd64/buildamdbase.sh | 4 + base/amd64/docker-compose.yml | 5 + base/docker-compose.yml | 5 + base/rpi/Dockerfile | 29 ++ base/rpi/buildrpibase.sh | 5 + base/rpi/jack.c | 354 ++++++++++++++++++ base/upload_flac_s3.py | 102 +++++ base/upload_s3.py | 100 +++++ config_audio.json | 23 -- dashcast.conf | 3 - mseed/Dockerfile | 26 ++ mseed/README.md | 20 + mseed/docker-compose.yml | 37 ++ mseed/files.txt | 300 +++++++++++++++ mseed/mseedpull.py | 230 ++++++++++++ mseed/streamfiles.sh | 41 ++ node/Dockerfile | 37 ++ node/docker-compose.amd64-build.yml | 27 ++ node/docker-compose.amd64-pull.yml | 26 ++ .../docker-compose.rpi-build.yml | 0 node/docker-compose.rpi-pull.yml | 28 ++ .../samples}/haro-strait_2005.wav | Bin node/stream.sh | 110 ++++++ stream.sh | 159 -------- 28 files changed, 1539 insertions(+), 282 deletions(-) delete mode 100644 Dockerfile create mode 100644 base/DockerCommon create mode 100644 base/amd64/Dockerfile create mode 100755 base/amd64/buildamdbase.sh create mode 100644 base/amd64/docker-compose.yml create mode 100644 base/docker-compose.yml create mode 100644 base/rpi/Dockerfile create mode 100755 base/rpi/buildrpibase.sh create mode 100644 base/rpi/jack.c create mode 100644 base/upload_flac_s3.py create mode 100644 base/upload_s3.py delete mode 100644 config_audio.json delete mode 100644 dashcast.conf create mode 100644 mseed/Dockerfile create mode 100644 mseed/README.md create mode 100644 mseed/docker-compose.yml create mode 100644 mseed/files.txt create mode 100644 mseed/mseedpull.py create mode 100755 mseed/streamfiles.sh create mode 100644 node/Dockerfile create mode 100644 node/docker-compose.amd64-build.yml create mode 100644 node/docker-compose.amd64-pull.yml rename docker-compose.yml => node/docker-compose.rpi-build.yml (100%) create mode 100644 node/docker-compose.rpi-pull.yml rename {samples => node/samples}/haro-strait_2005.wav (100%) create mode 100755 node/stream.sh delete mode 100755 stream.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d0af72b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,85 +0,0 @@ -# Node Dockerfile for hydrophone streaming - -# Use official debian image, but pull the armhf (v7+) image explicitly because -# Docker currently has a bug where armel is used instead when relying on -# multiarch manifest: https://github.com/moby/moby/issues/34875 -# When this is fixed, this can be changed to just `FROM debian:stretch-slim` -FROM arm32v7/debian:stretch-slim -MAINTAINER Orcasound - -# Upgrade OS -RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" - -# Set default command to bash as a placeholder -CMD ["/bin/bash"] - -# Make sure we're the root user -USER root - -WORKDIR /root - -############################### Install GPAC ############################## - -# Install required libraries -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - software-properties-common \ - curl \ - gnupg \ - wget \ - git - -# Install inotify-tools and rsync -RUN apt-get update && apt-get install -y --no-install-recommends inotify-tools rsync - -# Install ffmpeg -RUN \ - apt-get update && \ - apt-get install -y --no-install-recommends libx264-dev ffmpeg - -# Install ALSA and GPAC -RUN apt-get update && apt-get install -y --no-install-recommends \ - alsa-utils \ - gpac - -# Install npm and http-server for testing -# Based on https://nodesource.com/blog/installing-node-js-tutorial-debian-linux/ -RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \ - apt-get update && apt-get install -y --no-install-recommends nodejs - -RUN npm install -g \ - http-server \ - jsonlint - -# Install test-engine-live-tools -RUN git clone https://github.com/ebu/test-engine-live-tools.git && \ - cd test-engine-live-tools && \ - npm install - -# Install misc tools -RUN apt-get update && apt-get install -y --no-install-recommends \ - # General tools - htop \ - nano \ - sox \ - tmux \ - wget - -############################### Install s3fs ################################### - -RUN apt-get update && apt-get install -y --no-install-recommends s3fs - -############################### Copy files ##################################### - -COPY . . - -################################## TODO ######################################## -# Do the following: -# - Add pisound driver curl command -# - Add other audio drivers and configure via CLI if possible? -# - Remove "misc tools" and other installs no longer needed (upon Resin.io deployment)? - -################################# Miscellaneous ################################ - -# Clean up APT when done. -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/README.md b/README.md index 3528cad..8877c99 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Orcasound's orcanode +# Orcasound's orcastream -This software captures local audio data and streams it to AWS S3 buckets -- both as lossy (AAC-encoded) data in HLS segments for live-listening and as a lossless (FLAC-encoded) for archiving and/or acoustic analysis. There are branches for both arm32v7 and amd64 architectures, though the majority of initial development has been on the ARM-based Raspberry Pi. The current branches use ffmpeg+s3fs called by a bash script; an improved upload script is under development using ffmpeg+boto within a Python script, possibly with redis handling queueing. +This software contains audio tools and scripts for capturing, reformatting, transcoding and uploading audio for Orcasound. There is a base set of tools and a couple of specific projects, orcanode and orcamseed. Orcanode is streaming using Intel (amd64) or Raspberry Pi (arm32v7) platforms using a soundcard. While any soundcard should work, the most common one in use is the pisound board on either a Raspberry Pi 3B+ or 4. The other project orcamseed is for converting mseed format data to be streamed on Orcanode. This is mainly used for the [OOI](https://oceanobservatories.org/ "OOI") network. See the README in each of those directories for more info. ## Background & motivation @@ -22,18 +22,11 @@ An ARM or X86 device with a sound card (or other audio input devices) connected ### Installing -Choose the branch that is appropriate for your architecture. Clone that branch and create an .env file that contains the following: +Create a base docker image for your architecture by running the script in /base/rpi or /base/amd64 as appropriate. You will need to create a .env file as appropriate for your projects. Common to to all projects are the need for AWS keys ``` AWSACCESSKEYID=YourAWSaccessKey AWSSECRETACCESSKEY=YourAWSsecretAccessKey - -NODE_NAME=YourNodeName -NODE_TYPE=hls-only -AUDIO_HW_ID=1,0 -CHANNELS=2 -FLAC_DURATION=30 -SEGMENT_DURATION=10 SYSLOG_URL=syslog+tls://syslog-a.logdna.com:YourLogDNAPort SYSLOG_STRUCTURED_DATA='logdna@YourLogDNAnumber key="YourLogDNAKey" tag="docker" @@ -41,14 +34,14 @@ SYSLOG_STRUCTURED_DATA='logdna@YourLogDNAnumber key="YourLogDNAKey" tag="docker" * NODE_NAME should indicate your device and it's location, ideally in the form `device_location` (e.g. we call our Raspberry Pi staging device in Seattle `rpi_seattle`. * NODE_TYPE determines what audio data formats will be generated and transferred to their respective AWS buckets. -* AUDIO_HW_ID is the card, device providing the audio data. +* AUDIO_HW_ID is the card, device providing the audio data. Note: you can find your sound device by using the command "arecord -l". It's preferred to use the logical name i.e. pisound, USB, etc, instead of the "0,0" or "1,0" format which can change on reboots. * CHANNELS indicates the number of audio channels to expect (1 or 2). * FLAC_DURATION is the amount of seconds you want in each archived lossless file. * SEGMENT_DURATION is the amount of seconds you want in each streamed lossy segment. ## Running local tests -In the repository directory (where you also put your .env file) run `docker-compose up -d`. Watch what happens using `htop`. If you want to verify files are being written to /tmp or /mnt directories, get the name of your streaming service using `docker-compose ps` (in this case `orcanode_streaming_1`) and then do `docker exec -it orcanode_streaming_1 /bin/bash` to get a bash shell within the running container. +In the repository directory (where you also put your .env file) first copy the compose file you want to docker-compose.yml. For example if you are raspberry pi and you want to use the prebuilt image then copy docker-compose.rpi-pull.yml to docker-compose. Then run `docker-compose up -d`. Watch what happens using `htop`. If you want to verify files are being written to /tmp or /mnt directories, get the name of your streaming service using `docker-compose ps` (in this case `orcanode_streaming_1`) and then do `docker exec -it orcanode_streaming_1 /bin/bash` to get a bash shell within the running container. ### Running an end-to-end test diff --git a/base/DockerCommon b/base/DockerCommon new file mode 100644 index 0000000..2cce3af --- /dev/null +++ b/base/DockerCommon @@ -0,0 +1,34 @@ +# Install misc tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + # General tools + htop \ + nano \ + sox \ + tmux \ + wget \ + curl \ + git + +# Upgrade OS +RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" + +# Set default command to bash as a placeholder +CMD ["/bin/bash"] + +# Make sure we're the root user +USER root + +WORKDIR /root + + +############################### Install boto and inotify libraies ################################### + +RUN apt-get update && apt-get install -y python3-pip +RUN pip3 install -U boto3 inotify + +############################## Copy common scripts ################################################## + +COPY . . + +# Clean up APT when done. +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/base/amd64/Dockerfile b/base/amd64/Dockerfile new file mode 100644 index 0000000..217c3e3 --- /dev/null +++ b/base/amd64/Dockerfile @@ -0,0 +1,14 @@ +# Node Dockerfile for hydrophone streaming + +# Use official debian image, but pull the armhf (v7+) image explicitly because +# Docker currently has a bug where armel is used instead when relying on +# multiarch manifest: https://github.com/moby/moby/issues/34875 +# When this is fixed, this can be changed to just `FROM debian:stretch-slim` +FROM python:3.6-slim-buster +# FROM arm32v7/debian:buster-slim +MAINTAINER Orcasound + +####################### Install FFMPEG ##################################################### + +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg + diff --git a/base/amd64/buildamdbase.sh b/base/amd64/buildamdbase.sh new file mode 100755 index 0000000..3b7d885 --- /dev/null +++ b/base/amd64/buildamdbase.sh @@ -0,0 +1,4 @@ +#/bin/bash +cd .. +cat amd64/Dockerfile DockerCommon >./Dockerfile +docker-compose build --force-rm diff --git a/base/amd64/docker-compose.yml b/base/amd64/docker-compose.yml new file mode 100644 index 0000000..da8f3c5 --- /dev/null +++ b/base/amd64/docker-compose.yml @@ -0,0 +1,5 @@ +version: "3" +services: + pull: + image: orcastream/orcabase + build: ./amd64 diff --git a/base/docker-compose.yml b/base/docker-compose.yml new file mode 100644 index 0000000..4f16039 --- /dev/null +++ b/base/docker-compose.yml @@ -0,0 +1,5 @@ +version: "3" +services: + pull: + image: orcastream/orcabase + build: . diff --git a/base/rpi/Dockerfile b/base/rpi/Dockerfile new file mode 100644 index 0000000..4d6c412 --- /dev/null +++ b/base/rpi/Dockerfile @@ -0,0 +1,29 @@ +# Node Dockerfile for hydrophone streaming + +# Use official debian image, but pull the armhf (v7+) image explicitly because +# Docker currently has a bug where armel is used instead when relying on +# multiarch manifest: https://github.com/moby/moby/issues/34875 +# When this is fixed, this can be changed to just `FROM debian:stretch-slim` +# FROM python:3.6-slim-buster +FROM arm32v7/debian:buster-slim +MAINTAINER Orcasound + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git\ + build-essential \ + software-properties-common \ + curl \ + gnupg \ + wget + +####################### Build ffmpeg with Jack ##################################################### +# Note this doesn't work with amd64 because of the --arch-arme1 command + +RUN git clone git://source.ffmpeg.org/ffmpeg.git +RUN apt-get update && apt-get install -y --no-install-recommends libomxil-bellagio-dev libjack-dev +RUN cd ffmpeg && ./configure --arch=armel --target-os=linux --enable-gpl --enable-nonfree --enable-libjack +RUN cd ffmpeg && make -j4 +# Hack to patch jack.c with slightly longer timeout +COPY ./jack.c ./ffmpeg/libavdevice/jack.c +RUN cd ffmpeg && make +RUN cd ffmpeg && make install diff --git a/base/rpi/buildrpibase.sh b/base/rpi/buildrpibase.sh new file mode 100755 index 0000000..0386b8b --- /dev/null +++ b/base/rpi/buildrpibase.sh @@ -0,0 +1,5 @@ +#/bin/bash +cd .. +cp rpi/jack.c ./jack.c +cat rpi/Dockerfile DockerCommon >./Dockerfile +docker-compose build --force-rm diff --git a/base/rpi/jack.c b/base/rpi/jack.c new file mode 100644 index 0000000..8e46754 --- /dev/null +++ b/base/rpi/jack.c @@ -0,0 +1,354 @@ +/* + * JACK Audio Connection Kit input device + * Copyright (c) 2009 Samalyse + * Author: Olivier Guilyardi + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" +#include +#include + +#include "libavutil/internal.h" +#include "libavutil/log.h" +#include "libavutil/fifo.h" +#include "libavutil/opt.h" +#include "libavutil/time.h" +#include "libavcodec/avcodec.h" +#include "libavformat/avformat.h" +#include "libavformat/internal.h" +#include "timefilter.h" +#include "avdevice.h" + +/** + * Size of the internal FIFO buffers as a number of audio packets + */ +#define FIFO_PACKETS_NUM 16 + +typedef struct JackData { + AVClass *class; + jack_client_t * client; + int activated; + sem_t packet_count; + jack_nframes_t sample_rate; + jack_nframes_t buffer_size; + jack_port_t ** ports; + int nports; + TimeFilter * timefilter; + AVFifoBuffer * new_pkts; + AVFifoBuffer * filled_pkts; + int pkt_xrun; + int jack_xrun; +} JackData; + +static int process_callback(jack_nframes_t nframes, void *arg) +{ + /* Warning: this function runs in realtime. One mustn't allocate memory here + * or do any other thing that could block. */ + + int i, j; + JackData *self = arg; + float * buffer; + jack_nframes_t latency, cycle_delay; + AVPacket pkt; + float *pkt_data; + double cycle_time; + + if (!self->client) + return 0; + + /* The approximate delay since the hardware interrupt as a number of frames */ + cycle_delay = jack_frames_since_cycle_start(self->client); + + /* Retrieve filtered cycle time */ + cycle_time = ff_timefilter_update(self->timefilter, + av_gettime() / 1000000.0 - (double) cycle_delay / self->sample_rate, + self->buffer_size); + + /* Check if an empty packet is available, and if there's enough space to send it back once filled */ + if ((av_fifo_size(self->new_pkts) < sizeof(pkt)) || (av_fifo_space(self->filled_pkts) < sizeof(pkt))) { + self->pkt_xrun = 1; + return 0; + } + + /* Retrieve empty (but allocated) packet */ + av_fifo_generic_read(self->new_pkts, &pkt, sizeof(pkt), NULL); + + pkt_data = (float *) pkt.data; + latency = 0; + + /* Copy and interleave audio data from the JACK buffer into the packet */ + for (i = 0; i < self->nports; i++) { + jack_latency_range_t range; + jack_port_get_latency_range(self->ports[i], JackCaptureLatency, &range); + latency += range.max; + buffer = jack_port_get_buffer(self->ports[i], self->buffer_size); + for (j = 0; j < self->buffer_size; j++) + pkt_data[j * self->nports + i] = buffer[j]; + } + + /* Timestamp the packet with the cycle start time minus the average latency */ + pkt.pts = (cycle_time - (double) latency / (self->nports * self->sample_rate)) * 1000000.0; + + /* Send the now filled packet back, and increase packet counter */ + av_fifo_generic_write(self->filled_pkts, &pkt, sizeof(pkt), NULL); + sem_post(&self->packet_count); + + return 0; +} + +static void shutdown_callback(void *arg) +{ + JackData *self = arg; + self->client = NULL; +} + +static int xrun_callback(void *arg) +{ + JackData *self = arg; + self->jack_xrun = 1; + ff_timefilter_reset(self->timefilter); + return 0; +} + +static int supply_new_packets(JackData *self, AVFormatContext *context) +{ + AVPacket pkt; + int test, pkt_size = self->buffer_size * self->nports * sizeof(float); + + /* Supply the process callback with new empty packets, by filling the new + * packets FIFO buffer with as many packets as possible. process_callback() + * can't do this by itself, because it can't allocate memory in realtime. */ + while (av_fifo_space(self->new_pkts) >= sizeof(pkt)) { + if ((test = av_new_packet(&pkt, pkt_size)) < 0) { + av_log(context, AV_LOG_ERROR, "Could not create packet of size %d\n", pkt_size); + return test; + } + av_fifo_generic_write(self->new_pkts, &pkt, sizeof(pkt), NULL); + } + return 0; +} + +static int start_jack(AVFormatContext *context) +{ + JackData *self = context->priv_data; + jack_status_t status; + int i, test; + + /* Register as a JACK client, using the context url as client name. */ + self->client = jack_client_open(context->url, JackNullOption, &status); + if (!self->client) { + av_log(context, AV_LOG_ERROR, "Unable to register as a JACK client\n"); + return AVERROR(EIO); + } + + sem_init(&self->packet_count, 0, 0); + + self->sample_rate = jack_get_sample_rate(self->client); + self->ports = av_malloc_array(self->nports, sizeof(*self->ports)); + if (!self->ports) + return AVERROR(ENOMEM); + self->buffer_size = jack_get_buffer_size(self->client); + + /* Register JACK ports */ + for (i = 0; i < self->nports; i++) { + char str[16]; + snprintf(str, sizeof(str), "input_%d", i + 1); + self->ports[i] = jack_port_register(self->client, str, + JACK_DEFAULT_AUDIO_TYPE, + JackPortIsInput, 0); + if (!self->ports[i]) { + av_log(context, AV_LOG_ERROR, "Unable to register port %s:%s\n", + context->url, str); + jack_client_close(self->client); + return AVERROR(EIO); + } + } + + /* Register JACK callbacks */ + jack_set_process_callback(self->client, process_callback, self); + jack_on_shutdown(self->client, shutdown_callback, self); + jack_set_xrun_callback(self->client, xrun_callback, self); + + /* Create time filter */ + self->timefilter = ff_timefilter_new (1.0 / self->sample_rate, self->buffer_size, 1.5); + if (!self->timefilter) { + jack_client_close(self->client); + return AVERROR(ENOMEM); + } + + /* Create FIFO buffers */ + self->filled_pkts = av_fifo_alloc_array(FIFO_PACKETS_NUM, sizeof(AVPacket)); + /* New packets FIFO with one extra packet for safety against underruns */ + self->new_pkts = av_fifo_alloc_array((FIFO_PACKETS_NUM + 1), sizeof(AVPacket)); + if (!self->new_pkts) { + jack_client_close(self->client); + return AVERROR(ENOMEM); + } + if ((test = supply_new_packets(self, context))) { + jack_client_close(self->client); + return test; + } + + return 0; + +} + +static void free_pkt_fifo(AVFifoBuffer **fifo) +{ + AVPacket pkt; + while (av_fifo_size(*fifo)) { + av_fifo_generic_read(*fifo, &pkt, sizeof(pkt), NULL); + av_packet_unref(&pkt); + } + av_fifo_freep(fifo); +} + +static void stop_jack(JackData *self) +{ + if (self->client) { + if (self->activated) + jack_deactivate(self->client); + jack_client_close(self->client); + } + sem_destroy(&self->packet_count); + free_pkt_fifo(&self->new_pkts); + free_pkt_fifo(&self->filled_pkts); + av_freep(&self->ports); + ff_timefilter_destroy(self->timefilter); +} + +static int audio_read_header(AVFormatContext *context) +{ + JackData *self = context->priv_data; + AVStream *stream; + int test; + + if ((test = start_jack(context))) + return test; + + stream = avformat_new_stream(context, NULL); + if (!stream) { + stop_jack(self); + return AVERROR(ENOMEM); + } + + stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; +#if HAVE_BIGENDIAN + stream->codecpar->codec_id = AV_CODEC_ID_PCM_F32BE; +#else + stream->codecpar->codec_id = AV_CODEC_ID_PCM_F32LE; +#endif + stream->codecpar->sample_rate = self->sample_rate; + stream->codecpar->channels = self->nports; + + avpriv_set_pts_info(stream, 64, 1, 1000000); /* 64 bits pts in us */ + return 0; +} + +static int audio_read_packet(AVFormatContext *context, AVPacket *pkt) +{ + JackData *self = context->priv_data; + struct timespec timeout = {0, 0}; + int test; + + /* Activate the JACK client on first packet read. Activating the JACK client + * means that process_callback() starts to get called at regular interval. + * If we activate it in audio_read_header(), we're actually reading audio data + * from the device before instructed to, and that may result in an overrun. */ + if (!self->activated) { + if (!jack_activate(self->client)) { + self->activated = 1; + av_log(context, AV_LOG_INFO, + "JACK client registered and activated (rate=%dHz, buffer_size=%d frames)\n", + self->sample_rate, self->buffer_size); + } else { + av_log(context, AV_LOG_ERROR, "Unable to activate JACK client\n"); + return AVERROR(EIO); + } + } + + /* Wait for a packet coming back from process_callback(), if one isn't available yet */ + timeout.tv_sec = av_gettime() / 1000000 + 3; + if (sem_timedwait(&self->packet_count, &timeout)) { + if (errno == ETIMEDOUT) { + av_log(context, AV_LOG_ERROR, + "Input error: timed out when waiting for JACK process callback output\n"); + } else { + char errbuf[128]; + int ret = AVERROR(errno); + av_strerror(ret, errbuf, sizeof(errbuf)); + av_log(context, AV_LOG_ERROR, "Error while waiting for audio packet: %s\n", + errbuf); + } + if (!self->client) + av_log(context, AV_LOG_ERROR, "Input error: JACK server is gone\n"); + + return AVERROR(EIO); + } + + if (self->pkt_xrun) { + av_log(context, AV_LOG_WARNING, "Audio packet xrun\n"); + self->pkt_xrun = 0; + } + + if (self->jack_xrun) { + av_log(context, AV_LOG_WARNING, "JACK xrun\n"); + self->jack_xrun = 0; + } + + /* Retrieve the packet filled with audio data by process_callback() */ + av_fifo_generic_read(self->filled_pkts, pkt, sizeof(*pkt), NULL); + + if ((test = supply_new_packets(self, context))) + return test; + + return 0; +} + +static int audio_read_close(AVFormatContext *context) +{ + JackData *self = context->priv_data; + stop_jack(self); + return 0; +} + +#define OFFSET(x) offsetof(JackData, x) +static const AVOption options[] = { + { "channels", "Number of audio channels.", OFFSET(nports), AV_OPT_TYPE_INT, { .i64 = 2 }, 1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM }, + { NULL }, +}; + +static const AVClass jack_indev_class = { + .class_name = "JACK indev", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, + .category = AV_CLASS_CATEGORY_DEVICE_AUDIO_INPUT, +}; + +AVInputFormat ff_jack_demuxer = { + .name = "jack", + .long_name = NULL_IF_CONFIG_SMALL("JACK Audio Connection Kit"), + .priv_data_size = sizeof(JackData), + .read_header = audio_read_header, + .read_packet = audio_read_packet, + .read_close = audio_read_close, + .flags = AVFMT_NOFILE, + .priv_class = &jack_indev_class, +}; diff --git a/base/upload_flac_s3.py b/base/upload_flac_s3.py new file mode 100644 index 0000000..cb91019 --- /dev/null +++ b/base/upload_flac_s3.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Based on https://github.com/gergnz/s3autoloader/blob/master/s3autoloader.py +# Needs to replace this code +# while true; do +# inotifywait -r -e close_write,create /tmp/$NODE_NAME /tmp/flac/$NODE_NAME +# echo "Running rsync on $NODE_NAME..." +# nice -n -5 rsync -rtv /tmp/flac/$NODE_NAME /mnt/dev-archive-orcasound-net +# nice -n -5 rsync -rtv /tmp/$NODE_NAME /mnt/dev-streaming-orcasound-net +# done +# # +# +# Version 1 - just to hls +# Version 2 - + flac +# +# +# + +from boto3.s3.transfer import S3Transfer +import inotify.adapters +import logging +import logging.handlers +import boto3 +import os +import sys + +NODE = os.environ["NODE_NAME"] +BASEPATH = os.path.join("/tmp", NODE) +PATH = os.path.join(BASEPATH, "flac") +# Paths to watch is /tmp/NODE_NAME an /tmp/flac/NODE_NAME +# "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" +# "/tmp/flac/$NODE_NAME" +# s3.Bucket(name='dev-archive-orcasound-net') // flac +# s3.Bucket(name='dev-streaming-orcasound-net') // hls + + +REGION = os.environ["REGION"] +LOGLEVEL = logging.DEBUG + +log = logging.getLogger(__name__) + +log.setLevel(LOGLEVEL) + +handler = logging.StreamHandler(sys.stdout) + +formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s') +handler.setFormatter(formatter) + +log.addHandler(handler) + +BUCKET = "" +if "BUCKET_TYPE" in os.environ: + if(os.environ["BUCKET_TYPE"] == "prod"): + print("using production bucket") + BUCKET = 'archive-orcasound-net' + elif (os.environ["BUCKET_TYPE"] == "custom"): + print("using custom bucket") + BUCKET = os.environ["BUCKET_ARCHIVE"] + else: + print("using dev bucket") + BUCKET = "dev-archive-orcasound-net" + + log.debug("archive bucket set to ", BUCKET) + + +def s3_copy_file(path, filename): + log.debug('uploading file '+filename+' from '+path+' to bucket '+BUCKET) + try: + resource = boto3.resource('s3', REGION) # Doesn't seem like we have to specify region + # transfer = S3Transfer(client) + uploadfile = os.path.join(path, filename) + log.debug('upload file: ' + uploadfile) + uploadpath = os.path.relpath(path, "/tmp") + uploadkey = os.path.join(uploadpath, filename, ) + log.debug('upload key: ' + uploadkey) + resource.meta.client.upload_file(uploadfile, BUCKET, uploadkey, + ExtraArgs={'ACL': 'public-read'}) # TODO have to build filename into correct key. + os.remove(path+'/'+filename) # maybe not necessary since we write to /tmp and reboot every so often + except: + e = sys.exc_info()[0] + log.critical('error uploading to S3: '+str(e)) + +def _main(): + #s3_copy_file(PATH, 'latest.txt') + i = inotify.adapters.InotifyTree(PATH) + # TODO we should ideally block block_duration_s on the watch about the rate at which we write files, maybe slightly less + try: + for event in i.event_gen(yield_nones=False): + (header, type_names, path, filename) = event + if type_names[0] == 'IN_CLOSE_WRITE': + if 'tmp' not in filename: + log.debug('Recieved a new file ' + filename) + s3_copy_file(path, filename) + if type_names[0] == 'IN_MOVED_TO': + log.debug('Recieved a new file ' + filename) + s3_copy_file(path, filename) + finally: + log.debug('all done') + + +if __name__ == '__main__': + _main() + diff --git a/base/upload_s3.py b/base/upload_s3.py new file mode 100644 index 0000000..63375e0 --- /dev/null +++ b/base/upload_s3.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Based on https://github.com/gergnz/s3autoloader/blob/master/s3autoloader.py +# Needs to replace this code +# while true; do +# inotifywait -r -e close_write,create /tmp/$NODE_NAME /tmp/flac/$NODE_NAME +# echo "Running rsync on $NODE_NAME..." +# nice -n -5 rsync -rtv /tmp/flac/$NODE_NAME /mnt/dev-archive-orcasound-net +# nice -n -5 rsync -rtv /tmp/$NODE_NAME /mnt/dev-streaming-orcasound-net +# done +# # +# +# Version 1 - just to hls +# Version 2 - + flac +# +# +# + +from boto3.s3.transfer import S3Transfer +import inotify.adapters +import logging +import logging.handlers +import boto3 +import os +import sys + +NODE = os.environ["NODE_NAME"] +BASEPATH = os.path.join("/tmp", NODE) +PATH = os.path.join(BASEPATH, "hls") +# Paths to watch is /tmp/NODE_NAME an /tmp/flac/NODE_NAME +# "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" +# "/tmp/flac/$NODE_NAME" +# s3.Bucket(name='dev-archive-orcasound-net') // flac +# s3.Bucket(name='dev-streaming-orcasound-net') // hls + + +REGION = os.environ["REGION"] +LOGLEVEL = logging.DEBUG + +log = logging.getLogger(__name__) + +log.setLevel(LOGLEVEL) + +handler = logging.StreamHandler(sys.stdout) + +formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s') +handler.setFormatter(formatter) + +log.addHandler(handler) + +BUCKET = "" +if "BUCKET_TYPE" in os.environ: + if(os.environ["BUCKET_TYPE"] == "prod"): + print("using production bucket") + BUCKET = 'streaming-orcasound-net' + elif (os.environ["BUCKET_TYPE"] == "custom"): + print("using custom bucket") + BUCKET = os.environ["BUCKET_STREAMING"] + else: + BUCKET = "dev-streaming-orcasound-net" + + log.debug("hls bucket set to "+BUCKET) + +def s3_copy_file(path, filename): + log.debug('uploading file '+filename+' from '+path+' to bucket '+BUCKET) + try: + resource = boto3.resource('s3', REGION) # Doesn't seem like we have to specify region + # transfer = S3Transfer(client) + uploadfile = os.path.join(path, filename) + log.debug('upload file: ' + uploadfile) + uploadpath = os.path.relpath(path, "/tmp") + uploadkey = os.path.join(uploadpath, filename, ) + log.debug('upload key: ' + uploadkey) + resource.meta.client.upload_file(uploadfile, BUCKET, uploadkey, + ExtraArgs={'ACL': 'public-read'}) # TODO have to build filename into correct key. + os.remove(path+'/'+filename) # maybe not necessary since we write to /tmp and reboot every so often + except: + e = sys.exc_info()[0] + log.critical('error uploading to S3: '+str(e)) + +def _main(): + s3_copy_file(BASEPATH, 'latest.txt') + i = inotify.adapters.InotifyTree(PATH) + # TODO we should ideally block block_duration_s on the watch about the rate at which we write files, maybe slightly less + try: + for event in i.event_gen(yield_nones=False): + (header, type_names, path, filename) = event + if type_names[0] == 'IN_CLOSE_WRITE': + if 'tmp' not in filename: + log.debug('Recieved a new file ' + filename) + s3_copy_file(path, filename) + if type_names[0] == 'IN_MOVED_TO': + log.debug('Recieved a new file ' + filename) + s3_copy_file(path, filename) + finally: + log.debug('all done') + + +if __name__ == '__main__': + _main() + diff --git a/config_audio.json b/config_audio.json deleted file mode 100644 index ee52225..0000000 --- a/config_audio.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "segmentDir": "/tmp/dash_segment_input_dir", - "outputDir": "/tmp/dash_output_dir", - "mp4box": "MP4Box", - "ffmpeg": "ffmpeg", - "encoding": { - "commandPrefix": [ "-re", "-i", "-", "-threads", "0", "-y" ], - "representations": { - "audio": [ - "-map", "0:0", "-vn", "-acodec", "aac", "-strict", "-2", "-ar", "48000", "-ac", "2", - "-f", "segment", "-segment_time", "15", "-segment_format", "mpegts" - ] - } - }, - "packaging": { - "mp4box_opts": [ - "-dash-ctx", "/tmp/dash_output_dir/dash-live.txt", "-dash", "15000", "-rap", "-ast-offset", "12", - "-no-frags-default", "-bs-switching", "no","-mpd-refresh", "5", "-min-buffer", "15000", "-url-template", "-time-shift", - "-1", "-mpd-title", "MPEG-DASH live stream", "-mpd-info-url", "http://www.orcasound.net/", "-keep-utc", "-segment-name", - "live_$RepresentationID$_", "-out", "/tmp/dash_output_dir/live", "-dynamic", "-subsegs-per-sidx", "-1", "-tmp", "/tmp/" - ] - } -} diff --git a/dashcast.conf b/dashcast.conf deleted file mode 100644 index d88e39f..0000000 --- a/dashcast.conf +++ /dev/null @@ -1,3 +0,0 @@ -[a1] -type=audio -bitrate=128000 diff --git a/mseed/Dockerfile b/mseed/Dockerfile new file mode 100644 index 0000000..1a09609 --- /dev/null +++ b/mseed/Dockerfile @@ -0,0 +1,26 @@ +# Node Dockerfile for hydrophone streaming + +# Use official debian image, but pull the armhf (v7+) image explicitly because +# Docker currently has a bug where armel is used instead when relying on +# multiarch manifest: https://github.com/moby/moby/issues/34875 +# When this is fixed, this can be changed to just `FROM debian:stretch-slim` +FROM orcastream/orcabase:latest +MAINTAINER Orcasound + +RUN pip3 install numpy +RUN pip3 install obspy + +############################### Copy files ##################################### + +COPY . . + +################################## TODO ######################################## +# Do the following: +# - Add pisound driver curl command +# - Add other audio drivers and configure via CLI if possible? +# - Remove "misc tools" and other installs no longer needed (upon Resin.io deployment)? + +################################# Miscellaneous ################################ + +# Clean up APT when done. +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/mseed/README.md b/mseed/README.md new file mode 100644 index 0000000..be314fe --- /dev/null +++ b/mseed/README.md @@ -0,0 +1,20 @@ +### Installing + +After first creating a base image and a baseline .env file you will need to add certain specific env variable for mseed. Then you can use docker-compose build to build your image and docker-compose up to run it. Your .env file should look like this. + +``` +AWSACCESSKEYID=YourAWSaccessKey +AWSSECRETACCESSKEY=YourAWSsecretAccessKey + +SYSLOG_URL=syslog+tls://syslog-a.logdna.com:YourLogDNAPort +SYSLOG_STRUCTURED_DATA='logdna@YourLogDNAnumber key="YourLogDNAKey" tag="docker" +``` + +You will need to add the following variables to your baseline .env file to be able to pull and parse the mseed files. + +* STREAM_DELAY This is how many hours your stream will be delayed from the OOI websites, which are not updated in real time. +* DELAY_SEGMENT This is how many hours will be buffered locally after your delay. +* BASE_URL This is the root URL that your mseed files will be pulled from +* TIME_PREFIX This is a unique file prefix for each OOI site which will be ignored when checking time filetimes +* TIME_POSTFIX This is the portion after the timestamp. Nominally should be .mseed + diff --git a/mseed/docker-compose.yml b/mseed/docker-compose.yml new file mode 100644 index 0000000..0cbb5cc --- /dev/null +++ b/mseed/docker-compose.yml @@ -0,0 +1,37 @@ +version: "3" +services: + pull: + image: orcastream/orcamseed + build: ./ + # command: tail -F README.md + command: python3 mseedpull.py + restart: always + env_file: .env + volumes: + - data:/root/data + stream: + image: orcastream/orcamseed + build: ./ + # command: tail -F README.md + command: ./streamfiles.sh + restart: always + env_file: .env + volumes: + - data:/root/data + privileged: true + logspout: + image: gliderlabs/logspout + command: ${SYSLOG_URL} + restart: always + hostname: ${NODE_NAME} + env_file: .env + environment: + - SYSLOG_HOSTNAME=${NODE_NAME} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ports: + - "8000:8000" + + +volumes: + data: \ No newline at end of file diff --git a/mseed/files.txt b/mseed/files.txt new file mode 100644 index 0000000..4fab502 --- /dev/null +++ b/mseed/files.txt @@ -0,0 +1,300 @@ +ffconcat version 1.0 +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' +file 'data/dummy.ts' diff --git a/mseed/mseedpull.py b/mseed/mseedpull.py new file mode 100644 index 0000000..d37394c --- /dev/null +++ b/mseed/mseedpull.py @@ -0,0 +1,230 @@ +# OOI Conversion script for continious streaming audio +# The OOI website doesn't live post so this script continually +# looks for old files (based on delay) and then converts them +# to WAV files, which could then be posted to Orcasound website +# via FFMPEG and other scripts +# +# The code below is based on some code from Val Veirs, +# Elijah Blaisdell, Scott Veirs and Valentina Staneva from Democracy Lab +# Hackathon, Jan 9, 2021. +# +# +# +""" +!wget 'https://rawdata.oceanobservatories.org/files/RS01SBPS/PC01A/08-HYDBBA103/2021/01/09/OO-HYVM2--YDH-2021-01-09T00:15:00.000015.mseed' +""" + +from obspy import read +import requests +from html.parser import HTMLParser +import time +from datetime import datetime, timedelta +import os +import shutil +import dateutil.parser +import logging +import logging.handlers +import sys + +DELAY = int(os.environ["STREAM_DELAY"]) +# DELAY = 6.5 +SEGMENT = int(os.environ["DELAY_SEGMENT"]) # maybe change to "buffer" +# SEGMENT = 1 +# TODO Should put this in env variable +#BASE_URL = os.environ["BASE_URL"] +BASE_URL = 'https://rawdata.oceanobservatories.org/files/RS01SBPS/PC01A/08-HYDBBA103/' + +# Format of date in filename is ISO 8601 extended format +# To parse the start time of the file +# import dateutil.parser +# >>> dateutil.parser.isoparse('2021-01-09T00:15:00.000015') +# datetime.datetime(2021, 1, 9, 0, 15, 0, 15) +TIME_PREFIX = os.environ['TIME_PREFIX'] +TIME_POSTFIX = os.environ['TIME_POSTFIX'] +LOGLEVEL = logging.DEBUG + +log = logging.getLogger(__name__) + +log.setLevel(LOGLEVEL) + +handler = logging.StreamHandler(sys.stdout) + +formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s') +handler.setFormatter(formatter) + +log.addHandler(handler) + +filesdone = [] # files that have already been converted + + +def getFileTime(filestring, prefix=TIME_PREFIX, postfix=TIME_POSTFIX): + y = filestring.replace(prefix, '') + z = y.replace(postfix, '') + return(dateutil.parser.isoparse(z)) + + +def getFileUrls(): + class MyHTMLParser(HTMLParser): + def _init_(self, url): + self.url = url + + def handle_data(self, data): + if 'HYVM2' in data: + datetimestr = getFileTime(data) + filelist.append({'datetime': datetimestr, 'url': url, 'filepath': data}) + dates = [] + filelist = [] + now = datetime.utcnow() + # TODO This only deals with a delay of 24 hours. To generalize we need to + # divide delta by 24 to figure how the maximum number of days. + datestr = (now - timedelta(hours=DELAY)).strftime('%Y/%m/%d') + log.debug("now-delay: " + datestr) + datenowstr = (now).strftime('%Y/%m/%d') + log.debug("now: " + datenowstr) + dates.append(datestr) + if (datestr != datenowstr): + dates.append(datenowstr) + for datestr in dates: + url = BASE_URL + '{}'.format(datestr) + log.debug("fetching: "+url) + r = requests.get(url) + if r == 'Response [404]': + # Day folder does not exist yet or website down + print("website not responding or file not posted") + parser = MyHTMLParser() + parser.feed(str(r.content)) + return filelist + + +def fetchAndConvert(files): + convertedfiles = [] + toconvert = 0 + now = datetime.utcnow() + maxdelay = timedelta(hours=DELAY) + # mindelay = maxdelay - timedelta(hours=SEGMENT) + mindelay = maxdelay - timedelta(minutes=SEGMENT) + for file in files: + filepath = file['filepath'] + filetime = file['datetime'] + filedelay = now - filetime + if filepath not in filesdone: + if (filedelay < maxdelay and filedelay > mindelay): + toconvert += 1 + log.debug(f'files to convert: {toconvert}') + for file in files: + filetime = file['datetime'] + url = file['url'] + filepath = file['filepath'] + filedelay = now - filetime + if filepath not in filesdone: + full_url = f'{url}/{filepath}' + if (filedelay < maxdelay and filedelay > mindelay): + # reading from url + hydro = read(full_url) # load file into obspy object + log.debug('read url') + file['duration'] = hydro[0].meta['endtime'] - hydro[0].meta['starttime'] + # increasing amplitude + hydro[0].data = hydro[0].data * 1e4 + sampling_rate = hydro[0].meta['sampling_rate'] + # writing to wav file + wavfilename = (filepath[:-12]+'wav').replace(':', '-') # TODO Could be tmp filename + tsname = (filepath[:-12]+'ts').replace(':', '-') + hydro.write(wavfilename, framerate=sampling_rate, format='WAV') + log.debug('converted wav') + # TODO fix this -ar to actually use sampling_rate + if os.path.exists(tsname): + os.remove(tsname) + os.system('ffmpeg -i {filename} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfilename}'.format(filename=wavfilename, tsfilename=tsname)) + log.debug('made mpegts') + if os.path.exists(wavfilename): + os.remove(wavfilename) + file['samplerate'] = sampling_rate + file['tsfilename'] = tsname + filesdone.append(filepath) + convertedfiles.append(file) + toconvert -= 1 + log.debug(f'files to convert: {toconvert}') + return(convertedfiles) + +def queueFiles(files): + delay = timedelta(hours=DELAY) + now = datetime.utcnow() + played = 0 + deleted = 0 + for idx, entry in enumerate(files): + duration = timedelta(seconds=entry['duration']) + age = now - entry['datetime'] + tsfilename = entry['tsfilename'] + filepath = entry['filepath'] + if (delay + duration < age): # in the past + log.debug('deleting old entry: ' + tsfilename) + if os.path.exists(tsfilename): + os.remove(tsfilename) + filesdone.remove(filepath) + del files[idx] + deleted += 1 + if ((delay + duration >= age) and (age > delay)): + # should be playing next + log.debug('playing : ' + tsfilename) + if os.path.exists(tsfilename): + shutil.move(tsfilename, '/root/data/dummy.ts') + played += 1 + filesdone.remove(filepath) + del files[idx] + deleted += 1 + return(played, deleted, files) + + +def main_loop(): + starttime = time.time() + convertedfiles = [] + files = [] + while True: + # TODO this converts correctly but after queue files it + # get overwritten by fetchandconver + # you need to change it fetchandconvert appends the exisitng list + # and all timedate stamps are only converted once at most. + log.debug("checking") + files = getFileUrls() + log.debug(f'number of URLS: {len(files)}') + convertedfiles.extend(fetchAndConvert(files)) + log.debug(f'number of converted files: {len(files)}') + played, deleted, convertedfiles = queueFiles(convertedfiles) + log.debug(f'played: {played}, deleted: {deleted}') + time.sleep(150.0 - ((time.time() - starttime) % 150.0)) + + +main_loop() + +# todo - try encoding first to aac .ts and then stream looping +# ffmpeg -i dummy.wav -f mpegts -acodec aac dummy.ts + +# splitting into 10 second .ts files + +# os.system('ffmpeg -i {filename} -f segment -segment_list "live.m3u8" -segment_time 10 -segment_format mpegts -ar 48000 -ac 2 -acodec aac "live/live%03d.ts"'.format(filename=wavfilename)) +# To encode hls forever +# +# ffmpeg -re -stream_loop -1 -i list.txt -f segment -segment_list \ +# "./tmp/live.m3u8" -segment_list_flags +live -segment_time 10 \ +# -segment_format mpegts -ar 64000 -ac 2 -threads 3 -acodec aac \ +# "./tmp/live%03d.ts" + +# list.txt contents below +# +# ffconcat version 1.0 +# file 'dummy.wav' +# file 'list.txt' +# +# + + + + + + + + + + + + diff --git a/mseed/streamfiles.sh b/mseed/streamfiles.sh new file mode 100755 index 0000000..2599ffd --- /dev/null +++ b/mseed/streamfiles.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Script for live DASH/HLS streaming lossy audio as AAC and/or archiving lossless audio as FLAC +if [ -z ${NODE_NAME+x} ]; then echo "NODE_NAME is unset"; else echo "node name is set to '$NODE_NAME'"; fi +if [ -z ${NODE_LOOPBACK+x} ]; then echo "NODE_LOOPBACK is unset"; else echo "node loopback is set to '$NODE_LOOPBACK'"; fi + + +# Get current timestamp +timestamp=$(date +%s) + +#### Set up local output directories +mkdir -p /tmp/$NODE_NAME +mkdir -p /tmp/$NODE_NAME/hls +mkdir -p /tmp/$NODE_NAME/hls/$timestamp +# Output timestamp for this (latest) stream +echo $timestamp > /tmp/$NODE_NAME/latest.txt +mkdir -p /root/data +# Create a starting dummy file so you will always at least get a tone +# sox -n -r 64000 /root/data/dummy.wav synth 60 sine 500 +# rm dummy.ts +# ffmpeg -i dummy.wav -f mpegts -ar 64000 -acodec aac dummy.ts +# force new file +rm ./data/dummy.ts + +while [ ! -f ./data/dummy.ts ] +do + echo "waiting for dummy.ts" + sleep 30 +done + +if [ $NODE_LOOPBACK = "hls" ]; then + sleep 20 + ffplay -nodisp /tmp/$NODE_NAME/hls/$timestamp/live.m3u8 +fi + +echo "starting ffmpeg" + +ffmpeg -re -stream_loop -1 -i files.txt -flush_packets 0 -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_time 10 -segment_format mpegts -ar 64000 -ac 1 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" & + +python3 upload_s3.py + +echo "all done" diff --git a/node/Dockerfile b/node/Dockerfile new file mode 100644 index 0000000..20b0ce9 --- /dev/null +++ b/node/Dockerfile @@ -0,0 +1,37 @@ +# Node Dockerfile for hydrophone streaming +# use base image for project + +FROM orcastream/orcabase:latest +MAINTAINER Orcasound + +###### hack to get ffmpeg to build +# RUN apt-get update && apt-get install -y --no-install-recommends libraspberrypi-dev raspberrypi-kernel-headers +# RUN git clone https://github.com/raspberrypi/userland.git +# RUN cd userland/host_applications/linux/apps/hello_pi && ./rebuild.sh + + +###### Install Jack ################################# + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends jack-capture +RUN apt-get update && apt-get install -y --no-install-recommends jackd1 + +# Install ALSA and GPAC +#RUN apt-get update && apt-get install -y --no-install-recommends \ +# alsa-utils \ +# gpac + +############################### Copy files ##################################### + +COPY . . + +################################## TODO ######################################## +# Do the following: +# - Add pisound driver curl command +# - Add other audio drivers and configure via CLI if possible? +# - Remove "misc tools" and other installs no longer needed (upon Resin.io deployment)? + +################################# Miscellaneous ################################ + +# Clean up APT when done. +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/node/docker-compose.amd64-build.yml b/node/docker-compose.amd64-build.yml new file mode 100644 index 0000000..9aa661f --- /dev/null +++ b/node/docker-compose.amd64-build.yml @@ -0,0 +1,27 @@ +version: "3" +services: + streaming: + image: orcasound/orcanode + build: ./ + command: ./stream.sh + restart: always + env_file: .env + ports: + - "1234:1234" + - "8080:8080" + devices: + - "/dev/snd:/dev/snd" + privileged: true + + logspout: + image: gliderlabs/logspout + command: ${SYSLOG_URL} + restart: always + hostname: ${NODE_NAME} + env_file: .env + environment: + - SYSLOG_HOSTNAME=${NODE_NAME} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ports: + - "8000:8000" diff --git a/node/docker-compose.amd64-pull.yml b/node/docker-compose.amd64-pull.yml new file mode 100644 index 0000000..348de05 --- /dev/null +++ b/node/docker-compose.amd64-pull.yml @@ -0,0 +1,26 @@ +version: "3" +services: + streaming: + image: orcasound/orcanode:amd64 + command: ./stream.sh + restart: always + env_file: .env + ports: + - "1234:1234" + - "8080:8080" + devices: + - "/dev/snd:/dev/snd" + privileged: true + + logspout: + image: gliderlabs/logspout + command: ${SYSLOG_URL} + restart: always + hostname: ${NODE_NAME} + env_file: .env + environment: + - SYSLOG_HOSTNAME=${NODE_NAME} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ports: + - "8000:8000" diff --git a/docker-compose.yml b/node/docker-compose.rpi-build.yml similarity index 100% rename from docker-compose.yml rename to node/docker-compose.rpi-build.yml diff --git a/node/docker-compose.rpi-pull.yml b/node/docker-compose.rpi-pull.yml new file mode 100644 index 0000000..0849953 --- /dev/null +++ b/node/docker-compose.rpi-pull.yml @@ -0,0 +1,28 @@ +version: "3" +services: + streaming: + image: orcasound/orcanode:arm32v7 + command: ./stream.sh + restart: always + env_file: .env + ports: + - "1234:1234" + - "8080:8080" + devices: + - "/dev/snd:/dev/snd" + privileged: true + + logspout: + # Use unofficial logspout that's been compiled for armhf + # TODO: Build an updated version of this image. Looks unmaintained. + image: emdem/raspi-logspout + command: ${SYSLOG_URL} + restart: always + hostname: ${NODE_NAME} + env_file: .env + environment: + - SYSLOG_HOSTNAME=${NODE_NAME} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ports: + - "8000:8000" diff --git a/samples/haro-strait_2005.wav b/node/samples/haro-strait_2005.wav similarity index 100% rename from samples/haro-strait_2005.wav rename to node/samples/haro-strait_2005.wav diff --git a/node/stream.sh b/node/stream.sh new file mode 100755 index 0000000..c929c3d --- /dev/null +++ b/node/stream.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# Script for live DASH/HLS streaming lossy audio as AAC and/or archiving lossless audio as FLAC + +# Get current timestamp +timestamp=$(date +%s) + +if [ -z ${NODE_NAME+x} ]; then echo "NODE_NAME is unset"; else echo "node name is set to '$NODE_NAME'"; fi +if [ -z ${SAMPLE_RATE+x} ]; then echo "SAMPLE_RATE is unset"; else echo "sample rate is set to '$SAMPLE_RATE'"; fi +if [ -z ${AUDIO_HW_ID+x} ]; then echo "AUDIO_HW_ID is unset"; else echo "sound card is set to '$AUDIO_HW_ID'"; fi +if [ -z ${CHANNELS+x} ]; then echo "CHANNELS is unset"; else echo "Number of audio channels is set to '$CHANNELS'"; fi +if [ -z ${NODE_TYPE+x} ]; then echo "NODE_TYPE is unset"; else echo "node type is set to '$NODE_TYPE'"; fi +if [ -z ${STREAM_RATE+x} ]; then echo "STREAM_RATE is unset"; else echo "stream rate is set to '$STREAM_RATE'"; fi +if [ -z ${SEGMENT_DURATION+x} ]; then echo "SEGMENT_DURATION is unset"; else echo "segment duration is set to '$SEGMENT_DURATION'"; fi +if [ -z ${NODE_LOOPBACK+x} ]; then echo "NODE_LOOPBACK is unset"; else echo "node loopback is set to '$NODE_LOOPBACK'"; fi + + + +#### Set up local output directories +mkdir -p /tmp/$NODE_NAME +mkdir -p /tmp/$NODE_NAME/flac +mkdir -p /tmp/$NODE_NAME/hls +mkdir -p /tmp/$NODE_NAME/hls/$timestamp +#mkdir -p /tmp/$NODE_NAME/dash +#mkdir -p /tmp/$NODE_NAME/dash/$timestamp +#ln /tmp/$NODE_NAME/dash/$timestamp /tmp/dash_output_dir + +# Output timestamp for this (latest) stream +echo $timestamp > /tmp/$NODE_NAME/latest.txt + +STREAM_RATE=48000 + +if [ -z ${SAMPLE_RATE+48000}]; then + echo "setting sampling rate to 48000" +else + echo "sample rate is set to $SAMPLE_RATE"; +fi + +# Setup jack +echo @audio - memlock 256000 >> /etc/security/limits.conf +echo @audio - rtprio 75 >> /etc/security/limits.co +JACK_NO_AUDIO_RESERVATION=1 jackd -t 2000 -P 75 -d alsa -d hw:$AUDIO_HW_ID -r $SAMPLE_RATE -p 1024 -n 10 -s & + +#### Generate stream segments and manifests, and/or lossless archive + +echo "Node started at $timestamp" +echo "Node is named $NODE_NAME and is of type $NODE_TYPE" +## NODE_TYPE set in .env filt to one of: "research"; "debug" (DASH-only); "hls-only"; or default (FLAC+HLS+DASH) + +if [ $NODE_TYPE = "research" ]; then + #SAMPLE_RATE=192000 + ## Setup Jack Audio outside for now + # sudo echo @audio - memlock 256000 >> /etc/security/limits.conf + # sudo echo @audio - rtprio 75 >> /etc/security/limits.co + # sudo JACK_NO_AUDIO_RESERVATION=1 jackd -t 2000 -P 75 -d alsa -d hw:pisound -r 192000 -p 1024 -n 10 -s + echo "Sampling $CHANNELS channels from $AUDIO_HW_ID at $SAMPLE_RATE Hz with bitrate of 32 bits/sample..." + echo "Asking ffmpeg to write $FLAC_DURATION second $SAMPLE_RATE Hz FLAC files..." + ## Streaming HLS with FLAC archive + nice -n -10 ffmpeg -f jack -i ffjack \ + -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/tmp/$NODE_NAME/flac/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" \ + -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format \ + mpegts -ar $STREAM_RATE -ac 2 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" >/dev/null 2>/dev/null & +elif [ $NODE_TYPE = "debug" ]; then + echo "Sampling $CHANNELS channels from $AUDIO_HW_ID at $SAMPLE_RATE Hz with bitrate of 32 bits/sample..." + echo "Asking ffmpeg to stream DASH via mpegts at $STREAM_RATE Hz..." + ## Streaming DASH only via mpegts + nice -n -10 ffmpeg -t 0 -f jack -i ffjack -f mpegts udp://127.0.0.1:1234 & + #### Stream with test engine live tools + ## May need to adjust segment length in config_audio.json to match $SEGMENT_DURATION... + nice -n -7 ./test-engine-live-tools/bin/live-stream -c ./config_audio.json udp://127.0.0.1:1234 & +elif [ $NODE_TYPE = "hls-only" ]; then + echo "Sampling $CHANNELS channels from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." + echo "Asking ffmpeg to stream only HLS segments at $STREAM_RATE Hz......" + ## Streaming HLS only via mpegts + nice -n -10 ffmpeg -f jack -i ffjack -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" & +elif [ $NODE_TYPE = "dev-virt-s3" ]; then + SAMPLE_RATE=48000 + STREAM_RATE=48000 + echo "Sampling from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." + echo "Asking ffmpeg to stream only HLS segments at $STREAM_RATE Hz......" + ## Streaming HLS only via mpegts + nice -n -10 ffmpeg -re -fflags +genpts -stream_loop -1 -i "samples/haro-strait_2005.wav" \ + -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts \ + -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" & +else + echo "unsupported please pick hls-only, research, or dev-virt-s3" +fi + +# takes a second for ffmpeg to make ffjack connection before we can connect +sleep 3 +jack_connect system:capture_1 ffjack:input_1 +jack_connect system:capture_2 ffjack:input_2 + +if [ $NODE_LOOPBACK = "true" ]; then + jack_connect system:capture_1 system:playback_1 + jack_connect system:capture_2 system:playback_2 +fi + +if [ $NODE_LOOPBACK = "hls" ]; then + sleep 20 + ffplay -nodisp /tmp/$NODE_NAME/hls/$timestamp/live.m3u8 +fi + +if [ $NODE_TYPE = "research" ]; then + python3 upload_s3.py & + python3 upload_flac_s3.py +else + python3 upload_s3.py +fi + +echo "all done" diff --git a/stream.sh b/stream.sh deleted file mode 100755 index 0e4360b..0000000 --- a/stream.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash -# Script for live DASH/HLS streaming lossy audio as AAC and/or archiving lossless audio as FLAC -# Some environmental variables set by local .env file; others here: - -SEGMENT_DURATION=10 -FLAC_DURATION=10 -LAG_SEGMENTS=6 -LAG=$(( LAG_SEGMENTS*SEGMENT_DURATION )) -CHOP_M3U8_LINES=$(( LAG_SEGMENTS*(-2) )) - -# Get current timestamp -timestamp=$(date +%s) - -#### Set up local output directories -##mkdir -p /tmp/flac/ -##mkdir -p /tmp/flac/$NODE_NAME -mkdir -p /tmp/m3u8tmp -mkdir -p /tmp/m3u8tmp/$timestamp -mkdir -p /tmp/$NODE_NAME -mkdir -p /tmp/$NODE_NAME/hls -mkdir -p /tmp/$NODE_NAME/hls/$timestamp -#mkdir -p /tmp/$NODE_NAME/dash -#mkdir -p /tmp/$NODE_NAME/dash/$timestamp -#ln /tmp/$NODE_NAME/dash/$timestamp /tmp/dash_output_dir - -# Output timestamp for this (latest) stream -echo $timestamp > /tmp/$NODE_NAME/latest.txt - - -#### Set up /tmp, /mnt directories and start s3fs, with architecture depending on the node-type - - ## Could move the latest copy up to where subdirs are made, and also add dev vs other logic there... - - if [ $NODE_TYPE = "dev-stable" ] || [ $NODE_TYPE = "dev-virt-s3" ] ; then - mkdir -p /mnt/dev-archive-orcasound-net - mkdir -p /mnt/dev-streaming-orcasound-net - s3fs -o default_acl=public-read --debug -o dbglevel=info dev-archive-orcasound-net /mnt/dev-archive-orcasound-net/ - s3fs -o default_acl=public-read --debug -o dbglevel=info dev-streaming-orcasound-net /mnt/dev-streaming-orcasound-net/ - mkdir -p /mnt/dev-archive-orcasound-net/$NODE_NAME - mkdir -p /mnt/dev-streaming-orcasound-net/$NODE_NAME - mkdir -p /mnt/dev-streaming-orcasound-net/$NODE_NAME/hls - mkdir -p /mnt/dev-streaming-orcasound-net/$NODE_NAME/hls/$timestamp - cp /tmp/$NODE_NAME/latest.txt /mnt/dev-streaming-orcasound-net/$NODE_NAME/latest.txt - else - mkdir -p /mnt/archive-orcasound-net - mkdir -p /mnt/streaming-orcasound-net - s3fs -o default_acl=public-read --debug -o dbglevel=info archive-orcasound-net /mnt/archive-orcasound-net/ - s3fs -o default_acl=public-read --debug -o dbglevel=info streaming-orcasound-net /mnt/streaming-orcasound-net/ - mkdir -p /mnt/archive-orcasound-net/$NODE_NAME - mkdir -p /mnt/streaming-orcasound-net/$NODE_NAME - mkdir -p /mnt/streaming-orcasound-net/$NODE_NAME/hls - mkdir -p /mnt/streaming-orcasound-net/$NODE_NAME/hls/$timestamp - cp /tmp/$NODE_NAME/latest.txt /mnt/streaming-orcasound-net/$NODE_NAME/latest.txt - fi - - -#### Generate stream segments and manifests, and/or lossless archive - -echo "Node started at $timestamp" -echo "Node is named $NODE_NAME and is of type $NODE_TYPE" -## NODE_TYPE set in .env filt to one of: -## "research" -- writes FLAC files to archive bucket; -## "dash-only" -- streams MPEG-DASH segments to streaming bucket; -## "hls-only" -- streams HLS segments to streaming bucket; -## "dev-virt-s3" -- loops local .wav file to virtual S3 bucket; -## "dev-stable" -- streams FLAC and/or HLS to dev S3 buckets; -## or default (FLAC+HLS+DASH), e.g. for improving Orcasite app/player features/encoding/browser-compatibilty etc... - -if [ $NODE_TYPE = "research" ]; then - SAMPLE_RATE=48000 - STREAM_RATE=48000 ## Is it efficient to specify this so mpegts isn't hit by 4x the uncompressed data? - echo "Sampling from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." - echo "Asking ffmpeg to write HLS and $FLAC_DURATION second $SAMPLE_RATE Hz FLAC files..." - ## Streaming HLS with FLAC archive via /tmp (and rsync) - ##nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -ac $CHANNELS -ar $SAMPLE_RATE -sample_fmt s32 -acodec flac \ - -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/tmp/flac/$NODE_NAME/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" \ - -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format \ - mpegts -ar $STREAM_RATE -ac $CHANNELS -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" & - ## Streaming HLS segments and FLAC archive direct to /mnt directories, but live.m3u8 via /tmp - nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -ac $CHANNELS -ar $SAMPLE_RATE -sample_fmt s32 -acodec flac -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/mnt/archive-orcasound-net/$NODE_NAME/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -acodec aac "/mnt/streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live%03d.ts" & - -elif [ $NODE_TYPE = "dash-only" ]; then - SAMPLE_RATE=48000 - STREAM_RATE=48000 - echo "Sampling from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." - echo "Asking ffmpeg to stream DASH via mpegts at $STREAM_RATE Hz..." - ## Streaming DASH only via mpegts - nice -n -10 ffmpeg -t 0 -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -ac $CHANNELS -f mpegts udp://127.0.0.1:1234 & - #### Stream with test engine live tools - ## May need to adjust segment length in config_audio.json to match $SEGMENT_DURATION... - nice -n -7 ./test-engine-live-tools/bin/live-stream -c ./config_audio.json udp://127.0.0.1:1234 & - -elif [ $NODE_TYPE = "hls-only" ]; then - SAMPLE_RATE=48000 - STREAM_RATE=48000 - echo "Sampling from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." - echo "Asking ffmpeg to stream only HLS segments at $STREAM_RATE Hz......" - ## Streaming HLS only with .ts segments to /mnt, but live.m3u8 to /tmp - nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/mnt/streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live%03d.ts" & - ## Streaming HLS only via mpegts (the old way with .ts segments via /tmp dirs) - ##nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -ac $CHANNELS -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" & - -elif [ $NODE_TYPE = "dev-virt-s3" ]; then - SAMPLE_RATE=48000 - STREAM_RATE=48000 - echo "Sampling from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." - echo "Asking ffmpeg to stream only HLS segments at $STREAM_RATE Hz......" - ## Streaming HLS only via mpegts - nice -n -10 ffmpeg -re -fflags +genpts -stream_loop -1 -i "samples/haro-strait_2005.wav" \ - -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts \ - -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/mnt/dev-streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live%03d.ts" & - -elif [ $NODE_TYPE = "dev-stable" ]; then - SAMPLE_RATE=48000 - STREAM_RATE=48000 - echo "Sampling from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." - ## Streaming HLS only to S3FS via /mnt directory - nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/mnt/dev-streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live%03d.ts" & - ## Streaming HLS and FLAC directly to S3FS via /mnt directories - ## echo "Asking ffmpeg to write $FLAC_DURATION second $SAMPLE_RATE Hz FLAC files..." - ## nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -ac $CHANNELS -ar $SAMPLE_RATE -sample_fmt s32 -acodec flac -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/mnt/dev-archive-orcasound-net/$NODE_NAME/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -acodec aac "/mnt/dev-streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live%03d.ts" & - -## Default NODE_TYPE settings -else - SAMPLE_RATE=48000 - STREAM_RATE=48000 - echo "Sampling from $AUDIO_HW_ID at $SAMPLE_RATE Hz..." - echo "Asking ffmpeg to write $FLAC_DURATION second $SAMPLE_RATE Hz lo-res flac files while streaming in both DASH and HLS..." - ## Streaming DASH/HLS with low-res flac archive (the old way via tmp directories) - ##nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -ac $CHANNELS -ar $SAMPLE_RATE -sample_fmt s32 -acodec flac \ - -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/tmp/flac/$NODE_NAME/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" \ - -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format \ - mpegts -ac $CHANNELS -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ - -f mpegts -ac $CHANNELS udp://127.0.0.1:1234 & - ## Streaming HLS segments and FLAC archive direct to /mnt directories, but live.m3u8 via /tmp - nice -n -10 ffmpeg -f alsa -ac 2 -ar $SAMPLE_RATE -thread_queue_size 1024 -i hw:$AUDIO_HW_ID -ac $CHANNELS -ar $SAMPLE_RATE -sample_fmt s32 -acodec flac -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/mnt/archive-orcasound-net/$NODE_NAME/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" -f segment -segment_list "/tmp/m3u8tmp/$timestamp/live.m3u8" -segment_list_flags +live -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -acodec aac "/mnt/streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live%03d.ts" & - #### Stream with test engine live tools - ## May need to adjust segment length in config_audio.json to match $SEGMENT_DURATION... - nice -n -7 ./test-engine-live-tools/bin/live-stream -c ./config_audio.json udp://127.0.0.1:1234 & -fi - -sleep $LAG - -while true; do - ##inotifywait -r -e close_write /tmp/$NODE_NAME /tmp/flac/$NODE_NAME - echo "In while loop copying aged m3u8 for $NODE_NAME with lag of $LAG_SEGMENTS segments, or $LAG seconds..." - head -n $CHOP_M3U8_LINES /tmp/m3u8tmp/$timestamp/live.m3u8 > /tmp/$NODE_NAME/hls/$timestamp/live.m3u8 - if [ $NODE_TYPE = "dev-stable" ] || [ $NODE_TYPE = "dev-virt-s3" ] ; then - cp /tmp/$NODE_NAME/hls/$timestamp/live.m3u8 /mnt/dev-streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live.m3u8 - ##mv /tmp/$NODE_NAME/hls/$timestamp/live*.ts /mnt/dev-streaming-orcasound-net/$NODE_NAME/hls/$timestamp - ##nice -n -5 rsync -avW --progress --inplace --size-only /tmp/flac/$NODE_NAME /mnt/dev-archive-orcasound-net - ##nice -n -5 rsync -avW --progress --inplace --size-only --exclude='*.tmp' --exclude '.live*' /tmp/$NODE_NAME /mnt/dev-streaming-orcasound-net - else - cp /tmp/$NODE_NAME/hls/$timestamp/live.m3u8 /mnt/streaming-orcasound-net/$NODE_NAME/hls/$timestamp/live.m3u8 - ##nice -n -5 rsync -avW --progress --inplace --size-only /tmp/flac/$NODE_NAME /mnt/archive-orcasound-net - ##nice -n -5 rsync -avW --progress --inplace --size-only --exclude='*.tmp' --exclude '.live*' /tmp/$NODE_NAME /mnt/streaming-orcasound-net - fi -sleep $SEGMENT_DURATION -done From ab6ef3748f83a8c8222ec54f52652f94b9a0bc7a Mon Sep 17 00:00:00 2001 From: Scott Veirs Date: Tue, 1 Jun 2021 16:30:56 -0700 Subject: [PATCH 02/14] Add example .env file for node streaming development --- README.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8877c99..e35c624 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,25 @@ An ARM or X86 device with a sound card (or other audio input devices) connected ### Installing -Create a base docker image for your architecture by running the script in /base/rpi or /base/amd64 as appropriate. You will need to create a .env file as appropriate for your projects. Common to to all projects are the need for AWS keys +Create a base docker image for your architecture by running the script in /base/rpi or /base/amd64 as appropriate. You will need to create a .env file as appropriate for your projects. Here is an example of an .env file (tested/working as of June, 2021) without the keys that are common to all Orcasound projects: + +``` +AWS_METADATA_SERVICE_TIMEOUT=5 +AWS_METADATA_SERVICE_NUM_ATTEMPTS=0 +REGION=us-west-2 +BUCKET_TYPE=dev +NODE_TYPE=hls-only +NODE_NAME=rpi_YOURNODENAME_test +NODE_LOOPBACK=true +SAMPLE_RATE=48000 +AUDIO_HW_ID=pisound +CHANNELS=1 +FLAC_DURATION=30 +SEGMENT_DURATION=10 +LC_ALL=C.UTF-8 +``` + +except that the following fields are excised and will be need added if you are integrating with the audio and logging data streaming systems of Orcasound. (You can request keys via the #hydrophone-nodes channel in the Orcasound Slack. As of June, 2021, we are continuing to use AWS S3 for storage and LogDNA for live-logging and troubleshooting.) ``` AWSACCESSKEYID=YourAWSaccessKey @@ -32,6 +50,8 @@ SYSLOG_URL=syslog+tls://syslog-a.logdna.com:YourLogDNAPort SYSLOG_STRUCTURED_DATA='logdna@YourLogDNAnumber key="YourLogDNAKey" tag="docker" ``` +Here are explanations of some of the .env fields: + * NODE_NAME should indicate your device and it's location, ideally in the form `device_location` (e.g. we call our Raspberry Pi staging device in Seattle `rpi_seattle`. * NODE_TYPE determines what audio data formats will be generated and transferred to their respective AWS buckets. * AUDIO_HW_ID is the card, device providing the audio data. Note: you can find your sound device by using the command "arecord -l". It's preferred to use the logical name i.e. pisound, USB, etc, instead of the "0,0" or "1,0" format which can change on reboots. @@ -39,6 +59,8 @@ SYSLOG_STRUCTURED_DATA='logdna@YourLogDNAnumber key="YourLogDNAKey" tag="docker" * FLAC_DURATION is the amount of seconds you want in each archived lossless file. * SEGMENT_DURATION is the amount of seconds you want in each streamed lossy segment. + + ## Running local tests In the repository directory (where you also put your .env file) first copy the compose file you want to docker-compose.yml. For example if you are raspberry pi and you want to use the prebuilt image then copy docker-compose.rpi-pull.yml to docker-compose. Then run `docker-compose up -d`. Watch what happens using `htop`. If you want to verify files are being written to /tmp or /mnt directories, get the name of your streaming service using `docker-compose ps` (in this case `orcanode_streaming_1`) and then do `docker exec -it orcanode_streaming_1 /bin/bash` to get a bash shell within the running container. From 5c68103f257979c8c9d8c0724bf59e59871dc2e5 Mon Sep 17 00:00:00 2001 From: Scott Veirs Date: Tue, 3 Aug 2021 10:16:17 -0700 Subject: [PATCH 03/14] Clarify intro; add emphasis re directory structure, Pisound link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e35c624..0f01a49 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Orcasound's orcastream -This software contains audio tools and scripts for capturing, reformatting, transcoding and uploading audio for Orcasound. There is a base set of tools and a couple of specific projects, orcanode and orcamseed. Orcanode is streaming using Intel (amd64) or Raspberry Pi (arm32v7) platforms using a soundcard. While any soundcard should work, the most common one in use is the pisound board on either a Raspberry Pi 3B+ or 4. The other project orcamseed is for converting mseed format data to be streamed on Orcanode. This is mainly used for the [OOI](https://oceanobservatories.org/ "OOI") network. See the README in each of those directories for more info. +This software contains audio tools and scripts for capturing, reformatting, transcoding and uploading audio for Orcasound. The directory structure reflects that we have developed a **base** set of tools and a couple of specific projects, orcanode and orcamseed (in the node and mseed directories). Orcasound hydrophone nodes stream by running the **node** code on Intel (amd64) or Raspberry Pi (arm32v7) platforms using a soundcard. While any soundcard should work, the most common one in use is the [Pisound](https://blokas.io/pisound/) board on either a Raspberry Pi 3B+ or 4. The other project (in the **mseed** directory) is for converting mseed format data to be streamed on Orcanode. This is mainly used for streaming audio date from the [OOI](https://oceanobservatories.org/ "OOI") (NSF-funded Ocean Observatory Initiative) hydrophones off the coast of Oregon. See the README in each of those directories for more info. ## Background & motivation From 96ca94d509e13c357a3a5f8428b48719b643b664 Mon Sep 17 00:00:00 2001 From: mcshicks Date: Wed, 29 Sep 2021 10:29:01 -0700 Subject: [PATCH 04/14] Add Support for Arm64 (#33) * arm64: install ffmpeg instaed of building * No logspout for arm64 There is no logspout for arm64 so comment this out. * Create docker-compose.arm64-pull.yml New version to support arm64 dockerhub image Co-authored-by: JoyceLiao --- base/arm64/Dockerfile | 13 + base/arm64/buildrpibase.sh | 5 + base/arm64/jack.c | 354 ++++++++++++++++++++++++++++ node/docker-compose.arm64-build.yml | 27 +++ node/docker-compose.arm64-pull.yml | 28 +++ 5 files changed, 427 insertions(+) create mode 100644 base/arm64/Dockerfile create mode 100755 base/arm64/buildrpibase.sh create mode 100644 base/arm64/jack.c create mode 100644 node/docker-compose.arm64-build.yml create mode 100644 node/docker-compose.arm64-pull.yml diff --git a/base/arm64/Dockerfile b/base/arm64/Dockerfile new file mode 100644 index 0000000..e644a91 --- /dev/null +++ b/base/arm64/Dockerfile @@ -0,0 +1,13 @@ +# Node Dockerfile for hydrophone streaming + +# Use official debian image, but pull the armhf (v7+) image explicitly because +# Docker currently has a bug where armel is used instead when relying on +# multiarch manifest: https://github.com/moby/moby/issues/34875 +# When this is fixed, this can be changed to just `FROM debian:stretch-slim` +FROM python:3.6-slim-buster +# FROM arm32v7/debian:buster-slim +MAINTAINER Orcasound + +####################### Install FFMPEG ##################################################### + +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg diff --git a/base/arm64/buildrpibase.sh b/base/arm64/buildrpibase.sh new file mode 100755 index 0000000..573012b --- /dev/null +++ b/base/arm64/buildrpibase.sh @@ -0,0 +1,5 @@ +#/bin/bash +cd .. +cp arm64/jack.c ./jack.c +cat arm64/Dockerfile DockerCommon >./Dockerfile +docker-compose build --force-rm diff --git a/base/arm64/jack.c b/base/arm64/jack.c new file mode 100644 index 0000000..8e46754 --- /dev/null +++ b/base/arm64/jack.c @@ -0,0 +1,354 @@ +/* + * JACK Audio Connection Kit input device + * Copyright (c) 2009 Samalyse + * Author: Olivier Guilyardi + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" +#include +#include + +#include "libavutil/internal.h" +#include "libavutil/log.h" +#include "libavutil/fifo.h" +#include "libavutil/opt.h" +#include "libavutil/time.h" +#include "libavcodec/avcodec.h" +#include "libavformat/avformat.h" +#include "libavformat/internal.h" +#include "timefilter.h" +#include "avdevice.h" + +/** + * Size of the internal FIFO buffers as a number of audio packets + */ +#define FIFO_PACKETS_NUM 16 + +typedef struct JackData { + AVClass *class; + jack_client_t * client; + int activated; + sem_t packet_count; + jack_nframes_t sample_rate; + jack_nframes_t buffer_size; + jack_port_t ** ports; + int nports; + TimeFilter * timefilter; + AVFifoBuffer * new_pkts; + AVFifoBuffer * filled_pkts; + int pkt_xrun; + int jack_xrun; +} JackData; + +static int process_callback(jack_nframes_t nframes, void *arg) +{ + /* Warning: this function runs in realtime. One mustn't allocate memory here + * or do any other thing that could block. */ + + int i, j; + JackData *self = arg; + float * buffer; + jack_nframes_t latency, cycle_delay; + AVPacket pkt; + float *pkt_data; + double cycle_time; + + if (!self->client) + return 0; + + /* The approximate delay since the hardware interrupt as a number of frames */ + cycle_delay = jack_frames_since_cycle_start(self->client); + + /* Retrieve filtered cycle time */ + cycle_time = ff_timefilter_update(self->timefilter, + av_gettime() / 1000000.0 - (double) cycle_delay / self->sample_rate, + self->buffer_size); + + /* Check if an empty packet is available, and if there's enough space to send it back once filled */ + if ((av_fifo_size(self->new_pkts) < sizeof(pkt)) || (av_fifo_space(self->filled_pkts) < sizeof(pkt))) { + self->pkt_xrun = 1; + return 0; + } + + /* Retrieve empty (but allocated) packet */ + av_fifo_generic_read(self->new_pkts, &pkt, sizeof(pkt), NULL); + + pkt_data = (float *) pkt.data; + latency = 0; + + /* Copy and interleave audio data from the JACK buffer into the packet */ + for (i = 0; i < self->nports; i++) { + jack_latency_range_t range; + jack_port_get_latency_range(self->ports[i], JackCaptureLatency, &range); + latency += range.max; + buffer = jack_port_get_buffer(self->ports[i], self->buffer_size); + for (j = 0; j < self->buffer_size; j++) + pkt_data[j * self->nports + i] = buffer[j]; + } + + /* Timestamp the packet with the cycle start time minus the average latency */ + pkt.pts = (cycle_time - (double) latency / (self->nports * self->sample_rate)) * 1000000.0; + + /* Send the now filled packet back, and increase packet counter */ + av_fifo_generic_write(self->filled_pkts, &pkt, sizeof(pkt), NULL); + sem_post(&self->packet_count); + + return 0; +} + +static void shutdown_callback(void *arg) +{ + JackData *self = arg; + self->client = NULL; +} + +static int xrun_callback(void *arg) +{ + JackData *self = arg; + self->jack_xrun = 1; + ff_timefilter_reset(self->timefilter); + return 0; +} + +static int supply_new_packets(JackData *self, AVFormatContext *context) +{ + AVPacket pkt; + int test, pkt_size = self->buffer_size * self->nports * sizeof(float); + + /* Supply the process callback with new empty packets, by filling the new + * packets FIFO buffer with as many packets as possible. process_callback() + * can't do this by itself, because it can't allocate memory in realtime. */ + while (av_fifo_space(self->new_pkts) >= sizeof(pkt)) { + if ((test = av_new_packet(&pkt, pkt_size)) < 0) { + av_log(context, AV_LOG_ERROR, "Could not create packet of size %d\n", pkt_size); + return test; + } + av_fifo_generic_write(self->new_pkts, &pkt, sizeof(pkt), NULL); + } + return 0; +} + +static int start_jack(AVFormatContext *context) +{ + JackData *self = context->priv_data; + jack_status_t status; + int i, test; + + /* Register as a JACK client, using the context url as client name. */ + self->client = jack_client_open(context->url, JackNullOption, &status); + if (!self->client) { + av_log(context, AV_LOG_ERROR, "Unable to register as a JACK client\n"); + return AVERROR(EIO); + } + + sem_init(&self->packet_count, 0, 0); + + self->sample_rate = jack_get_sample_rate(self->client); + self->ports = av_malloc_array(self->nports, sizeof(*self->ports)); + if (!self->ports) + return AVERROR(ENOMEM); + self->buffer_size = jack_get_buffer_size(self->client); + + /* Register JACK ports */ + for (i = 0; i < self->nports; i++) { + char str[16]; + snprintf(str, sizeof(str), "input_%d", i + 1); + self->ports[i] = jack_port_register(self->client, str, + JACK_DEFAULT_AUDIO_TYPE, + JackPortIsInput, 0); + if (!self->ports[i]) { + av_log(context, AV_LOG_ERROR, "Unable to register port %s:%s\n", + context->url, str); + jack_client_close(self->client); + return AVERROR(EIO); + } + } + + /* Register JACK callbacks */ + jack_set_process_callback(self->client, process_callback, self); + jack_on_shutdown(self->client, shutdown_callback, self); + jack_set_xrun_callback(self->client, xrun_callback, self); + + /* Create time filter */ + self->timefilter = ff_timefilter_new (1.0 / self->sample_rate, self->buffer_size, 1.5); + if (!self->timefilter) { + jack_client_close(self->client); + return AVERROR(ENOMEM); + } + + /* Create FIFO buffers */ + self->filled_pkts = av_fifo_alloc_array(FIFO_PACKETS_NUM, sizeof(AVPacket)); + /* New packets FIFO with one extra packet for safety against underruns */ + self->new_pkts = av_fifo_alloc_array((FIFO_PACKETS_NUM + 1), sizeof(AVPacket)); + if (!self->new_pkts) { + jack_client_close(self->client); + return AVERROR(ENOMEM); + } + if ((test = supply_new_packets(self, context))) { + jack_client_close(self->client); + return test; + } + + return 0; + +} + +static void free_pkt_fifo(AVFifoBuffer **fifo) +{ + AVPacket pkt; + while (av_fifo_size(*fifo)) { + av_fifo_generic_read(*fifo, &pkt, sizeof(pkt), NULL); + av_packet_unref(&pkt); + } + av_fifo_freep(fifo); +} + +static void stop_jack(JackData *self) +{ + if (self->client) { + if (self->activated) + jack_deactivate(self->client); + jack_client_close(self->client); + } + sem_destroy(&self->packet_count); + free_pkt_fifo(&self->new_pkts); + free_pkt_fifo(&self->filled_pkts); + av_freep(&self->ports); + ff_timefilter_destroy(self->timefilter); +} + +static int audio_read_header(AVFormatContext *context) +{ + JackData *self = context->priv_data; + AVStream *stream; + int test; + + if ((test = start_jack(context))) + return test; + + stream = avformat_new_stream(context, NULL); + if (!stream) { + stop_jack(self); + return AVERROR(ENOMEM); + } + + stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; +#if HAVE_BIGENDIAN + stream->codecpar->codec_id = AV_CODEC_ID_PCM_F32BE; +#else + stream->codecpar->codec_id = AV_CODEC_ID_PCM_F32LE; +#endif + stream->codecpar->sample_rate = self->sample_rate; + stream->codecpar->channels = self->nports; + + avpriv_set_pts_info(stream, 64, 1, 1000000); /* 64 bits pts in us */ + return 0; +} + +static int audio_read_packet(AVFormatContext *context, AVPacket *pkt) +{ + JackData *self = context->priv_data; + struct timespec timeout = {0, 0}; + int test; + + /* Activate the JACK client on first packet read. Activating the JACK client + * means that process_callback() starts to get called at regular interval. + * If we activate it in audio_read_header(), we're actually reading audio data + * from the device before instructed to, and that may result in an overrun. */ + if (!self->activated) { + if (!jack_activate(self->client)) { + self->activated = 1; + av_log(context, AV_LOG_INFO, + "JACK client registered and activated (rate=%dHz, buffer_size=%d frames)\n", + self->sample_rate, self->buffer_size); + } else { + av_log(context, AV_LOG_ERROR, "Unable to activate JACK client\n"); + return AVERROR(EIO); + } + } + + /* Wait for a packet coming back from process_callback(), if one isn't available yet */ + timeout.tv_sec = av_gettime() / 1000000 + 3; + if (sem_timedwait(&self->packet_count, &timeout)) { + if (errno == ETIMEDOUT) { + av_log(context, AV_LOG_ERROR, + "Input error: timed out when waiting for JACK process callback output\n"); + } else { + char errbuf[128]; + int ret = AVERROR(errno); + av_strerror(ret, errbuf, sizeof(errbuf)); + av_log(context, AV_LOG_ERROR, "Error while waiting for audio packet: %s\n", + errbuf); + } + if (!self->client) + av_log(context, AV_LOG_ERROR, "Input error: JACK server is gone\n"); + + return AVERROR(EIO); + } + + if (self->pkt_xrun) { + av_log(context, AV_LOG_WARNING, "Audio packet xrun\n"); + self->pkt_xrun = 0; + } + + if (self->jack_xrun) { + av_log(context, AV_LOG_WARNING, "JACK xrun\n"); + self->jack_xrun = 0; + } + + /* Retrieve the packet filled with audio data by process_callback() */ + av_fifo_generic_read(self->filled_pkts, pkt, sizeof(*pkt), NULL); + + if ((test = supply_new_packets(self, context))) + return test; + + return 0; +} + +static int audio_read_close(AVFormatContext *context) +{ + JackData *self = context->priv_data; + stop_jack(self); + return 0; +} + +#define OFFSET(x) offsetof(JackData, x) +static const AVOption options[] = { + { "channels", "Number of audio channels.", OFFSET(nports), AV_OPT_TYPE_INT, { .i64 = 2 }, 1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM }, + { NULL }, +}; + +static const AVClass jack_indev_class = { + .class_name = "JACK indev", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, + .category = AV_CLASS_CATEGORY_DEVICE_AUDIO_INPUT, +}; + +AVInputFormat ff_jack_demuxer = { + .name = "jack", + .long_name = NULL_IF_CONFIG_SMALL("JACK Audio Connection Kit"), + .priv_data_size = sizeof(JackData), + .read_header = audio_read_header, + .read_packet = audio_read_packet, + .read_close = audio_read_close, + .flags = AVFMT_NOFILE, + .priv_class = &jack_indev_class, +}; diff --git a/node/docker-compose.arm64-build.yml b/node/docker-compose.arm64-build.yml new file mode 100644 index 0000000..6d6440b --- /dev/null +++ b/node/docker-compose.arm64-build.yml @@ -0,0 +1,27 @@ +version: "3" +services: + streaming: + image: orcasound/orcanode + build: ./ + command: ./stream.sh + restart: always + env_file: .env + ports: + - "1234:1234" + - "8080:8080" + devices: + - "/dev/snd:/dev/snd" + privileged: true + +# logspout: +# image: gliderlabs/logspout +# command: ${SYSLOG_URL} +# restart: always +# hostname: ${NODE_NAME} +# env_file: .env +# environment: +# - SYSLOG_HOSTNAME=${NODE_NAME} +# volumes: +# - /var/run/docker.sock:/var/run/docker.sock +# ports: +# - "8000:8000" diff --git a/node/docker-compose.arm64-pull.yml b/node/docker-compose.arm64-pull.yml new file mode 100644 index 0000000..d448026 --- /dev/null +++ b/node/docker-compose.arm64-pull.yml @@ -0,0 +1,28 @@ +version: "3" +services: + streaming: + image: orcasound/orcanode:arm64 + command: ./stream.sh + restart: always + env_file: .env + ports: + - "1234:1234" + - "8080:8080" + devices: + - "/dev/snd:/dev/snd" + privileged: true + + +# Not working on arm64 +# logspout: +# image: gliderlabs/logspout +# command: ${SYSLOG_URL} +# restart: always +# hostname: ${NODE_NAME} +# env_file: .env +# environment: +# - SYSLOG_HOSTNAME=${NODE_NAME} +# volumes: +# - /var/run/docker.sock:/var/run/docker.sock +# ports: +# - "8000:8000" From 6063e1bd9a1cb6ca8554df693e4fa424f548fe9f Mon Sep 17 00:00:00 2001 From: Scott Veirs Date: Sat, 2 Oct 2021 14:47:47 -0700 Subject: [PATCH 05/14] Bring README up to date, add links and fix broken ones. --- README.md | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0f01a49..9339a20 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # Orcasound's orcastream -This software contains audio tools and scripts for capturing, reformatting, transcoding and uploading audio for Orcasound. The directory structure reflects that we have developed a **base** set of tools and a couple of specific projects, orcanode and orcamseed (in the node and mseed directories). Orcasound hydrophone nodes stream by running the **node** code on Intel (amd64) or Raspberry Pi (arm32v7) platforms using a soundcard. While any soundcard should work, the most common one in use is the [Pisound](https://blokas.io/pisound/) board on either a Raspberry Pi 3B+ or 4. The other project (in the **mseed** directory) is for converting mseed format data to be streamed on Orcanode. This is mainly used for streaming audio date from the [OOI](https://oceanobservatories.org/ "OOI") (NSF-funded Ocean Observatory Initiative) hydrophones off the coast of Oregon. See the README in each of those directories for more info. +This software contains audio tools and scripts for capturing, reformatting, transcoding and uploading audio for Orcasound. The directory structure reflects that we have developed a **base** set of tools and a couple of specific projects, orcanode and orcamseed (in the node and mseed directories). Orcasound hydrophone nodes stream by running the **node** code on Intel (amd64) or [Raspberry Pi](https://www.raspberrypi.org/) (arm32v7) platforms using a soundcard. While any soundcard should work, the most common one in use is the [Pisound](https://blokas.io/pisound/) board on either a Raspberry Pi 3B+ or 4. The other project (in the **mseed** directory) is for converting mseed format data to be streamed via Orcanode through the Orcasound human & machine detection pipeline. This is mainly used for streaming audio data from the [OOI](https://oceanobservatories.org/ "OOI") (NSF-funded Ocean Observatory Initiative) hydrophones off the coast of Oregon. See the README in each of those directories for more info. ## Background & motivation -This code was developed for source nodes on the [Orcasound](http://orcasound.net) hydrophone network (WA, USA) -- thus the repository names! Our primary motivation is to make it easy for lots of folks to listen for whales using their favorite device/OS/browser. +This code was developed for source nodes on the [Orcasound](http://orcasound.net) hydrophone network (WA, USA) -- thus the repository names begin with "orca"! Our primary motivation is to make it easy for lots of folks to listen for whales using their favorite device/OS/browser. -We also aspire to use open-source software as much as possible. A long-term goal is to stream lossless FLAC-encoded data within DASH segments to a player that works optimally on as many listening devices as possible. +We also aspire to use open source software as much as possible. We rely heavily on [FFmpeg](https://www.ffmpeg.org/). One of our long-term goals is to stream lossless FLAC-encoded data within DASH segments to a player that works optimally on as many listening devices as possible. ## Getting Started -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See the deployment section (below) for notes on how to deploy the project on a live system like [live.orcaound.net](https://live.orcaound.net). If you want to set up your hardware to host a hydrophone within the Orcasound network, take a look at [how to join Orcasound](http://www.orcasound.net/join/) and [our prototype built from a Raspberry Pi3b with the Pisound Hat](http://www.orcasound.net/2018/04/27/orcasounds-new-live-audio-solution-from-hydrophone-to-headphone-with-a-raspberry-pi-computer-and-hls-dash-streaming-software/). -Audio data is acquired within a Docker container by ALSA/FFmpeg, written to /tmp directories, transferred to /mnt directories by rsync, and transferred to AWS S3 buckets by s3fs. Errors/etc are logged to LogDNA via a separate Docker container. +The general scheme is to acquire audio data from a sound card within a Docker container via ALSA or Jack and FFmpeg, and then stream the audio data with minimal latency to cloud-based storage (as of Oct 2021, we use AWS S3 buckets). Errors/etc are logged to LogDNA via a separate Docker container. ### Prerequisites @@ -22,7 +22,7 @@ An ARM or X86 device with a sound card (or other audio input devices) connected ### Installing -Create a base docker image for your architecture by running the script in /base/rpi or /base/amd64 as appropriate. You will need to create a .env file as appropriate for your projects. Here is an example of an .env file (tested/working as of June, 2021) without the keys that are common to all Orcasound projects: +Create a base docker image for your architecture by running the script in /base/rpi or /base/amd64 as appropriate. You will need to create a .env file as appropriate for your projects. Here is an example of an .env file (tested/working as of June, 2021)... ``` AWS_METADATA_SERVICE_TIMEOUT=5 @@ -40,7 +40,7 @@ SEGMENT_DURATION=10 LC_ALL=C.UTF-8 ``` -except that the following fields are excised and will be need added if you are integrating with the audio and logging data streaming systems of Orcasound. (You can request keys via the #hydrophone-nodes channel in the Orcasound Slack. As of June, 2021, we are continuing to use AWS S3 for storage and LogDNA for live-logging and troubleshooting.) +... except that the following fields are excised and will need to be added if you are integrating with the audio and logging systems of Orcasound: ``` AWSACCESSKEYID=YourAWSaccessKey @@ -50,6 +50,8 @@ SYSLOG_URL=syslog+tls://syslog-a.logdna.com:YourLogDNAPort SYSLOG_STRUCTURED_DATA='logdna@YourLogDNAnumber key="YourLogDNAKey" tag="docker" ``` +(You can request keys via the #hydrophone-nodes channel in the Orcasound Slack. As of October, 2021, we are continuing to use AWS S3 for storage and LogDNA for live-logging and troubleshooting.) + Here are explanations of some of the .env fields: * NODE_NAME should indicate your device and it's location, ideally in the form `device_location` (e.g. we call our Raspberry Pi staging device in Seattle `rpi_seattle`. @@ -60,31 +62,32 @@ Here are explanations of some of the .env fields: * SEGMENT_DURATION is the amount of seconds you want in each streamed lossy segment. - ## Running local tests -In the repository directory (where you also put your .env file) first copy the compose file you want to docker-compose.yml. For example if you are raspberry pi and you want to use the prebuilt image then copy docker-compose.rpi-pull.yml to docker-compose. Then run `docker-compose up -d`. Watch what happens using `htop`. If you want to verify files are being written to /tmp or /mnt directories, get the name of your streaming service using `docker-compose ps` (in this case `orcanode_streaming_1`) and then do `docker exec -it orcanode_streaming_1 /bin/bash` to get a bash shell within the running container. +At the root of the repository directory (where you also put your .env file) first copy the compose file you want to `docker-compose.yml`. For example, if you have a Raspberry Pi and you want to use the prebuilt image, then copy `docker-compose.rpi-pull.yml` to `docker-compose.yml`. Then run `docker-compose up -d`. Watch what happens using `htop`. If you want to verify files are being written to /tmp or /mnt directories, get the name of your streaming service using `docker-compose ps` (in this case `orcanode_streaming_1`) and then do `docker exec -it orcanode_streaming_1 /bin/bash` to get a bash shell within the running container. ### Running an end-to-end test -Once you've verified files are making it to your S3 bucket (with public read access), you can test the stream using a browser-based reference player. For example, with [Bitmovin HLS/MPEG/DASH player] you can use the drop-down menu to select HLS and then paste the URL for your current S3-based m3u8 manifest file into it to listen to the stream. +Once you've verified files are making it to your S3 bucket (with public read access), you can test the stream using a browser-based reference player. For example, with [Bitmovin HLS/MPEG/DASH player](https://bitmovin.com/demos/stream-test?format=hls&manifest=) you can use select HLS and then paste the URL for your current S3-based manifest (`.m3u8` file) to listen to the stream (and observe buffer levels and bitrate in real-time). Your URL should look something like this: ``` https://s3-us-west-2.amazonaws.com/dev-streaming-orcasound-net/rpi_seattle/hls/1526661120/live.m3u8 ``` -For end-to-end tests of Orcasound nodes, this schematic describes how sources map to the .dev, .beta, and .live subdomains of orcasound.net -- +For end-to-end tests of Orcasound nodes, this schematic describes how sources map to the `dev`, `beta`, and `live` subdomains of orcasound.net -- ![Schematic of Orcasound source-subdomain mapping](http://orcasound.net/img/orcasound-app/Orcasound-software-evolution-model.png "Orcasound software evolution model") -- and you can monitor your development stream via the web-app using this URL structure: -```dev.orcasound.net/dynamic/node_name``` so for node_name = rpi_orcasound_lab the test URL would be [dev.orcasound.net/dynamic/rpi_orcasound_lab](http://dev.orcasound.net/dynamic/rpi_orcasound_lab). +```dev.orcasound.net/dynamic/node_name``` + +For example, with node_name = rpi_orcasound_lab the test URL would be [dev.orcasound.net/dynamic/rpi_orcasound_lab](http://dev.orcasound.net/dynamic/rpi_orcasound_lab). ## Deployment -If you would like to add a node to the Orcasound hydrophone network, contact Scott for guidance on how to participate. +If you would like to add a node to the Orcasound hydrophone network, contact admin@orcasound.net for guidance on how to participate. ## Built With @@ -94,16 +97,16 @@ If you would like to add a node to the Orcasound hydrophone network, contact Sco ## Contributing -Please read [CONTRIBUTING.md](https://github.com/orcasound/orcanode/blob/master/CONTRIBUTING) for details on our code of conduct, and the process for submitting pull requests to us. +Please read [CONTRIBUTING.md](https://github.com/orcasound/orcanode/blob/master/CONTRIBUTING) for details on our code of conduct, and the process for submitting pull requests. ## Authors +* **Steve Hicks** - *Raspberry Pi expert* - [Steve on Github](https://github.com/mcshicks) * **Paul Cretu** - *Lead developer* - [Paul on Github](https://github.com/paulcretu) * **Scott Veirs** - *Project manager* - [Scott on Github](https://github.com/scottveirs) -* **Steve Hicks** - *Raspberry Pi expert* - [Steve on Github](https://github.com/mcshicks) * **Val Veirs** - *Hydrophone expert* - [Val on Github](https://github.com/veirs) -See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project. +See also the list of [orcanode contributors](https://github.com/orcasound/orcanode/graphs/contributors) who have helped this project and the [Orcasound Hacker Hall of Fame] who have advanced both Orcasound open source code and the hydrophone network in the habitat of the endangered Southern Resident killer whales. ## License @@ -111,6 +114,6 @@ This project is licensed under the GNU Affero General Public License v3.0 - see ## Acknowledgments -* Thanks to the backers of the 2017 Kickstarter that funded the development of this open-source code. +* Thanks to the backers of the 2017 Kickstarter that funded the development of this open source code. * Thanks to the makers of the Raspberry Pi and the Pisound HAT. * Thanks to the many friends and backers who helped improve maintain nodes and improve the [Orcasound app](https://github.com/orcasound/orcasite). From 884ea73f88e024a7499bfdfdbac8776bd95f9cd7 Mon Sep 17 00:00:00 2001 From: Evan-Scallan <83608345+evanjscallan@users.noreply.github.com> Date: Tue, 15 Mar 2022 22:45:53 -0500 Subject: [PATCH 06/14] typo fix on 'README.md' (Line 13). Fixed broken link by changing 'https://live.orcaound.net' to 'https://live.orcasound.net' (#37) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9339a20..d5b2ce6 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ We also aspire to use open source software as much as possible. We rely heavily ## Getting Started -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See the deployment section (below) for notes on how to deploy the project on a live system like [live.orcaound.net](https://live.orcaound.net). +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See the deployment section (below) for notes on how to deploy the project on a live system like [live.orcasound.net](https://live.orcasound.net). If you want to set up your hardware to host a hydrophone within the Orcasound network, take a look at [how to join Orcasound](http://www.orcasound.net/join/) and [our prototype built from a Raspberry Pi3b with the Pisound Hat](http://www.orcasound.net/2018/04/27/orcasounds-new-live-audio-solution-from-hydrophone-to-headphone-with-a-raspberry-pi-computer-and-hls-dash-streaming-software/). From 590ed6ba61d6cbd58d76ec8e9b8d42ef140b7606 Mon Sep 17 00:00:00 2001 From: karan mishra Date: Thu, 16 Jun 2022 20:05:46 +0530 Subject: [PATCH 07/14] Fetch data using ooipy --- base/Dockerfile | 48 +++++++++++++++++++++++++++++++++++ mseed/Dockerfile | 2 +- mseed/docker-compose.yml | 32 ++++++++++++------------ mseed/mseedpull.py | 4 +-- mseed/ooipypull.py | 54 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 19 deletions(-) create mode 100644 base/Dockerfile create mode 100644 mseed/ooipypull.py diff --git a/base/Dockerfile b/base/Dockerfile new file mode 100644 index 0000000..371fa5d --- /dev/null +++ b/base/Dockerfile @@ -0,0 +1,48 @@ +# Node Dockerfile for hydrophone streaming + +# Use official debian image, but pull the armhf (v7+) image explicitly because +# Docker currently has a bug where armel is used instead when relying on +# multiarch manifest: https://github.com/moby/moby/issues/34875 +# When this is fixed, this can be changed to just `FROM debian:stretch-slim` +FROM python:3.6-slim-buster +# FROM arm32v7/debian:buster-slim +MAINTAINER Orcasound + +####################### Install FFMPEG ##################################################### + +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg + +# Install misc tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + # General tools + htop \ + nano \ + sox \ + tmux \ + wget \ + curl \ + git + +# Upgrade OS +RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" + +# Set default command to bash as a placeholder +CMD ["/bin/bash"] + +# Make sure we're the root user +USER root + +WORKDIR /root + + +############################### Install boto and inotify libraies ################################### + +RUN apt-get update && apt-get install -y python3-pip +RUN pip3 install -U boto3 inotify + +############################## Copy common scripts ################################################## + +COPY . . + +# Clean up APT when done. +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/mseed/Dockerfile b/mseed/Dockerfile index 1a09609..d5c5d9b 100644 --- a/mseed/Dockerfile +++ b/mseed/Dockerfile @@ -5,7 +5,7 @@ # multiarch manifest: https://github.com/moby/moby/issues/34875 # When this is fixed, this can be changed to just `FROM debian:stretch-slim` FROM orcastream/orcabase:latest -MAINTAINER Orcasound +# MAINTAINER Orcasound RUN pip3 install numpy RUN pip3 install obspy diff --git a/mseed/docker-compose.yml b/mseed/docker-compose.yml index 0cbb5cc..087b998 100644 --- a/mseed/docker-compose.yml +++ b/mseed/docker-compose.yml @@ -3,8 +3,8 @@ services: pull: image: orcastream/orcamseed build: ./ - # command: tail -F README.md - command: python3 mseedpull.py + command: tail -F README.md + #command: python3 mseedpull.py restart: always env_file: .env volumes: @@ -12,25 +12,25 @@ services: stream: image: orcastream/orcamseed build: ./ - # command: tail -F README.md - command: ./streamfiles.sh + command: tail -F README.md + # command: ./streamfiles.sh restart: always env_file: .env volumes: - data:/root/data privileged: true - logspout: - image: gliderlabs/logspout - command: ${SYSLOG_URL} - restart: always - hostname: ${NODE_NAME} - env_file: .env - environment: - - SYSLOG_HOSTNAME=${NODE_NAME} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - ports: - - "8000:8000" + # logspout: + # image: gliderlabs/logspout + # command: ${SYSLOG_URL} + # restart: always + # hostname: ${NODE_NAME} + # env_file: .env + # environment: + # - SYSLOG_HOSTNAME=${NODE_NAME} + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # ports: + # - "8000:8000" volumes: diff --git a/mseed/mseedpull.py b/mseed/mseedpull.py index d37394c..01c9a29 100644 --- a/mseed/mseedpull.py +++ b/mseed/mseedpull.py @@ -27,7 +27,7 @@ import sys DELAY = int(os.environ["STREAM_DELAY"]) -# DELAY = 6.5 +#DELAY = 6.5 SEGMENT = int(os.environ["DELAY_SEGMENT"]) # maybe change to "buffer" # SEGMENT = 1 # TODO Should put this in env variable @@ -181,7 +181,7 @@ def main_loop(): files = [] while True: # TODO this converts correctly but after queue files it - # get overwritten by fetchandconver + # get overwritten by fetchandconvert # you need to change it fetchandconvert appends the exisitng list # and all timedate stamps are only converted once at most. log.debug("checking") diff --git a/mseed/ooipypull.py b/mseed/ooipypull.py new file mode 100644 index 0000000..3cd46a4 --- /dev/null +++ b/mseed/ooipypull.py @@ -0,0 +1,54 @@ +import ooipy +import os +import datetime +import shutil +import logging +import logging.handlers +import sys + + +LOGLEVEL = logging.DEBUG +NODE = os.environ["NODE_NAME"] +BASEPATH = os.path.join('/tmp', NODE) +PATH = os.path.join(BASEPATH, 'hls') + +log = logging.getLogger(__name__) + +log.setLevel(LOGLEVEL) + +handler = logging.StreamHandler(sys.stdout) + +formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s') +handler.setFormatter(formatter) + +log.addHandler(handler) + +def fetchData(start_time, segment_length, end_time, node): + os.makedirs(BASEPATH, exist_ok=True) + os.makedirs(PATH, exist_ok=True) + while start_time < end_time: + segment_end = min(start_time + segment_length, end_time) + hydrophone_data = ooipy.request.hydrophone_request.get_acoustic_data( + start_time, segment_end, node, verbose=True, data_gap_mode=2 + ) + if hydrophone_data is None: + print(f"Could not get data from {start_time} to {segment_end}") + start_time = segment_end + continue + print(f"data: {hydrophone_data}") + datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] + wav_name = f"{datestr}.wav" + hydrophone_data.wav_write(wav_name) + sub_directory = start_time.strftime("%Y-%m-%d") + file_path = os.path.join(PATH, sub_directory) + if not os.path.exists(file_path): + os.makedirs(file_path) + shutil.move(wav_name, file_path) + # os.remove(wav_name) + start_time = segment_end + +start_time = datetime.datetime(2021, 4, 27) +end_time = datetime.datetime(2022, 4, 30) +segment_length = datetime.timedelta(minutes = 5) + +fetchData(start_time, segment_length, end_time, 'PC01A') From cf995c8253b6327fccd8ba44248858ed41a71111 Mon Sep 17 00:00:00 2001 From: karan mishra Date: Fri, 17 Jun 2022 19:32:23 +0530 Subject: [PATCH 08/14] remove conflict markers --- mseed/docker-compose.yml | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/mseed/docker-compose.yml b/mseed/docker-compose.yml index 072a6b5..1e25020 100644 --- a/mseed/docker-compose.yml +++ b/mseed/docker-compose.yml @@ -4,11 +4,7 @@ services: image: orcastream/orcamseed build: ./ command: tail -F README.md -<<<<<<< HEAD - #command: python3 mseedpull.py -======= # command: python3 mseedpull.py ->>>>>>> cf82246904a1e725e1fc180ebfb053cd96cf8475 restart: always env_file: .env volumes: @@ -16,31 +12,13 @@ services: stream: image: orcastream/orcamseed build: ./ - command: tail -F README.md -<<<<<<< HEAD - # command: ./streamfiles.sh -======= + # command: tail -F README.md command: ./streamfiles.sh ->>>>>>> cf82246904a1e725e1fc180ebfb053cd96cf8475 restart: always env_file: .env volumes: - data:/root/data privileged: true -<<<<<<< HEAD - # logspout: - # image: gliderlabs/logspout - # command: ${SYSLOG_URL} - # restart: always - # hostname: ${NODE_NAME} - # env_file: .env - # environment: - # - SYSLOG_HOSTNAME=${NODE_NAME} - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # ports: - # - "8000:8000" -======= logspout: image: gliderlabs/logspout command: ${SYSLOG_URL} @@ -53,7 +31,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock ports: - "8000:8000" ->>>>>>> cf82246904a1e725e1fc180ebfb053cd96cf8475 volumes: From 96c09386858fa68720806c63c34f973ee889ae9f Mon Sep 17 00:00:00 2001 From: karan mishra Date: Sat, 18 Jun 2022 23:16:17 +0530 Subject: [PATCH 09/14] add requirements to DockerFile --- mseed/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mseed/Dockerfile b/mseed/Dockerfile index 1a09609..42131ef 100644 --- a/mseed/Dockerfile +++ b/mseed/Dockerfile @@ -9,6 +9,8 @@ MAINTAINER Orcasound RUN pip3 install numpy RUN pip3 install obspy +RUN pip3 install ooipy +RUN pip3 install ffmpeg-python ############################### Copy files ##################################### From 4a4a5b6d768f8678a4b6c7fbe07aec82a2c0c73b Mon Sep 17 00:00:00 2001 From: karan mishra Date: Mon, 20 Jun 2022 18:42:37 +0530 Subject: [PATCH 10/14] convert to .ts --- base/Dockerfile | 48 ---------------------------------------------- mseed/ooipypull.py | 30 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 57 deletions(-) delete mode 100644 base/Dockerfile diff --git a/base/Dockerfile b/base/Dockerfile deleted file mode 100644 index 371fa5d..0000000 --- a/base/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# Node Dockerfile for hydrophone streaming - -# Use official debian image, but pull the armhf (v7+) image explicitly because -# Docker currently has a bug where armel is used instead when relying on -# multiarch manifest: https://github.com/moby/moby/issues/34875 -# When this is fixed, this can be changed to just `FROM debian:stretch-slim` -FROM python:3.6-slim-buster -# FROM arm32v7/debian:buster-slim -MAINTAINER Orcasound - -####################### Install FFMPEG ##################################################### - -RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg - -# Install misc tools -RUN apt-get update && apt-get install -y --no-install-recommends \ - # General tools - htop \ - nano \ - sox \ - tmux \ - wget \ - curl \ - git - -# Upgrade OS -RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" - -# Set default command to bash as a placeholder -CMD ["/bin/bash"] - -# Make sure we're the root user -USER root - -WORKDIR /root - - -############################### Install boto and inotify libraies ################################### - -RUN apt-get update && apt-get install -y python3-pip -RUN pip3 install -U boto3 inotify - -############################## Copy common scripts ################################################## - -COPY . . - -# Clean up APT when done. -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/mseed/ooipypull.py b/mseed/ooipypull.py index 3cd46a4..2bae482 100644 --- a/mseed/ooipypull.py +++ b/mseed/ooipypull.py @@ -12,6 +12,8 @@ BASEPATH = os.path.join('/tmp', NODE) PATH = os.path.join(BASEPATH, 'hls') +ffmpeg_cmd = "ffmpeg -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE_NAME/hls/$timestamp/live.m3u8' -segment_list_flags -segment_time 10 -segment_format mpegts -ar 64000 -ac 1 -acodec aac '/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts'" + log = logging.getLogger(__name__) log.setLevel(LOGLEVEL) @@ -37,18 +39,28 @@ def fetchData(start_time, segment_length, end_time, node): continue print(f"data: {hydrophone_data}") datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] - wav_name = f"{datestr}.wav" - hydrophone_data.wav_write(wav_name) sub_directory = start_time.strftime("%Y-%m-%d") file_path = os.path.join(PATH, sub_directory) + wav_name = f"{datestr}.wav" + hydrophone_data.wav_write(wav_name) + ts_name = f"{datestr}.ts" + with open("files.txt", "w") as f: + f.write('file ' + "'.." + file_path + "/" + ts_name + "'" + "\n") + os.system('ffmpeg -i {wavfile} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfile}'.format(wavfile=wav_name, tsfile=ts_name)) + os.system("ffmpeg -f concat -safe 0 -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE/hls/$sub_directory/live.m3u8'") if not os.path.exists(file_path): - os.makedirs(file_path) - shutil.move(wav_name, file_path) - # os.remove(wav_name) + os.makedirs(file_path) + shutil.move(ts_name, file_path) + os.remove(wav_name) start_time = segment_end -start_time = datetime.datetime(2021, 4, 27) -end_time = datetime.datetime(2022, 4, 30) -segment_length = datetime.timedelta(minutes = 5) +def _main(): + + start_time = datetime.datetime(2021, 4, 27) + end_time = datetime.datetime(2022, 4, 30) + segment_length = datetime.timedelta(seconds = 10) + + fetchData(start_time, segment_length, end_time, 'PC01A') + -fetchData(start_time, segment_length, end_time, 'PC01A') +_main() From a4ef1659bd62095490798f8485a768e25cde1acc Mon Sep 17 00:00:00 2001 From: karan mishra Date: Wed, 22 Jun 2022 23:54:10 +0530 Subject: [PATCH 11/14] generate .m3u8 manifest file --- mseed/ooipypull.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mseed/ooipypull.py b/mseed/ooipypull.py index 2bae482..6865b56 100644 --- a/mseed/ooipypull.py +++ b/mseed/ooipypull.py @@ -44,13 +44,15 @@ def fetchData(start_time, segment_length, end_time, node): wav_name = f"{datestr}.wav" hydrophone_data.wav_write(wav_name) ts_name = f"{datestr}.ts" - with open("files.txt", "w") as f: - f.write('file ' + "'.." + file_path + "/" + ts_name + "'" + "\n") - os.system('ffmpeg -i {wavfile} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfile}'.format(wavfile=wav_name, tsfile=ts_name)) - os.system("ffmpeg -f concat -safe 0 -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE/hls/$sub_directory/live.m3u8'") + #os.system('ffmpeg -i {wavfile} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfile}'.format(wavfile=wav_name, tsfile=ts_name)) + #os.system("ffmpeg -f concat -safe 0 -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE/hls/$sub_directory/live.m3u8'") + os.system("ffmpeg -i {wavfile} -f segment -segment_list './live.m3u8' -strftime 1 -segment_time 10 -segment_format mpegts -ac 1 -acodec aac {tsfile}".format(wavfile=wav_name, tsfile=ts_name)) if not os.path.exists(file_path): os.makedirs(file_path) shutil.move(ts_name, file_path) + if not os.path.exists(os.path.join(file_path, 'live.m3u8')): + shutil.move('/root/live.m3u8', file_path) + shutil.copy('/root/live.m3u8', file_path) os.remove(wav_name) start_time = segment_end From 7733be6cc837431092f4201336fee1a7f5b5ab90 Mon Sep 17 00:00:00 2001 From: karan mishra Date: Wed, 29 Jun 2022 21:11:06 +0530 Subject: [PATCH 12/14] resolve shutil error --- mseed/docker-compose.yml | 24 ++++++++++++------------ mseed/ooipypull.py | 8 +++++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/mseed/docker-compose.yml b/mseed/docker-compose.yml index 1e25020..2ac0461 100644 --- a/mseed/docker-compose.yml +++ b/mseed/docker-compose.yml @@ -19,18 +19,18 @@ services: volumes: - data:/root/data privileged: true - logspout: - image: gliderlabs/logspout - command: ${SYSLOG_URL} - restart: always - hostname: ${NODE_NAME} - env_file: .env - environment: - - SYSLOG_HOSTNAME=${NODE_NAME} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - ports: - - "8000:8000" + # logspout: + # image: gliderlabs/logspout + # command: ${SYSLOG_URL} + # restart: always + # hostname: ${NODE_NAME} + # env_file: .env + # environment: + # - SYSLOG_HOSTNAME=${NODE_NAME} + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # ports: + # - "8000:8000" volumes: diff --git a/mseed/ooipypull.py b/mseed/ooipypull.py index 6865b56..7d59331 100644 --- a/mseed/ooipypull.py +++ b/mseed/ooipypull.py @@ -49,13 +49,15 @@ def fetchData(start_time, segment_length, end_time, node): os.system("ffmpeg -i {wavfile} -f segment -segment_list './live.m3u8' -strftime 1 -segment_time 10 -segment_format mpegts -ac 1 -acodec aac {tsfile}".format(wavfile=wav_name, tsfile=ts_name)) if not os.path.exists(file_path): os.makedirs(file_path) - shutil.move(ts_name, file_path) - if not os.path.exists(os.path.join(file_path, 'live.m3u8')): - shutil.move('/root/live.m3u8', file_path) + shutil.move(os.path.join('/root', ts_name), os.path.join(file_path, ts_name)) shutil.copy('/root/live.m3u8', file_path) os.remove(wav_name) start_time = segment_end + with open('latest.txt', 'w') as f: + f.write(sub_directory) + shutil.copy('/root/latest.txt', BASEPATH) + def _main(): start_time = datetime.datetime(2021, 4, 27) From ce581e09e713f9dd9f6879941dfdf23c9bd15304 Mon Sep 17 00:00:00 2001 From: karan mishra Date: Fri, 1 Jul 2022 23:27:27 +0530 Subject: [PATCH 13/14] add prefix to .ts filenames --- mseed/ooipypull.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/mseed/ooipypull.py b/mseed/ooipypull.py index 7d59331..a7ae36c 100644 --- a/mseed/ooipypull.py +++ b/mseed/ooipypull.py @@ -6,14 +6,13 @@ import logging.handlers import sys - LOGLEVEL = logging.DEBUG +PREFIX = os.environ["TIME_PREFIX"] +DELAY = os.environ["DELAY_SEGMENT"] NODE = os.environ["NODE_NAME"] BASEPATH = os.path.join('/tmp', NODE) PATH = os.path.join(BASEPATH, 'hls') -ffmpeg_cmd = "ffmpeg -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE_NAME/hls/$timestamp/live.m3u8' -segment_list_flags -segment_time 10 -segment_format mpegts -ar 64000 -ac 1 -acodec aac '/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts'" - log = logging.getLogger(__name__) log.setLevel(LOGLEVEL) @@ -41,9 +40,9 @@ def fetchData(start_time, segment_length, end_time, node): datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] sub_directory = start_time.strftime("%Y-%m-%d") file_path = os.path.join(PATH, sub_directory) - wav_name = f"{datestr}.wav" + wav_name = "{date}.wav".format(date=datestr) hydrophone_data.wav_write(wav_name) - ts_name = f"{datestr}.ts" + ts_name = "{prefix}{date}.ts".format(prefix=PREFIX, date=datestr) #os.system('ffmpeg -i {wavfile} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfile}'.format(wavfile=wav_name, tsfile=ts_name)) #os.system("ffmpeg -f concat -safe 0 -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE/hls/$sub_directory/live.m3u8'") os.system("ffmpeg -i {wavfile} -f segment -segment_list './live.m3u8' -strftime 1 -segment_time 10 -segment_format mpegts -ac 1 -acodec aac {tsfile}".format(wavfile=wav_name, tsfile=ts_name)) @@ -60,11 +59,10 @@ def fetchData(start_time, segment_length, end_time, node): def _main(): - start_time = datetime.datetime(2021, 4, 27) - end_time = datetime.datetime(2022, 4, 30) segment_length = datetime.timedelta(seconds = 10) + fixed_delay = datetime.timedelta(hours=8) - fetchData(start_time, segment_length, end_time, 'PC01A') + fetchData(datetime.datetime.utcnow() - fixed_delay, segment_length, datetime.datetime.utcnow(), 'PC01A') _main() From c2f5f5842cb65c449fd890f7932c46f72c734a62 Mon Sep 17 00:00:00 2001 From: karan mishra Date: Sun, 3 Jul 2022 01:29:43 +0530 Subject: [PATCH 14/14] fetch for lags --- mseed/ooipypull.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/mseed/ooipypull.py b/mseed/ooipypull.py index a7ae36c..8175c48 100644 --- a/mseed/ooipypull.py +++ b/mseed/ooipypull.py @@ -29,6 +29,16 @@ def fetchData(start_time, segment_length, end_time, node): os.makedirs(PATH, exist_ok=True) while start_time < end_time: segment_end = min(start_time + segment_length, end_time) + #paths and filenames + datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] + sub_directory = start_time.strftime("%Y-%m-%d") + file_path = os.path.join(PATH, sub_directory) + wav_name = "{date}.wav".format(date=datestr) + ts_name = "{prefix}{date}.ts".format(prefix=PREFIX, date=datestr) + #fetch if file doesn't already exist + if(os.path.exists(os.path.join(file_path, ts_name))): + print("EXISTS") + continue hydrophone_data = ooipy.request.hydrophone_request.get_acoustic_data( start_time, segment_end, node, verbose=True, data_gap_mode=2 ) @@ -37,14 +47,7 @@ def fetchData(start_time, segment_length, end_time, node): start_time = segment_end continue print(f"data: {hydrophone_data}") - datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] - sub_directory = start_time.strftime("%Y-%m-%d") - file_path = os.path.join(PATH, sub_directory) - wav_name = "{date}.wav".format(date=datestr) hydrophone_data.wav_write(wav_name) - ts_name = "{prefix}{date}.ts".format(prefix=PREFIX, date=datestr) - #os.system('ffmpeg -i {wavfile} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfile}'.format(wavfile=wav_name, tsfile=ts_name)) - #os.system("ffmpeg -f concat -safe 0 -i files.txt -flush_packets 0 -f segment -segment_list '/tmp/$NODE/hls/$sub_directory/live.m3u8'") os.system("ffmpeg -i {wavfile} -f segment -segment_list './live.m3u8' -strftime 1 -segment_time 10 -segment_format mpegts -ac 1 -acodec aac {tsfile}".format(wavfile=wav_name, tsfile=ts_name)) if not os.path.exists(file_path): os.makedirs(file_path) @@ -57,12 +60,26 @@ def fetchData(start_time, segment_length, end_time, node): f.write(sub_directory) shutil.copy('/root/latest.txt', BASEPATH) + def _main(): segment_length = datetime.timedelta(seconds = 10) fixed_delay = datetime.timedelta(hours=8) - fetchData(datetime.datetime.utcnow() - fixed_delay, segment_length, datetime.datetime.utcnow(), 'PC01A') + while True: + end_time = datetime.datetime.utcnow() + start_time = end_time - datetime.timedelta(hours=8) + + #near live fetch + fetchData(start_time, segment_length, end_time, 'PC01A') + + #delayed fetch + fetchData(end_time-datetime.timedelta(hours=24), segment_length, end_time, 'PC01A') + + start_time, end_time = end_time, datetime.datetime.utcnow() + + + _main()