yt-extract-sound.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fc3e3cd0-79a1-44e9-a8b1-7408ba6b9035",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pytube\n",
      "  Downloading pytube-15.0.0-py3-none-any.whl.metadata (5.0 kB)\n",
      "Downloading pytube-15.0.0-py3-none-any.whl (57 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: pytube\n",
      "Successfully installed pytube-15.0.0\n",
      "Collecting moviepy\n",
      "  Downloading moviepy-1.0.3.tar.gz (388 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.3/388.3 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting decorator<5.0,>=4.0.2 (from moviepy)\n",
      "  Downloading decorator-4.4.2-py2.py3-none-any.whl.metadata (4.2 kB)\n",
      "Requirement already satisfied: tqdm<5.0,>=4.11.2 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from moviepy) (4.65.0)\n",
      "Requirement already satisfied: requests<3.0,>=2.8.1 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from moviepy) (2.31.0)\n",
      "Collecting proglog<=1.0.0 (from moviepy)\n",
      "  Downloading proglog-0.1.10-py3-none-any.whl.metadata (639 bytes)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from moviepy) (1.26.4)\n",
      "Collecting imageio<3.0,>=2.5 (from moviepy)\n",
      "  Downloading imageio-2.34.0-py3-none-any.whl.metadata (4.9 kB)\n",
      "Collecting imageio_ffmpeg>=0.2.0 (from moviepy)\n",
      "  Downloading imageio-ffmpeg-0.4.9.tar.gz (17 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: pillow>=8.3.2 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from imageio<3.0,>=2.5->moviepy) (10.2.0)\n",
      "Requirement already satisfied: setuptools in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from imageio_ffmpeg>=0.2.0->moviepy) (68.2.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/lily/anaconda3/envs/tts-yt/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (2024.2.2)\n",
      "Downloading decorator-4.4.2-py2.py3-none-any.whl (9.2 kB)\n",
      "Downloading imageio-2.34.0-py3-none-any.whl (313 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m313.4/313.4 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading proglog-0.1.10-py3-none-any.whl (6.1 kB)\n",
      "Building wheels for collected packages: moviepy, imageio_ffmpeg\n",
      "  Building wheel for moviepy (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for moviepy: filename=moviepy-1.0.3-py3-none-any.whl size=110721 sha256=a7aba27d2b2d26d8225159edf7035c145b31af3cf4ae2f93dd502cb943d306aa\n",
      "  Stored in directory: /Users/lily/Library/Caches/pip/wheels/29/15/e4/4f790bec6acd51a00b67e8ee1394f0bc6e0135c315f8ff399a\n",
      "  Building wheel for imageio_ffmpeg (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for imageio_ffmpeg: filename=imageio_ffmpeg-0.4.9-py3-none-any.whl size=16367 sha256=a179343d3f256e7c0f3642affd66a82bbdbff54f0a313fea0796dd2baff0005d\n",
      "  Stored in directory: /Users/lily/Library/Caches/pip/wheels/a0/a2/a5/801b8c9041c8149b486509ade5eb48e3c474d17d31f3746860\n",
      "Successfully built moviepy imageio_ffmpeg\n",
      "Installing collected packages: proglog, imageio_ffmpeg, imageio, decorator, moviepy\n",
      "  Attempting uninstall: decorator\n",
      "    Found existing installation: decorator 5.1.1\n",
      "    Uninstalling decorator-5.1.1:\n",
      "      Successfully uninstalled decorator-5.1.1\n",
      "Successfully installed decorator-4.4.2 imageio-2.34.0 imageio_ffmpeg-0.4.9 moviepy-1.0.3 proglog-0.1.10\n"
     ]
    }
   ],
   "source": [
    "!pip install pytube\n",
    "!pip install moviepy\n",
    "!pip install pydub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "1d95804d-51a1-4b65-ab7d-5ed342677350",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydub import AudioSegment\n",
    "from pytube import YouTube\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "0d741a6d-b829-4911-ba89-44533bd28f15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully open_aws_dynamodb_session with environment variables. at 2024-03-05 21:41:50\n",
      "\n",
      "Unique video ids: ['ICMfseFhE-E', 'C_xsXnRd_uc', 'xKAYvtoWxGs'] written to file\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "import os\n",
    "from datetime import datetime\n",
    "from dotenv import load_dotenv\n",
    "import random\n",
    "\n",
    "\n",
    "def current_timestamp():\n",
    "    \"\"\"Returns the current timestamp formatted for readability.\"\"\"\n",
    "    return datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "def get_env_variables():\n",
    "    \"\"\"Fetch all necessary configurations from environment variables.\"\"\"\n",
    "    return {\n",
    "        'DEVELOPER_KEY': os.getenv('DEVELOPER_KEY'),\n",
    "        'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'),\n",
    "        'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),\n",
    "        'SEARCH_CACHE_TABLE': os.getenv('SEARCH_CACHE_TABLE'),\n",
    "        'RESULTS_TABLE_NAME': os.getenv('RESULTS_TABLE_NAME'),\n",
    "        'SEARCH_QUERY': os.getenv('SEARCH_QUERY'),\n",
    "        'MAX_RESULTS': int(os.getenv('MAX_RESULTS', 50)),\n",
    "        'ORDER': os.getenv('ORDER', 'viewCount'),\n",
    "        'VIDEO_DURATION': os.getenv('VIDEO_DURATION', 'medium'),\n",
    "        'PUBLISHED_AFTER': os.getenv('PUBLISHED_AFTER', '2010-01-01T00:00:00Z'),\n",
    "        'PUBLISHED_BEFORE': os.getenv('PUBLISHED_BEFORE', '2024-12-31T23:59:59Z'),\n",
    "        'RELEVANCE_LANGUAGE': os.getenv('RELEVANCE_LANGUAGE', 'en'),\n",
    "        # 'VIDEO_CATEGORY_ID': os.getenv('VIDEO_CATEGORY_ID', '10'),\n",
    "        'AWS_REGION': os.getenv('AWS_REGION')  # Add AWS region to the environment variables\n",
    "    }\n",
    "\n",
    "def open_aws_dynamodb_session(options):\n",
    "    \"\"\"\n",
    "    Configures a boto3 session using AWS credentials read from environment variable.\n",
    "    Returns a boto3 DynamoDB resource configured with these credentials.\n",
    "    \"\"\"\n",
    "    try:\n",
    "\n",
    "        # Configure the boto3 session with the read credentials\n",
    "        session = boto3.Session(\n",
    "            aws_access_key_id=options['AWS_ACCESS_KEY_ID'],\n",
    "            aws_secret_access_key=options['AWS_SECRET_ACCESS_KEY'],\n",
    "            region_name=options['AWS_REGION']  # Specify your AWS region\n",
    "        )\n",
    "        dynamodb_resource = session.resource('dynamodb')\n",
    "        print(f\"Successfully open_aws_dynamodb_session with environment variables. at {current_timestamp()}\\n\")\n",
    "        # Return the configured DynamoDB resource\n",
    "        return dynamodb_resource\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Error open_aws_dynamodb_session from environment variables: {e} at {current_timestamp()}\\n\")\n",
    "        return None\n",
    "\n",
    "def convert_duration_to_minutes(duration):\n",
    "    if 'H' in duration:\n",
    "        hours = int(duration.split('H')[0][2:])\n",
    "    else:\n",
    "        hours = 0\n",
    "    if 'M' in duration:\n",
    "        minutes = int(duration.split('M')[0].split('T')[-1])\n",
    "    else:\n",
    "        minutes = 0\n",
    "    if 'S' in duration:\n",
    "        seconds = int(duration.split('S')[0].split('M')[-1])\n",
    "    else:\n",
    "        seconds = 0\n",
    "    return hours * 3600 + minutes * 60 + seconds\n",
    "\n",
    "\n",
    "def retrieve_video_ids(table_name, search_term, max_duration_seconds=15, dynamodb=None):\n",
    "    if dynamodb is None:\n",
    "        dynamodb = boto3.resource('dynamodb')\n",
    "\n",
    "    table = dynamodb.Table(table_name)\n",
    "\n",
    "    response = table.scan(\n",
    "        ProjectionExpression='#videoId, #title, #duration',  # Adjust 'title' and 'duration' based on your column names\n",
    "        ExpressionAttributeNames={\n",
    "            '#videoId': 'videoId',\n",
    "            '#title': 'title',\n",
    "            '#duration': 'duration'\n",
    "        }\n",
    "    )\n",
    "\n",
    "    items = response.get('Items', [])\n",
    "\n",
    "    # Filter items based on search term and duration\n",
    "    filtered_items = [\n",
    "        item for item in items\n",
    "        if search_term.lower() in item.get('title', '').lower() and\n",
    "        convert_duration_to_minutes(item.get('duration', '')) < max_duration_minutes\n",
    "    ]\n",
    "\n",
    "    # Extract video IDs from filtered items, limiting to the specified total\n",
    "    video_ids = [item['videoId'] for item in filtered_items]\n",
    "\n",
    "    return video_ids\n",
    "\n",
    "def compare_video_ids_with_file(video_ids, file_path):\n",
    "    # Read video IDs from the file\n",
    "    with open(file_path, 'r') as file:\n",
    "        # Iterate through each line and extract video IDs\n",
    "        file_video_ids = [line.strip().split(',') for line in file]\n",
    "\n",
    "    # Check if any line in the file exactly matches the given video IDs\n",
    "    return any(set(video_ids) == set(line) for line in file_video_ids)\n",
    "\n",
    "def write_video_ids_to_file(video_ids, file_path):\n",
    "    with open(file_path, 'a') as file:\n",
    "        file.write(','.join(video_ids) + '\\n')\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    load_dotenv()\n",
    "    options = get_env_variables()\n",
    "    if options:\n",
    "        table_name = options['RESULTS_TABLE_NAME']\n",
    "        dynamodb = open_aws_dynamodb_session(options)\n",
    "        \n",
    "        if dynamodb:\n",
    "            file_path = 'batched_videos_list.txt'\n",
    "            search_term = 'meditation'\n",
    "            max_duration_seconds = 500\n",
    "            result_limit = 3\n",
    "            first_pass_check = True\n",
    "\n",
    "            # search for a comprehensive list of video ids from dynamodb\n",
    "            video_ids = retrieve_video_ids(table_name, search_term, max_duration_seconds, dynamodb)            \n",
    "            for _ in range(10):  # Run the loop 20 times\n",
    "                \n",
    "                # if both are false the new record is unique\n",
    "                if first_pass_check or compare_video_ids_with_file(video_ids, file_path):\n",
    "                    if not first_pass_check:\n",
    "                        print('Exact match found in records, running again')\n",
    "                    first_pass_check = False\n",
    "\n",
    "                    #randomly shuffle the list and return a limited list of video ids\n",
    "                    random.shuffle(video_ids)\n",
    "                    video_ids = video_ids[:result_limit]\n",
    "\n",
    "            # save record to txt file\n",
    "            write_video_ids_to_file(video_ids, file_path)\n",
    "            print(f\"Unique video ids: {video_ids} written to file\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "25718a02-50c9-4366-b66c-4a198d5b131f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ICMfseFhE-E', 'C_xsXnRd_uc', 'xKAYvtoWxGs']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "video_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "0ba620f2-4db4-4fc6-b535-9b0d8e509d8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import asyncio\n",
    "from pytube import YouTube\n",
    "import os\n",
    "import nest_asyncio\n",
    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "def replace_mp4_with_wav(file_name):\n",
    "    return re.sub(r'\\.mp4$', '.wav', file_name)\n",
    "\n",
    "async def download_and_process_video(video_id, parent_output_folder, output_folder):\n",
    "    video_url = f'https://www.youtube.com/watch?v=' + video_id\n",
    "\n",
    "    # Create a YouTube object\n",
    "    yt = YouTube(video_url)\n",
    "\n",
    "    # Filter and select the audio stream\n",
    "    audio_stream = yt.streams.filter(only_audio=True, file_extension='mp4').first()\n",
    "\n",
    "    # Define the output path for the downloaded audio\n",
    "    output_path = f'{parent_output_folder}/{output_folder}/{video_id}'\n",
    "\n",
    "    # Download the audio stream to the specified output path\n",
    "    audio_stream.download(output_path=output_path)\n",
    "\n",
    "    # Get the list of files in the downloaded folder\n",
    "    files = os.listdir(output_path)\n",
    "\n",
    "    resulting_mp4_path = f'{output_path}.mp4'\n",
    "\n",
    "    # Move the downloaded file to the parent folder and rename it to the video ID\n",
    "    os.rename(f'{output_path}/{files[0]}' , resulting_mp4_path)\n",
    "\n",
    "    # Remove the now-empty subfolder\n",
    "    os.rmdir(output_path)\n",
    "\n",
    "    # Load the MP4 file using Pydub\n",
    "    audio = AudioSegment.from_file(resulting_mp4_path, format=\"mp4\")\n",
    "\n",
    "    wav_file_path = replace_mp4_with_wav(resulting_mp4_path)\n",
    "    \n",
    "    # Export the audio to a WAV file\n",
    "    audio.export(wav_file_path, format=\"wav\")\n",
    "\n",
    "    # Delete .mp4 file\n",
    "    os.remove(resulting_mp4_path)\n",
    "\n",
    "async def main():\n",
    "    video_ids = ['ICMfseFhE-E', 'C_xsXnRd_uc', 'xKAYvtoWxGs']\n",
    "    parent_output_folder = 'audio_mp4s'\n",
    "    # Generate timestamp\n",
    "    timestamp = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "    output_folder = f'{search_term}_{timestamp}'\n",
    "    os.makedirs(f'{parent_output_folder}/{output_folder}', exist_ok=True)\n",
    "\n",
    "    # Create a list of tasks\n",
    "    tasks = [download_and_process_video(video_id, parent_output_folder, output_folder) for video_id in video_ids]\n",
    "\n",
    "    # Run the tasks asynchronously\n",
    "    await asyncio.gather(*tasks)\n",
    "\n",
    "# Run the event loop\n",
    "await main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "094e4e84-bcd6-4fab-a300-c1e7274749ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_first_15_seconds(audio_path, output_path):\n",
    "    # Load audio file\n",
    "    audio = AudioSegment.from_wav(audio_path)\n",
    "\n",
    "    # Extract the first 15 seconds\n",
    "    first_15_seconds = audio[:15000]  # 15 seconds in milliseconds\n",
    "\n",
    "    # Save the snippet\n",
    "    first_15_seconds.export(output_path, format='wav')\n",
    "\n",
    "# Example usage\n",
    "audio_file_path = 'video_1vvyyhteIv4/video_1vvyyhteIv4.wav'\n",
    "output_file_path = 'video_1vvyyhteIv4/first_15_1vvyyhteIv4.wav'\n",
    "extract_first_15_seconds(audio_file_path, output_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6086eafa-64bc-41e7-89e9-addb9c073ae5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}