First commit

.env.example

@@ -0,0 +1,7 @@
+COOKIE=COOKIE_HERE
+AUTHORIZATION=AUTHORIZATION_HERE
+
+DOWNLOADS_DIR=./downloads
+DATABASE_DIR=./database
+
+INTERVAL=24

.gitignore

@@ -0,0 +1,4 @@
+.idea
+.env
+database
+downloads

docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.1'
+
+services:
+  downloader:
+    build: src/
+    restart: always
+
+    volumes:
+      - ${DOWNLOADS_DIR:-./downloads}:/downloads
+      - ${DATABASE_PATH:-./database}:/database
+
+    environment:
+      COOKIE: ${COOKIE}
+      AUTHORIZATION: ${AUTHORIZATION}
+      INTERVAL: ${INTERVAL:-24}

readme.md

@@ -0,0 +1,62 @@
+# YTMusic History
+
+A nice way to always keep track of what you like to listen - even if copyright laws jump into the matter.
+
+## Requirements
+
+1. Docker
+2. Docker Compose
+3. A working brain
+4. A browser with a logged in Google account 
+
+## Installation
+
+#### Clone the repo
+
+    git clone https://github.com/LucaTheHacker/YTMusic-History
+
+#### CD into the folder
+
+    cd YTMusic-History
+
+#### Copy the example config
+
+    cp .env.example .env
+
+#### Edit the config filling the required fields, use a text editor for that.
+
+Open https://music.youtube.com/library, and open the developer console.  
+Developer console > Network tab > filter by XHR.  
+Click on the history logo (the clock with the arrow), one request with endpoint starting with "browse" should appear.  
+Select the headers tab and scroll to "Request Headers". 
+Copy the content of COOKIE and AUTHENTICATION in the .env file.  
+
+COOKIE is the cookie you get from your browser.  
+AUTHENTICATION is the authentication token you get from your browser.  
+DOWNLOAD_DIR is the directory where the songs will be downloaded.  
+DATABASE_PATH is the path where the database will be stored.  
+INTERVAL is the interval between each run, in hours, 24 is recommended.
+
+#### Start the containers
+
+    docker compose up -d
+
+## Why
+
+I lost many remixes due to copyright strikes on YouTube.  
+Tired of that, I decided to download every song I listen to, and keep it in a safe place.  
+This is the result, and this also records the listening history (max 1 view per day), so that in 10 years I can cringe
+at myself for my music taste.
+
+## Workarounds
+
+YouTube, thanks to their asshole-being, does not consent to stream music from two different devices
+(Because who thinks I may have left my phone playing music at no volume whilst I'm at my computer!).
+At the same time, downloading all the songs with an account would cause continuous "song paused due to stream on another device" errors.
+Whilst using the same account on two different devices is forbidden, YouTube doesn't allow to download some songs without logging in,
+even if they're not 18+ or restricted in any way.
+To work around this pile of garbage, a first try with no account is made, and if the song is not downloadable it's sent to the authenticated queue.
+This should reduce the amount of music stops by a lot.
+
+Thanks Google. 
+Spero che domani mattina ve sveja San Pietro.

src/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3-bookworm
+MAINTAINER Dametto Luca
+
+RUN apt -y update
+RUN apt -y install python3-dev ffmpeg
+ADD requirements.txt .
+RUN python -m pip install -r requirements.txt
+RUN apt -y remove python3-dev
+
+COPY .. /home/
+WORKDIR /home/
+CMD ["python", "/home/main.py"]

src/database.py

@@ -0,0 +1,50 @@
+import sqlite3
+
+
+class Database:
+    def __init__(self):
+        self.conn = sqlite3.connect('/database/database.db')
+        self.cur = self.conn.cursor()
+
+        if not self.is_setup():
+            self.setup()
+
+    def setup(self):
+        self.cur.execute('''
+            CREATE TABLE IF NOT EXISTS songs (
+                video_id TEXT,
+                name TEXT,
+                artist TEXT,
+                album TEXT
+            )
+        ''')
+
+        self.cur.execute('''CREATE UNIQUE INDEX IF NOT EXISTS video_id_index ON songs (video_id)''')
+
+        self.cur.execute('''
+            CREATE TABLE IF NOT EXISTS view (
+                video_id TEXT,
+                date TEXT
+            )
+        ''')
+
+        self.cur.execute('''CREATE UNIQUE INDEX IF NOT EXISTS video_id_date_index ON view (video_id, date)''')
+
+    def is_setup(self) -> bool:
+        self.cur.execute('''SELECT name FROM sqlite_master WHERE type='table' AND name='songs' ''')
+        return self.cur.fetchone() is not None
+
+    def add_song(self, video_id: str, name: str, artist: str, album: str) -> bool:
+        try:
+            self.cur.execute('''INSERT INTO songs VALUES (?, ?, ?, ?)''', (video_id, name, artist, album))
+            self.conn.commit()
+            return True
+        except sqlite3.IntegrityError:
+            return False
+
+    def add_view(self, video_id: str, date: str):
+        try:
+            self.cur.execute('''INSERT INTO view VALUES (?, ?)''', (video_id, date))
+            self.conn.commit()
+        except sqlite3.IntegrityError:
+            pass

src/main.py

@@ -0,0 +1,81 @@
+from datetime import datetime
+import time
+import sys
+import os
+
+from database import Database
+from yt_download import yt_download
+from yt_request import fetch_history
+
+db = Database()
+
+
+def data_from_element(element):
+    col = element['musicResponsiveListItemRenderer']['flexColumns']
+    x = 'musicResponsiveListItemFlexColumnRenderer'
+
+    name = col[0][x]['text']['runs'][0]['text']
+    video_id = col[0][x]['text']['runs'][0]['navigationEndpoint']['watchEndpoint']['videoId']
+
+    album = None
+    if 'runs' in col[2][x]['text']:
+        album = col[2][x]['text']['runs'][0]['text']
+    if album is not None and album == name:
+        album = None
+
+    artist = ''.join([obj['text'] for obj in col[1][x]['text']['runs']]).split("•")[0].strip()
+
+    return {
+        'name': name,
+        'video_id': video_id,
+        'album': album,
+        'artist': artist,
+    }
+
+
+def download():
+    data = fetch_history()
+
+    auth_required_queue = []
+    i = 0
+    for group in data:
+        for element in group['musicShelfRenderer']['contents']:
+            data = data_from_element(element)
+            print(data)
+
+            if db.add_song(data['name'], data['video_id'], data['album'], data['artist']):
+                try:
+                    # Download song if it hasn't been added to db yet
+                    yt_download(data)
+                except Exception as e:
+                    print(f"Failure on {data['video_id']} {data['name']}: {e}, adding to auth_queue", file=sys.stderr)
+                    auth_required_queue.append(data)
+
+            if i == 0:
+                db.add_view(data['video_id'], datetime.now().strftime('%d/%m/%Y'))
+
+        i += 1
+
+    for data in auth_required_queue:
+        try:
+            yt_download(data, auth=True)
+        except Exception as e:
+            print(f"Failure on {data['video_id']} {data['name']}: {e}, skipping", file=sys.stderr)
+            pass
+
+
+if __name__ == '__main__':
+    if os.getenv("COOKIE") is None or len(os.getenv("COOKIE")) < 8:
+        print("Please set the COOKIE environment variable to your YouTube Music cookie", file=sys.stderr)
+        exit(1)
+
+    if os.getenv("AUTHORIZATION") is None or len(os.getenv("AUTHORIZATION")) < 8:
+        print("Please set the AUTHORIZATION environment variable to your YouTube Music authorization", file=sys.stderr)
+        exit(1)
+
+    print("YTMusic History Downloader started")
+    while True:
+        start = datetime.now()
+        download()
+        print(f"Downloaded in {datetime.now() - start}")
+        time.sleep(60 * 60 * int(os.getenv('INTERVAL', 24)))

src/requirements.txt

@@ -0,0 +1,2 @@
+yt_dlp
+requests

src/yt_download.py

@@ -0,0 +1,37 @@
+from http.cookiejar import Cookie
+from yt_dlp import YoutubeDL
+import os
+
+opts = {
+    'format': 'bestaudio/best',
+    'outtmpl': '/downloads/%(id)s.%(ext)s',
+    'postprocessors': [{
+        'key': 'FFmpegExtractAudio',
+        'preferredcodec': 'mp3',
+    }],
+    'quiet': True,
+    'no_warnings': True,
+    'noprogress': True,
+    'ignoreerrors': True,
+}
+
+
+def cookie_converter(s: str):
+    return [obj.split("=", maxsplit=1) for obj in s.split("; ")]
+
+
+def yt_download(data, auth=False):
+    try:
+        with YoutubeDL(opts) as ydl:
+            if auth:
+                for x in cookie_converter(os.getenv("COOKIE")):
+                    ydl.cookiejar.set_cookie(Cookie(
+                        name=x[0], value=x[1], domain='.youtube.com',
+                        version=0, port=None, path='/', secure=True, expires=None, discard=False,
+                        comment=None, comment_url=None, rest={'HttpOnly': None},
+                        domain_initial_dot=True, port_specified=False, domain_specified=True, path_specified=False))
+
+            print(f'Downloading {data["name"]} by {data["artist"]}')
+            ydl.download([f'https://music.youtube.com/watch?v={data["video_id"]}'])
+    except Exception as e:
+        raise e

src/yt_request.py

@@ -0,0 +1,46 @@
+import os
+import sys
+
+import requests
+
+headers = {
+    'authority': 'music.youtube.com',
+    'accept': '*/*',
+    'accept-language': 'it-IT,it;q=0.9',
+    'authorization': os.getenv("AUTHORIZATION"),
+    'content-type': 'application/json',
+    'cookie': os.getenv("COOKIE"),
+    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+    'x-origin': 'https://music.youtube.com',
+    'x-youtube-bootstrap-logged-in': 'true',
+    'x-youtube-client-name': '67',
+    'x-youtube-client-version': '1.20230829.05.00'
+}
+
+
+def fetch_history():
+    r = requests.post("https://music.youtube.com/youtubei/v1/browse?prettyPrint=false", headers=headers, json={
+        "context": {
+            "client": {
+                "hl": "it",
+                "gl": "IT",
+                "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36,gzip(gfe)",
+                "clientName": "WEB_REMIX",
+                "clientVersion": "1.20230829.05.00",
+                "platform": "DESKTOP",
+                "acceptHeader": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "timeZone": "Europe/Rome"
+            },
+            "user": {
+                "lockedSafetyMode": False
+            }
+        },
+        "browseId": "FEmusic_history"
+    })
+
+    if r.status_code != 200:
+        print("Error while fetching YTMusic history, please check credentials", file=sys.stderr)
+        print(r.text, file=sys.stderr)
+        return []
+
+    return r.json()['contents']['singleColumnBrowseResultsRenderer']['tabs'][0]['tabRenderer']['content']['sectionListRenderer']['contents']