diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3114110 --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +COOKIE=COOKIE_HERE +AUTHORIZATION=AUTHORIZATION_HERE + +DOWNLOADS_DIR=./downloads +DATABASE_DIR=./database + +INTERVAL=24 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0684c3d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +.env +database +downloads \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6e19f05 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.1' + +services: + downloader: + build: src/ + restart: always + + volumes: + - ${DOWNLOADS_DIR:-./downloads}:/downloads + - ${DATABASE_PATH:-./database}:/database + + environment: + COOKIE: ${COOKIE} + AUTHORIZATION: ${AUTHORIZATION} + INTERVAL: ${INTERVAL:-24} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..c6c3484 --- /dev/null +++ b/readme.md @@ -0,0 +1,62 @@ +# YTMusic History + +A nice way to always keep track of what you like to listen - even if copyright laws jump into the matter. + +## Requirements + +1. Docker +2. Docker Compose +3. A working brain +4. A browser with a logged in Google account + +## Installation + +#### Clone the repo + + git clone https://github.com/LucaTheHacker/YTMusic-History + +#### CD into the folder + + cd YTMusic-History + +#### Copy the example config + + cp .env.example .env + +#### Edit the config filling the required fields, use a text editor for that. + +Open https://music.youtube.com/library, and open the developer console. +Developer console > Network tab > filter by XHR. +Click on the history logo (the clock with the arrow), one request with endpoint starting with "browse" should appear. +Select the headers tab and scroll to "Request Headers". +Copy the content of COOKIE and AUTHENTICATION in the .env file. + +COOKIE is the cookie you get from your browser. +AUTHENTICATION is the authentication token you get from your browser. +DOWNLOAD_DIR is the directory where the songs will be downloaded. +DATABASE_PATH is the path where the database will be stored. +INTERVAL is the interval between each run, in hours, 24 is recommended. + +#### Start the containers + + docker compose up -d + +## Why + +I lost many remixes due to copyright strikes on YouTube. +Tired of that, I decided to download every song I listen to, and keep it in a safe place. +This is the result, and this also records the listening history (max 1 view per day), so that in 10 years I can cringe +at myself for my music taste. + +## Workarounds + +YouTube, thanks to their asshole-being, does not consent to stream music from two different devices +(Because who thinks I may have left my phone playing music at no volume whilst I'm at my computer!). +At the same time, downloading all the songs with an account would cause continuous "song paused due to stream on another device" errors. +Whilst using the same account on two different devices is forbidden, YouTube doesn't allow to download some songs without logging in, +even if they're not 18+ or restricted in any way. +To work around this pile of garbage, a first try with no account is made, and if the song is not downloadable it's sent to the authenticated queue. +This should reduce the amount of music stops by a lot. + +Thanks Google. +Spero che domani mattina ve sveja San Pietro. \ No newline at end of file diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..d581310 --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3-bookworm +MAINTAINER Dametto Luca + +RUN apt -y update +RUN apt -y install python3-dev ffmpeg +ADD requirements.txt . +RUN python -m pip install -r requirements.txt +RUN apt -y remove python3-dev + +COPY .. /home/ +WORKDIR /home/ +CMD ["python", "/home/main.py"] diff --git a/src/database.py b/src/database.py new file mode 100644 index 0000000..e7858ef --- /dev/null +++ b/src/database.py @@ -0,0 +1,50 @@ +import sqlite3 + + +class Database: + def __init__(self): + self.conn = sqlite3.connect('/database/database.db') + self.cur = self.conn.cursor() + + if not self.is_setup(): + self.setup() + + def setup(self): + self.cur.execute(''' + CREATE TABLE IF NOT EXISTS songs ( + video_id TEXT, + name TEXT, + artist TEXT, + album TEXT + ) + ''') + + self.cur.execute('''CREATE UNIQUE INDEX IF NOT EXISTS video_id_index ON songs (video_id)''') + + self.cur.execute(''' + CREATE TABLE IF NOT EXISTS view ( + video_id TEXT, + date TEXT + ) + ''') + + self.cur.execute('''CREATE UNIQUE INDEX IF NOT EXISTS video_id_date_index ON view (video_id, date)''') + + def is_setup(self) -> bool: + self.cur.execute('''SELECT name FROM sqlite_master WHERE type='table' AND name='songs' ''') + return self.cur.fetchone() is not None + + def add_song(self, video_id: str, name: str, artist: str, album: str) -> bool: + try: + self.cur.execute('''INSERT INTO songs VALUES (?, ?, ?, ?)''', (video_id, name, artist, album)) + self.conn.commit() + return True + except sqlite3.IntegrityError: + return False + + def add_view(self, video_id: str, date: str): + try: + self.cur.execute('''INSERT INTO view VALUES (?, ?)''', (video_id, date)) + self.conn.commit() + except sqlite3.IntegrityError: + pass diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..bc4691f --- /dev/null +++ b/src/main.py @@ -0,0 +1,81 @@ +from datetime import datetime +import time +import sys +import os + +from database import Database +from yt_download import yt_download +from yt_request import fetch_history + +db = Database() + + +def data_from_element(element): + col = element['musicResponsiveListItemRenderer']['flexColumns'] + x = 'musicResponsiveListItemFlexColumnRenderer' + + name = col[0][x]['text']['runs'][0]['text'] + video_id = col[0][x]['text']['runs'][0]['navigationEndpoint']['watchEndpoint']['videoId'] + + album = None + if 'runs' in col[2][x]['text']: + album = col[2][x]['text']['runs'][0]['text'] + if album is not None and album == name: + album = None + + artist = ''.join([obj['text'] for obj in col[1][x]['text']['runs']]).split("•")[0].strip() + + return { + 'name': name, + 'video_id': video_id, + 'album': album, + 'artist': artist, + } + + +def download(): + data = fetch_history() + + auth_required_queue = [] + i = 0 + for group in data: + for element in group['musicShelfRenderer']['contents']: + data = data_from_element(element) + print(data) + + if db.add_song(data['name'], data['video_id'], data['album'], data['artist']): + try: + # Download song if it hasn't been added to db yet + yt_download(data) + except Exception as e: + print(f"Failure on {data['video_id']} {data['name']}: {e}, adding to auth_queue", file=sys.stderr) + auth_required_queue.append(data) + + if i == 0: + db.add_view(data['video_id'], datetime.now().strftime('%d/%m/%Y')) + + i += 1 + + for data in auth_required_queue: + try: + yt_download(data, auth=True) + except Exception as e: + print(f"Failure on {data['video_id']} {data['name']}: {e}, skipping", file=sys.stderr) + pass + + +if __name__ == '__main__': + if os.getenv("COOKIE") is None or len(os.getenv("COOKIE")) < 8: + print("Please set the COOKIE environment variable to your YouTube Music cookie", file=sys.stderr) + exit(1) + + if os.getenv("AUTHORIZATION") is None or len(os.getenv("AUTHORIZATION")) < 8: + print("Please set the AUTHORIZATION environment variable to your YouTube Music authorization", file=sys.stderr) + exit(1) + + print("YTMusic History Downloader started") + while True: + start = datetime.now() + download() + print(f"Downloaded in {datetime.now() - start}") + time.sleep(60 * 60 * int(os.getenv('INTERVAL', 24))) diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..9047b07 --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1,2 @@ +yt_dlp +requests \ No newline at end of file diff --git a/src/yt_download.py b/src/yt_download.py new file mode 100644 index 0000000..21e4cde --- /dev/null +++ b/src/yt_download.py @@ -0,0 +1,37 @@ +from http.cookiejar import Cookie +from yt_dlp import YoutubeDL +import os + +opts = { + 'format': 'bestaudio/best', + 'outtmpl': '/downloads/%(id)s.%(ext)s', + 'postprocessors': [{ + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + }], + 'quiet': True, + 'no_warnings': True, + 'noprogress': True, + 'ignoreerrors': True, +} + + +def cookie_converter(s: str): + return [obj.split("=", maxsplit=1) for obj in s.split("; ")] + + +def yt_download(data, auth=False): + try: + with YoutubeDL(opts) as ydl: + if auth: + for x in cookie_converter(os.getenv("COOKIE")): + ydl.cookiejar.set_cookie(Cookie( + name=x[0], value=x[1], domain='.youtube.com', + version=0, port=None, path='/', secure=True, expires=None, discard=False, + comment=None, comment_url=None, rest={'HttpOnly': None}, + domain_initial_dot=True, port_specified=False, domain_specified=True, path_specified=False)) + + print(f'Downloading {data["name"]} by {data["artist"]}') + ydl.download([f'https://music.youtube.com/watch?v={data["video_id"]}']) + except Exception as e: + raise e diff --git a/src/yt_request.py b/src/yt_request.py new file mode 100644 index 0000000..2381795 --- /dev/null +++ b/src/yt_request.py @@ -0,0 +1,46 @@ +import os +import sys + +import requests + +headers = { + 'authority': 'music.youtube.com', + 'accept': '*/*', + 'accept-language': 'it-IT,it;q=0.9', + 'authorization': os.getenv("AUTHORIZATION"), + 'content-type': 'application/json', + 'cookie': os.getenv("COOKIE"), + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + 'x-origin': 'https://music.youtube.com', + 'x-youtube-bootstrap-logged-in': 'true', + 'x-youtube-client-name': '67', + 'x-youtube-client-version': '1.20230829.05.00' +} + + +def fetch_history(): + r = requests.post("https://music.youtube.com/youtubei/v1/browse?prettyPrint=false", headers=headers, json={ + "context": { + "client": { + "hl": "it", + "gl": "IT", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36,gzip(gfe)", + "clientName": "WEB_REMIX", + "clientVersion": "1.20230829.05.00", + "platform": "DESKTOP", + "acceptHeader": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "timeZone": "Europe/Rome" + }, + "user": { + "lockedSafetyMode": False + } + }, + "browseId": "FEmusic_history" + }) + + if r.status_code != 200: + print("Error while fetching YTMusic history, please check credentials", file=sys.stderr) + print(r.text, file=sys.stderr) + return [] + + return r.json()['contents']['singleColumnBrowseResultsRenderer']['tabs'][0]['tabRenderer']['content']['sectionListRenderer']['contents']