From 388be1016db8567d29b61c77d921850681e4c7c3 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 8 Aug 2024 20:49:05 +0800 Subject: [PATCH 1/4] fix: generate uuid for request --- scripts/get_blog.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/get_blog.py b/scripts/get_blog.py index 8381362..2588ec2 100644 --- a/scripts/get_blog.py +++ b/scripts/get_blog.py @@ -3,12 +3,17 @@ from datetime import datetime import os import re +import uuid strapi_blog_url = "" strapi_upload_url = "" API_KEY = "" +def generate_code(): + new_code = str(uuid.uuid4()) + return new_code + def fetch_strapi_blogs(): response = requests.get(f"{strapi_blog_url}?_limit=100", headers={"Authorization": f"Bearer {API_KEY}"}) if response.status_code == 200: @@ -56,14 +61,23 @@ def upload_image_to_strapi(image_path): print(f"Failed to upload image to Strapi: {response.content}") return None +code = generate_code() url = "https://taiko.mirror.xyz/api/graphql" headers = { 'accept': '*/*', + 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8', 'content-type': 'application/json', - 'cookie': '', + 'cookie': f'anonymousUserId={code};', 'origin': 'https://taiko.mirror.xyz', 'priority': 'u=1, i', 'referer': 'https://taiko.mirror.xyz/', + 'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"macOS"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' } payload = { @@ -290,6 +304,7 @@ def upload_image_to_strapi(image_path): featured_image_url = post.get('featuredImage', {}).get('url', 'N/A') formatted_date = datetime.fromtimestamp(published_at).strftime('%Y-%m-%d') if published_at != 'N/A' else 'N/A' slug = create_slug(title) + print(link) if link not in strapi_link: print("Unpublished blog: "+ title) # Download featured image From ab0d98041f87619b4bbdde4278cf6956bf5f50a6 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 8 Aug 2024 20:49:46 +0800 Subject: [PATCH 2/4] fix: print --- scripts/get_blog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/get_blog.py b/scripts/get_blog.py index 2588ec2..437745f 100644 --- a/scripts/get_blog.py +++ b/scripts/get_blog.py @@ -304,7 +304,6 @@ def upload_image_to_strapi(image_path): featured_image_url = post.get('featuredImage', {}).get('url', 'N/A') formatted_date = datetime.fromtimestamp(published_at).strftime('%Y-%m-%d') if published_at != 'N/A' else 'N/A' slug = create_slug(title) - print(link) if link not in strapi_link: print("Unpublished blog: "+ title) # Download featured image From 4b6c15e1fe521c01f7d87a75fea112afbc5dae1e Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 8 Aug 2024 20:59:40 +0800 Subject: [PATCH 3/4] fix: payload --- scripts/get_blog.py | 207 +++----------------------------------------- 1 file changed, 13 insertions(+), 194 deletions(-) diff --git a/scripts/get_blog.py b/scripts/get_blog.py index 437745f..fc32242 100644 --- a/scripts/get_blog.py +++ b/scripts/get_blog.py @@ -84,208 +84,27 @@ def upload_image_to_strapi(image_path): "operationName": "ProjectPage", "variables": { "projectAddress": "taiko.mirror.xyz", - "limit": 100 + "limit": 10 }, "query": """ query ProjectPage($projectAddress: String!, $limit: Int, $cursor: Int) { - projectFeed(projectAddress: $projectAddress, limit: $limit, cursor: $cursor) { - _id - domain - ens - theme { - accent - colorMode - __typename + projectFeed(projectAddress: $projectAddress, limit: $limit, cursor: $cursor) { + ...projectPage } - displayName - ens - address - ...projectPage - ...publicationLayoutProject - __typename - } } - fragment projectPage on ProjectType { - _id - address - avatarURL - description - displayName - domain - ens - twitterUsername - externalUrl - theme { - accent - colorMode - __typename - } - headerImage { - id - url - __typename - } - posts { - ... on crowdfund { - _id - id - __typename - } - ... on entry { - _id - id - body - digest - title - publishedAtTimestamp - writingNFT { - _id - optimisticNumSold - proxyAddress - purchases { - numSold - __typename - } - __typename - } - collectorBubbles { - _id - address - project { - ...projectDetails - __typename + posts { + ... on entry { + _id + id + digest + title + publishedAtTimestamp + featuredImage { + url + } } - __typename - } - featuredImage { - mimetype - url - __typename - } - publisher { - ...publisherDetails - __typename - } - settings { - ...entrySettingsDetails - __typename - } - __typename } - ... on SubscriberEditionType { - _id - __typename - } - __typename - } - __typename - } - - fragment projectDetails on ProjectType { - _id - address - avatarURL - description - displayName - domain - ens - gaTrackingID - ga4TrackingID - mailingListURL - twitterUsername - wnftChainId - externalUrl - headerImage { - ...mediaAsset - __typename - } - theme { - ...themeDetails - __typename - } - __typename - } - - fragment mediaAsset on MediaAssetType { - id - cid - mimetype - sizes { - ...mediaAssetSizes - __typename - } - url - __typename - } - - fragment mediaAssetSizes on MediaAssetSizesType { - og { - ...mediaAssetSize - __typename - } - lg { - ...mediaAssetSize - __typename - } - md { - ...mediaAssetSize - __typename - } - sm { - ...mediaAssetSize - __typename - } - __typename - } - - fragment mediaAssetSize on MediaAssetSizeType { - src - height - width - __typename - } - - fragment themeDetails on UserProfileThemeType { - accent - colorMode - __typename - } - - fragment publisherDetails on PublisherType { - project { - ...projectDetails - __typename - } - member { - ...projectDetails - __typename - } - __typename - } - - fragment entrySettingsDetails on EntrySettingsType { - description - metaImage { - ...mediaAsset - __typename - } - title - __typename - } - - fragment publicationLayoutProject on ProjectType { - _id - avatarURL - displayName - domain - address - ens - gaTrackingID - ga4TrackingID - mailingListURL - description - __typename } """ } From de48835b546c59bcf697c8b26e2cc154d4ebfa8d Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 8 Aug 2024 23:18:10 +0800 Subject: [PATCH 4/4] feat: use cronjob --- scripts/blog.yaml | 220 ++++++++++++++++++++++++++++++++++++++++++++ scripts/get_blog.py | 169 ---------------------------------- 2 files changed, 220 insertions(+), 169 deletions(-) create mode 100644 scripts/blog.yaml delete mode 100644 scripts/get_blog.py diff --git a/scripts/blog.yaml b/scripts/blog.yaml new file mode 100644 index 0000000..757386f --- /dev/null +++ b/scripts/blog.yaml @@ -0,0 +1,220 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: strapi-update-cronjob +spec: + schedule: "0 0 * * *" # Run daily at midnight + jobTemplate: + spec: + template: + spec: + containers: + - name: strapi-update-container + image: python:3.9 # Use a Python Docker image + command: ["/bin/sh", "/scripts/entrypoint.sh"] + volumeMounts: + - name: script-volume + mountPath: /scripts + restartPolicy: OnFailure + volumes: + - name: script-volume + configMap: + name: strapi-update-script +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: strapi-update-script +data: + entrypoint.sh: | + #!/bin/sh + set -e + + # Install the required Python packages + pip install requests python-dateutil Pillow + + # Run the Python script + python /scripts/strapi_update_script.py + strapi_update_script.py: | + import requests + import json + from datetime import datetime + import os + import re + import uuid + + # Strapi API URL and API key + strapi_blog_url = "" + strapi_upload_url = "" + API_KEY = "" + + # Function to generate a unique UUID code + def generate_code(): + new_code = str(uuid.uuid4()) + return new_code + + # Function to fetch blogs from Strapi + def fetch_strapi_blogs(): + response = requests.get(f"{strapi_blog_url}?_limit=100", headers={"Authorization": f"Bearer {API_KEY}"}) + if response.status_code == 200: + return response.json()['results'] + else: + print(f"Failed to fetch blogs from Strapi: {response.text}") + return [] + + # Function to create entries in Strapi + def create_entries(data_array): + for data in data_array: + response = requests.post( + strapi_blog_url, + json=data, + headers={"Content-Type": "application/json", "Authorization": f"bearer {API_KEY}",}, + ) + if response.status_code == 200: + print(f"Entry created successfully: {data}") + else: + print(f"Failed to create entry: {data}") + print(f"Response: {response.text}") + + # Function to create a slug from a title + def create_slug(title): + slug = title.lower().replace(" ", "-") + slug = re.sub(r'[^A-Za-z0-9-_.~]', '', slug) + return slug + + # Helper function to download an image + def download_image(url, filename): + response = requests.get(url) + if response.status_code == 200: + with open(filename, 'wb') as file: + file.write(response.content) + print(f"Downloaded image: {filename}") + else: + print(f"Failed to download image: {url}") + + # Function to upload an image to Strapi + def upload_image_to_strapi(image_path): + files = {'files': open(image_path, 'rb')} + headers = {'Authorization': f'Bearer {API_KEY}'} + response = requests.post(strapi_upload_url, headers=headers, files=files) + if response.status_code == 200: + print(f"Uploaded image: {response.json()[0]['id']}") + return response.json()[0]['id'] + else: + print(f"Failed to upload image to Strapi: {response.content}") + return None + + # Generate the code for anonymousUserId + anonymous_user_id = generate_code() + + # Define the URL and headers for the request + url = "https://taiko.mirror.xyz/api/graphql" + headers = { + 'accept': '*/*', + 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8', + 'content-type': 'application/json', + 'cookie': f'anonymousUserId={anonymous_user_id};', + 'origin': 'https://taiko.mirror.xyz', + 'priority': 'u=1, i', + 'referer': 'https://taiko.mirror.xyz/', + 'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"macOS"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' + } + + # Define the payload for the request + payload = { + "operationName": "ProjectPage", + "variables": { + "projectAddress": "taiko.mirror.xyz", + "limit": 10 + }, + "query": """ + query ProjectPage($projectAddress: String!, $limit: Int, $cursor: Int) { + projectFeed(projectAddress: $projectAddress, limit: $limit, cursor: $cursor) { + ...projectPage + } + } + fragment projectPage on ProjectType { + posts { + ... on entry { + _id + id + digest + title + publishedAtTimestamp + featuredImage { + url + } + } + } + } + """ + } + + # Fetch existing blogs from Strapi + strapi_blogs = fetch_strapi_blogs() + strapi_link = [blog['link'] for blog in strapi_blogs] + + # Make the request to taiko.mirror.xyz + response = requests.post(url, headers=headers, json=payload) + data = response.json() + transformed_data_list = [] + + # Process each post + for post in data['data']['projectFeed']['posts']: + post_id = post['_id'] + link = f"https://taiko.mirror.xyz/{post_id}" + title = post.get('title', 'N/A') + published_at = post.get('publishedAtTimestamp', 'N/A') + featured_image_url = post.get('featuredImage', {}).get('url', 'N/A') + formatted_date = datetime.fromtimestamp(published_at).strftime('%Y-%m-%d') if published_at != 'N/A' else 'N/A' + slug = create_slug(title) + if link not in strapi_link: + print("Unpublished blog: "+ title) + # Download featured image + imgid = "placeholder_imgid" + if featured_image_url != 'N/A': + img_filename = os.path.basename(featured_image_url.split('?')[0]) # Remove query parameters from URL + download_image(featured_image_url, img_filename) + imgid = upload_image_to_strapi(img_filename) + + # Transform the data into the desired format + transformed_data = { + "category": { + "disconnect": [], + "connect": [{"id": 1, "position": {"end": True}}] + }, + "slug": slug, + "link": f"https://taiko.mirror.xyz/{post_id}", + "date": formatted_date, + "content": [ + { + "type": "paragraph", + "children": [ + {"type": "text", "text": title} + ] + } + ], + "howToApply": [ + { + "type": "paragraph", + "children": [ + {"type": "text", "text": "Visit our github"} + ] + } + ], + "title": title, + "timeToRead": "6 min", + "image": imgid, + } + + # Add to the list + transformed_data_list.append(transformed_data) + + # Create new entries in Strapi + create_entries(transformed_data_list) diff --git a/scripts/get_blog.py b/scripts/get_blog.py deleted file mode 100644 index fc32242..0000000 --- a/scripts/get_blog.py +++ /dev/null @@ -1,169 +0,0 @@ -import requests -import json -from datetime import datetime -import os -import re -import uuid - - -strapi_blog_url = "" -strapi_upload_url = "" -API_KEY = "" - -def generate_code(): - new_code = str(uuid.uuid4()) - return new_code - -def fetch_strapi_blogs(): - response = requests.get(f"{strapi_blog_url}?_limit=100", headers={"Authorization": f"Bearer {API_KEY}"}) - if response.status_code == 200: - return response.json()['results'] - else: - print(f"Failed to fetch blogs from Strapi: {response.text}") - return [] - -def create_entries(data_array): - for data in data_array: - response = requests.post( - strapi_blog_url, - json=data, - headers={"Content-Type": "application/json", "Authorization": f"bearer {API_KEY}",}, - ) - if response.status_code == 200: - print(f"Entry created successfully: {data}") - else: - print(f"Failed to create entry: {data}") - print(f"Response: {response.text}") - -def create_slug(title): - slug = title.lower().replace(" ", "-") - slug = re.sub(r'[^A-Za-z0-9-_.~]', '', slug) - return slug - -# Helper function to download an image -def download_image(url, filename): - response = requests.get(url) - if response.status_code == 200: - with open(filename, 'wb') as file: - file.write(response.content) - print(f"Downloaded image: {filename}") - else: - print(f"Failed to download image: {url}") - -def upload_image_to_strapi(image_path): - files = {'files': open(image_path, 'rb')} - headers = {'Authorization': f'Bearer {API_KEY}'} - response = requests.post(strapi_upload_url, headers=headers, files=files) - if response.status_code == 200: - print(f"Uploaded image: {response.json()[0]['id']}") - return response.json()[0]['id'] - else: - print(f"Failed to upload image to Strapi: {response.content}") - return None - -code = generate_code() -url = "https://taiko.mirror.xyz/api/graphql" -headers = { - 'accept': '*/*', - 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8', - 'content-type': 'application/json', - 'cookie': f'anonymousUserId={code};', - 'origin': 'https://taiko.mirror.xyz', - 'priority': 'u=1, i', - 'referer': 'https://taiko.mirror.xyz/', - 'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"macOS"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' -} - -payload = { - "operationName": "ProjectPage", - "variables": { - "projectAddress": "taiko.mirror.xyz", - "limit": 10 - }, - "query": """ - query ProjectPage($projectAddress: String!, $limit: Int, $cursor: Int) { - projectFeed(projectAddress: $projectAddress, limit: $limit, cursor: $cursor) { - ...projectPage - } - } - fragment projectPage on ProjectType { - posts { - ... on entry { - _id - id - digest - title - publishedAtTimestamp - featuredImage { - url - } - } - } - } - """ -} -strapi_blogs = fetch_strapi_blogs() -strapi_link = [blog['link'] for blog in strapi_blogs] -response = requests.post(url, headers=headers, json=payload) -data = response.json() -transformed_data_list = [] - -# Process each post -for post in data['data']['projectFeed']['posts']: - post_id = post['_id'] - link = f"https://taiko.mirror.xyz/{post_id}" - title = post.get('title', 'N/A') - published_at = post.get('publishedAtTimestamp', 'N/A') - featured_image_url = post.get('featuredImage', {}).get('url', 'N/A') - formatted_date = datetime.fromtimestamp(published_at).strftime('%Y-%m-%d') if published_at != 'N/A' else 'N/A' - slug = create_slug(title) - if link not in strapi_link: - print("Unpublished blog: "+ title) - # Download featured image - imgid = "placeholder_imgid" - if featured_image_url != 'N/A': - img_filename = os.path.basename(featured_image_url.split('?')[0]) # Remove query parameters from URL - download_image(featured_image_url, img_filename) - imgid = upload_image_to_strapi(img_filename) - - # Transform the data into the desired format - transformed_data = { - "category": { - "disconnect": [], - "connect": [{"id": 1, "position": {"end": True}}] - }, - "slug": slug, - "link": f"https://taiko.mirror.xyz/{post_id}", - "date": formatted_date, - "content": [ - { - "type": "paragraph", - "children": [ - {"type": "text", "text": title} - ] - } - ], - "howToApply": [ - { - "type": "paragraph", - "children": [ - {"type": "text", "text": "Visit our github"} - ] - } - ], - "title": title, - "timeToRead": "6 min", - "image": imgid, - } - - # Add to the list - transformed_data_list.append(transformed_data) - - -create_entries(transformed_data_list) \ No newline at end of file