forked from podverse/podverse-ops
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
319 lines (264 loc) · 20.2 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
ifeq ($(UNAME),Darwin)
SHELL := /opt/local/bin/bash
OS_X := true
else ifneq (,$(wildcard /etc/redhat-release))
RHEL := true
else
OS_DEB := true
SHELL := /bin/bash
endif
.PHONY: say_hello
say_hello:
@echo "Hello Podverse"
.PHONY: local_validate_init
local_validate_init: config/podverse-api-local.env config/podverse-db-local.env config/podverse-web-local.env
config/podverse-api-local.env:
@echo "Missing: $@"
@echo "Copying from example file"
cp ./[email protected] ./$@
config/podverse-db-local.env:
@echo "Missing: $@"
@echo "Copying from example file"
cp ./[email protected] ./$@
config/podverse-web-local.env:
@echo "Missing: $@"
@echo "Copying from example file"
cp ./[email protected] ./$@
.PHONY: local_nginx_proxy
local_nginx_proxy:
@echo 'Generate new cert'
test -d proxy/local/certs || mkdir -p proxy/local/certs
cd proxy/local/certs && openssl genrsa -out podverse-server.key 4096
cd proxy/local/certs && openssl rsa -in podverse-server.key -out podverse-server.key.insecure
cd proxy/local/certs && openssl req -new -sha256 -key podverse-server.key -subj "/C=US/ST=Jefferson/L=Grand/O=EXA/OU=MPL/CN=podverse.local" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:podverse.local,DNS:www.podverse.local,DNS:api.podverse.local")) -out podverse-server.csr
cd proxy/local/certs && openssl x509 -req -days 365 -in podverse-server.csr -signkey podverse-server.key -out podverse-server.crt
.PHONY: local_up_db
local_up_db:
docker-compose -f docker-compose/local/docker-compose.yml up podverse_db -d
.PHONY: local_init_materialized_views
local_init_materialized_views:
# init
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_db psql -U postgres -d postgres -a -f /opt/migrations/0032_mediaRefs_videos_materialized_view.sql
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_db psql -U postgres -d postgres -a -f /opt/migrations/0037_episodes_most_recent_materialized_view.sql
# have to call REFRESH one time for each table without CONCURRENTLY
# the _refresh make commands use CONCURRENTLY.
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_db psql -U postgres -d postgres -c 'REFRESH MATERIALIZED VIEW "mediaRefs_videos"'
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_db psql -U postgres -d postgres -c 'REFRESH MATERIALIZED VIEW "episodes_most_recent"'
# This materialized view is used when querying for only mediaRefs for video podcasts.
.PHONY: local_materialized_view_mediaRefs_refresh
local_materialized_view_mediaRefs_refresh:
docker-compose -f docker-compose/local/docker-compose.yml run --name refreshMediaRefsVideosMaterializedView --rm podverse_api_worker npm run scripts:refreshMediaRefsVideosMaterializedView
# This materialized view is used for sorting all episodes by recency.
# It is limited to the past ~21 days currently.
.PHONY: local_materialized_view_episodes_refresh
local_materialized_view_episodes_refresh:
docker-compose -f docker-compose/local/docker-compose.yml run --name refreshEpisodesMostRecentMaterializedView --rm podverse_api_worker npm run scripts:refreshEpisodesMostRecentMaterializedView
.PHONY: local_up_manticore_server
local_up_manticore_server:
docker-compose -f docker-compose/local/docker-compose.yml up podverse_manticore -d
.PHONY: local_manticore_indexes_init
local_manticore_indexes_init:
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_media_ref --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_playlist --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_podcast --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_01 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_02 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_03 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_04 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_05 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_06 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_07 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_08 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_09 --verbose
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_10 --verbose
# after initializing the indexes, manticore must be restarted to be able to use the indexes
docker restart podverse_manticore_local
.PHONY: local_manticore_indexes_rotate
local_manticore_indexes_rotate:
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_author --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_media_ref --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_playlist --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_podcast --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_01 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_02 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_03 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_04 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_05 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_06 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_07 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_08 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_09 --rotate --verbose;
docker-compose -f docker-compose/local/docker-compose.yml exec podverse_manticore gosu manticore indexer idx_episode_10 --rotate --verbose;
.PHONY: local_up_api
local_up_api:
docker-compose -f docker-compose/local/docker-compose.yml up podverse_api -d
.PHONY: local_up_api_no_cache
local_up_api_no_cache:
docker-compose -f docker-compose/local/docker-compose.yml build --no-cache podverse_api podverse_api_worker
.PHONY: local_up_web
local_up_web:
docker-compose -f docker-compose/local/docker-compose.yml up podverse_web -d
.PHONY: local_up_web_no_cache
local_up_web_no_cache:
docker-compose -f docker-compose/local/docker-compose.yml build --no-cache podverse_web
.PHONY: local_up_maintenance_mode
local_up_maintenance_mode:
docker-compose -f docker-compose/local/docker-compose.yml up podverse_maintenance_mode_web -d
.PHONY: local_up_proxy
local_up_proxy:
docker-compose -f docker-compose/local/docker-compose.yml up podverse_nginx_proxy -d
.PHONY: local_down_docker_compose
local_down_docker_compose:
docker-compose -f docker-compose/local/docker-compose.yml down
# This will add a predefined list of ~30 feed urls to the AWS SQS queue based on their Podcast Index id.
# This is useful for seeding the database with podcast data.
# You may need a valid Podcast Index API keys for the
# PODCAST_INDEX_AUTH_KEY and PODCAST_INDEX_SECRET_KEY env vars.
# Sign up for a key here: https://api.podcastindex.org/
# TODO: is a Podcast Index API actually required by our services?
# TODO: Research how to validate key after it has been required, or post error message explaining the key is missing.
# TODO: Use environment variables to insert the ids as a parameter at the end of the command.
.PHONY: local_add_podcast_index_seed_feeds_to_queue
local_add_podcast_index_seed_feeds_to_queue:
docker-compose -f docker-compose/local/docker-compose.yml run --name addPodcastIndexSeedFeedsToQueue --rm podverse_api_worker npm run scripts:addFeedsByPodcastIndexIdToQueue -- 5718023,387129,3662287,160817,150842,878147,487548,167137,465231,767934,577105,54545,650774,955598,3758236,203827,879740,393504,575694,921030,41504,5341434,757675,174725,920666,1333070,227573,5465405,5498327,5495489,556715,5485175,202764,830124,66844,4169501
.PHONY: local_add_podcast_index_seed_feeds_to_queue_small
local_add_podcast_index_seed_feeds_to_queue_small:
docker-compose -f docker-compose/local/docker-compose.yml run --name addPodcastIndexSeedFeedsToQueue --rm podverse_api_worker npm run scripts:addFeedsByPodcastIndexIdToQueue -- 5718023,387129,3662287,160817
.PHONY: local_add_podcast_index_seed_feeds_with_live_items_to_queue
local_add_podcast_index_seed_feeds_with_live_items_to_queue:
docker-compose -f docker-compose/local/docker-compose.yml run --name addPodcastIndexSeedFeedsToQueue --rm podverse_api_worker npm run scripts:addFeedsByPodcastIndexIdToQueue -- 4935828,5495489,162612,5461087,486332,480983,3727160,5496786,901876,5498327,4207213,5710520,5465405,5485175,574891,920666,540927,4432692,5718023,41504,3756449,150842,937170,946122,5373053,624721,5700613,288180,955598
# This will run 3 parsers that pull from different SQS queues.
# The priority queue is used in most cases, but the non-priority queue
# is used for less time sensitive jobs (like doing a full sync with Podcast Index).
# The live queue is used for live streams.
# TODO: Do an environment variable check for AWS credentials.
.PHONY: local_up_parsers
local_up_parsers:
docker-compose -f docker-compose/local/docker-compose.yml run -d --name --rm podverse_api_parser_1 podverse_api_worker npm run scripts:parseFeedUrlsFromQueue -- 60000 priority
docker-compose -f docker-compose/local/docker-compose.yml run -d --name --rm podverse_api_parser_2 podverse_api_worker npm run scripts:parseFeedUrlsFromQueue -- 60000
docker-compose -f docker-compose/local/docker-compose.yml run -d --name --rm podverse_api_parser_3_live podverse_api_worker npm run scripts:parseFeedUrlsFromQueue -- 60000 live
.PHONY: local_down_parsers
local_down_parsers:
# I think down may not work like this, and it's a known issue?
# https://github.com/docker/compose/issues/9627#issuecomment-1196514436
# docker-compose -f docker-compose/local/docker-compose.yml down podverse_api_parser_1 podverse_api_parser_2 podverse_api_parser_3_live
docker stop podverse_api_parser_1 podverse_api_parser_2 podverse_api_parser_3_live
# This will request the new feeds added to Podcast Index over the past X milliseconds
# (as defined in the podverse-api-xxxxx.env), and then parse and add them to the database
# one-by-one immediately (without sending the feeds to an SQS queue).
# TODO: Add environment variable check. Make the time-range the request is for human readable in an echo.
.PHONY: local_add_podcast_index_new_feeds
local_add_podcast_index_new_feeds:
docker-compose -f docker-compose/local/docker-compose.yml run --name addNewFeedUrls --rm podverse_api_worker npm run scripts:addNewFeedsFromPodcastIndex
.PHONY: local_podping_liveitem_listener
local_podping_liveitem_listener:
docker-compose -f docker-compose/local/docker-compose.yml run -d --name runLiveItemListener podverse_api_worker npm run scripts:podping:runLiveItemListener
.PHONY: local_down_podping_liveitem_listener
local_down_podping_liveitem_listener:
# docker-compose -f docker-compose/local/docker-compose.yml down runLiveItemListener
docker stop runLiveItemListener
docker rm runLiveItemListener
# This will request the updated feeds according to Podcast Index over the past X milliseconds
# (as set as a parameter in seconds to the end of the command), and then add them to the priority SQS queue.
# If that podcastIndexId is not already available in the database,
# this process will not add it as a new feed. It only handles updating existing podcasts.
.PHONY: local_add_podcast_index_recently_updated_feed_urls
local_add_podcast_index_recently_updated_feed_urls:
docker-compose -f docker-compose/local/docker-compose.yml run --name addRecentlyUpdated --rm podverse_api_worker npm run scripts:addRecentlyUpdatedFeedUrlsToPriorityQueue $$(date -v-5M +%s)
# This will request a list of all dead feeds/podcasts from Podcast Index,
# and then set "isPublic=false" to those podcasts in our database.
.PHONY: local_hide_dead_podcasts_from_podcast_index
local_hide_dead_podcasts_from_podcast_index:
docker-compose -f docker-compose/local/docker-compose.yml run --name hideDeadPodcasts --rm podverse_api_worker npm run scripts:hideDeadPodcasts
# If an episode has "isPublic=false" and does not have any mediaRefs or playlists
# associated with it, then it will be considered "dead" and deleted from the database.
.PHONY: local_remove_dead_episodes
local_remove_dead_episodes:
docker-compose -f docker-compose/local/docker-compose.yml run --name removeDeadEpisodes --rm podverse_api_worker npm run scripts:removeDeadEpisodes
# This query gets the Value for Value aka <podcast:value> tag information
# from Podcast Index. Ideally <podcast:value> tag info is available in the RSS feed,
# but Podcast Index has a service called Podcaster Wallet, which serves as a "shim"
# so podcasters can share their <podcast:value> tag info without adding it to their RSS feed.
# To handle this, a few times per day we request from Podcast Index
# a full list of the podcasts that use Podcaster Wallet. Then, in our API,
# when a podcast is requested uses Podcaster Wallet, we make a request to
# Podcast Index to get the <podcast:value> shim data, so we can load it in our apps.
.PHONY: local_update_value_tags_from_podcast_index
local_update_value_tags_from_podcast_index:
docker-compose -f docker-compose/local/docker-compose.yml run --name updateValueTagEnabledPodcastIdsFromPI --rm podverse_api_worker npm run scripts:podcastindex:updateValueTagEnabledPodcastIdsFromPI
# The stats queries are running for me...but the Matomo API
# does not seem to return the data from the past hour.
# I just emailed Matomo support to ask if there is a delay before
# the data becomes available, or if I'm doing something wrong...
.PHONY: local_stats_queries
local_stats_queries:
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- clips hour
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- clips day
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- clips week
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- clips month
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- clips year
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- clips allTime
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- episodes hour
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- episodes day
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- episodes week
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- episodes month
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- episodes year
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- episodes allTime
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- podcasts hour
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- podcasts day
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- podcasts week
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- podcasts month
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- podcasts year
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- podcasts allTime
.PHONY: local_stats_queries_short
local_stats_queries_short:
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- clips week
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- episodes week
docker-compose -f docker-compose/local/docker-compose.yml run --rm podverse_api_worker npm run scripts:queryUniquePageviews -- podcasts week
.PHONY: local_up local_up_db local_up_manticore_server local_up_api local_up_web
local_up: local_up_db local_up_manticore_server local_up_api local_up_web
.PHONY: local_down local_down_docker_compose
local_down: local_down_docker_compose
.PHONY: local_refresh local_down local_up
local_refresh: local_down local_up
proxy/local/certs:
mkdir -p $@
proxy/local/certs/podverse-server.key:
openssl genrsa -out $@ 4096
proxy/local/certs/podverse-server.key.insecure: proxy/local/certs/podverse-server.key
openssl rsa -in $< -out $@
proxy/local/certs/podverse-server.csr: proxy/local/certs/podverse-server.key
openssl req -new -sha256 -key $< -subj "/C=US/ST=Jefferson/L=Grand/O=Podverse/OU=Inra/CN=podverse.local" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:podverse.local,DNS:www.podverse.local,DNS:api.podverse.local")) -out $@
proxy/local/certs/podverse-server.crt: proxy/local/certs/podverse-server.csr
openssl x509 -req -days 365 -in $< -signkey proxy/local/certs/podverse-server.key -out $@
.PHONY: local_nginx_proxy_cert
local_nginx_proxy_cert: proxy/local/certs proxy/local/certs/podverse-server.key proxy/local/certs/podverse-server.key.insecure proxy/local/certs/podverse-server.csr proxy/local/certs/podverse-server.crt
@echo 'Generate new cert'
.PHONY: local_git_sub_init
local_git_sub_init:
git submodule update --init --recursive
.PHONY: stage_clean_manticore
stage_clean_manticore:
@echo "Cleaning Manticore"
rm -rf ./manticore/data
.PHONY: prod_cron_init
prod_cron_init:
crontab cronjobs/prod-podverse-workers
.PHONY: prod_srv_docker-compose_up
prod_srv_docker-compose_up:
docker-compose -f docker-compose/prod/srv/docker-compose.yml up -d
.PHONY: prod_srv_docker-compose_up-no_dettach
prod_srv_docker-compose_up-no_dettach:
docker-compose -f docker-compose/prod/srv/docker-compose.yml up
.PHONY: sanbox_db_docker-compose_up
sanbox_db_docker-compose_up:
docker-compose -f docker-compose/sandbox/db/docker-compose.yml up -d
.PHONY: sanbox_db_docker-compose_up-no_dettach
sanbox_db_docker-compose_up-no_dettach:
docker-compose -f docker-compose/sandbox/db/docker-compose.yml up
.PHONY: sanbox_srv_docker-compose_up
sanbox_srv_docker-compose_up:
docker-compose -f docker-compose/sandbox/srv/docker-compose.yml up -d
.PHONY: sanbox_srv_docker-compose_up-no_dettach
sanbox_srv_docker-compose_up-no_dettach:
docker-compose -f docker-compose/sandbox/srv/docker-compose.yml up