Skip to content

Commit

Permalink
Centralized CMS caching 1 (#1830)
Browse files Browse the repository at this point in the history
* just enough of the cms caching pr to prove that redis is working

* fix mix lock

* test that logger is working for endpoints

* convert to test cache

* set log level to notice

* catch redis connection error and log

* PR feedback

* empty line

* make test that redis disconnect is logged

* default on basic auth for testing

* force ssl

* try to go through api

* missing dep

* reset router info

* just slap those creds in there

* runtime config issue for prod

* log basic auth so we can make sure what we are getting from drupal

* more deployment checking changes

* hard code redis host to see if connection works

* remove hard coded redis host
  • Loading branch information
anthonyshull authored Dec 20, 2023
1 parent 6258f82 commit de71c7a
Show file tree
Hide file tree
Showing 15 changed files with 113 additions and 19 deletions.
8 changes: 8 additions & 0 deletions .envrc.template
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,11 @@ export WIREMOCK_TRIP_PLAN_PROXY_URL=http://otp-local.mbtace.com
# You can also log in manually with `docker login`.
# export DOCKER_USERNAME=
# export DOCKER_PASSWORD=

# You can optionally set a Redis host. It will default to 127.0.0.1
# export REDIS_HOST=

# These credentials control access to resetting cache entries for the CMS.
# You can set them to be whatever you want, but they'll need to match those on the Drupal side.
# export CMS_BASIC_AUTH_USERNAME=
# export CMS_BASIC_AUTH_PASSWORD=
5 changes: 0 additions & 5 deletions apps/alerts/config/runtime.exs

This file was deleted.

5 changes: 5 additions & 0 deletions apps/cms/lib/cache.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
defmodule CMS.Cache do
use Nebulex.Cache,
otp_app: :cms,
adapter: NebulexRedisAdapter
end
1 change: 1 addition & 0 deletions apps/cms/lib/cms.ex
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ defmodule CMS do

# Define workers and child supervisors to be supervised
children = [
CMS.Cache,
CMS.Repo
]

Expand Down
14 changes: 8 additions & 6 deletions apps/cms/mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,18 @@ defmodule CMS.Mixfile do
# Type "mix help deps" for more examples and options
defp deps do
[
{:httpoison, ">= 0.0.0"},
{:poison, ">= 0.0.0", override: true},
{:timex, ">= 0.0.0"},
{:plug, "~> 1.14.2"},
{:html_sanitize_ex, "1.3.0"},
{:bypass, "~> 1.0", only: :test},
{:quixir, "~> 0.9", only: :test},
{:html_sanitize_ex, "1.3.0"},
{:httpoison, ">= 0.0.0"},
{:mock, "~> 0.3.3", only: :test},
{:nebulex, "2.5.2"},
{:nebulex_redis_adapter, "2.3.1"},
{:phoenix_html, "~> 3.0"},
{:plug, "~> 1.14.2"},
{:poison, ">= 0.0.0", override: true},
{:quixir, "~> 0.9", only: :test},
{:repo_cache, in_umbrella: true},
{:timex, ">= 0.0.0"},
{:util, in_umbrella: true}
]
end
Expand Down
4 changes: 2 additions & 2 deletions apps/site/config/config.exs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
use Mix.Config
import Config

# Configures the endpoint
config :site, SiteWeb.Endpoint,
Expand Down Expand Up @@ -81,4 +81,4 @@ config :site,

# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
import_config "#{config_env()}.exs"
2 changes: 1 addition & 1 deletion apps/site/config/dev.exs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use Mix.Config
import Config

# For development, we disable any cache and enable
# debugging and code reloading.
Expand Down
2 changes: 1 addition & 1 deletion apps/site/config/prod.exs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use Mix.Config
import Config

# For production, we configure the host to read the PORT
# from the system environment. Therefore, you will need
Expand Down
5 changes: 2 additions & 3 deletions apps/site/config/test.exs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use Mix.Config
import Config

# We don't run a server during test. If one is required,
# you can enable the server option below.
Expand All @@ -13,8 +13,7 @@ config :site, :secure_pipeline,
rewrite_on: [:x_forwarded_proto]
]

# Print only warnings and errors during test
config :logger, level: :warn
config :logger, level: :notice

# Don't fetch tz data in test mode; can cause a race if we're doing TZ
# operations while it updates.
Expand Down
28 changes: 28 additions & 0 deletions apps/site/lib/site_web/controllers/cms_controller.ex
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,34 @@ defmodule SiteWeb.CMSController do
|> handle_page_response(conn)
end

def reset_cache_key(conn, %{"object" => object, "id" => id}) do
Logger.notice("cms.cache.delete redis_host=#{System.get_env("REDIS_HOST")}")

try do
CMS.Cache.delete("/cms/#{object}/#{id}")

Logger.notice("cms.cache.delete path=/cms/#{object}/#{id}")
rescue
e in Redix.ConnectionError -> Logger.warning("cms.cache.delete error=redis-#{e.reason}")
end

send_resp(conn, 202, "") |> halt()
end

def reset_cache_key(conn, %{"id" => id}) do
Logger.notice("cms.cache.delete redis_host=#{System.get_env("REDIS_HOST")}")

try do
CMS.Cache.delete("/cms/#{id}")

Logger.notice("cms.cache.delete path=/cms/#{id}")
rescue
e in Redix.ConnectionError -> Logger.warning("cms.cache.delete error=redis-#{e.reason}")
end

send_resp(conn, 202, "") |> halt()
end

@spec handle_page_response(Page.t() | {:error, API.error()}, Conn.t()) ::
Plug.Conn.t()
defp handle_page_response(%{__struct__: struct} = page, conn)
Expand Down
13 changes: 13 additions & 0 deletions apps/site/lib/site_web/router.ex
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ defmodule SiteWeb.Router do
get("/_health", HealthController, :index)
end

scope "/cms", SiteWeb do
pipe_through([:basic_auth])

patch("/:object/:id", CMSController, :reset_cache_key)
patch("/:id", CMSController, :reset_cache_key)
end

# redirect 't.mbta.com' and 'beta.mbta.com' to 'https://www.mbta.com'
scope "/", SiteWeb, host: "t." do
# no pipe
Expand Down Expand Up @@ -264,6 +271,12 @@ defmodule SiteWeb.Router do
get("/*path", CMSController, :page)
end

defp basic_auth(conn, _) do
opts = Application.get_env(:site, SiteWeb.Router)[:cms_basic_auth]

Plug.BasicAuth.basic_auth(conn, opts)
end

defp optional_disable_indexing(conn, _) do
if Application.get_env(:site, :allow_indexing) do
conn
Expand Down
13 changes: 13 additions & 0 deletions apps/site/test/site_web/controllers/cms_controller_test.exs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
defmodule SiteWeb.CMSControllerTest do
use SiteWeb.ConnCase, async: false

import ExUnit.CaptureLog

alias Plug.Conn

describe "GET - page" do
Expand Down Expand Up @@ -182,4 +184,15 @@ defmodule SiteWeb.CMSControllerTest do
assert html_response(conn, 500) =~ "Something went wrong on our end."
end
end

describe "PATCH /cms/*" do
test "it logs that no redis connection was made", %{conn: conn} do
assert capture_log(fn ->
conn
|> put_req_header("content-type", "application/json")
|> put_req_header("authorization", "Basic " <> Base.encode64("username:password"))
|> patch("/cms/foo/bar")
end) =~ "cms.cache.delete error=redis-closed"
end
end
end
27 changes: 27 additions & 0 deletions config/runtime.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import Config

config :cms, CMS.Cache,
conn_opts: [
host: System.get_env("REDIS_HOST", "127.0.0.1"),
port: 6379
],
stats: false,
telemetry: false

if config_env() == :test do
config :site, SiteWeb.Router,
cms_basic_auth: [
username: "username",
password: "password"
]
else
config :site, SiteWeb.Router,
cms_basic_auth: [
username: System.fetch_env!("CMS_BASIC_AUTH_USERNAME"),
password: System.fetch_env!("CMS_BASIC_AUTH_PASSWORD")
]
end

if config_env() == :prod do
config :alerts, bus_stop_change_bucket: System.get_env("S3_PREFIX_BUSCHANGE")
end
3 changes: 3 additions & 0 deletions mix.lock
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@
"mint_web_socket": {:hex, :mint_web_socket, "1.0.3", "aab42fff792a74649916236d0b01f560a0b3f03ca5dea693c230d1c44736b50e", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "ca3810ca44cc8532e3dce499cc17f958596695d226bb578b2fbb88c09b5954b0"},
"mochiweb": {:hex, :mochiweb, "2.22.0", "f104d6747c01a330c38613561977e565b788b9170055c5241ac9dd6e4617cba5", [:rebar3], [], "hexpm", "cbbd1fd315d283c576d1c8a13e0738f6dafb63dc840611249608697502a07655"},
"mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "a280d1f7b6f4bbcbd9282616e57502721781c66ee5b540720efabeaf627cc7eb"},
"nebulex": {:hex, :nebulex, "2.5.2", "2d358813ccb2eeea525e3a29c270ad123d3337e97ed9159d9113cf128108bd4c", [:mix], [{:decorator, "~> 1.4", [hex: :decorator, repo: "hexpm", optional: true]}, {:shards, "~> 1.1", [hex: :shards, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "61a122302cf42fa61eca22515b1df21aaaa1b98cf462f6dd0998de9797aaf1c7"},
"nebulex_redis_adapter": {:hex, :nebulex_redis_adapter, "2.3.1", "ea57629ee21f78332ca8d0356e6777d2fdbd6755b7453e298557091b6f8811f6", [:mix], [{:crc, "~> 0.10", [hex: :crc, repo: "hexpm", optional: true]}, {:jchash, "~> 0.1", [hex: :jchash, repo: "hexpm", optional: true]}, {:nebulex, "~> 2.5", [hex: :nebulex, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:redix, "~> 1.2", [hex: :redix, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "8384bf249af7c0b9903578b5c75a18157562863054952dbaea55dfe7255b75e7"},
"nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"},
"nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"},
"nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"},
Expand All @@ -81,6 +83,7 @@
"ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
"recaptcha": {:git, "https://github.com/samueljseay/recaptcha.git", "8ea13f63990ca18725ac006d30e55d42c3a58457", [ref: "8ea13f63990ca18725ac006d30e55d42c3a58457"]},
"recon": {:hex, :recon, "2.5.1", "430ffa60685ac1efdfb1fe4c97b8767c92d0d92e6e7c3e8621559ba77598678a", [:mix, :rebar3], [], "hexpm", "5721c6b6d50122d8f68cccac712caa1231f97894bab779eff5ff0f886cb44648"},
"redix": {:hex, :redix, "1.3.0", "f4121163ff9d73bf72157539ff23b13e38422284520bb58c05e014b19d6f0577", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "60d483d320c77329c8cbd3df73007e51b23f3fae75b7693bc31120d83ab26131"},
"req": {:hex, :req, "0.3.12", "f84c2f9e7cc71c81d7cbeacf7c61e763e53ab5f3065703792a4ab264b4f22672", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c91103d4d1c8edeba90c84e0ba223a59865b673eaab217bfd17da3aa54ab136c"},
"rstar": {:git, "https://github.com/armon/erl-rstar.git", "a406b2cce609029bf65b9ccfbe93a0416c0ee0cd", []},
"sentry": {:hex, :sentry, "7.0.4", "a9a00b480becfca5b897a1b383f88e21fa21abc876e2b1b30060040f76a9c776", [:mix], [{:hackney, "~> 1.8 or 1.6.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "0aa7e405388af5634aefad7f3e11f188e8c13d0003ae7100b956eb01c8b002a5"},
Expand Down
2 changes: 1 addition & 1 deletion rel/config.exs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ environment :prod do

set(
overlays: [
{:copy, "apps/alerts/config/runtime.exs", "etc/runtime.exs"}
{:copy, "config/runtime.exs", "etc/runtime.exs"}
]
)

Expand Down

0 comments on commit de71c7a

Please sign in to comment.