fix: ketelhuis

ckuijjer · Dec 22, 2023 · 6a8ff03 · 6a8ff03
1 parent 6d5384d
commit 6a8ff03
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -73,3 +73,13 @@ aws dynamodb scan --table-name expatcinema-scrapers-analytics --profile casper >
 ### Favicon
 
 Use https://favicongrabber.com/ to grab a favicon for the cinema.json file
+
+## Troubleshooting
+
+When running a puppeteer based scraper locally, e.g. `yarn tsx scrapers/ketelhuis.ts` and getting an error like
+
+```
+Error: Failed to launch the browser process! spawn /tmp/localChromium/chromium/mac_arm-1205129/chrome-mac/Chromium.app/Contents/MacOS/Chromium ENOENT
+````
+
+you need to install Chromium locally, run `yarn install-chromium` to do so and update `LOCAL_CHROMIUM_EXECUTABLE_PATH` in `browser.ts` to point to the Chromium executable
diff --git a/cloud/browser.ts b/cloud/browser.ts
@@ -9,7 +9,7 @@ import { Logger } from '@aws-lambda-powertools/logger'
 // see https://github.com/Sparticuz/chromium#running-locally--headlessheadful-mode
 // for how to install a locally running chromium
 const LOCAL_CHROMIUM_EXECUTABLE_PATH =
-  '/tmp/localChromium/chromium/mac_arm-1205129/chrome-mac/Chromium.app/Contents/MacOS/Chromium'
+  '/tmp/localChromium/chromium/mac_arm-1240459/chrome-mac/Chromium.app/Contents/MacOS/Chromium'
 
 const createBrowserSingleton = () => {
   let instance: Browser

diff --git a/cloud/package.json b/cloud/package.json
@@ -22,7 +22,8 @@
     "tail-cloudwatch-logs": "lumigo-cli tail-cloudwatch-logs -n /aws/lambda/expatcinema-dev --profile casper --region eu-west-1",
     "prettify-log": "awk -F'\\t' '{if ($5 && $5 ~ /^{/ && !system(\"echo \\x27\" $5 \"\\x27 | jq -e > /dev/null 2>&1\")) { print $1\"\\t\"$2\"\\t\"$3\"\\t\"$4 system(\"echo \\x27\" $5 \"\\x27 | jq\") } else { print $0 }}'",
     "prettify-log:test": "cat tail-cloudwatch-logs.example.log | yarn prettify-log",
-    "log": "yarn tail-cloudwatch-logs|yarn prettify-log"
+    "log": "yarn tail-cloudwatch-logs|yarn prettify-log",
+    "install-local-chromium": "npx @puppeteer/browsers install chromium@latest --path /tmp/localChromium"
   },
   "author": "Casper Kuijjer",
   "license": "ISC",

diff --git a/cloud/scrapers/ketelhuis.ts b/cloud/scrapers/ketelhuis.ts
@@ -53,7 +53,18 @@ const extractFromMainPage = async () => {
   )
   logger.info('scraped /deutsches-kino', { deutschesKinoResults })
 
-  const results = [...expatCinemaResults, ...deutschesKinoResults]
+  const italianCineclubResults = await xray(
+    'https://www.ketelhuis.nl/specials/italian-cineclub/',
+    '.c-default-page-content a[href^="https://www.ketelhuis.nl/films/"]',
+    selector,
+  )
+  logger.info('scraped /italian-cineclub', { italianCineclubResults })
+
+  const results = [
+    ...expatCinemaResults,
+    ...deutschesKinoResults,
+    ...italianCineclubResults,
+  ]
   const uniqueResults = R.uniq(results)
 
   logger.info('results', { results })
@@ -66,9 +77,11 @@ const extractFromMainPage = async () => {
   return extracted
 }
 
-const hasEnglishSubtitles = ({ metadata, mainContent }) =>
-  metadata?.includes('English subtitles') ||
-  mainContent?.includes('Engels ondertiteld')
+const hasEnglishSubtitles = ({ metadata, mainContent, title }) =>
+  title?.toLowerCase().includes('english subs') ||
+  metadata?.toLowerCase().includes('english subtitles') ||
+  metadata?.toLowerCase().includes('engels ondertiteld') ||
+  mainContent?.toLowerCase().includes('engels ondertiteld')
 
 const splitFirstDate = (date: string) => {
   if (date === 'Vandaag') {