VNC (#52)

* Some base

* Testing

* Working vnc but not working HEADFULL

* Working!

* Not working solution in HEADLESS=false

* Working VNC!

* Puppeteer version update

* Simplified Dockerfile

* Docs

* Changes due to self-review

* Dependencies

.dockerignore

@@ -1,3 +1,3 @@
 node_modules
 .gitignore
-.git
+.git

Dockerfile

@@ -1,10 +1,14 @@
 FROM node:18
 
+ENV USER=root
+ENV DISPLAY=:1
+ENV RESOLUTION=1080x720
+
 # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
 # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer
 # installs, work.
 RUN apt-get update \
-    && apt-get install -y wget gnupg \
+    && apt-get install -y wget gnupg tightvncserver xfce4 xfce4-goodies xfonts-base dbus-x11 \
     && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg \
     && sh -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
     && apt-get update \
@@ -20,11 +24,17 @@ ENV NODE_PATH=/app/node_modules
 RUN groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
     && mkdir -p /home/pptruser/Downloads \
     && chown -R pptruser:pptruser /home/pptruser \
-    && chown -R pptruser:pptruser /app
+    && chown -R pptruser:pptruser /app \
+    && chmod +x start_vnc.sh \
+    && chmod +x start_container.sh
 
 USER pptruser
 
 RUN yarn install
 
+# puppeteer-service
 EXPOSE 3000
-CMD npm start
+# VNC-server
+EXPOSE 5901
+
+CMD ./start_container.sh

README.md

@@ -18,27 +18,35 @@ To start service run the docker container.
 Since the Dockerfile adds a pptr user as a non-privileged user, it may not have all the necessary privileges.
 So you should use `docker run --cap-add=SYS_ADMIN` option.
 ```shell script
-$ docker run -d -p 3000:3000 --name scrapy-puppeter-service --cap-add SYS_ADMIN isprascrawlers/scrapy-puppeteer-service 
+$ docker run -d -p 3000:3000 --name scrapy-puppeteer-service --cap-add SYS_ADMIN isprascrawlers/scrapy-puppeteer-service 
 ```
 
 To run example which shows how to deploy several instances of service with load balancer use this command.
 ```shell script
 $ docker-compose up -d
 ```
 
+To run headfull puppeteer in container provide `HEADLESS=false` environment variable to the container.
+In this case a VNC server at `localhost:5901` with password `password` is started.
+You may change the password providing `VNC_SERVER` environment variable.
+
+```shell script
+$ docker run -d -p 3000:3000 -p 5901:5901 -e HEADLESS=false -e VNC_PASSWORD=puppeteer --name scrapy-puppeteer-service --cap-add SYS_ADMIN scrapy-puppeteer-service
+```
+
 ## API
 
 Here is the list of implemented methods that could be used to connect to puppeteer.
 For All requests puppeteer browser creates new incognito browser context and new page in it.
 If your want to reuse your browser context simple send context_id in your query. 
 All request return their context ids in response. 
-Also you could reuse your browser page and more actions with it.
+Also, you could reuse your browser page and more actions with it.
 In order to do so you should send in your request pageId that is returned in your previous request,
 that would make service reuse current page and return again its pageId. 
 If you want to close the page you are working with you should send in query param "closePage" with non-empty value.
 If you want your requests on page make through proxy, just add to normal request "proxy" param. 
 Proxy username and password params are optional.
-Also you can add extra http headers to each request that is made on page.
+Also, you can add extra http headers to each request that is made on page.
 ```json5
 {
   //request params  
@@ -74,9 +82,10 @@ This POST method allows to goto a page with a specific url in puppeteer.
 
 Params: 
 
-url - the url which puppeteer should navigate to.      
-navigationOptions - [possible options to use for request.](https://pptr.dev/api/puppeteer.page.goto#remarks)      
-waitOptions - [wait for selector](https://pptr.dev/api/puppeteer.page.waitforselector), [xpath](https://pptr.dev/api/puppeteer.page.waitforxpath), or timeout after navigation completes.
+url - the url which puppeteer should navigate to. \
+navigationOptions - [possible options to use for request.](https://pptr.dev/api/puppeteer.page.goto#remarks) \
+waitOptions - [wait for selector](https://pptr.dev/api/puppeteer.page.waitforselector), [xpath](https://pptr.dev/api/puppeteer.page.waitforxpath), or timeout after navigation completes. \
+harRecording - whether to start writing HAR or not.
 
 Example request body
 ```json5
@@ -98,7 +107,8 @@ Example request body
         "options": { // <object> Options to wait for elements (see https://pptr.dev/api/puppeteer.waitforselectoroptions)
             "timeout": 10000
         } 
-    }
+    },
+    "harRecording": true,
 }
 ```
 
@@ -205,7 +215,7 @@ async function action(page, request) {
 
 This POST method returns screenshots of current page more.  
 Description of options you can see on [puppeteer GitHub](https://github.com/GoogleChrome/puppeteer/blob/v1.19.0/docs/api.md#pagescreenshotoptions).
-The path options is omitted in options. Also the only possibly encoding is `base64`.
+The path options is omitted in options. Also, the only possible encoding is `base64`.
 
 Example request body:
 ```json5

app.js

@@ -76,7 +76,10 @@ async function setupBrowser() {
             {
                 headless: HEADLESS,
                 defaultViewport: { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT },
-                timeout: CONNECT_TIMEOUT
+                timeout: CONNECT_TIMEOUT,
+                args: [
+                    "--no-sandbox",
+                ]
             });
         browser.on('disconnected', setupBrowser);
         app.set('browser', browser);

helpers/utils.js

@@ -132,7 +132,7 @@ async function newContext(browser, options = {}) {
     }
 
     try {
-        const context = await browser.createIncognitoBrowserContext(options);
+        const context = await browser.createBrowserContext(options);
         limitContext.incContextCounter();
         timeoutContext.setContextTimeout(context);
         return context;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapy-puppeteer-service",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "scripts": {
     "start": "node ./bin/www"
@@ -20,7 +20,7 @@
     "fingerprint-injector": "^2.1.30",
     "morgan": "~1.10.0",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^20.1.2",
+    "puppeteer": "^22.15.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "puppeteer-extra-plugin-stealth": "^2.11.2",

routes/goto.js

@@ -24,7 +24,8 @@ async function action(page, request) {
 //         "selector": <string> Wait for element by selector (see https://pptr.dev/api/puppeteer.page.waitforselector)
 //         "xpath": <string> Wait for element by xpath (see https://pptr.dev/api/puppeteer.page.waitforxpath)
 //         "options": <object> Options to wait for elements (see https://pptr.dev/api/puppeteer.waitforselectoroptions)
-//     }
+//     },
+//     "harRecording": true,
 // }
 //
 router.post('/', async function (req, res, next) {

start_container.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Start VNC server if puppeteer is headfull
+if [ "${HEADLESS}" == "false" ]; then
+  printf "%s\n%s\nn" "${VNC_PASSWORD:=password}" "${VNC_PASSWORD:=password}" | vncpasswd
+  ./start_vnc.sh &
+fi
+
+# Start scrapy-puppeteer-service
+yarn start
+
+# Wait for any process to exit
+wait -n
+
+# Exit with status of process that exited first
+exit $?

start_vnc.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo "Starting VNC server at $RESOLUTION..."
+vncserver -kill :1 || true
+vncserver -geometry "${RESOLUTION}" &
+echo "VNC server started at ${RESOLUTION}! ^-^"
+
+tail -f /dev/null