Update package versions and remove cloud-run-proxy

README.md

@@ -32,6 +32,8 @@ Run and Eventarc.
     adds a local cache of ClamAV definition files which is updated on a schedule.
 * 2023-06-20 v2.1.0 Resolve #46 where docker build failed, and #50 where pip3
     installs failed
+* 2023-11-22 v2.2.0 Add support for using environmental variables in the config.json file.
+* 2023-11-23 v2.3.0 Remove need for cloud-run-proxy, and update versions of packages. Fixes multiple CVEs.
 
 ## Upgrading from v1.x to v2.x
 

cloudrun-malware-scanner/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM node:20
+FROM node:21
 WORKDIR /app
 COPY .  /app
 COPY config.json  /app
@@ -71,12 +71,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     gcloud --version && \
     truncate -s 0 /var/log/apt/*.log /var/log/*.log
 
-# Get the cloud-run-proxy tool
-RUN CLOUD_RUN_PROXY_VERSION=0.3.0 && \
-    curl -s -L \
-        https://github.com/GoogleCloudPlatform/cloud-run-proxy/releases/download/v${CLOUD_RUN_PROXY_VERSION}/cloud-run-proxy_${CLOUD_RUN_PROXY_VERSION}_linux_amd64.tar.gz | \
-        tar -zxf - cloud-run-proxy
-
 # Get all required node.js dependencies
 RUN npm install --omit=dev
 

cloudrun-malware-scanner/bootstrap.sh

@@ -47,19 +47,6 @@ apt-get -qqy install --no-install-recommends clamav-daemon clamav-freshclam
 export PATH="$PATH:$HOME/.local/bin" # add pipx locations to path.
 pipx install cvdupdate
 
-# Set up an authentication proxy server to point to GCS CVD Mirror bucket, and to
-# restart with a fresh token every 50 mins because access tokens expire after 1hr
-AUTH_SERVER_ADDRESS=127.0.0.1:8001
-while true ; do
-    Log INFO main "Restarting authentication proxy service"
-   ./cloud-run-proxy \
-       -host https://storage.googleapis.com/  \
-       -token  "$(gcloud auth print-access-token)" \
-       -bind "${AUTH_SERVER_ADDRESS}" \
-       -server-up-time 50m
-done &
-
-
 # Ensure clamav services are shut down, as we do not have the config files set up yet.
 service clamav-daemon stop &
 service clamav-freshclam stop &
@@ -88,6 +75,30 @@ if ! gsutil ls "gs://${CVD_MIRROR_BUCKET}/" > /dev/null ; then
     exit 1
 fi
 
+# Start the reverse proxy which adds authentication
+# to requests to GCS REST API, allowing freshclam to access the GCS
+# CVD mirror bucket as if it was an unauthenticated HTPP server
+#
+export PROXY_PORT=8888
+PROXY_SERVER_ADDRESS=127.0.0.1:${PROXY_PORT}
+npm run start-proxy "${CONFIG_FILE}" &
+
+# wait for it to startup before freshclam connects to it...
+PROXY_CHECK_ATTEMPTS=${PROXY_CHECK_ATTEMPTS:-12}
+PROXY_CHECK_INTERVAL=${PROXY_CHECK_INTERVAL:-5}
+attempts=0
+while [[ $attempts -lt  "${PROXY_CHECK_ATTEMPTS}" ]]; do
+  attempts=$((attempts + 1))
+  Log INFO main "Waiting for proxy server to start...${attempts}"
+  sleep ${SLEEP_INTERVAL}
+  # Query a known file to verify if the proxy is up and running.
+  curl -s -I "http://${PROXY_SERVER_ADDRESS}/${CVD_MIRROR_BUCKET}/cvds/state.json" > /dev/null 2>&1 && break
+done
+if [[ $attempts -eq ${PROXY_CHECK_ATTEMPTS} ]] ; then
+  Log ERROR main "Proxy server did not start after $((PROXY_CHECK_ATTEMPTS * PROXY_CHECK_INTERVAL)) secs"
+  exit 1;
+fi
+
 # This function is used to update clam and freshclam config files.
 # Use by specifying the config file on the command line and
 # piping the config file updates in.
@@ -149,7 +160,7 @@ EOF
 updateClamConfigFile /etc/clamav/freshclam.conf << EOF
 # DatabaseMirror specifies to which mirror(s) freshclam should connect.
 # Set to the authentication proxy service which proxys to the GCS API.
-DatabaseMirror http://${AUTH_SERVER_ADDRESS}/${CVD_MIRROR_BUCKET}/cvds
+DatabaseMirror http://${PROXY_SERVER_ADDRESS}/${CVD_MIRROR_BUCKET}/cvds
 
 # Number of database checks per day.
 # Once per half hour, which is fine as we are using a local mirror.
@@ -173,4 +184,4 @@ service clamav-freshclam force-reload &
 
 # Run node server process
 Log INFO main "Starting malware-scanner service"
-npm start "${CONFIG_FILE}"
+npm run start "${CONFIG_FILE}"

cloudrun-malware-scanner/config.js

@@ -0,0 +1,131 @@
+/*
+* Copyright 2022 Google LLC
+
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+
+*     https://www.apache.org/licenses/LICENSE-2.0
+
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+const {Storage} = require('@google-cloud/storage');
+const {logger} = require('./logger.js');
+const pkgJson = require('./package.json');
+
+
+/**
+ * Configuration object.
+ *
+ * Values are read from the JSON configuration file.
+ * See {@link readAndVerifyConfig}.
+ *
+ * @typedef {{
+ *    buckets: Array<
+ *      {
+ *        unscanned: string,
+ *        clean: string,
+ *        quarantined: string
+ *       }>,
+ *    ClamCvdMirrorBucket: string
+ *  }}
+ */
+const Config = null;
+
+const storage = new Storage({userAgent: `${pkgJson.name}/${pkgJson.version}`});
+
+/**
+ * Read configuration from JSON configuration file, verify
+ * and return a Config object
+ *
+ * @async
+ * @param {string} configFile
+ * @return {Config}
+ */
+async function readAndVerifyConfig(configFile) {
+  logger.info(`Using configuration file: ${configFile}`);
+
+
+  /** @type {Config} */
+  let config;
+
+  try {
+    config = require(configFile);
+    delete config.comments;
+  } catch (e) {
+    logger.fatal({err: e}, `Unable to read JSON file from ${configFile}`);
+    throw new Error(`Invalid configuration ${configFile}`);
+  }
+
+  if (config.buckets.length === 0) {
+    logger.fatal(`No buckets configured for scanning in ${configFile}`);
+    throw new Error('No buckets configured');
+  }
+
+  logger.info('BUCKET_CONFIG: ' + JSON.stringify(config, null, 2));
+
+  // Check buckets are specified and exist.
+  let success = true;
+  for (let x = 0; x < config.buckets.length; x++) {
+    const buckets = config.buckets[x];
+    for (const bucketType of ['unscanned', 'clean', 'quarantined']) {
+      if (!(await checkBucketExists(
+          buckets[bucketType], `config.buckets[${x}].${bucketType}`))) {
+        success = false;
+      }
+    }
+    if (buckets.unscanned === buckets.clean ||
+        buckets.unscanned === buckets.quarantined ||
+        buckets.clean === buckets.quarantined) {
+      logger.fatal(
+          `Error in ${configFile} buckets[${x}]: bucket names are not unique`);
+      success = false;
+    }
+  }
+  if (!(await checkBucketExists(
+      config.ClamCvdMirrorBucket, 'ClamCvdMirrorBucket'))) {
+    success = false;
+  }
+
+  if (!success) {
+    throw new Error('Invalid configuration');
+  }
+  return config;
+}
+
+
+/**
+ * Check that given bucket exists. Returns true on success
+ *
+ * @param {string} bucketName
+ * @param {string} configName
+ * @return {Promise<boolean>}
+ */
+async function checkBucketExists(bucketName, configName) {
+  if (!bucketName) {
+    logger.fatal(`Error in config: no "${configName}" bucket defined`);
+    success = false;
+  }
+  // Check for bucket existence by listing files in bucket, will throw
+  // an exception if the bucket is not readable.
+  // This is used in place of Bucket.exists() to avoid the need for
+  // Project/viewer permission.
+  try {
+    await storage.bucket(bucketName)
+        .getFiles({maxResults: 1, prefix: 'zzz', autoPaginate: false});
+    return true;
+  } catch (e) {
+    logger.fatal(`Error in config: cannot view files in "${configName}" : ${
+      bucketName} : ${e.message}`);
+    logger.debug({err: e});
+    return false;
+  }
+}
+
+exports.Config = Config;
+exports.readAndVerifyConfig = readAndVerifyConfig;

cloudrun-malware-scanner/gcs-proxy-server.js

@@ -0,0 +1,140 @@
+/*
+* Copyright 2022 Google LLC
+
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+
+*     https://www.apache.org/licenses/LICENSE-2.0
+
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+const {GoogleAuth} = require('google-auth-library');
+const {logger} = require('./logger.js');
+// eslint-disable-next-line no-unused-vars
+const {Config, readAndVerifyConfig} = require('./config.js');
+const httpProxy = require('http-proxy');
+const {nextTick} = require('process');
+
+const TOKEN_REFRESH_THRESHOLD_MILLIS = 60000;
+
+const googleAuth = new GoogleAuth();
+
+
+// access token for GCS requests - will be refreshed shortly before it expires
+let accessToken;
+let accessTokenRefreshTimeout;
+let clamCvdMirrorBucket = 'uninitialized';
+
+/**
+ * Check to see when access token expires and refresh it just before.
+ * This is required because proxy requires access token to be available
+ * synchronously, but getAccessToken() is async.
+ * So a 'current' access token needs to be available.
+ */
+async function accessTokenRefresh() {
+  if (accessTokenRefreshTimeout) {
+    clearTimeout(accessTokenRefreshTimeout);
+    accessTokenRefreshTimeout = null;
+  }
+
+  const client = await googleAuth.getClient();
+  if (!client.credentials?.expiry_date ||
+      client.credentials.expiry_date <=
+          new Date().getTime() + TOKEN_REFRESH_THRESHOLD_MILLIS) {
+    accessToken = await googleAuth.getAccessToken();
+    logger.info(`Access token expires at ${
+        new Date(client.credentials.expiry_date).toISOString()}`);
+  }
+  const nextCheckDate =
+      new Date(client.credentials.expiry_date - TOKEN_REFRESH_THRESHOLD_MILLIS);
+  logger.debug(
+      `Next access token refresh check at ${nextCheckDate.toISOString()}`);
+  accessTokenRefreshTimeout = setTimeout(
+      accessTokenRefresh, nextCheckDate.getTime() - new Date().getTime());
+}
+
+/**
+ * Handle any internal proxy errors by returning a 500
+ */
+function handleProxyError(err, req, res) {
+  logger.error(
+      `Failed to proxy to GCS for path ${req.url}, returning code 500: ${err}`);
+  res.writeHead(500, {
+    'Content-Type': 'text/plain',
+  });
+  res.end(`Failed to proxy to GCS: internal error\n`);
+}
+
+/**
+ * Handle proxy requests - check path, and add Authorization header.
+ */
+function handleProxyReq(proxyReq, req, res) {
+  if (proxyReq.path.startsWith('/' + clamCvdMirrorBucket + '/')) {
+    logger.info(`Proxying request for ${proxyReq.path} to GCS`);
+    proxyReq.setHeader('Authorization', 'Bearer ' + accessToken);
+  } else {
+    logger.error(`Denying Proxy request for ${proxyReq.path} to GCS`);
+    res.writeHead(403, {
+      'Content-Type': 'text/plain',
+    });
+    res.end('Failed to proxy to GCS - unauthorzied path: status 403\n');
+  }
+}
+
+/**
+ * Set up a reverse proxy to add authentication to HTTP requests from
+ * freshclam and proxy it to the GCS API
+ */
+async function setupGcsReverseProxy() {
+  const proxy = httpProxy.createProxyServer({
+    target: 'https://storage.googleapis.com/',
+    changeOrigin: true,
+    autoRewrite: true,
+    secure: true,
+    ws: false,
+  });
+
+  proxy.on('error', handleProxyError);
+  proxy.on('proxyReq', handleProxyReq);
+
+  const PROXY_PORT = process.env.PROXY_PORT || 8888;
+
+  proxy.listen(PROXY_PORT, 'localhost');
+  logger.info(`GCS authenticating reverse proxy listenting on port ${
+      PROXY_PORT} for requests to ${clamCvdMirrorBucket}`);
+}
+
+/**
+ * Perform async setup and start the app.
+ *
+ * @async
+ */
+async function run() {
+  let configFile;
+  if (process.argv.length >= 3) {
+    configFile = process.argv[2];
+  } else {
+    configFile = './config.json';
+  }
+
+  /** @type {Config} */
+  const config = await readAndVerifyConfig(configFile);
+
+  clamCvdMirrorBucket = config.ClamCvdMirrorBucket;
+
+  await accessTokenRefresh();
+  await setupGcsReverseProxy(config.ClamCvdMirrorBucket);
+}
+
+// Start the service, exiting on error.
+run().catch((e) => {
+  logger.fatal(e);
+  logger.fatal('Exiting');
+  process.exit(1);
+});