Add opensearch-dashboards and fluent-bit to retrieve docker logs (#170)

* Migration to opensearch

* Implement indexes update on version change

* Add opensearch-dashboard and logstash to retrieve docker logs

* Replace logstash with fluent-bit to retrieve docker logs

* Add running fluent-bit logging to OCI

* Change fluent-bit supply to package delivery

* Implement index cleanup every 30 days via lua script

* Implement index cleanup via exec on timer

* Optimize indexes update on version change

* Add a check that fluent-bit has installed successfully

* Add the dashboard location and authorization for it

* Move dashboards location to router

* Migration to opensearch in docspace.profiles.yml

* Replace the naming with dashboards

* Add password generation for /dashboards/

config/nginx/onlyoffice.conf

@@ -157,6 +157,19 @@ server {
 
 	}
 
+	location ^~ /dashboards/ {
+		auth_basic "Restricted Access";
+		auth_basic_user_file /etc/nginx/.htpasswd_dashboards;
+
+		rewrite ^/dashboards(/.*)$ $1 break;
+		proxy_pass http://127.0.0.1:5601;
+		proxy_redirect off;
+		proxy_buffering off;
+
+		proxy_set_header Connection "Keep-Alive";
+		proxy_set_header Proxy-Connection "Keep-Alive";
+	}
+
 	location / {
 		proxy_pass http://127.0.0.1:5001;
 		proxy_redirect off;

install/OneClickInstall/install-Docker.sh

@@ -60,6 +60,7 @@ INSTALL_RABBITMQ="true";
 INSTALL_MYSQL_SERVER="true";
 INSTALL_DOCUMENT_SERVER="true";
 INSTALL_ELASTICSEARCH="true";
+INSTALL_FLUENT_BIT="true";
 INSTALL_PRODUCT="true";
 UPDATE="false";
 
@@ -372,6 +373,13 @@ while [ "$1" != "" ]; do
 			fi
 		;;
 
+		-ifb | --installfluentbit )
+			if [ "$2" != "" ]; then
+				INSTALL_FLUENT_BIT=$2
+				shift
+			fi
+		;;
+
 		-rdsh | --redishost )
 			if [ "$2" != "" ]; then
 				REDIS_HOST=$2
@@ -463,6 +471,20 @@ while [ "$1" != "" ]; do
 			fi
 		;;
 
+		-du | --dashboadrsusername )
+			if [ "$2" != "" ]; then
+				DASHBOARDS_USERNAME=$2
+				shift
+			fi
+		;;
+
+		-dp | --dashboadrspassword )
+			if [ "$2" != "" ]; then
+				DASHBOARDS_PASSWORD=$2
+				shift
+			fi
+		;;
+
 		-noni | --noninteractive )
 			if [ "$2" != "" ]; then
 				NON_INTERACTIVE=$2
@@ -496,6 +518,9 @@ while [ "$1" != "" ]; do
 			echo "      -irds, --installredis             install or update redis (true|false)"
 			echo "      -imysql, --installmysql           install or update mysql (true|false)"		
 			echo "      -ies, --installelastic            install or update elasticsearch (true|false)"
+			echo "      -ifb, --installfluentbit          install or update fluent-bit (true|false)"
+			echo "      -du, --dashboadrsusername         login for authorization in /dashboards/"
+			echo "      -dp, --dashboadrspassword         password for authorization in /dashboards/"
 			echo "      -espr, --elasticprotocol          the protocol for the connection to elasticsearch (default value http)"
 			echo "      -esh, --elastichost               the IP address or hostname of the elasticsearch"
 			echo "      -esp, --elasticport               elasticsearch port number (default value 9200)"
@@ -1137,6 +1162,9 @@ set_docspace_params() {
 	RABBIT_PASSWORD=${RABBIT_PASSWORD:-$(get_env_parameter "RABBIT_PASSWORD" "${CONTAINER_NAME}")};
 	RABBIT_VIRTUAL_HOST=${RABBIT_VIRTUAL_HOST:-$(get_env_parameter "RABBIT_VIRTUAL_HOST" "${CONTAINER_NAME}")};
 
+	DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME:-$(get_env_parameter "DASHBOARDS_USERNAME" "${CONTAINER_NAME}")};
+	DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD:-$(get_env_parameter "DASHBOARDS_PASSWORD" "${CONTAINER_NAME}")};
+
 	CERTIFICATE_PATH=${CERTIFICATE_PATH:-$(get_env_parameter "CERTIFICATE_PATH")};
 	CERTIFICATE_KEY_PATH=${CERTIFICATE_KEY_PATH:-$(get_env_parameter "CERTIFICATE_KEY_PATH")};
 	DHPARAM_PATH=${DHPARAM_PATH:-$(get_env_parameter "DHPARAM_PATH")};
@@ -1285,6 +1313,38 @@ install_elasticsearch () {
 	fi
 }
 
+install_fluent_bit () {
+	if [ "$INSTALL_FLUENT_BIT" == "true" ]; then
+		curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
+
+		if systemctl list-unit-files --type=service | grep -q "fluent-bit.service"; then
+			sed -i "s/OPENSEARCH_SCHEME/$(get_env_parameter "ELK_SHEME")/g" "${BASE_DIR}/config/fluent-bit.conf"
+			sed -i "s/OPENSEARCH_HOST/${ELK_HOST:-127.0.0.1}/g" "${BASE_DIR}/config/fluent-bit.conf"
+			sed -i "s/OPENSEARCH_PORT/$(get_env_parameter "ELK_PORT")/g" ${BASE_DIR}/config/fluent-bit.conf
+			sed -i "s/OPENSEARCH_INDEX/${OPENSEARCH_INDEX:-"${PACKAGE_SYSNAME}-fluent-bit"}/g" ${BASE_DIR}/config/fluent-bit.conf
+			[ ! -z "${ELK_HOST}" ] && sed -i "s/ELK_CONTAINER_NAME/ELK_HOST/g" ${BASE_DIR}/dashboards.yml
+			cp -rf ${BASE_DIR}/config/fluent-bit.conf /etc/fluent-bit/fluent-bit.conf
+			systemctl restart fluent-bit
+
+			DOCKER_DAEMON_FILE="/etc/docker/daemon.json"
+			if [[ ! -f "${DOCKER_DAEMON_FILE}" ]]; then
+				echo "{\"log-driver\": \"fluentd\", \"log-opts\": { \"fluentd-address\": \"127.0.0.1:24224\" }}" > "${DOCKER_DAEMON_FILE}"
+				systemctl restart docker
+			elif ! grep -q "log-driver" ${DOCKER_DAEMON_FILE}; then
+				sed -i 's!{!& "log-driver": "fluentd", "log-opts": { "fluentd-address": "127.0.0.1:24224" },!' "${DOCKER_DAEMON_FILE}"
+				systemctl restart docker
+			fi
+
+			reconfigure DASHBOARDS_USERNAME "${DASHBOARDS_USERNAME:-"onlyoffice"}"
+			reconfigure DASHBOARDS_PASSWORD "${DASHBOARDS_PASSWORD:-$(get_random_str 20)}"
+
+			docker-compose -f ${BASE_DIR}/dashboards.yml up -d
+		else
+			echo "The installation of the fluent-bit service was unsuccessful."
+		fi
+	fi
+}
+
 install_product () {
 	DOCKER_TAG="${DOCKER_TAG:-$(get_available_version ${IMAGE_NAME})}"
 	reconfigure DOCKER_TAG ${DOCKER_TAG}
@@ -1402,15 +1462,17 @@ start_installation () {
 
 	download_files
 
+	install_elasticsearch
+
+	install_fluent_bit
+
 	install_mysql_server
-
-	install_document_server
 
 	install_rabbitmq
 
 	install_redis
 
-	install_elasticsearch
+	install_document_server
 
 	install_product
 

install/docker/.env

@@ -8,18 +8,22 @@
     CONTAINER_PREFIX=${PRODUCT}-
     MYSQL_VERSION=8.3.0
     MYSQL_IMAGE=mysql:${MYSQL_VERSION}
-    ELK_VERSION=2.11.1
     SERVICE_PORT=5050
     DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/4testing-documentserver-ee:latest
     DOCKERFILE=Dockerfile.app
     APP_DOTNET_ENV=""
     EXTERNAL_PORT="80"
 
-# elasticsearch #
+# opensearch stack #
+    ELK_VERSION=2.11.1
     ELK_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch
     ELK_SHEME=http
     ELK_HOST=""
     ELK_PORT=9200
+    DASHBOARDS_VERSION=2.11.1
+    DASHBOARDS_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch-dashboards
+    DASHBOARDS_USERNAME=onlyoffice
+    DASHBOARDS_PASSWORD=onlyoffice
 
 # app service environment #
     ENV_EXTENSION=none

install/docker/Dockerfile.app

@@ -173,6 +173,7 @@ RUN sed -i 's/127.0.0.1:5010/$service_api_system/' /etc/nginx/conf.d/onlyoffice.
     if [[ -z "${SERVICE_CLIENT}" ]] ; then sed -i 's/127.0.0.1:5001/$service_client/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
     if [[ -z "${SERVICE_MANAGEMENT}" ]] ; then sed -i 's/127.0.0.1:5015/$service_management/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
     sed -i 's/127.0.0.1:5033/$service_healthchecks/' /etc/nginx/conf.d/onlyoffice.conf && \
+    sed -i 's/127.0.0.1:5601/$dashboards_host:5601/' /etc/nginx/conf.d/onlyoffice.conf && \
     sed -i 's/$public_root/\/var\/www\/public\//' /etc/nginx/conf.d/onlyoffice.conf && \
     sed -i 's/http:\/\/172.*/$document_server;/' /etc/nginx/conf.d/onlyoffice.conf && \
     sed -i '/client_body_temp_path/ i \ \ \ \ $MAP_HASH_BUCKET_SIZE' /etc/nginx/nginx.conf.template && \

install/docker/config/docspace-logs

@@ -14,7 +14,7 @@ else
   echo "Error: yml files not found." && exit 1
 fi
 
-FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "db")
+FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "dashboards" "db")
 
 LOG_DIR="${DOCKERCOMPOSE}/logs"
 mkdir -p ${LOG_DIR}

install/docker/config/fluent-bit.conf

@@ -0,0 +1,25 @@
+[SERVICE]
+    Flush               1
+    Log_Level           info
+    Daemon              off
+
+[INPUT]
+    Name                forward
+    Listen              127.0.0.1
+    Port                24224
+
+[INPUT]
+    Name                exec
+    Interval_Sec        86400
+    Command             curl -s -X POST 'OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query' -H 'Content-Type: application/json' -d "{\"query\": {\"range\": {\"@timestamp\": {\"lt\": \"$(date -u -d '30 days ago' '+%Y-%m-%dT%H:%M:%S')\"}}}}"
+
+[OUTPUT]
+    Name                opensearch
+    Match               *
+    Host                OPENSEARCH_HOST
+    Port                OPENSEARCH_PORT
+    Replace_Dots        On
+    Suppress_Type_Name  On
+    Time_Key            @timestamp  
+    Type                _doc
+    Index               OPENSEARCH_INDEX

install/docker/config/nginx/templates/upstream.conf.template

@@ -83,3 +83,9 @@ map $SERVICE_CLIENT $service_client {
     "" 127.0.0.1:5001;
     default $SERVICE_CLIENT;
 }
+
+map $DASHBOARDS_CONTAINER_NAME $dashboards_host {
+    volatile;
+    default onlyoffice-opensearch-dashboards;
+    ~^(.*)$ $1;
+}

install/docker/dashboards.yml

@@ -0,0 +1,17 @@
+version: "3"
+services:
+  onlyoffice-opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:${DASHBOARDS_VERSION}
+    container_name: ${DASHBOARDS_CONTAINER_NAME}
+    restart: always
+    environment:
+      - OPENSEARCH_HOSTS=${ELK_SHEME}://${ELK_CONTAINER_NAME}:${ELK_PORT}
+      - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true"
+      - "SERVER_BASEPATH=/dashboards"
+    expose:
+      - "5601"
+
+networks:
+  default:
+    name: ${NETWORK_NAME}
+    external: true

install/docker/docspace.profiles.yml

@@ -279,6 +279,7 @@ services:
       - REDIS_HOST=${REDIS_HOST}
       - REDIS_PORT=${REDIS_PORT}
       - SERVICE_PORT=${SERVICE_PORT}
+      - DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
     volumes:
       - router_log:/var/log/nginx
 

install/docker/docspace.yml

@@ -223,6 +223,9 @@ services:
       - REDIS_PORT=${REDIS_PORT}
       - REDIS_PASSWORD=${REDIS_PASSWORD}
       - SERVICE_PORT=${SERVICE_PORT}
+      - DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
+      - DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME}
+      - DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD}
     volumes:
       - router_log:/var/log/nginx
 

install/docker/opensearch.yml

@@ -23,6 +23,9 @@ services:
     expose:
       - "9200"
       - "9600" # required for Performance Analyzer
+    ports:
+      - 127.0.0.1:9200:9200
+
 networks:
   default:
     name: ${NETWORK_NAME}

install/docker/prepare-nginx-router.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 WRONG_PORTAL_NAME_URL=${WRONG_PORTAL_NAME_URL:-""}
 REDIS_HOST=${REDIS_HOST:-"${REDIS_CONTAINER_NAME}"}
 REDIS_PORT=${REDIS_PORT:-"6379"}
@@ -9,3 +9,4 @@ sed -i "s~\(redis_host =\).*~\1 \"$REDIS_HOST\"~" /etc/nginx/conf.d/onlyoffice.c
 sed -i "s~\(redis_port =\).*~\1 $REDIS_PORT~" /etc/nginx/conf.d/onlyoffice.conf
 sed -i "s~\(redis_pass =\).*~\1 \"$REDIS_PASSWORD\"~" /etc/nginx/conf.d/onlyoffice.conf
 sed -i "s~\(\"wrongPortalNameUrl\":\).*,~\1 \"${WRONG_PORTAL_NAME_URL}\",~g" /var/www/public/scripts/config.json
+echo "${DASHBOARDS_USERNAME:-onlyoffice}:$(openssl passwd -6 -stdin <<< "${DASHBOARDS_PASSWORD:-onlyoffice}")" > /etc/nginx/.htpasswd_dashboards