Merge remote-tracking branch 'upstream/4.4.x' into thuenen_4.x

.env.sample

@@ -152,24 +152,180 @@ services:
       - rabbitmq:/var/lib/rabbitmq
     restart: unless-stopped
 
-volumes:
-  statics:
-    name: ${COMPOSE_PROJECT_NAME}-statics
-  nginx-confd:
-    name: ${COMPOSE_PROJECT_NAME}-nginxconfd
-  nginx-certificates:
-    name: ${COMPOSE_PROJECT_NAME}-nginxcerts
-  geoserver-data-dir:
-    name: ${COMPOSE_PROJECT_NAME}-gsdatadir
-  dbdata:
-    name: ${COMPOSE_PROJECT_NAME}-dbdata
-  dbbackups:
-    name: ${COMPOSE_PROJECT_NAME}-dbbackups
-  backup-restore:
-    name: ${COMPOSE_PROJECT_NAME}-backup-restore
-  data:
-    name: ${COMPOSE_PROJECT_NAME}-data
-  tmp:
-    name: ${COMPOSE_PROJECT_NAME}-tmp
-  rabbitmq:
-    name: ${COMPOSE_PROJECT_NAME}-rabbitmq
+RESOLVER=127.0.0.11
+
+# #################
+# geoserver
+# #################
+GEOSERVER_LB_HOST_IP=geoserver
+GEOSERVER_LB_PORT=8080
+GEOSERVER_WEB_UI_LOCATION={siteurl}/geoserver/
+GEOSERVER_PUBLIC_LOCATION={siteurl}/geoserver/
+GEOSERVER_LOCATION=http://${GEOSERVER_LB_HOST_IP}:${GEOSERVER_LB_PORT}/geoserver/
+GEOSERVER_ADMIN_USER=admin
+GEOSERVER_ADMIN_PASSWORD={geoserverpwd}
+
+OGC_REQUEST_TIMEOUT=30
+OGC_REQUEST_MAX_RETRIES=1
+OGC_REQUEST_BACKOFF_FACTOR=0.3
+OGC_REQUEST_POOL_MAXSIZE=10
+OGC_REQUEST_POOL_CONNECTIONS=10
+
+# Java Options & Memory
+ENABLE_JSONP=true
+outFormat=text/javascript
+GEOSERVER_JAVA_OPTS=-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL={siteurl}/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine
+
+# #################
+# Security
+# #################
+# Admin Settings
+#
+# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time
+# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT,
+# otherwise the invoke updateadmin task will be skipped and the current password already stored
+# in DB will honored.
+
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD={geonodepwd}
+ADMIN_EMAIL={email}
+
+# EMAIL Notifications
+EMAIL_ENABLE=False
+DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
+DJANGO_EMAIL_HOST=localhost
+DJANGO_EMAIL_PORT=25
+DJANGO_EMAIL_HOST_USER=
+DJANGO_EMAIL_HOST_PASSWORD=
+DJANGO_EMAIL_USE_TLS=False
+DJANGO_EMAIL_USE_SSL=False
+DEFAULT_FROM_EMAIL='{email}' # eg Company <[email protected]>
+
+# Session/Access Control
+LOCKDOWN_GEONODE=False
+X_FRAME_OPTIONS="SAMEORIGIN"
+SESSION_EXPIRED_CONTROL_ENABLED=True
+DEFAULT_ANONYMOUS_VIEW_PERMISSION=True
+DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True
+
+CORS_ALLOW_ALL_ORIGINS=True
+GEOSERVER_CORS_ENABLED=True
+GEOSERVER_CORS_ALLOWED_ORIGINS=*
+GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS
+GEOSERVER_CORS_ALLOWED_HEADERS=*
+
+# Users Registration
+ACCOUNT_OPEN_SIGNUP=True
+ACCOUNT_EMAIL_REQUIRED=True
+ACCOUNT_APPROVAL_REQUIRED=False
+ACCOUNT_CONFIRM_EMAIL_ON_GET=False
+ACCOUNT_EMAIL_VERIFICATION=none
+ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False
+ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False
+ACCOUNT_AUTHENTICATION_METHOD=username_email
+AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True
+AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True
+
+# OAuth2
+OAUTH2_API_KEY=
+OAUTH2_CLIENT_ID={clientid}
+OAUTH2_CLIENT_SECRET={clientsecret}
+
+# GeoNode APIs
+API_LOCKDOWN=False
+TASTYPIE_APIKEY=
+
+# #################
+# Production and
+# Monitoring
+# #################
+DEBUG={debug}
+
+SECRET_KEY='{secret_key}'
+
+STATIC_ROOT=/mnt/volumes/statics/static/
+MEDIA_ROOT=/mnt/volumes/statics/uploaded/
+ASSETS_ROOT=/mnt/volumes/statics/assets/
+GEOIP_PATH=/mnt/volumes/statics/geoip.db
+
+CACHE_BUSTING_STATIC_ENABLED=False
+
+MEMCACHED_ENABLED=False
+MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache
+MEMCACHED_LOCATION=memcached:11211
+MEMCACHED_LOCK_EXPIRE=3600
+MEMCACHED_LOCK_TIMEOUT=10
+#
+# Options for memcached binary, e.g. -vvv to log all requests and cache hits
+#
+MEMCACHED_OPTIONS=
+
+MAX_DOCUMENT_SIZE=200
+CLIENT_RESULTS_LIMIT=5
+API_LIMIT_PER_PAGE=1000
+
+# GIS Client
+GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore
+MAPBOX_ACCESS_TOKEN=
+BING_API_KEY=
+GOOGLE_API_KEY=
+
+# Monitoring
+MONITORING_ENABLED=False
+MONITORING_DATA_TTL=365
+USER_ANALYTICS_ENABLED=True
+USER_ANALYTICS_GZIP=True
+CENTRALIZED_DASHBOARD_ENABLED=False
+MONITORING_SERVICE_NAME=local-geonode
+MONITORING_HOST_NAME=geonode
+
+# Other Options/Contribs
+MODIFY_TOPICCATEGORY=True
+AVATAR_GRAVATAR_SSL=True
+EXIF_ENABLED=True
+CREATE_LAYER=True
+FAVORITE_ENABLED=True
+
+# Advanced Workflow
+RESOURCE_PUBLISHING=False
+ADMIN_MODERATE_UPLOADS=False
+
+# LDAP
+LDAP_ENABLED=False
+LDAP_SERVER_URL=ldap://<the_ldap_server>
+LDAP_BIND_DN=uid=ldapinfo,cn=users,dc=ad,dc=example,dc=org
+LDAP_BIND_PASSWORD=<something_secret>
+LDAP_USER_SEARCH_DN=dc=ad,dc=example,dc=org
+LDAP_USER_SEARCH_FILTERSTR=(&(uid=%(user)s)(objectClass=person))
+LDAP_GROUP_SEARCH_DN=cn=groups,dc=ad,dc=example,dc=org
+LDAP_GROUP_SEARCH_FILTERSTR=(|(cn=abt1)(cn=abt2)(cn=abt3)(cn=abt4)(cn=abt5)(cn=abt6))
+LDAP_GROUP_PROFILE_MEMBER_ATTR=uniqueMember
+
+# CELERY
+
+# expressed in KB
+# CELERY__MAX_MEMORY_PER_CHILD="200000"
+# ##
+# Note right autoscale value must coincide with worker concurrency value
+# CELERY__AUTOSCALE_VALUES="15,10"
+# CELERY__WORKER_CONCURRENCY="10"
+# ##
+# CELERY__OPTS="--without-gossip --without-mingle -Ofair -B -E"
+# CELERY__BEAT_SCHEDULE="/mnt/volumes/statics/celerybeat-schedule"
+# CELERY__LOG_LEVEL="INFO"
+# CELERY__LOG_FILE="/var/log/celery.log"
+# CELERY__WORKER_NAME="worker1@%h"
+
+# PostgreSQL
+POSTGRESQL_MAX_CONNECTIONS=200
+
+# Common containers restart policy
+RESTART_POLICY_CONDITION="on-failure"
+RESTART_POLICY_DELAY="5s"
+RESTART_POLICY_MAX_ATTEMPTS="3"
+RESTART_POLICY_WINDOW=120s
+
+DEFAULT_MAX_UPLOAD_SIZE=5368709120
+DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5
+
+# FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration

geonode/assets/management/commands/migrate_file_to_assets.py

@@ -96,9 +96,19 @@ def handle(self, **options):
             try:
                 logger.info("Moving file to the asset folder")
 
-                dest = shutil.move(source, handler._create_asset_dir())
-
-                logger.info("Fixing perms")
+                if len(asset.location) == 1:
+                    # In older installations, all documents are stored in a single folder.
+                    # Instead of moving the entire folder, we can simply move the individual document.
+                    # This approach prevents the risk of breaking the other documents
+                    # that are stored in the same folder
+                    # oldpath = {MEDIA_ROOT}/documents/document/file.extension
+                    dest = shutil.move(asset.location[0], handler._create_asset_dir())
+                else:
+                    dest = shutil.move(source, handler._create_asset_dir())
+
+                logger.info(f"New destination path: {dest}")
+
+                logger.info("Fixing file/folder perms if required")
                 if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None:
                     os.chmod(os.path.dirname(dest), settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS)
 
@@ -111,7 +121,10 @@ def handle(self, **options):
 
             logger.info("Updating location field with new folder value")
 
-            asset.location = [x.replace(source, dest) for x in asset.location]
+            if len(asset.location) == 1:
+                asset.location = [dest]
+            else:
+                asset.location = [x.replace(source, dest) for x in asset.location]
             asset.save()
 
             logger.info("Checking if geoserver should be updated")

geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py

@@ -1,4 +1,5 @@
 # Generated by Django 4.2.9 on 2024-03-12 11:55
+import itertools
 import logging
 import os
 
@@ -10,7 +11,7 @@
 
 from geonode.base.models import Link
 from geonode.assets.models import LocalAsset
-from geonode.utils import build_absolute_uri
+from geonode.utils import build_absolute_uri, get_supported_datasets_file_types
 
 logger = logging.getLogger(__name__)
 
@@ -54,10 +55,14 @@ def get_ext(filename):
                 ext = get_ext(files[0])
             else:
                 ext = None
+                supported_file_types = get_supported_datasets_file_types()
                 for file in files:
-                    for filetype in settings.SUPPORTED_DATASET_FILE_TYPES:
+                    for filetype in supported_file_types:
                         file_ext = get_ext(file)
-                        if file_ext in filetype["ext"]:
+                        _ext = list(
+                            itertools.chain.from_iterable(y for y in [x["required_ext"] for x in filetype["formats"]])
+                        )
+                        if file_ext in _ext:
                             ext = filetype["id"]
                             break
                     if ext: