支持使用环境变量设置部分 django settings (hhyo#1543)

* 支持使用环境变量设置部分 settings

* add some test related settings

* update chart

* update docker compose settings

* add environment example

* update environ

* update chart

* update goinception config

* smaller config

* remove id iin config model definition

* update sql_config , add primary key id

* 提升版本为 1.9.0

* add multiline flag when compling auto review

* remove newline when trying to review

* remove unused import

* add qcluster sync mode config

* add CSRF_TRUSTED_ORIGINS settings

.dockerignore

@@ -0,0 +1,4 @@
+venv
+env
+.env
+local_settings.py

.env.list

@@ -0,0 +1,6 @@
+DATABASE_URL="mysql://root:@127.0.0.1:3306/archery"
+CACHE_URL="redis://127.0.0.1:6379/0"
+DINGDING_CACHE_URL="redis://127.0.0.1:6379/1"
+ENABLE_LDAP="true"
+AUTH_LDAP_ALWAYS_UPDATE_USER="true"
+AUTH_LDAP_USER_ATTR_MAP="username=cn,display=displayname,email=email"

.gitignore

@@ -11,4 +11,6 @@ sql/migrations/
 venv
 env
 sonar-project.properties
-.scannerwork
+.scannerwork
+.env
+local_settings.py

archery/__init__.py

@@ -1,2 +1,2 @@
-version = (1, 8, 5)
+version = (1, 9, 0)
 display_version = '.'.join(str(i) for i in version)

archery/settings.py

@@ -3,17 +3,43 @@
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import os
+from typing import List
 from datetime import timedelta
+import environ
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
+env = environ.Env(
+    DEBUG=(bool, False),
+    ALLOWED_HOSTS=(List[str], ["*"]),
+    SECRET_KEY=(str, 'hfusaf2m4ot#7)fkw#di2bu6(cv0@opwmafx5n#6=3d%x^hpl6'),
+    DATABASE_URL=(str, "mysql://root:@127.0.0.1:3306/archery"),
+    CACHE_URL=(str, "redis://127.0.0.1:6379/0"),
+    DINGDING_CACHE_URL=(str, "redis://127.0.0.1:6379/1"),
+    ENABLE_LDAP=(bool, False),
+    AUTH_LDAP_ALWAYS_UPDATE_USER=(bool, True),
+    AUTH_LDAP_USER_ATTR_MAP=(dict, {
+        "username": "cn",
+        "display": "displayname",
+        "email": "mail"
+    }),
+    Q_CLUISTER_SYNC=(bool, False) # qcluster 同步模式, debug 时可以调整为 True
+    # CSRF_TRUSTED_ORIGINS=subdomain.example.com,subdomain.example2.com subdomain.example.com
+    CSRF_TRUSTED_ORIGINS=(list, []) 
+)
+
+
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'hfusaf2m4ot#7)fkw#di2bu6(cv0@opwmafx5n#6=3d%x^hpl6'
+SECRET_KEY = env("SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = env("DEBUG")
+
+ALLOWED_HOSTS = env("ALLOWED_HOSTS")
 
-ALLOWED_HOSTS = ['*']
+# https://docs.djangoproject.com/en/4.0/ref/settings/#csrf-trusted-origins
+CSRF_TRUSTED_ORIGINS = env("CSRF_TRUSTED_ORIGINS")
 
 # 解决nginx部署跳转404
 USE_X_FORWARDED_HOST = True
@@ -124,20 +150,17 @@
 # 该项目本身的mysql数据库地址
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.mysql',
-        'NAME': 'archery',
-        'USER': 'root',
-        'PASSWORD': '',
-        'HOST': '127.0.0.1',
-        'PORT': '3306',
-        'OPTIONS': {
-            'init_command': "SET sql_mode='STRICT_TRANS_TABLES'",
-            'charset': 'utf8mb4'
-        },
-        'TEST': {
-            'NAME': 'test_archery',
-            'CHARSET': 'utf8mb4',
-        },
+        **env.db(),
+        **{
+            'OPTIONS': {
+                'init_command': "SET sql_mode='STRICT_TRANS_TABLES'",
+                'charset': 'utf8mb4'
+            },
+            'TEST': {
+                'NAME': 'test_archery',
+                'CHARSET': 'utf8mb4',
+            }
+        }
     }
 }
 
@@ -153,27 +176,13 @@
     'queue_limit': 50,
     'label': 'Django Q',
     'django_redis': 'default',
-    'sync': False  # 本地调试可以修改为True，使用同步模式
+    'sync': env("Q_CLUISTER_SYNC")  # 本地调试可以修改为True，使用同步模式
 }
 
 # 缓存配置
 CACHES = {
-    "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379/0",
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            "PASSWORD": ""
-        }
-    },
-    "dingding": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379/1",
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            "PASSWORD": ""
-        }
-    }
+    "default": env.cache(),
+    "dingding": env.cache_url("DINGDING_CACHE_URL")
 }
 
 # https://docs.djangoproject.com/en/3.2/ref/settings/#std-setting-DEFAULT_AUTO_FIELD
@@ -223,7 +232,7 @@
 }
 
 # LDAP
-ENABLE_LDAP = False
+ENABLE_LDAP = env("ENABLE_LDAP", False)
 if ENABLE_LDAP:
     import ldap
     from django_auth_ldap.config import LDAPSearch
@@ -233,20 +242,17 @@
         'django.contrib.auth.backends.ModelBackend',  # django系统中手动创建的用户也可使用，优先级靠后。注意这2行的顺序
     )
 
-    AUTH_LDAP_SERVER_URI = "ldap://xxx"
-    AUTH_LDAP_USER_DN_TEMPLATE = "cn=%(user)s,ou=xxx,dc=xxx,dc=xxx"
-    # ldap认证的另一种方式,使用时注释AUTH_LDAP_USER_DN_TEMPLATE
-    """
-    AUTH_LDAP_BIND_DN = "cn=xxx,ou=xxx,dc=xxx,dc=xxx"
-    AUTH_LDAP_BIND_PASSWORD = "***********"
-    AUTH_LDAP_USER_SEARCH = LDAPSearch('ou=xxx,dc=xxx,dc=xxx',ldap.SCOPE_SUBTREE, '(cn=%(user)s)',)
-    """
-    AUTH_LDAP_ALWAYS_UPDATE_USER = True  # 每次登录从ldap同步用户信息
-    AUTH_LDAP_USER_ATTR_MAP = {  # key为archery.sql_users字段名，value为ldap中字段名，用户同步信息
-        "username": "cn",
-        "display": "displayname",
-        "email": "mail"
-    }
+    AUTH_LDAP_SERVER_URI = env("AUTH_LDAP_SERVER_URI", default="ldap://xxx")
+    AUTH_LDAP_USER_DN_TEMPLATE = env("AUTH_LDAP_USER_DN_TEMPLATE", default=None)
+    if not AUTH_LDAP_USER_DN_TEMPLATE:
+        del AUTH_LDAP_USER_DN_TEMPLATE
+        AUTH_LDAP_BIND_DN = env("AUTH_LDAP_BIND_DN", default="cn=xxx,ou=xxx,dc=xxx,dc=xxx")
+        AUTH_LDAP_BIND_PASSWORD = env("AUTH_LDAP_BIND_PASSWORD", default="***********")
+        AUTH_LDAP_USER_SEARCH_BASE = env("AUTH_LDAP_USER_SEARCH_BASE", default="ou=xxx,dc=xxx,dc=xxx")
+        AUTH_LDAP_USER_SEARCH_FILTER = env("AUTH_LDAP_USER_SEARCH_FILTER", default='(cn=%(user)s)')
+        AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, AUTH_LDAP_USER_SEARCH_FILTER)
+    AUTH_LDAP_ALWAYS_UPDATE_USER = env("AUTH_LDAP_ALWAYS_UPDATE_USER", default=True)  # 每次登录从ldap同步用户信息
+    AUTH_LDAP_USER_ATTR_MAP = env("AUTH_LDAP_USER_ATTR_MAP")
 
 # LOG配置
 LOGGING = {
@@ -315,3 +321,8 @@
 PKEY_ROOT = os.path.join(MEDIA_ROOT, 'keys')
 if not os.path.exists(PKEY_ROOT):
     os.mkdir(PKEY_ROOT)
+
+try:
+    from local_settings import *
+except ImportError:
+    print("import local settings failed, ignored")

docs/docs.md

@@ -1,4 +1,4 @@
-# MySQL数据库设计规范（仅供参考）
+# MySQL数据库设计规范（仅供参考）
 ## 目录
    1. 规范背景与目的
    2. 设计规范

requirements.txt

@@ -33,3 +33,4 @@ drf-spectacular==0.22.0
 pyotp==2.6.0
 pillow==9.0.1
 qrcode==7.3.1
+django-environ

sql/models.py

@@ -1,11 +1,9 @@
 # -*- coding: UTF-8 -*-
 from django.db import models
 from django.contrib.auth.models import AbstractUser
-from django.conf import settings
 from mirage import fields
 from django.utils.translation import gettext as _
 from mirage.crypto import Crypto
-import os
 
 
 class ResourceGroup(models.Model):
@@ -630,7 +628,7 @@ class Config(models.Model):
     """
     配置信息表
     """
-    item = models.CharField('配置项', max_length=200, primary_key=True)
+    item = models.CharField('配置项', max_length=100, unique=True)
     value = fields.EncryptedCharField(verbose_name='配置项值', max_length=500)
     description = models.CharField('描述', max_length=200, default='', blank=True)
 

sql/utils/sql_review.py

@@ -35,7 +35,7 @@ def is_auto_review(workflow_id):
         for review_row in json.loads(review_content):
             review_result = ReviewResult(**review_row)
             # 去除SQL注释 https://github.com/hhyo/Archery/issues/949
-            sql = remove_comments(review_result.sql)
+            sql = remove_comments(review_result.sql).replace("\n","").replace("\r", "")
             # 正则匹配
             if p.match(sql):
                 auto_review = False

sql/views.py

@@ -448,7 +448,7 @@ def dbaprinciples(request):
     """SQL文档页面"""
     #  读取MD文件
     file = os.path.join(settings.BASE_DIR, 'docs/docs.md')
-    with open(file, 'r') as f:
+    with open(file, 'r', encoding="utf-8") as f:
         md = f.read().replace('\n', '\\n')
     return render(request, 'dbaprinciples.html', {'md': md})
 

src/charts/Chart.yaml

@@ -1,9 +1,9 @@
-aiVersion: v2
+apiVersion: v2
 appVersion: "1.0"
 home: https://archerydms.com/
 description: Archery Helm chart for Kubernetes
 name: archery
-version: 0.1.1
+version: 0.2.0
 sources:
   - https://github.com/hhyo/Archery
 
@@ -15,6 +15,8 @@ dependencies:
   - name: redis
     version: ~15.7.1
     repository: https://charts.bitnami.com/bitnami
+    condition: redis.embedded
   - name: mysql
     version: ~8.8.20
     repository: https://charts.bitnami.com/bitnami
+    condition: mysql.embedded

src/charts/charts/goinception/templates/NOTES.txt

@@ -1,11 +1,5 @@
 1. Get the application URL by running these commands:
-{{- if .Values.ingress.enabled }}
-{{- range $host := .Values.ingress.hosts }}
-  {{- range $.Values.ingress.paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }}
-  {{- end }}
-{{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
+{{- if contains "NodePort" .Values.service.type }}
   export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "goinception.fullname" . }})
   export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
   echo http://$NODE_IP:$NODE_PORT

src/charts/charts/goinception/templates/deployment.yaml

@@ -19,6 +19,20 @@ spec:
         app.kubernetes.io/name: {{ include "goinception.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
+      initContainers:
+        - name: override-configs
+          image: busybox:1.28
+          {{- with .Values.initEnv }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command: ['sh', '-c',
+                    'sed "s/BACKUP_PASSWORD_PLACEHOLDER/${BACKUP_PASSWORD}/g" /etc/goinception-template/config.toml.template > /etc/goinception/config.toml']
+          volumeMounts:
+            - name: goinception-config-volume
+              mountPath: /etc/goinception
+            - name: goinception-config-template
+              mountPath: /etc/goinception-template
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
@@ -32,12 +46,17 @@ spec:
               port: goinception
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          command:
+            - "/usr/local/bin/dumb-init"
+            - "/goInception"
+            - "--config=/etc/goinception/config.toml"
           volumeMounts:
             - name: goinception-config-volume
-              subPath: config.toml
-              mountPath: /etc/config.toml
+              mountPath: /etc/goinception
       volumes:
         - name: goinception-config-volume
+          emptyDir: {}
+        - name: goinception-config-template
           configMap:
             name: goinception-config
       {{- with .Values.nodeSelector }}

src/charts/charts/goinception/values.yaml

@@ -17,25 +17,18 @@ service:
   port: 4000
   targetPort: 4000
 
-ingress:
-  enabled: false
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  paths: []
-  hosts:
-    - chart-example.local
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
+initEnv:
+  - name: BACKUP_PASSWORD
+    value: "my-password"
+# - name: xxx
+#   value: xxx
 
 resources: {}
 
 configMap:
   enabled: true
   data:
-    config.toml: |-
+    config.toml.template: |-
       host="0.0.0.0"
       advertise_address=""
       port=4000
@@ -91,7 +84,7 @@ configMap:
       backup_host="archery-mysql"
       backup_port=3306
       backup_user="root"
-      backup_password="MYSQL_ROOT_PASSWORD"
+      backup_password="BACKUP_PASSWORD_PLACEHOLDER"
       #安全更新是否开启.
       #-1表示不做操作,基于远端数据库[默认值]
       #0表示关闭安全更新