From 8aeda2dc512f23fa95b6cde46892fd957e314cef Mon Sep 17 00:00:00 2001 From: Leo Q Date: Sat, 11 Jun 2022 12:26:56 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=BD=BF=E7=94=A8=E7=8E=AF?= =?UTF-8?q?=E5=A2=83=E5=8F=98=E9=87=8F=E8=AE=BE=E7=BD=AE=E9=83=A8=E5=88=86?= =?UTF-8?q?=20django=20settings=20(#1543)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 支持使用环境变量设置部分 settings * add some test related settings * update chart * update docker compose settings * add environment example * update environ * update chart * update goinception config * smaller config * remove id iin config model definition * update sql_config , add primary key id * 提升版本为 1.9.0 * add multiline flag when compling auto review * remove newline when trying to review * remove unused import * add qcluster sync mode config * add CSRF_TRUSTED_ORIGINS settings --- .dockerignore | 4 + .env.list | 6 + .gitignore | 4 +- archery/__init__.py | 2 +- archery/settings.py | 109 +++--- docs/docs.md | 2 +- requirements.txt | 1 + sql/models.py | 4 +- sql/utils/sql_review.py | 2 +- sql/views.py | 2 +- src/charts/Chart.yaml | 6 +- .../charts/goinception/templates/NOTES.txt | 8 +- .../goinception/templates/deployment.yaml | 23 +- .../charts/goinception/templates/ingress.yaml | 40 --- src/charts/charts/goinception/values.yaml | 21 +- src/charts/charts/inception/.helmignore | 22 -- src/charts/charts/inception/Chart.yaml | 5 - .../charts/inception/templates/NOTES.txt | 21 -- .../charts/inception/templates/_helpers.tpl | 32 -- .../charts/inception/templates/configMap.yaml | 15 - .../inception/templates/deployment.yaml | 54 --- .../charts/inception/templates/ingress.yaml | 40 --- .../charts/inception/templates/service.yaml | 19 - src/charts/charts/inception/values.yaml | 77 ---- src/charts/templates/configMap.yaml | 65 +++- src/charts/templates/deployment.yaml | 85 ++++- src/charts/templates/ingress.yaml | 3 + src/charts/values.yaml | 335 ++---------------- src/docker-compose/archery/settings.py | 290 +-------------- src/docker-compose/docker-compose.yml | 6 +- src/init_sql/v1.9.0.sql | 3 + 31 files changed, 298 insertions(+), 1008 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.list delete mode 100644 src/charts/charts/goinception/templates/ingress.yaml delete mode 100644 src/charts/charts/inception/.helmignore delete mode 100644 src/charts/charts/inception/Chart.yaml delete mode 100644 src/charts/charts/inception/templates/NOTES.txt delete mode 100644 src/charts/charts/inception/templates/_helpers.tpl delete mode 100644 src/charts/charts/inception/templates/configMap.yaml delete mode 100644 src/charts/charts/inception/templates/deployment.yaml delete mode 100644 src/charts/charts/inception/templates/ingress.yaml delete mode 100644 src/charts/charts/inception/templates/service.yaml delete mode 100644 src/charts/charts/inception/values.yaml create mode 100644 src/init_sql/v1.9.0.sql diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..3b640464c5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +venv +env +.env +local_settings.py \ No newline at end of file diff --git a/.env.list b/.env.list new file mode 100644 index 0000000000..89bd1bddb5 --- /dev/null +++ b/.env.list @@ -0,0 +1,6 @@ +DATABASE_URL="mysql://root:@127.0.0.1:3306/archery" +CACHE_URL="redis://127.0.0.1:6379/0" +DINGDING_CACHE_URL="redis://127.0.0.1:6379/1" +ENABLE_LDAP="true" +AUTH_LDAP_ALWAYS_UPDATE_USER="true" +AUTH_LDAP_USER_ATTR_MAP="username=cn,display=displayname,email=email" diff --git a/.gitignore b/.gitignore index fbacca3aa9..09b51e97cd 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ sql/migrations/ venv env sonar-project.properties -.scannerwork \ No newline at end of file +.scannerwork +.env +local_settings.py \ No newline at end of file diff --git a/archery/__init__.py b/archery/__init__.py index 6308328e17..980bd0c637 100644 --- a/archery/__init__.py +++ b/archery/__init__.py @@ -1,2 +1,2 @@ -version = (1, 8, 5) +version = (1, 9, 0) display_version = '.'.join(str(i) for i in version) diff --git a/archery/settings.py b/archery/settings.py index 2e81089bfb..4c2c6e259e 100644 --- a/archery/settings.py +++ b/archery/settings.py @@ -3,17 +3,43 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os +from typing import List from datetime import timedelta +import environ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +environ.Env.read_env(os.path.join(BASE_DIR, '.env')) +env = environ.Env( + DEBUG=(bool, False), + ALLOWED_HOSTS=(List[str], ["*"]), + SECRET_KEY=(str, 'hfusaf2m4ot#7)fkw#di2bu6(cv0@opwmafx5n#6=3d%x^hpl6'), + DATABASE_URL=(str, "mysql://root:@127.0.0.1:3306/archery"), + CACHE_URL=(str, "redis://127.0.0.1:6379/0"), + DINGDING_CACHE_URL=(str, "redis://127.0.0.1:6379/1"), + ENABLE_LDAP=(bool, False), + AUTH_LDAP_ALWAYS_UPDATE_USER=(bool, True), + AUTH_LDAP_USER_ATTR_MAP=(dict, { + "username": "cn", + "display": "displayname", + "email": "mail" + }), + Q_CLUISTER_SYNC=(bool, False) # qcluster 同步模式, debug 时可以调整为 True + # CSRF_TRUSTED_ORIGINS=subdomain.example.com,subdomain.example2.com subdomain.example.com + CSRF_TRUSTED_ORIGINS=(list, []) +) + + # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'hfusaf2m4ot#7)fkw#di2bu6(cv0@opwmafx5n#6=3d%x^hpl6' +SECRET_KEY = env("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = env("DEBUG") + +ALLOWED_HOSTS = env("ALLOWED_HOSTS") -ALLOWED_HOSTS = ['*'] +# https://docs.djangoproject.com/en/4.0/ref/settings/#csrf-trusted-origins +CSRF_TRUSTED_ORIGINS = env("CSRF_TRUSTED_ORIGINS") # 解决nginx部署跳转404 USE_X_FORWARDED_HOST = True @@ -124,20 +150,17 @@ # 该项目本身的mysql数据库地址 DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'archery', - 'USER': 'root', - 'PASSWORD': '', - 'HOST': '127.0.0.1', - 'PORT': '3306', - 'OPTIONS': { - 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", - 'charset': 'utf8mb4' - }, - 'TEST': { - 'NAME': 'test_archery', - 'CHARSET': 'utf8mb4', - }, + **env.db(), + **{ + 'OPTIONS': { + 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", + 'charset': 'utf8mb4' + }, + 'TEST': { + 'NAME': 'test_archery', + 'CHARSET': 'utf8mb4', + } + } } } @@ -153,27 +176,13 @@ 'queue_limit': 50, 'label': 'Django Q', 'django_redis': 'default', - 'sync': False # 本地调试可以修改为True，使用同步模式 + 'sync': env("Q_CLUISTER_SYNC") # 本地调试可以修改为True，使用同步模式 } # 缓存配置 CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/0", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "PASSWORD": "" - } - }, - "dingding": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/1", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "PASSWORD": "" - } - } + "default": env.cache(), + "dingding": env.cache_url("DINGDING_CACHE_URL") } # https://docs.djangoproject.com/en/3.2/ref/settings/#std-setting-DEFAULT_AUTO_FIELD @@ -223,7 +232,7 @@ } # LDAP -ENABLE_LDAP = False +ENABLE_LDAP = env("ENABLE_LDAP", False) if ENABLE_LDAP: import ldap from django_auth_ldap.config import LDAPSearch @@ -233,20 +242,17 @@ 'django.contrib.auth.backends.ModelBackend', # django系统中手动创建的用户也可使用，优先级靠后。注意这2行的顺序 ) - AUTH_LDAP_SERVER_URI = "ldap://xxx" - AUTH_LDAP_USER_DN_TEMPLATE = "cn=%(user)s,ou=xxx,dc=xxx,dc=xxx" - # ldap认证的另一种方式,使用时注释AUTH_LDAP_USER_DN_TEMPLATE - """ - AUTH_LDAP_BIND_DN = "cn=xxx,ou=xxx,dc=xxx,dc=xxx" - AUTH_LDAP_BIND_PASSWORD = "***********" - AUTH_LDAP_USER_SEARCH = LDAPSearch('ou=xxx,dc=xxx,dc=xxx',ldap.SCOPE_SUBTREE, '(cn=%(user)s)',) - """ - AUTH_LDAP_ALWAYS_UPDATE_USER = True # 每次登录从ldap同步用户信息 - AUTH_LDAP_USER_ATTR_MAP = { # key为archery.sql_users字段名，value为ldap中字段名，用户同步信息 - "username": "cn", - "display": "displayname", - "email": "mail" - } + AUTH_LDAP_SERVER_URI = env("AUTH_LDAP_SERVER_URI", default="ldap://xxx") + AUTH_LDAP_USER_DN_TEMPLATE = env("AUTH_LDAP_USER_DN_TEMPLATE", default=None) + if not AUTH_LDAP_USER_DN_TEMPLATE: + del AUTH_LDAP_USER_DN_TEMPLATE + AUTH_LDAP_BIND_DN = env("AUTH_LDAP_BIND_DN", default="cn=xxx,ou=xxx,dc=xxx,dc=xxx") + AUTH_LDAP_BIND_PASSWORD = env("AUTH_LDAP_BIND_PASSWORD", default="***********") + AUTH_LDAP_USER_SEARCH_BASE = env("AUTH_LDAP_USER_SEARCH_BASE", default="ou=xxx,dc=xxx,dc=xxx") + AUTH_LDAP_USER_SEARCH_FILTER = env("AUTH_LDAP_USER_SEARCH_FILTER", default='(cn=%(user)s)') + AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, AUTH_LDAP_USER_SEARCH_FILTER) + AUTH_LDAP_ALWAYS_UPDATE_USER = env("AUTH_LDAP_ALWAYS_UPDATE_USER", default=True) # 每次登录从ldap同步用户信息 + AUTH_LDAP_USER_ATTR_MAP = env("AUTH_LDAP_USER_ATTR_MAP") # LOG配置 LOGGING = { @@ -315,3 +321,8 @@ PKEY_ROOT = os.path.join(MEDIA_ROOT, 'keys') if not os.path.exists(PKEY_ROOT): os.mkdir(PKEY_ROOT) + +try: + from local_settings import * +except ImportError: + print("import local settings failed, ignored") diff --git a/docs/docs.md b/docs/docs.md index 56eef3f20d..36a5a8262d 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -1,4 +1,4 @@ -# MySQL数据库设计规范（仅供参考） +# MySQL数据库设计规范（仅供参考） ## 目录 1. 规范背景与目的 2. 设计规范 diff --git a/requirements.txt b/requirements.txt index c6716969c8..fa17ba6808 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,3 +33,4 @@ drf-spectacular==0.22.0 pyotp==2.6.0 pillow==9.0.1 qrcode==7.3.1 +django-environ diff --git a/sql/models.py b/sql/models.py index 9753f5c68e..cf6c66fd2c 100755 --- a/sql/models.py +++ b/sql/models.py @@ -1,11 +1,9 @@ # -*- coding: UTF-8 -*- from django.db import models from django.contrib.auth.models import AbstractUser -from django.conf import settings from mirage import fields from django.utils.translation import gettext as _ from mirage.crypto import Crypto -import os class ResourceGroup(models.Model): @@ -630,7 +628,7 @@ class Config(models.Model): """ 配置信息表 """ - item = models.CharField('配置项', max_length=200, primary_key=True) + item = models.CharField('配置项', max_length=100, unique=True) value = fields.EncryptedCharField(verbose_name='配置项值', max_length=500) description = models.CharField('描述', max_length=200, default='', blank=True) diff --git a/sql/utils/sql_review.py b/sql/utils/sql_review.py index 7dfa52fba8..7cd38eb59a 100644 --- a/sql/utils/sql_review.py +++ b/sql/utils/sql_review.py @@ -35,7 +35,7 @@ def is_auto_review(workflow_id): for review_row in json.loads(review_content): review_result = ReviewResult(**review_row) # 去除SQL注释 https://github.com/hhyo/Archery/issues/949 - sql = remove_comments(review_result.sql) + sql = remove_comments(review_result.sql).replace("\n","").replace("\r", "") # 正则匹配 if p.match(sql): auto_review = False diff --git a/sql/views.py b/sql/views.py index f793707947..1a9ea11766 100644 --- a/sql/views.py +++ b/sql/views.py @@ -448,7 +448,7 @@ def dbaprinciples(request): """SQL文档页面""" # 读取MD文件 file = os.path.join(settings.BASE_DIR, 'docs/docs.md') - with open(file, 'r') as f: + with open(file, 'r', encoding="utf-8") as f: md = f.read().replace('\n', '\\n') return render(request, 'dbaprinciples.html', {'md': md}) diff --git a/src/charts/Chart.yaml b/src/charts/Chart.yaml index d8fc91a5a4..1e5fbde822 100644 --- a/src/charts/Chart.yaml +++ b/src/charts/Chart.yaml @@ -1,9 +1,9 @@ -aiVersion: v2 +apiVersion: v2 appVersion: "1.0" home: https://archerydms.com/ description: Archery Helm chart for Kubernetes name: archery -version: 0.1.1 +version: 0.2.0 sources: - https://github.com/hhyo/Archery @@ -15,6 +15,8 @@ dependencies: - name: redis version: ~15.7.1 repository: https://charts.bitnami.com/bitnami + condition: redis.embedded - name: mysql version: ~8.8.20 repository: https://charts.bitnami.com/bitnami + condition: mysql.embedded diff --git a/src/charts/charts/goinception/templates/NOTES.txt b/src/charts/charts/goinception/templates/NOTES.txt index 69d76cadf1..3bbc3aa400 100644 --- a/src/charts/charts/goinception/templates/NOTES.txt +++ b/src/charts/charts/goinception/templates/NOTES.txt @@ -1,11 +1,5 @@ 1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range $.Values.ingress.paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} +{{- if contains "NodePort" .Values.service.type }} export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "goinception.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT diff --git a/src/charts/charts/goinception/templates/deployment.yaml b/src/charts/charts/goinception/templates/deployment.yaml index ce4b8c61bf..ced7bb9639 100644 --- a/src/charts/charts/goinception/templates/deployment.yaml +++ b/src/charts/charts/goinception/templates/deployment.yaml @@ -19,6 +19,20 @@ spec: app.kubernetes.io/name: {{ include "goinception.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} spec: + initContainers: + - name: override-configs + image: busybox:1.28 + {{- with .Values.initEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + command: ['sh', '-c', + 'sed "s/BACKUP_PASSWORD_PLACEHOLDER/${BACKUP_PASSWORD}/g" /etc/goinception-template/config.toml.template > /etc/goinception/config.toml'] + volumeMounts: + - name: goinception-config-volume + mountPath: /etc/goinception + - name: goinception-config-template + mountPath: /etc/goinception-template containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -32,12 +46,17 @@ spec: port: goinception resources: {{- toYaml .Values.resources | nindent 12 }} + command: + - "/usr/local/bin/dumb-init" + - "/goInception" + - "--config=/etc/goinception/config.toml" volumeMounts: - name: goinception-config-volume - subPath: config.toml - mountPath: /etc/config.toml + mountPath: /etc/goinception volumes: - name: goinception-config-volume + emptyDir: {} + - name: goinception-config-template configMap: name: goinception-config {{- with .Values.nodeSelector }} diff --git a/src/charts/charts/goinception/templates/ingress.yaml b/src/charts/charts/goinception/templates/ingress.yaml deleted file mode 100644 index e54ba1e434..0000000000 --- a/src/charts/charts/goinception/templates/ingress.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "goinception.fullname" . -}} -{{- $ingressPaths := .Values.ingress.paths -}} -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - app.kubernetes.io/name: {{ include "goinception.name" . }} - helm.sh/chart: {{ include "goinception.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: -{{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} -{{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ . | quote }} - http: - paths: - {{- range $ingressPaths }} - - path: {{ . }} - backend: - serviceName: {{ $fullName }} - servicePort: http - {{- end }} - {{- end }} -{{- end }} diff --git a/src/charts/charts/goinception/values.yaml b/src/charts/charts/goinception/values.yaml index b11ac5a454..a6b53e6519 100644 --- a/src/charts/charts/goinception/values.yaml +++ b/src/charts/charts/goinception/values.yaml @@ -17,25 +17,18 @@ service: port: 4000 targetPort: 4000 -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - paths: [] - hosts: - - chart-example.local - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local +initEnv: + - name: BACKUP_PASSWORD + value: "my-password" +# - name: xxx +# value: xxx resources: {} configMap: enabled: true data: - config.toml: |- + config.toml.template: |- host="0.0.0.0" advertise_address="" port=4000 @@ -91,7 +84,7 @@ configMap: backup_host="archery-mysql" backup_port=3306 backup_user="root" - backup_password="MYSQL_ROOT_PASSWORD" + backup_password="BACKUP_PASSWORD_PLACEHOLDER" #安全更新是否开启. #-1表示不做操作,基于远端数据库[默认值] #0表示关闭安全更新 diff --git a/src/charts/charts/inception/.helmignore b/src/charts/charts/inception/.helmignore deleted file mode 100644 index 50af031725..0000000000 --- a/src/charts/charts/inception/.helmignore +++ /dev/null @@ -1,22 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/src/charts/charts/inception/Chart.yaml b/src/charts/charts/inception/Chart.yaml deleted file mode 100644 index 9b3ac9e451..0000000000 --- a/src/charts/charts/inception/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for Kubernetes -name: inception -version: 0.1.0 diff --git a/src/charts/charts/inception/templates/NOTES.txt b/src/charts/charts/inception/templates/NOTES.txt deleted file mode 100644 index f6a019c62a..0000000000 --- a/src/charts/charts/inception/templates/NOTES.txt +++ /dev/null @@ -1,21 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range $.Values.ingress.paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "inception.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get svc -w {{ include "inception.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "inception.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "inception.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl port-forward $POD_NAME 8080:80 -{{- end }} diff --git a/src/charts/charts/inception/templates/_helpers.tpl b/src/charts/charts/inception/templates/_helpers.tpl deleted file mode 100644 index b13a0cb71c..0000000000 --- a/src/charts/charts/inception/templates/_helpers.tpl +++ /dev/null @@ -1,32 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "inception.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "inception.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "inception.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/src/charts/charts/inception/templates/configMap.yaml b/src/charts/charts/inception/templates/configMap.yaml deleted file mode 100644 index bc019daf55..0000000000 --- a/src/charts/charts/inception/templates/configMap.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.configMap.enabled -}} -kind: ConfigMap -apiVersion: v1 -metadata: - name: inception-config - labels: - app.kubernetes.io/name: {{ include "inception.name" . }} - helm.sh/chart: {{ include "inception.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- with .Values.configMap.data }} -data: -{{- toYaml . | nindent 2 }} -{{- end }} -{{- end }} diff --git a/src/charts/charts/inception/templates/deployment.yaml b/src/charts/charts/inception/templates/deployment.yaml deleted file mode 100644 index 87abf3c459..0000000000 --- a/src/charts/charts/inception/templates/deployment.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "inception.fullname" . }} - labels: - app.kubernetes.io/name: {{ include "inception.name" . }} - helm.sh/chart: {{ include "inception.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app.kubernetes.io/name: {{ include "inception.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "inception.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: inception - containerPort: 6669 - protocol: TCP - livenessProbe: - tcpSocket: - port: inception - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: inception-config-volume - subPath: inc.cnf - mountPath: /etc/inc.cnf - volumes: - - name: inception-config-volume - configMap: - name: inception-config - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/src/charts/charts/inception/templates/ingress.yaml b/src/charts/charts/inception/templates/ingress.yaml deleted file mode 100644 index 033479fce2..0000000000 --- a/src/charts/charts/inception/templates/ingress.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "inception.fullname" . -}} -{{- $ingressPaths := .Values.ingress.paths -}} -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - app.kubernetes.io/name: {{ include "inception.name" . }} - helm.sh/chart: {{ include "inception.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: -{{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} -{{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ . | quote }} - http: - paths: - {{- range $ingressPaths }} - - path: {{ . }} - backend: - serviceName: {{ $fullName }} - servicePort: http - {{- end }} - {{- end }} -{{- end }} diff --git a/src/charts/charts/inception/templates/service.yaml b/src/charts/charts/inception/templates/service.yaml deleted file mode 100644 index 4dfd620a07..0000000000 --- a/src/charts/charts/inception/templates/service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "inception.fullname" . }} - labels: - app.kubernetes.io/name: {{ include "inception.name" . }} - helm.sh/chart: {{ include "inception.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: {{ .Values.service.targetPort }} - protocol: TCP - name: inception - selector: - app.kubernetes.io/name: {{ include "inception.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/src/charts/charts/inception/values.yaml b/src/charts/charts/inception/values.yaml deleted file mode 100644 index 915104e7e0..0000000000 --- a/src/charts/charts/inception/values.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Default values for inception. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: hhyo/inception - tag: latest - pullPolicy: IfNotPresent - -nameOverride: "" -fullnameOverride: "" - -service: - type: ClusterIP - port: 6669 - targetPort: 6669 - -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - paths: [] - hosts: - - chart-example.local - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -configMap: - enabled: true - data: - inc.cnf: |- - [inception] - general_log=1 - general_log_file=inception.log - port=6669 - socket=/tmp/inc.socket - character-set-client-handshake=0 - character-set-server=utf8mb4 - inception_language_code=zh-CN - inception_remote_system_password=MYSQL_ROOT_PASSWORD - inception_remote_system_user=root - inception_remote_backup_port=3306 - inception_remote_backup_host=archery-mysql - inception_support_charset=utf8,utf8mb4 - inception_enable_nullable=0 - inception_check_primary_key=1 - inception_check_column_comment=1 - inception_check_table_comment=1 - inception_osc_on=OFF - inception_osc_bin_dir=/usr/bin - inception_osc_min_table_size=10 - inception_osc_chunk_time=0.1 - inception_enable_blob_type=1 - inception_check_column_default_value=1 - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/src/charts/templates/configMap.yaml b/src/charts/templates/configMap.yaml index a0981286ea..2316e0bd3d 100644 --- a/src/charts/templates/configMap.yaml +++ b/src/charts/templates/configMap.yaml @@ -8,8 +8,67 @@ metadata: helm.sh/chart: {{ include "archery.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- with .Values.configMap.data }} data: -{{- toYaml . | nindent 2 }} -{{- end -}} + init-archery.sh: |- + #!/bin/bash + mkdir -p /opt/archery/downloads/{binlog2sql,log,schemasync} + cd /opt/archery + echo 切换python运行环境 + source /opt/venv4archery/bin/activate + + python3 manage.py makemigrations sql + python3 manage.py migrate + #数据初始化 + python3 manage.py dbshell=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} rules: {{- range .Values.ingress.hosts }} {{- $url := splitList "/" . }} diff --git a/src/charts/values.yaml b/src/charts/values.yaml index 86c5e076e2..54cdb75072 100644 --- a/src/charts/values.yaml +++ b/src/charts/values.yaml @@ -19,6 +19,7 @@ service: ingress: enabled: true + className: "nginx" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" @@ -32,9 +33,18 @@ ingress: # - secretName: chart-example-tls # hosts: # - chart-example.local + # subcharts redis redis: + embedded: true + # url: "redis://127.0.0.1:6379/0" + # urlSecret: "" + # urlSecretKey: "CACHE_URL" + # dingdingUrl: "redis://127.0.0.1:6379/1" + # dingdingUrlSecret: "" + # dingdingUrlSecretKey: "DINGDING_CACHE_URL" + # embedded 为 false, 会使用外部的 redis, 下方的 redis 就不会生效 architecture: standalone auth: password: "REDIS_PASSWORD" @@ -46,6 +56,11 @@ redis: # subcharts mysql mysql: + embedded: true + url: "mysql://root:@127.0.0.1:3306/archery" + urlSecret: "" + urlSecretKey: "DATABASE_URL" + # embedded 为 false时, 会使用外部的 mysql, 下方的 mysql 配置不会生效 architecture: standalone auth: database: archery @@ -118,10 +133,14 @@ resources: {} # requests: # cpu: 100m # memory: 128Mi +envs: [] + # - name: value + # value: xxx + volumeMounts: - name: archery-settings - subPath: settings.py - mountPath: /opt/archery/archery/settings.py + subPath: local_settings.py + mountPath: /opt/archery/local_settings.py - name: archery-settings subPath: soar.yaml mountPath: /etc/soar.yaml @@ -157,250 +176,15 @@ volumes: configMap: enabled: true + # admin password + superuser: + username: admin + password: archery # 请尽快修改 + email: "archery@example.com" data: - settings.py: |- + local_settings.py: |- # -*- coding: UTF-8 -*- - - # Build paths inside the project like this: os.path.join(BASE_DIR, ...) - import os - - BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - # SECURITY WARNING: keep the secret key used in production secret! - SECRET_KEY = 'hfusaf2m4ot#7)fkw#di2bu6(cv0@opwmafx5n#6=3d%x^hpl6' - - # SECURITY WARNING: don't run with debug turned on in production! - DEBUG = False - - ALLOWED_HOSTS = ['*'] - - # 解决nginx部署跳转404 - USE_X_FORWARDED_HOST = True - - # 请求限制 - DATA_UPLOAD_MAX_MEMORY_SIZE = 15728640 - - # Application definition - INSTALLED_APPS = ( - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django_q', - 'sql', - 'sql_api', - 'common', - ) - - MIDDLEWARE = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - 'common.middleware.check_login_middleware.CheckLoginMiddleware', - 'common.middleware.exception_logging_middleware.ExceptionLoggingMiddleware', - ) - - ROOT_URLCONF = 'archery.urls' - - TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'common/templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'common.utils.global_info.global_info', - ], - }, - }, - ] - - WSGI_APPLICATION = 'archery.wsgi.application' - - # Internationalization - LANGUAGE_CODE = 'zh-hans' - - TIME_ZONE = 'Asia/Shanghai' - - USE_I18N = True - - USE_TZ = False - - # 时间格式化 - USE_L10N = False - DATETIME_FORMAT = 'Y-m-d H:i:s' - DATE_FORMAT = 'Y-m-d' - - # Static files (CSS, JavaScript, Images) - STATIC_URL = '/static/' - STATIC_ROOT = os.path.join(BASE_DIR, 'static') - STATICFILES_DIRS = [os.path.join(BASE_DIR, 'common/static'), ] - - # 扩展django admin里users字段用到，指定了sql/models.py里的class users - AUTH_USER_MODEL = "sql.Users" - - # 密码校验 - AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - 'OPTIONS': { - 'min_length': 9, - } - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, - ] - - ############### 以下部分需要用户根据自己环境自行修改 ################### - - # SESSION 设置 - SESSION_COOKIE_AGE = 60 * 300 # 300分钟 - SESSION_SAVE_EVERY_REQUEST = True - SESSION_EXPIRE_AT_BROWSER_CLOSE = True # 关闭浏览器，则COOKIE失效 - - # 该项目本身的mysql数据库地址 - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'archery', - 'USER': 'root', - 'PASSWORD': 'MYSQL_ROOT_PASSWORD', - 'HOST': 'archery-mysql', - 'PORT': '3306', - 'OPTIONS': { - 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", - 'charset': 'utf8mb4' - }, - 'TEST': { - 'NAME': 'test_archery', - 'CHARSET': 'utf8mb4', - }, - } - } - - # Django-Q - Q_CLUSTER = { - 'name': 'archery', - 'workers': 4, - 'recycle': 500, - 'timeout': 60, - 'compress': True, - 'cpu_affinity': 1, - 'save_limit': 0, - 'queue_limit': 50, - 'label': 'Django Q', - 'django_redis': 'default', - 'sync': False # 本地调试可以修改为True，使用同步模式 - } - - # 缓存配置 - CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://archery-redis-master:6379/0", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "PASSWORD": "REDIS_PASSWORD" - } - }, - "dingding": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://archery-redis-master:6379/1", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "PASSWORD": "REDIS_PASSWORD" - } - } - } - - # LDAP - ENABLE_LDAP = False - if ENABLE_LDAP: - import ldap - from django_auth_ldap.config import LDAPSearch - - AUTHENTICATION_BACKENDS = ( - 'django_auth_ldap.backend.LDAPBackend', # 配置为先使用LDAP认证，如通过认证则不再使用后面的认证方式 - 'django.contrib.auth.backends.ModelBackend', # django系统中手动创建的用户也可使用，优先级靠后。注意这2行的顺序 - ) - - AUTH_LDAP_SERVER_URI = "ldap://xxx" - AUTH_LDAP_USER_DN_TEMPLATE = "cn=%(user)s,ou=xxx,dc=xxx,dc=xxx" - AUTH_LDAP_ALWAYS_UPDATE_USER = True # 每次登录从ldap同步用户信息 - AUTH_LDAP_USER_ATTR_MAP = { # key为archery.sql_users字段名，value为ldap中字段名，用户同步信息 - "username": "cn", - "display": "displayname", - "email": "mail" - } - - # LOG配置 - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'verbose': { - 'format': '[%(asctime)s][%(threadName)s:%(thread)d][task_id:%(name)s][%(filename)s:%(lineno)d][%(levelname)s]- %(message)s' - }, - }, - 'handlers': { - 'default': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': 'logs/archery.log', - 'maxBytes': 1024 * 1024 * 100, # 5 MB - 'backupCount': 5, - 'formatter': 'verbose', - }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'verbose' - } - }, - 'loggers': { - 'default': { # default日志 - 'handlers': ['console', 'default'], - 'level': 'DEBUG' - }, - 'django-q': { # django_q模块相关日志 - 'handlers': ['console', 'default'], - 'level': 'DEBUG', - 'propagate': False - }, - 'django_auth_ldap': { # django_auth_ldap模块相关日志 - 'handlers': ['console', 'default'], - 'level': 'DEBUG', - 'propagate': False - }, - # 'django.db': { # 打印SQL语句，方便开发 - # 'handlers': ['console', 'default'], - # 'level': 'DEBUG', - # 'propagate': False - # }, - 'django.request': { # 打印请求错误堆栈信息，方便开发 - 'handlers': ['console', 'default'], - 'level': 'DEBUG', - 'propagate': False - }, - } - } + # override your configs here soar.yaml: |- # 是否允许测试环境与线上环境配置相同 allow-online-as-test: false @@ -486,66 +270,3 @@ configMap: $slowquery_file > /tmp/analysis_slow_query.log echo `date +"%Y-%m-%d %H:%M:%S"`>last_analysis_time_$hostname - init-archery.sh: |- - #!/bin/bash - mkdir -p /opt/archery/downloads/{binlog2sql,log,schemasync} - cd /opt/archery - echo 切换python运行环境 - source /opt/venv4archery/bin/activate - - python3 manage.py makemigrations sql - python3 manage.py migrate - #数据初始化 - python3 manage.py dbshell