From f06fd3e4cd914c76e5cb6037c20286855e217894 Mon Sep 17 00:00:00 2001 From: liuyang Date: Fri, 4 Aug 2023 20:32:40 +0800 Subject: [PATCH] update --- deploy/meetingserver/deployment.yaml | 189 +------- deploy/vault-agent/vault-full-deploy.yaml | 524 ++++++++++++++++++++++ 2 files changed, 546 insertions(+), 167 deletions(-) create mode 100644 deploy/vault-agent/vault-full-deploy.yaml diff --git a/deploy/meetingserver/deployment.yaml b/deploy/meetingserver/deployment.yaml index e3fd82ad..888010de 100644 --- a/deploy/meetingserver/deployment.yaml +++ b/deploy/meetingserver/deployment.yaml @@ -12,12 +12,27 @@ spec: component: web-server template: metadata: + annotations: + vault.hashicorp.com/agent-inject: 'true' + vault.hashicorp.com/role: 'opengauss-meeting' + vault.hashicorp.com/agent-inject-secret-secrets.yaml: 'internal/data/opengauss/opengauss-meeting' + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-inject-template-secrets.yaml: | + {{- with secret "internal/data/opengauss/opengauss-meeting" -}} + {{ .Data.data.config }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-xarmor_pyrasp.ini: 'internal/data/opengauss/opengauss-meeting' + vault.hashicorp.com/agent-inject-template-xarmor_pyrasp.ini: | + {{- with secret "internal/data/opengauss/opengauss-meeting" -}} + {{ .Data.data.rasp }} + {{- end }} labels: app: meeting-server component: web-server spec: nodeSelector: meeting-server: "true" + serviceAccountName: opengauss-meeting containers: - name: meeting-server image: swr.cn-north-4.myhuaweicloud.com/opensourceway/opengauss/opengauss-meeting-server:87b3e08acb3e4bfa8a18d4c347e3282b8942af1c @@ -31,175 +46,15 @@ spec: cpu: 2000m memory: 2000Mi env: - - name: ZOOM_TOKEN - valueFrom: - secretKeyRef: - key: zoom_token - name: meeting-server-secrets - - name: GITEE_OAUTH_CLIENT_ID - valueFrom: - secretKeyRef: - key: gitee_oauth_client_id - name: meeting-server-secrets - - name: GITEE_OAUTH_CLIENT_SECRET - valueFrom: - secretKeyRef: - key: gitee_oauth_client_secret - name: meeting-server-secrets - - name: SECRET_KEY - valueFrom: - secretKeyRef: - key: secret_key - name: meeting-server-secrets - - name: GMAIL_USERNAME - valueFrom: - secretKeyRef: - key: gmail_username - name: meeting-server-secrets - - name: GMAIL_PASSWORD - valueFrom: - secretKeyRef: - key: gmail_password - name: meeting-server-secrets - - name: SMTP_SERVER_HOST - valueFrom: - secretKeyRef: - key: smtp_server_host - name: meeting-server-secrets - - name: BILI_UID - valueFrom: - secretKeyRef: - key: bili_uid - name: meeting-server-secrets - - name: SESSDATA - valueFrom: - secretKeyRef: - key: sessdata - name: meeting-server-secrets - - name: BILI_JCT - valueFrom: - secretKeyRef: - key: bili_jct - name: meeting-server-secrets - - name: ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: access_key_id - name: meeting-server-secrets - - name: SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: secret_access_key - name: meeting-server-secrets - - name: OBS_ENDPOINT - valueFrom: - secretKeyRef: - key: obs_endpoint - name: meeting-server-secrets - - name: OBS_BUCKETNAME - valueFrom: - secretKeyRef: - key: obs_bucketname - name: meeting-server-secrets - - name: REDIRECT_HOME_PAGE - valueFrom: - secretKeyRef: - key: redirect_home_page - name: meeting-server-secrets - - name: ZOOM_HOST_FIRST - valueFrom: - secretKeyRef: - key: zoom_host_first - name: meeting-server-secrets - - name: ZOOM_HOST_SECOND - valueFrom: - secretKeyRef: - key: zoom_host_second - name: meeting-server-secrets - - name: WELINK_HOST_1 - valueFrom: - secretKeyRef: - key: welink_host_1 - name: meeting-server-secrets - - name: WELINK_HOST_1_ACCOUNT - valueFrom: - secretKeyRef: - key: welink_host_1_account - name: meeting-server-secrets - - name: WELINK_HOST_1_PWD - valueFrom: - secretKeyRef: - key: welink_host_1_pwd - name: meeting-server-secrets - - name: DB_USER - valueFrom: - secretKeyRef: - key: db_user - name: meeting-server-secrets - - name: TZ - value: Asia/Shanghai - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - key: db_password - name: meeting-server-secrets - - name: DB_HOST - valueFrom: - secretKeyRef: - key: db_host - name: meeting-server-secrets - - name: DB_PORT - value: "3306" - - name: GITEE_OAUTH_REDIRECT - valueFrom: - secretKeyRef: - key: gitee_oauth_redirect - name: meeting-server-secrets - - name: QUERY_TOKEN - valueFrom: - secretKeyRef: - key: query_token - name: meeting-server-secrets - - name: ZOOM_ACCOUNT_FIRST - valueFrom: - secretKeyRef: - key: ZOOM_ACCOUNT_FIRST - name: meeting-server-secrets - - name: ZOOM_ACCOUNT_SECOND - valueFrom: - secretKeyRef: - key: ZOOM_ACCOUNT_SECOND - name: meeting-server-secrets - - name: SMTP_SENDER - valueFrom: - secretKeyRef: - key: SMTP_SENDER - name: meeting-server-secrets - - name: XARMOR_BACKEND_URL - valueFrom: - secretKeyRef: - key: xarmor_backend_url - name: meeting-server-secrets - - name: XARMOR_TENANT_ID - valueFrom: - secretKeyRef: - key: xarmor_tenant_id - name: meeting-server-secrets - - name: XARMOR_APP_ID - valueFrom: - secretKeyRef: - key: xarmor_app_id - name: meeting-server-secrets - - name: XARMOR_APP_TOKEN - valueFrom: - secretKeyRef: - key: xarmor_app_token - name: meeting-server-secrets + - name: TZ + value: Asia/Shanghai command: - /bin/sh - -c - | strip -R .note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5.11.3 + cp /vault/secrets/xarmor_pyrasp.ini /work/app-meeting-server/xarmor_pyrasp.ini + rm -rf /vault/secrets/xarmor_pyrasp.ini python manage.py collectstatic python manage.py makemigrations python manage.py migrate @@ -207,9 +62,9 @@ spec: exec uwsgi --ini /work/app-meeting-server/deploy/production/uwsgi.ini imagePullPolicy: IfNotPresent ports: - - containerPort: 8080 - name: http - protocol: TCP + - containerPort: 8080 + name: http + protocol: TCP readinessProbe: #就绪探针 initialDelaySeconds: 20 #延迟加载时间 periodSeconds: 5 #重试时间间隔 diff --git a/deploy/vault-agent/vault-full-deploy.yaml b/deploy/vault-agent/vault-full-deploy.yaml new file mode 100644 index 00000000..f3e41242 --- /dev/null +++ b/deploy/vault-agent/vault-full-deploy.yaml @@ -0,0 +1,524 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: vault +--- +# Source: vault/templates/injector-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-agent-injector + namespace: vault + labels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +--- +# Source: vault/templates/server-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault + namespace: vault + labels: + helm.sh/chart: vault-0.25.0 + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +--- +# Source: vault/templates/server-config-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: vault-config + namespace: vault + labels: + helm.sh/chart: vault-0.25.0 + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +data: + extraconfig-from-values.hcl: |- + disable_mlock = true + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + # Enable unauthenticated metrics access (necessary for Prometheus Operator) + #telemetry { + # unauthenticated_metrics_access = "true" + #} + } + storage "file" { + path = "/vault/data" + } + + # Example configuration for using auto-unseal, using Google Cloud KMS. The + # GKMS keys must already exist, and the cluster must have a service account + # that is authorized to access GCP KMS. + #seal "gcpckms" { + # project = "vault-helm-dev" + # region = "global" + # key_ring = "vault-helm-unseal-kr" + # crypto_key = "vault-helm-unseal-key" + #} + + # Example configuration for enabling Prometheus metrics in your config. + #telemetry { + # prometheus_retention_time = "30s" + # disable_hostname = true + #} +--- +# Source: vault/templates/injector-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vault-agent-injector-clusterrole + labels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: + - "get" + - "list" + - "watch" + - "patch" +--- +# Source: vault/templates/injector-clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vault-agent-injector-binding + labels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vault-agent-injector-clusterrole +subjects: +- kind: ServiceAccount + name: vault-agent-injector + namespace: vault +--- +# Source: vault/templates/server-clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vault-server-binding + labels: + helm.sh/chart: vault-0.25.0 + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: vault + namespace: vault +--- +# Source: vault/templates/injector-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: vault-agent-injector-svc + namespace: vault + labels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +spec: + ports: + - name: https + port: 443 + targetPort: 8080 + selector: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + component: webhook +--- +# Source: vault/templates/server-headless-service.yaml +# Service for Vault cluster +apiVersion: v1 +kind: Service +metadata: + name: vault-internal + namespace: vault + labels: + helm.sh/chart: vault-0.25.0 + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm + vault-internal: "true" + annotations: +spec: + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: "http" + port: 8200 + targetPort: 8200 + - name: https-internal + port: 8201 + targetPort: 8201 + selector: + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + component: server +--- +# Source: vault/templates/server-service.yaml +# Service for Vault cluster +apiVersion: v1 +kind: Service +metadata: + name: vault + namespace: vault + labels: + helm.sh/chart: vault-0.25.0 + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm + annotations: +spec: + # We want the servers to become available even if they're not ready + # since this DNS is also used for join operations. + publishNotReadyAddresses: true + ports: + - name: http + port: 8200 + targetPort: 8200 + - name: https-internal + port: 8201 + targetPort: 8201 + selector: + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + component: server +--- +# Source: vault/templates/injector-deployment.yaml +# Deployment for the injector +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vault-agent-injector + namespace: vault + labels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm + component: webhook +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + component: webhook + template: + metadata: + labels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + component: webhook + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: "vault" + component: webhook + topologyKey: kubernetes.io/hostname + serviceAccountName: "vault-agent-injector" + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 100 + fsGroup: 1000 + hostNetwork: false + imagePullSecrets: + - name: huawei-swr-image-pull-secret + containers: + - name: sidecar-injector + image: swr.cn-north-4.myhuaweicloud.com/opensourceway/common/vault-k8s:1.2.1 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + env: + - name: AGENT_INJECT_LISTEN + value: :8080 + - name: AGENT_INJECT_LOG_LEVEL + value: info + - name: AGENT_INJECT_VAULT_ADDR + value: http://vault.vault.svc:8200 + - name: AGENT_INJECT_VAULT_AUTH_PATH + value: auth/kubernetes + - name: AGENT_INJECT_VAULT_IMAGE + value: "swr.cn-north-4.myhuaweicloud.com/opensourceway/common/vault:1.14.0" + - name: AGENT_INJECT_TLS_AUTO + value: vault-agent-injector-cfg + - name: AGENT_INJECT_TLS_AUTO_HOSTS + value: vault-agent-injector-svc,vault-agent-injector-svc.vault,vault-agent-injector-svc.vault.svc + - name: AGENT_INJECT_LOG_FORMAT + value: standard + - name: AGENT_INJECT_REVOKE_ON_SHUTDOWN + value: "false" + - name: AGENT_INJECT_CPU_REQUEST + value: "250m" + - name: AGENT_INJECT_CPU_LIMIT + value: "500m" + - name: AGENT_INJECT_MEM_REQUEST + value: "64Mi" + - name: AGENT_INJECT_MEM_LIMIT + value: "128Mi" + - name: AGENT_INJECT_DEFAULT_TEMPLATE + value: "map" + - name: AGENT_INJECT_TEMPLATE_CONFIG_EXIT_ON_RETRY_FAILURE + value: "true" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - agent-inject + - 2>&1 + livenessProbe: + httpGet: + path: /health/ready + port: 8080 + scheme: HTTPS + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /health/ready + port: 8080 + scheme: HTTPS + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + startupProbe: + httpGet: + path: /health/ready + port: 8080 + scheme: HTTPS + failureThreshold: 12 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 +--- +# Source: vault/templates/server-statefulset.yaml +# StatefulSet to run the actual vault server cluster. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: vault + namespace: vault + labels: + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +spec: + serviceName: vault-internal + podManagementPolicy: Parallel + replicas: 1 + updateStrategy: + type: OnDelete + selector: + matchLabels: + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + component: server + template: + metadata: + labels: + helm.sh/chart: vault-0.25.0 + app.kubernetes.io/name: vault + app.kubernetes.io/instance: vault + component: server + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: vault + app.kubernetes.io/instance: "vault" + component: server + topologyKey: kubernetes.io/hostname + terminationGracePeriodSeconds: 10 + serviceAccountName: vault + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 100 + fsGroup: 1000 + hostNetwork: false + volumes: + - name: config + configMap: + name: vault-config + - name: home + emptyDir: {} + imagePullSecrets: + - name: huawei-swr-image-pull-secret + containers: + - name: vault + image: swr.cn-north-4.myhuaweicloud.com/opensourceway/common/vault:1.14.0 + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ec" + args: + - | + cp /vault/config/extraconfig-from-values.hcl /tmp/storageconfig.hcl; + [ -n "${HOST_IP}" ] && sed -Ei "s|HOST_IP|${HOST_IP?}|g" /tmp/storageconfig.hcl; + [ -n "${POD_IP}" ] && sed -Ei "s|POD_IP|${POD_IP?}|g" /tmp/storageconfig.hcl; + [ -n "${HOSTNAME}" ] && sed -Ei "s|HOSTNAME|${HOSTNAME?}|g" /tmp/storageconfig.hcl; + [ -n "${API_ADDR}" ] && sed -Ei "s|API_ADDR|${API_ADDR?}|g" /tmp/storageconfig.hcl; + [ -n "${TRANSIT_ADDR}" ] && sed -Ei "s|TRANSIT_ADDR|${TRANSIT_ADDR?}|g" /tmp/storageconfig.hcl; + [ -n "${RAFT_ADDR}" ] && sed -Ei "s|RAFT_ADDR|${RAFT_ADDR?}|g" /tmp/storageconfig.hcl; + /usr/local/bin/docker-entrypoint.sh vault server -config=/tmp/storageconfig.hcl + securityContext: + allowPrivilegeEscalation: false + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: VAULT_K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VAULT_K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: VAULT_ADDR + value: "http://127.0.0.1:8200" + - name: VAULT_API_ADDR + value: "http://$(POD_IP):8200" + - name: SKIP_CHOWN + value: "true" + - name: SKIP_SETCAP + value: "true" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VAULT_CLUSTER_ADDR + value: "https://$(HOSTNAME).vault-internal:8201" + - name: HOME + value: "/home/vault" + volumeMounts: + - name: data + mountPath: /vault/data + - name: config + mountPath: /vault/config + - name: home + mountPath: /home/vault + ports: + - containerPort: 8200 + name: http + - containerPort: 8201 + name: https-internal + - containerPort: 8202 + name: http-rep + readinessProbe: + # Check status; unsealed vault servers return 0 + # The exit code reflects the seal status: + # 0 - unsealed + # 1 - error + # 2 - sealed + exec: + command: ["/bin/sh", "-ec", "vault status -tls-skip-verify"] + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + lifecycle: + # Vault container doesn't receive SIGTERM from Kubernetes + # and after the grace period ends, Kube sends SIGKILL. This + # causes issues with graceful shutdowns such as deregistering itself + # from Consul (zombie services). + preStop: + exec: + command: [ + "/bin/sh", "-c", + # Adding a sleep here to give the pod eviction a + # chance to propagate, so requests will not be made + # to this pod while it's terminating + "sleep 5 && kill -SIGTERM $(pidof vault)", + ] + volumeClaimTemplates: + - metadata: + name: data + namespace: vault + annotations: + everest.io/disk-volume-type: SSD + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: csi-disk +--- +# Source: vault/templates/injector-mutating-webhook.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: vault-agent-injector-cfg + labels: + app.kubernetes.io/name: vault-agent-injector + app.kubernetes.io/instance: vault + app.kubernetes.io/managed-by: Helm +webhooks: + - name: vault.hashicorp.com + failurePolicy: Ignore + matchPolicy: Exact + sideEffects: None + timeoutSeconds: 30 + admissionReviewVersions: ["v1", "v1beta1"] + clientConfig: + service: + name: vault-agent-injector-svc + namespace: vault + path: "/mutate" + caBundle: "" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + objectSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: NotIn + values: + - vault-agent-injector + -- Gitee