diff --git a/clusters/d8-ugmk-prod/kustomization.yaml b/clusters/d8-ugmk-prod/kustomization.yaml index 7294059..3e1428b 100644 --- a/clusters/d8-ugmk-prod/kustomization.yaml +++ b/clusters/d8-ugmk-prod/kustomization.yaml @@ -12,6 +12,8 @@ resources: - ../../infrastructure/vault/d8-ugmk-prod - ../../infrastructure/postgresql/d8-ugmk-prod - ../../infrastructure/zitadel/d8-ugmk-prod + - ../../infrastructure/superset/d8-ugmk-prod + - ../../infrastructure/trino/d8-ugmk-prod - ../../infrastructure/istio-config/d8-ugmk-prod - ./istio-ingress @@ -50,4 +52,4 @@ resources: - ../../apps/issues/d8-ugmk-prod - ../../apps/pm/d8-ugmk-prod - ../../apps/message-hub/d8-ugmk-prod - - ../../apps/transmittal/d8-ugmk-prod \ No newline at end of file + - ../../apps/transmittal/d8-ugmk-prod diff --git a/infrastructure/istio-config/d8-ugmk-prod/istio-config.yaml b/infrastructure/istio-config/d8-ugmk-prod/istio-config.yaml index 3eaabec..39cdd66 100644 --- a/infrastructure/istio-config/d8-ugmk-prod/istio-config.yaml +++ b/infrastructure/istio-config/d8-ugmk-prod/istio-config.yaml @@ -22,6 +22,7 @@ spec: servers: - hosts: - sarex.uralmine.com + - bi.sarex-k8s.uralmine.com - sarex-login.uralmine.com - rabbitmq.sarex-k8s.uralmine.com - camunda-keycloak.sarex-k8s.uralmine.com @@ -518,6 +519,17 @@ spec: prefix: / service: rabbitmq.rabbitmq.svc.cluster.local port: 15672 + superset: + namespace: default + hosts: + - bi.sarex-k8s.uralmine.com + gateways: + - default/platform-gateway + routes: + - path: + prefix: / + service: superset.superset.svc.cluster.local + port: 8088 zitadel: namespace: default hosts: diff --git a/infrastructure/superset/d8-ugmk-prod/kustomization.yaml b/infrastructure/superset/d8-ugmk-prod/kustomization.yaml new file mode 100644 index 0000000..d27edd1 --- /dev/null +++ b/infrastructure/superset/d8-ugmk-prod/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base +patches: + - path: superset.yaml + - path: superset-namespace.yaml diff --git a/infrastructure/superset/d8-ugmk-prod/superset-namespace.yaml b/infrastructure/superset/d8-ugmk-prod/superset-namespace.yaml new file mode 100644 index 0000000..0000772 --- /dev/null +++ b/infrastructure/superset/d8-ugmk-prod/superset-namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: superset + labels: + istio-injection: disabled + security.deckhouse.io/pod-policy: privileged diff --git a/infrastructure/superset/d8-ugmk-prod/superset.yaml b/infrastructure/superset/d8-ugmk-prod/superset.yaml new file mode 100644 index 0000000..bde759f --- /dev/null +++ b/infrastructure/superset/d8-ugmk-prod/superset.yaml @@ -0,0 +1,129 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: superset + namespace: superset +spec: + interval: 5m + timeout: 20m + values: + fullnameOverride: superset + serviceAccount: + create: true + serviceAccountName: superset + imagePullSecrets: + - name: regcred + vault: + enabled: true + role: superset + authPath: auth/kubernetes + kvVersion: 2 + secrets: + - env: SUPERSET_SECRET_KEY + path: secrets/data/vault/apps/superset + key: SUPERSET_SECRET_KEY + - env: DB_PASS + path: secrets/data/apps/superset/postgres + key: password + - env: JWT_SECRET + path: secrets/data/vault/apps/superset + key: JWT_SECRET + stronghold: + enabled: false + extraEnv: + GUNICORN_TIMEOUT: "300" + SERVER_WORKER_AMOUNT: "4" + BABEL_DEFAULT_LOCALE: ru + extraEnvRaw: + - name: ENABLE_PROXY_FIX + value: "true" + supersetNode: + connections: + redis_host: superset-redis-headless + redis_port: "6379" + redis_user: "" + redis_cache_db: "1" + redis_celery_db: "0" + db_host: 10.222.255.162 + db_port: "5432" + db_user: superset + db_name: superset + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + memory: 2Gi + supersetWorker: + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + memory: 2Gi + init: + loadExamples: false + postgresql: + enabled: false + redis: + enabled: true + image: + registry: docker.io + repository: bitnamilegacy/redis + tag: 7.0.10-debian-11-r4 + architecture: standalone + auth: + enabled: false + existingSecret: "" + existingSecretKey: "" + password: "" + master: + persistence: + enabled: false + configOverrides: + feature_flags: | + FEATURE_FLAGS = { + "EMBEDDED_SUPERSET": True, + "ENABLE_TEMPLATE_PROCESSING": True, + } + GUEST_ROLE_NAME = "Gamma" + GUEST_TOKEN_JWT_AUDIENCE = "guest" + GUEST_TOKEN_JWT_SECRET = os.getenv("JWT_SECRET") + GUEST_TOKEN_JWT_EXP_SECONDS = 3600 + iframe_config: | + TALISMAN_ENABLED = True + TALISMAN_CONFIG = { + "frame_options": None, + "content_security_policy": { + "base-uri": ["'self'"], + "default-src": ["'self'"], + "img-src": ["'self'", "blob:", "data:", "https://apachesuperset.gateway.scarf.sh", "https://static.scarf.sh/"], + "worker-src": ["'self'", "blob:"], + "connect-src": ["'self'", "https://api.mapbox.com", "https://events.mapbox.com"], + "object-src": ["'none'"], + "style-src": ["'self'", "'unsafe-inline'"], + "script-src": ["'self'", "'strict-dynamic'"], + "frame-ancestors": ["'self'", "https://sarex.uralmine.com", "https://*.uralmine.com", "https://*.sarex-k8s.uralmine.com"], + }, + "content_security_policy_nonce_in": ["script-src"], + } + X_FRAME_OPTIONS = None + HTTP_HEADERS = { + "Content-Security-Policy": "frame-ancestors 'self' https://sarex.uralmine.com https://*.uralmine.com https://*.sarex-k8s.uralmine.com", + } + extend_timeout: | + SQLLAB_ASYNC_TIME_LIMIT_SEC = 300 + SUPERSET_WEBSERVER_TIMEOUT = 300 + SQLLAB_TIMEOUT = 600 + set_locale: | + BABEL_DEFAULT_LOCALE = "ru" + enable_oauth: "" + extraConfigs: + import_datasources.yaml: | + databases: + - database_name: trino + sqlalchemy_uri: trino://superset@trino.trino.svc.cluster.local:8080 + expose_in_sqllab: true + allow_ctas: true + allow_cvas: true + allow_dml: false diff --git a/infrastructure/trino/d8-ugmk-prod/kustomization.yaml b/infrastructure/trino/d8-ugmk-prod/kustomization.yaml new file mode 100644 index 0000000..d99c091 --- /dev/null +++ b/infrastructure/trino/d8-ugmk-prod/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base +patches: + - path: trino.yaml + - path: trino-namespace.yaml diff --git a/infrastructure/trino/d8-ugmk-prod/trino-namespace.yaml b/infrastructure/trino/d8-ugmk-prod/trino-namespace.yaml new file mode 100644 index 0000000..5584d59 --- /dev/null +++ b/infrastructure/trino/d8-ugmk-prod/trino-namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: trino + labels: + istio-injection: disabled diff --git a/infrastructure/trino/d8-ugmk-prod/trino.yaml b/infrastructure/trino/d8-ugmk-prod/trino.yaml new file mode 100644 index 0000000..68add67 --- /dev/null +++ b/infrastructure/trino/d8-ugmk-prod/trino.yaml @@ -0,0 +1,137 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: trino + namespace: trino +spec: + interval: 5m + timeout: 20m + values: + nameOverride: trino + coordinatorNameOverride: trino-coordinator + workerNameOverride: trino-worker + imagePullSecrets: + - name: regcred + serviceAccount: + create: true + name: trino + gateway: + enabled: false + virtualService: + enabled: false + vault: + enabled: true + role: trino + authPath: auth/kubernetes + kvVersion: 2 + secrets: + - env: TRINO_POSTGRES_HOST + path: secrets/data/vault/apps/trino + key: TRINO_POSTGRES_HOST + - env: TRINO_POSTGRES_PORT + path: secrets/data/vault/apps/trino + key: TRINO_POSTGRES_PORT + - env: TRINO_POSTGRES_USER + path: secrets/data/vault/apps/trino + key: TRINO_POSTGRES_USER + - env: TRINO_POSTGRES_PASSWORD + path: secrets/data/vault/apps/trino + key: TRINO_POSTGRES_PASSWORD + - env: TRINO_INTERNAL_SHARED_SECRET + path: secrets/data/vault/apps/trino + key: TRINO_INTERNAL_SHARED_SECRET + stronghold: + enabled: false + server: + workers: 1 + log: + trino: + level: INFO + config: + authenticationType: "" + query: + maxMemory: 20GB + autoscaling: + enabled: false + additionalLogProperties: [] + auth: + passwordAuthSecret: "" + env: [] + envFrom: [] + catalogs: + sarex_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/sarex_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + flows_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/flows_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + issues_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/issues_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + workspaces_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/workspaces_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + pm_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/pm_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + eav_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/eav_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + inspections_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/inspections_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + documentations_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/documentations_db?sslmode=disable + connection-user=${ENV:TRINO_POSTGRES_USER} + connection-password=${ENV:TRINO_POSTGRES_PASSWORD} + postgresql.array-mapping=AS_JSON + coordinator: + jvm: + maxHeapSize: 6G + config: + memory: + heapHeadroomPerNode: 1GB + query: + maxMemoryPerNode: 4GB + resources: + requests: + cpu: 500m + memory: 6Gi + limits: + memory: 8Gi + worker: + jvm: + maxHeapSize: 24G + config: + memory: + heapHeadroomPerNode: 4GB + query: + maxMemoryPerNode: 20GB + resources: + requests: + cpu: 500m + memory: 24Gi + limits: + memory: 32Gi