Merge remote-tracking branch 'origin/master'

clusters/d8-ugmk-prod/kustomization.yaml

@@ -12,6 +12,8 @@ resources:
   - ../../infrastructure/vault/d8-ugmk-prod
   - ../../infrastructure/postgresql/d8-ugmk-prod
   - ../../infrastructure/zitadel/d8-ugmk-prod
+  - ../../infrastructure/superset/d8-ugmk-prod
+  - ../../infrastructure/trino/d8-ugmk-prod
   - ../../infrastructure/istio-config/d8-ugmk-prod
   - ./istio-ingress
 

infrastructure/istio-config/d8-ugmk-prod/istio-config.yaml

@@ -22,6 +22,7 @@ spec:
               servers:
                 - hosts:
                     - sarex.uralmine.com
+                    - bi.sarex-k8s.uralmine.com
                     - sarex-login.uralmine.com
                     - rabbitmq.sarex-k8s.uralmine.com
                     - camunda-keycloak.sarex-k8s.uralmine.com
@@ -518,6 +519,17 @@ spec:
                     prefix: /
                   service: rabbitmq.rabbitmq.svc.cluster.local
                   port: 15672
+            superset:
+              namespace: default
+              hosts:
+                - bi.sarex-k8s.uralmine.com
+              gateways:
+                - default/platform-gateway
+              routes:
+                - path:
+                    prefix: /
+                  service: superset.superset.svc.cluster.local
+                  port: 8088
             zitadel:
               namespace: default
               hosts:

infrastructure/superset/d8-ugmk-prod/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+patches:
+  - path: superset.yaml
+  - path: superset-namespace.yaml

infrastructure/superset/d8-ugmk-prod/superset-namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: superset
+  labels:
+    istio-injection: disabled
+    security.deckhouse.io/pod-policy: privileged

infrastructure/superset/d8-ugmk-prod/superset.yaml

@@ -0,0 +1,129 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: superset
+  namespace: superset
+spec:
+  interval: 5m
+  timeout: 20m
+  values:
+    fullnameOverride: superset
+    serviceAccount:
+      create: true
+    serviceAccountName: superset
+    imagePullSecrets:
+      - name: regcred
+    vault:
+      enabled: true
+      role: superset
+      authPath: auth/kubernetes
+      kvVersion: 2
+      secrets:
+        - env: SUPERSET_SECRET_KEY
+          path: secrets/data/vault/apps/superset
+          key: SUPERSET_SECRET_KEY
+        - env: DB_PASS
+          path: secrets/data/apps/superset/postgres
+          key: password
+        - env: JWT_SECRET
+          path: secrets/data/vault/apps/superset
+          key: JWT_SECRET
+    stronghold:
+      enabled: false
+    extraEnv:
+      GUNICORN_TIMEOUT: "300"
+      SERVER_WORKER_AMOUNT: "4"
+      BABEL_DEFAULT_LOCALE: ru
+    extraEnvRaw:
+      - name: ENABLE_PROXY_FIX
+        value: "true"
+    supersetNode:
+      connections:
+        redis_host: superset-redis-headless
+        redis_port: "6379"
+        redis_user: ""
+        redis_cache_db: "1"
+        redis_celery_db: "0"
+        db_host: 10.222.255.162
+        db_port: "5432"
+        db_user: superset
+        db_name: superset
+      resources:
+        requests:
+          cpu: 500m
+          memory: 1Gi
+        limits:
+          memory: 2Gi
+    supersetWorker:
+      resources:
+        requests:
+          cpu: 500m
+          memory: 1Gi
+        limits:
+          memory: 2Gi
+    init:
+      loadExamples: false
+    postgresql:
+      enabled: false
+    redis:
+      enabled: true
+      image:
+        registry: docker.io
+        repository: bitnamilegacy/redis
+        tag: 7.0.10-debian-11-r4
+      architecture: standalone
+      auth:
+        enabled: false
+        existingSecret: ""
+        existingSecretKey: ""
+        password: ""
+      master:
+        persistence:
+          enabled: false
+    configOverrides:
+      feature_flags: |
+        FEATURE_FLAGS = {
+            "EMBEDDED_SUPERSET": True,
+            "ENABLE_TEMPLATE_PROCESSING": True,
+        }
+        GUEST_ROLE_NAME = "Gamma"
+        GUEST_TOKEN_JWT_AUDIENCE = "guest"
+        GUEST_TOKEN_JWT_SECRET = os.getenv("JWT_SECRET")
+        GUEST_TOKEN_JWT_EXP_SECONDS = 3600
+      iframe_config: |
+        TALISMAN_ENABLED = True
+        TALISMAN_CONFIG = {
+            "frame_options": None,
+            "content_security_policy": {
+                "base-uri": ["'self'"],
+                "default-src": ["'self'"],
+                "img-src": ["'self'", "blob:", "data:", "https://apachesuperset.gateway.scarf.sh", "https://static.scarf.sh/"],
+                "worker-src": ["'self'", "blob:"],
+                "connect-src": ["'self'", "https://api.mapbox.com", "https://events.mapbox.com"],
+                "object-src": ["'none'"],
+                "style-src": ["'self'", "'unsafe-inline'"],
+                "script-src": ["'self'", "'strict-dynamic'"],
+                "frame-ancestors": ["'self'", "https://sarex.uralmine.com", "https://*.uralmine.com", "https://*.sarex-k8s.uralmine.com"],
+            },
+            "content_security_policy_nonce_in": ["script-src"],
+        }
+        X_FRAME_OPTIONS = None
+        HTTP_HEADERS = {
+            "Content-Security-Policy": "frame-ancestors 'self' https://sarex.uralmine.com https://*.uralmine.com https://*.sarex-k8s.uralmine.com",
+        }
+      extend_timeout: |
+        SQLLAB_ASYNC_TIME_LIMIT_SEC = 300
+        SUPERSET_WEBSERVER_TIMEOUT = 300
+        SQLLAB_TIMEOUT = 600
+      set_locale: |
+        BABEL_DEFAULT_LOCALE = "ru"
+      enable_oauth: ""
+    extraConfigs:
+      import_datasources.yaml: |
+        databases:
+          - database_name: trino
+            sqlalchemy_uri: trino://superset@trino.trino.svc.cluster.local:8080
+            expose_in_sqllab: true
+            allow_ctas: true
+            allow_cvas: true
+            allow_dml: false

infrastructure/trino/d8-ugmk-prod/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+patches:
+  - path: trino.yaml
+  - path: trino-namespace.yaml

infrastructure/trino/d8-ugmk-prod/trino-namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: trino
+  labels:
+    istio-injection: disabled

infrastructure/trino/d8-ugmk-prod/trino.yaml

@@ -0,0 +1,137 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: trino
+  namespace: trino
+spec:
+  interval: 5m
+  timeout: 20m
+  values:
+    nameOverride: trino
+    coordinatorNameOverride: trino-coordinator
+    workerNameOverride: trino-worker
+    imagePullSecrets:
+      - name: regcred
+    serviceAccount:
+      create: true
+      name: trino
+    gateway:
+      enabled: false
+    virtualService:
+      enabled: false
+    vault:
+      enabled: true
+      role: trino
+      authPath: auth/kubernetes
+      kvVersion: 2
+      secrets:
+        - env: TRINO_POSTGRES_HOST
+          path: secrets/data/vault/apps/trino
+          key: TRINO_POSTGRES_HOST
+        - env: TRINO_POSTGRES_PORT
+          path: secrets/data/vault/apps/trino
+          key: TRINO_POSTGRES_PORT
+        - env: TRINO_POSTGRES_USER
+          path: secrets/data/vault/apps/trino
+          key: TRINO_POSTGRES_USER
+        - env: TRINO_POSTGRES_PASSWORD
+          path: secrets/data/vault/apps/trino
+          key: TRINO_POSTGRES_PASSWORD
+        - env: TRINO_INTERNAL_SHARED_SECRET
+          path: secrets/data/vault/apps/trino
+          key: TRINO_INTERNAL_SHARED_SECRET
+    stronghold:
+      enabled: false
+    server:
+      workers: 1
+      log:
+        trino:
+          level: INFO
+      config:
+        authenticationType: ""
+        query:
+          maxMemory: 20GB
+      autoscaling:
+        enabled: false
+    additionalLogProperties: []
+    auth:
+      passwordAuthSecret: ""
+    env: []
+    envFrom: []
+    catalogs:
+      sarex_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/sarex_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      flows_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/flows_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      issues_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/issues_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      workspaces_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/workspaces_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      pm_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/pm_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      eav_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/eav_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      inspections_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/inspections_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      documentations_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://${ENV:TRINO_POSTGRES_HOST}:${ENV:TRINO_POSTGRES_PORT}/documentations_db?sslmode=disable
+        connection-user=${ENV:TRINO_POSTGRES_USER}
+        connection-password=${ENV:TRINO_POSTGRES_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+    coordinator:
+      jvm:
+        maxHeapSize: 6G
+      config:
+        memory:
+          heapHeadroomPerNode: 1GB
+        query:
+          maxMemoryPerNode: 4GB
+      resources:
+        requests:
+          cpu: 500m
+          memory: 6Gi
+        limits:
+          memory: 8Gi
+    worker:
+      jvm:
+        maxHeapSize: 24G
+      config:
+        memory:
+          heapHeadroomPerNode: 4GB
+        query:
+          maxMemoryPerNode: 20GB
+      resources:
+        requests:
+          cpu: 500m
+          memory: 24Gi
+        limits:
+          memory: 32Gi