# We may need to label nodes with clickhouse=allow label for this example to run
# See ./label_nodes.sh for this purpose

apiVersion: "clickhouse.altinity.com/v1"
kind: "ClickHouseInstallation"
metadata:
  name: "clickhouse-installation-max"
  labels:
    label1: label1_value
    label2: label2_value
  annotations:
    annotation1: annotation1_value
    annotation2: annotation2_value

spec:

  stop: "no"

  troubleshoot: "no"

  namespaceDomainPattern:  "%s.svc.my.test"

  templating:
    policy: "manual"

  reconciling:
    policy: "nowait"
    configMapPropagationTimeout: 90
    cleanup:
      unknownObjects:
        statefulSet: Delete
        pvc: Delete
        configMap: Delete
        service: Delete
      reconcileFailedObjects:
        statefulSet: Retain
        pvc: Retain
        configMap: Retain
        service: Retain

  # List of templates used by a CHI
  useTemplates:
    - name: template1
      namespace: ns1
      useType: merge
    - name: template2
      # No namespace specified - use CHI namespace

  defaults:
    replicasUseFQDN: "no"
    distributedDDL:
      profile: default
    templates:
      hostTemplate: host-template-custom-ports
      podTemplate: clickhouse-v20.6
      dataVolumeClaimTemplate: default-volume-claim
      logVolumeClaimTemplate: default-volume-claim
      serviceTemplate: chi-service-template
      clusterServiceTemplate: cluster-service-template
      shardServiceTemplate: shard-service-template
      replicaServiceTemplate: replica-service-template

  configuration:
    zookeeper:
      nodes:
        - host: zookeeper-0.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
        - host: zookeeper-1.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
        - host: zookeeper-2.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
      session_timeout_ms: 30000
      operation_timeout_ms: 10000
      root: "/path/to/zookeeper/root/node"
      identity: "user:password"
    users:
      readonly/profile: readonly
      #     <users>
      #        <readonly>
      #          <profile>readonly</profile>
      #        </readonly>
      #     </users>
      test/networks/ip:
        - "127.0.0.1"
        - "::/0"
      #     <users>
      #        <test>
      #          <networks>
      #            <ip>127.0.0.1</ip>
      #            <ip>::/0</ip>
      #          </networks>
      #        </test>
      #     </users>
      test/profile: default
      test/quotas: default
    profiles:
      readonly/readonly: "1"
      #      <profiles>
      #        <readonly>
      #          <readonly>1</readonly>
      #        </readonly>
      #      </profiles>
      default/max_memory_usage: "1000000000"
    quotas:
      default/interval/duration: "3600"
      #     <quotas>
      #       <default>
      #         <interval>
      #           <duration>3600</duration>
      #         </interval>
      #       </default>
      #     </quotas>
    settings:
      compression/case/method: zstd
      #      <compression>
      #        <case>
      #          <method>zstd</method>
      #        </case>
      #      </compression>
      disable_internal_dns_cache: 1
      #      <disable_internal_dns_cache>1</disable_internal_dns_cache>
    files:
      dict1.xml: |
        <yandex>
            <!-- ref to file /etc/clickhouse-data/config.d/source1.csv -->
        </yandex>
      source1.csv: |
        a1,b1,c1,d1
        a2,b2,c2,d2

    clusters:

      - name: all-counts
        templates:
          podTemplate: clickhouse-v20.6
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        layout:
          shardsCount: 3
          replicasCount: 2

      - name: shards-only
        templates:
          podTemplate: clickhouse-v20.6
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        layout:
          shardsCount: 3
          # replicasCount not specified, assumed = 1, by default

      - name: replicas-only
        templates:
          podTemplate: clickhouse-v20.6
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        layout:
          # shardsCount not specified, assumed = 1, by default
          replicasCount: 3

      - name: customized
        templates:
          podTemplate: clickhouse-v20.6
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        layout:
          shards:
            - name: shard0
              replicasCount: 3
              weight: 1
              internalReplication: Disabled
              templates:
                podTemplate: clickhouse-v20.6
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim

            - name: shard1
              templates:
                podTemplate: clickhouse-v20.6
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim
              replicas:
                - name: replica0
                - name: replica1
                - name: replica2

            - name: shard2
              replicasCount: 3
              templates:
                podTemplate: clickhouse-v20.6
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim
                replicaServiceTemplate: replica-service-template
              replicas:
                - name: replica0
                  tcpPort: 9000
                  httpPort: 8123
                  interserverHTTPPort: 9009
                  templates:
                    podTemplate: clickhouse-v20.7
                    dataVolumeClaimTemplate: default-volume-claim
                    logVolumeClaimTemplate: default-volume-claim
                    replicaServiceTemplate: replica-service-template

  templates:
    hostTemplates:
      - name: host-template-custom-ports
        spec:
          tcpPort: 7000
          httpPort: 7001
          interserverHTTPPort: 7002

    serviceTemplates:
      - name: chi-service-template
        # generateName understands different sets of macroses,
        # depending on the level of the object, for which Service is being created:
        #
        # For CHI-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        #
        # For Cluster-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        #
        # For Shard-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        # 6. {shard} - shard name
        # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature)
        # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature)
        #
        # For Replica-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        # 6. {shard} - shard name
        # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature)
        # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature)
        # 9. {replica} - replica name
        # 10. {replicaID} - short hashed replica name (BEWARE, this is an experimental feature)
        # 11. {replicaIndex} - 0-based index of the replica in the shard (BEWARE, this is an experimental feature)
        # 12. {chiScopeIndex} - 0-based index of the host in the chi (BEWARE, this is an experimental feature)
        # 13. {chiScopeCycleIndex} - 0-based index of the host's cycle in the chi-scope (BEWARE, this is an experimental feature)
        # 14. {chiScopeCycleOffset} - 0-based offset of the host in the chi-scope cycle (BEWARE, this is an experimental feature)
        # 15. {clusterScopeIndex} - 0-based index of the host in the cluster (BEWARE, this is an experimental feature)
        # 16. {clusterScopeCycleIndex} - 0-based index of the host's cycle in the cluster-scope (BEWARE, this is an experimental feature)
        # 17. {clusterScopeCycleOffset} - 0-based offset of the host in the cluster-scope cycle (BEWARE, this is an experimental feature)
        # 18. {shardScopeIndex} - 0-based index of the host in the shard (BEWARE, this is an experimental feature)
        # 19. {replicaScopeIndex} - 0-based index of the host in the replica (BEWARE, this is an experimental feature)
        # 20. {clusterScopeCycleHeadPointsToPreviousCycleTail} - 0-based cluster-scope index of previous cycle tail
        generateName: "service-{chi}"
        # type ObjectMeta struct from k8s.io/meta/v1
        metadata:
          labels:
            custom.label: "custom.value"
          annotations:
            # For more details on Internal Load Balancer check
            # https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
            cloud.google.com/load-balancer-type: "Internal"
            service.beta.kubernetes.io/aws-load-balancer-internal: "true"
            service.beta.kubernetes.io/azure-load-balancer-internal: "true"
            service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
            service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true"

            # NLB Load Balancer
            service.beta.kubernetes.io/aws-load-balancer-type: "nlb"

        # type ServiceSpec struct from k8s.io/core/v1
        spec:
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
          type: LoadBalancer

      - name: replica-service-template
        # type ServiceSpec struct from k8s.io/core/v1
        spec:
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
            - name: interserver
              port: 9009
          type: ClusterIP
          clusterIP: None

      - name: preserve-client-source-ip
        # For more details on Preserving Client Source IP check
        # https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip
        spec:
          selector:
            app: example
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
            - name: interserver
              port: 9009
          externalTrafficPolicy: Local
          type: LoadBalancer

    volumeClaimTemplates:
      - name: default-volume-claim
        # type PersistentVolumeClaimSpec struct from k8s.io/core/v1
        spec:
          # 1. If storageClassName is not specified, default StorageClass
          # (must be specified by cluster administrator) would be used for provisioning
          # 2. If storageClassName is set to an empty string (‘’), no storage class will be used
          # dynamic provisioning is disabled for this PVC. Existing, “Available”, PVs
          # (that do not have a specified storageClassName) will be considered for binding to the PVC
          #storageClassName: gold
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi
      - name: volume-claim-retain-pvc
        # Keep PVC from being deleted
        # Retaining PVC will also keep backing PV from deletion. This is useful in case we need to keep data intact.
        reclaimPolicy: Retain
        # type PersistentVolumeClaimSpec struct from k8s.io/core/v1
        spec:
          # 1. If storageClassName is not specified, default StorageClass
          # (must be specified by cluster administrator) would be used for provisioning
          # 2. If storageClassName is set to an empty string (‘’), no storage class will be used
          # dynamic provisioning is disabled for this PVC. Existing, “Available”, PVs
          # (that do not have a specified storageClassName) will be considered for binding to the PVC
          #storageClassName: gold
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi

    podTemplates:
      # multiple pod templates makes possible to update version smoothly
      # pod template for ClickHouse v18.16.1
      - name: clickhouse-v20.7
        # We may need to label nodes with clickhouse=allow label for this example to run
        # See ./label_nodes.sh for this purpose
        zone:
          key: "clickhouse"
          values:
            - "allow"
        # Shortcut version for AWS installations
        #zone:
        #  values:
        #    - "us-east-1a"

        # Possible values for podDistribution are:
        # Unspecified - empty value
        # ClickHouseAntiAffinity - AntiAffinity by ClickHouse instances.
        #   Pod pushes away other ClickHouse pods, which allows one ClickHouse instance per node
        #   CH - (push away) - CH - (push away) - CH
        # ShardAntiAffinity - AntiAffinity by shard name.
        #   Pod pushes away other pods of the same shard (replicas of this shard), which allows one replica of a shard per node.
        #   Other shards are allowed - it does not push alll shards away, the same only.
        #   Used for data loss avoidance - keeps all copies of the shard on different nodes.
        #   shard1,replica1 - (push away) - shard1,replica2 - (push away) - shard1,replica3
        # ReplicaAntiAffinity - AntiAffinity by replica name.
        #   Pod pushes away other pods of the same replica (shards of this replica), which allows one shard of a replica per node.
        #   Other replicas are allowed - it does not push all replicas away, the same only.
        #   Used to evenly distribute load from "full cluster scan" queries.
        #   shard1,replica1 - (push away) - shard2,replica1 - (push away) - shard3,replica1
        # AnotherNamespaceAntiAffinity - AntiAffinity by "another" namespace.
        #   Pod pushes away pods from another namespace, which allows same-namespace pods per node
        #   ns1 - (push away) - ns2 - (push away) - ns3
        # AnotherClickHouseInstallationAntiAffinity - AntiAffinity by "another" ClickHouseInstallation name.
        #   Pod pushes away pods from another ClickHouseInstallation, which allows same-ClickHouseInstallation pods per node.
        #   CHI1 - (push away) - CHI2 - (push away) - CHI3
        # AnotherClusterAntiAffinity - AntiAffinity by "another" cluster name.
        #   Pod pushes away pods from another Cluster, which allows same-cluster pods per node
        #   cluster1 - (push away) - cluster2 - (push away) - cluster3
        # MaxNumberPerNode - AntiAffinity by cycle index.
        #   Pod pushes away pods from the same cycle, which allows to specify maximum number of ClickHouse instances per node.
        #   Used to setup circular replication.
        # NamespaceAffinity - Affinity by namespace.
        #   Pod attracts pods from the same namespace, which allows pods from same namespace per node.
        #   ns1 + (attracts) + ns1
        # ClickHouseInstallationAffinity - Affinity by ClickHouseInstallation name.
        #   Pod attracts pods from the same ClickHouseInstallation, which allows pods from the same CHI per node.
        #   CHI1 + (attracts) + CHI1
        # ClusterAffinity - Affinity by cluster name.
        #   Pod attracts pods from the same cluster, which allows pods from the same Cluster per node.
        #   cluster1 + (attracts) + cluster1
        # ShardAffinity - Affinity by shard name.
        #   Pod attracts pods from the same shard, which allows pods from the same Shard per node.
        #   shard1 + (attracts) + shard1
        # ReplicaAffinity - Affinity by replica name.
        #   Pod attracts pods from the same replica, which allows pods from the same Replica per node.
        #   replica1 + (attracts) + replica1
        # PreviousTailAffinity - Affinity to overlap cycles. Used to make cycle pod distribution
        #   cycle head + (attracts to) + previous cycle tail
        podDistribution:
          - type: ShardAntiAffinity
          - type: MaxNumberPerNode
            number: 2

        # type PodSpec struct {} from k8s.io/core/v1
        spec:
          containers:
            - name: clickhouse
              image: yandex/clickhouse-server:20.7
              volumeMounts:
                - name: default-volume-claim
                  mountPath: /var/lib/clickhouse
              resources:
                requests:
                  memory: "64Mi"
                  cpu: "100m"
                limits:
                  memory: "64Mi"
                  cpu: "100m"

      # pod template for ClickHouse v19.11.3.11
      - name: clickhouse-v20.6
        # type PodSpec struct {} from k8s.io/core/v1
        spec:
          containers:
            - name: clickhouse
              image: yandex/clickhouse-server:20.6
              volumeMounts:
                - name: default-volume-claim
                  mountPath: /var/lib/clickhouse
              resources:
                requests:
                  memory: "64Mi"
                  cpu: "100m"
                limits:
                  memory: "64Mi"
                  cpu: "100m"