Configuration

Overview

The KubermaticConfiguration CustomResourceDefinition is used for configuring the Kubermatic Kubernetes Platform (KKP) Operator and replaces what previously was done with the values.yaml for the KKP Helm chart.

The following is an example configuration, showing all possible options. Note that all fields that you don’t define explicitly are always defaulted to these values.

apiVersion: kubermatic.k8c.io/v1
kind: KubermaticConfiguration
metadata:
  name: <<mykubermatic>>
  namespace: kubermatic
spec:
  # API configures the frontend REST API used by the dashboard.
  api:
    # AccessibleAddons is a list of addons that should be enabled in the API.
    accessibleAddons:
      - cluster-autoscaler
      - node-exporter
      - kube-state-metrics
      - multus
      - hubble
    # DebugLog enables more verbose logging.
    debugLog: false
    # DockerRepository is the repository containing the Kubermatic REST API image.
    dockerRepository: quay.io/kubermatic/kubermatic
    # PProfEndpoint controls the port the API should listen on to provide pprof
    # data. This port is never exposed from the container and only available via port-forwardings.
    pprofEndpoint: :6600
    # Replicas sets the number of pod replicas for the API deployment.
    replicas: 2
    # Resources describes the requested and maximum allowed CPU/memory usage.
    resources:
      # Limits describes the maximum amount of compute resources allowed.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      limits:
        cpu: 250m
        memory: 1Gi
      # Requests describes the minimum amount of compute resources required.
      # If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
      # otherwise to an implementation-defined value.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      requests:
        cpu: 100m
        memory: 512Mi
  # Auth defines keys and URLs for Dex.
  auth:
    clientID: kubermatic
    issuerClientID: kubermaticIssuer
    issuerClientSecret: ""
    issuerCookieKey: ""
    issuerRedirectURL: https://example.com/api/v1/kubeconfig
    serviceAccountKey: ""
    skipTokenIssuerTLSVerify: false
    tokenIssuer: https://example.com/dex
  # CABundle references a ConfigMap in the same namespace as the KubermaticConfiguration.
  # This ConfigMap must contain a ca-bundle.pem with PEM-encoded certificates. This bundle
  # automatically synchronized into each seed and each usercluster. APIGroup and Kind are
  # currently ignored.
  caBundle:
    # APIGroup is the group for the resource being referenced.
    # If APIGroup is not specified, the specified Kind must be in the core API group.
    # For any other third-party types, APIGroup is required.
    apiGroup: null
    # Kind is the type of resource being referenced
    kind: ""
    # Name is the name of resource being referenced
    name: ca-bundle
  # ExposeStrategy is the strategy to expose the cluster with.
  # Note: The `seed_dns_overwrite` setting of a Seed's datacenter doesn't have any effect
  # if this is set to LoadBalancerStrategy.
  exposeStrategy: NodePort
  # FeatureGates are used to optionally enable certain features.
  featureGates: {}
  # ImagePullSecret is used to authenticate against Docker registries.
  imagePullSecret: ""
  # Ingress contains settings for making the API and UI accessible remotely.
  ingress:
    # CertificateIssuer is the name of a cert-manager Issuer or ClusterIssuer (default)
    # that will be used to acquire the certificate for the configured domain.
    # To use a namespaced Issuer, set the Kind to "Issuer" and manually create the
    # matching Issuer in Kubermatic's namespace.
    # Setting an empty name disables the automatic creation of certificates and disables
    # the TLS settings on the Kubermatic Ingress.
    certificateIssuer:
      # APIGroup is the group for the resource being referenced.
      # If APIGroup is not specified, the specified Kind must be in the core API group.
      # For any other third-party types, APIGroup is required.
      apiGroup: null
      # Kind is the type of resource being referenced
      kind: ClusterIssuer
      # Name is the name of resource being referenced
      name: ""
    # ClassName is the Ingress resource's class name, used for selecting the appropriate
    # ingress controller.
    className: nginx
    # Disable will prevent an Ingress from being created at all. This is mostly useful
    # during testing. If the Ingress is disabled, the CertificateIssuer setting can also
    # be left empty, as no Certificate resource will be created.
    disable: false
    # Domain is the base domain where the dashboard shall be available. Even with
    # a disabled Ingress, this must always be a valid hostname.
    domain: example.com
  # MasterController configures the master-controller-manager.
  masterController:
    # DebugLog enables more verbose logging.
    debugLog: false
    # DockerRepository is the repository containing the Kubermatic master-controller-manager image.
    dockerRepository: quay.io/kubermatic/kubermatic
    # PProfEndpoint controls the port the master-controller-manager should listen on to provide pprof
    # data. This port is never exposed from the container and only available via port-forwardings.
    pprofEndpoint: :6600
    # ProjectsMigrator configures the migrator for user projects.
    projectsMigrator:
      # DryRun makes the migrator only log the actions it would take.
      dryRun: false
    # Replicas sets the number of pod replicas for the master-controller-manager.
    replicas: 1
    # Resources describes the requested and maximum allowed CPU/memory usage.
    resources:
      # Limits describes the maximum amount of compute resources allowed.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      limits:
        cpu: 100m
        memory: 256Mi
      # Requests describes the minimum amount of compute resources required.
      # If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
      # otherwise to an implementation-defined value.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      requests:
        cpu: 50m
        memory: 128Mi
  # Proxy allows to configure Kubermatic to use proxies to talk to the
  # world outside of its cluster.
  proxy:
    # HTTP is the full URL to the proxy to use for plaintext HTTP
    # connections, e.g. "http://internalproxy.example.com:8080".
    http: ""
    # HTTPS is the full URL to the proxy to use for encrypted HTTPS
    # connections, e.g. "http://secureinternalproxy.example.com:8080".
    https: ""
    # NoProxy is a comma-separated list of hostnames / network masks
    # for which no proxy shall be used. If you make use of proxies,
    # this list should contain all local and cluster-internal domains
    # and networks, e.g. "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,mydomain".
    # The operator will always prepend the following elements to this
    # list if proxying is configured (i.e. HTTP/HTTPS are not empty):
    # "127.0.0.1/8", "localhost", ".local", ".local.", "kubernetes", ".default", ".svc"
    noProxy: ""
  # SeedController configures the seed-controller-manager.
  seedController:
    # BackupCleanupContainer is the container used for removing expired backups from the storage location.
    # This container is only relevant when the old, deprecated backup controllers are enabled.
    backupCleanupContainer: |
      name: cleanup-container
      image: quay.io/kubermatic/s3-storer:v0.1.6
      command:
      - /bin/sh
      - -c
      - |
        set -euo pipefail

        endpoint=minio.minio.svc.cluster.local:9000
        bucket=kubermatic-etcd-backups

        # by default, we keep the most recent backup for every user cluster
        s3-storeuploader delete-old-revisions \
          --ca-bundle=/etc/ca-bundle/ca-bundle.pem \
          --max-revisions 1 \
          --endpoint "$endpoint" \
          --bucket "$bucket" \
          --prefix $CLUSTER

        # alternatively, delete all backups for this cluster
        #s3-storeuploader delete-all \
        # --ca-bundle=/etc/ca-bundle/ca-bundle.pem \
        # --endpoint "$endpoint" \
        # --bucket "$bucket" \
        # --prefix $CLUSTER
      env:
      - name: ACCESS_KEY_ID
        valueFrom:
          secretKeyRef:
            name: kubermatic-s3-credentials
            key: ACCESS_KEY_ID
      - name: SECRET_ACCESS_KEY
        valueFrom:
          secretKeyRef:
            name: kubermatic-s3-credentials
            key: SECRET_ACCESS_KEY
    # BackupDeleteContainer is the container used for deleting etcd snapshots from a backup location.
    # This container is only relevant when the new backup/restore controllers are enabled.
    backupDeleteContainer: |
      name: delete-container
      image: d3fk/s3cmd@sha256:2061883abbf0ebcf0ea3d5d218558c9c229f212e9c08af4acdaa3758980eb67a
      command:
      - /bin/sh
      - -c
      - |
        s3cmd \
          --ca-certs=/etc/ca-bundle/ca-bundle.pem \
          --access_key=$ACCESS_KEY_ID \
          --secret_key=$SECRET_ACCESS_KEY \
          --host=$ENDPOINT \
          --host-bucket='%(bucket).'$ENDPOINT \
          del s3://$BUCKET_NAME/$CLUSTER-$BACKUP_TO_DELETE

        case $? in
        12)
          # backup no longer exists, which is fine
          exit 0
          ;;
        0)
          exit 0
          ;;
        *)
          exit $?
          ;;
        esac
    # BackupStoreContainer is the container used for shipping etcd snapshots to a backup location.
    backupStoreContainer: |
      name: store-container
      image: quay.io/kubermatic/s3-storer:v0.1.6
      command:
      - /bin/sh
      - -c
      - |
        set -euo pipefail

        endpoint=minio.minio.svc.cluster.local:9000
        bucket=kubermatic-etcd-backups

        s3-storeuploader store \
          --ca-bundle=/etc/ca-bundle/ca-bundle.pem \
          --file /backup/snapshot.db \
          --endpoint "$endpoint" \
          --bucket "$bucket" \
          --create-bucket \
          --prefix $CLUSTER

        s3-storeuploader delete-old-revisions \
          --ca-bundle=/etc/ca-bundle/ca-bundle.pem \
          --max-revisions 20 \
          --endpoint "$endpoint" \
          --bucket "$bucket" \
          --prefix $CLUSTER
      env:
      - name: ACCESS_KEY_ID
        valueFrom:
          secretKeyRef:
            name: kubermatic-s3-credentials
            key: ACCESS_KEY_ID
      - name: SECRET_ACCESS_KEY
        valueFrom:
          secretKeyRef:
            name: kubermatic-s3-credentials
            key: SECRET_ACCESS_KEY
      volumeMounts:
      - name: etcd-backup
        mountPath: /backup
    # DebugLog enables more verbose logging.
    debugLog: false
    # DockerRepository is the repository containing the Kubermatic seed-controller-manager image.
    dockerRepository: quay.io/kubermatic/kubermatic
    # MaximumParallelReconciles limits the number of cluster reconciliations
    # that are active at any given time.
    maximumParallelReconciles: 10
    # PProfEndpoint controls the port the seed-controller-manager should listen on to provide pprof
    # data. This port is never exposed from the container and only available via port-forwardings.
    pprofEndpoint: :6600
    # Replicas sets the number of pod replicas for the seed-controller-manager.
    replicas: 1
    # Resources describes the requested and maximum allowed CPU/memory usage.
    resources:
      # Limits describes the maximum amount of compute resources allowed.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      limits:
        cpu: 500m
        memory: 1Gi
      # Requests describes the minimum amount of compute resources required.
      # If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
      # otherwise to an implementation-defined value.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      requests:
        cpu: 200m
        memory: 512Mi
  # UI configures the dashboard.
  ui:
    # Config sets flags for various dashboard features.
    config: |-
      {
        "share_kubeconfig": false
      }
    # DockerRepository is the repository containing the Kubermatic dashboard image.
    dockerRepository: quay.io/kubermatic/dashboard
    # Replicas sets the number of pod replicas for the UI deployment.
    replicas: 2
    # Resources describes the requested and maximum allowed CPU/memory usage.
    resources:
      # Limits describes the maximum amount of compute resources allowed.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      limits:
        cpu: 250m
        memory: 128Mi
      # Requests describes the minimum amount of compute resources required.
      # If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
      # otherwise to an implementation-defined value.
      # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
      requests:
        cpu: 100m
        memory: 64Mi
  # UserCluster configures various aspects of the user-created clusters.
  userCluster:
    # Addons controls the optional additions installed into each user cluster.
    addons:
      # Default is the list of addons to be installed by default into each cluster.
      # Mutually exclusive with "defaultManifests".
      default: null
      # DefaultManifests is a list of addon manifests to install into all clusters.
      # Mutually exclusive with "default".
      defaultManifests: |-
        apiVersion: v1
        kind: List
        items:
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: canal
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: cilium
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: csi
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: kube-proxy
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: openvpn
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: rbac
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: kubeadm-configmap
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: kubelet-configmap
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: default-storage-class
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: pod-security-policy
            labels:
              addons.kubermatic.io/ensure: true
        - apiVersion: kubermatic.k8c.io/v1
          kind: Addon
          metadata:
            name: aws-node-termination-handler
            labels:
              addons.kubermatic.io/ensure: true
      # DockerRepository is the repository containing the Docker image containing
      # the possible addon manifests.
      dockerRepository: quay.io/kubermatic/addons
      # DockerTagSuffix is appended to the tag used for referring to the addons image.
      # If left empty, the tag will be the KKP version (e.g. "v2.15.0"), with a
      # suffix it becomes "v2.15.0-SUFFIX".
      dockerTagSuffix: ""
    # APIServerReplicas configures the replica count for the API-Server deployment inside user clusters.
    apiserverReplicas: 2
    # DisableAPIServerEndpointReconciling can be used to toggle the `--endpoint-reconciler-type` flag for
    # the Kubernetes API server.
    disableApiserverEndpointReconciling: false
    # DNATControllerDockerRepository is the repository containing the
    # dnat-controller image.
    dnatControllerDockerRepository: quay.io/kubermatic/kubeletdnat-controller
    # EtcdLauncherDockerRepository is the repository containing the Kubermatic
    # etcd-launcher image.
    etcdLauncherDockerRepository: quay.io/kubermatic/etcd-launcher
    # EtcdVolumeSize configures the volume size to use for each etcd pod inside user clusters.
    etcdVolumeSize: 5Gi
    # KubermaticDockerRepository is the repository containing the Kubermatic user-cluster-controller-manager image.
    kubermaticDockerRepository: quay.io/kubermatic/kubermatic
    # MachineController configures the Machine Controller
    machineController:
      # ImageRepository is used to override the Machine Controller image repository.
      # It is only for development, tests and PoC purposes. This field must not be set in production environments.
      imageRepository: ""
      # ImageTag is used to override the Machine Controller image.
      # It is only for development, tests and PoC purposes. This field must not be set in production environments.
      imageTag: ""
    # Monitoring can be used to fine-tune to in-cluster Prometheus.
    monitoring:
      # CustomRules can be used to inject custom recording and alerting rules. This field
      # must be a YAML-formatted string with a `group` element at its root, as documented
      # on https://prometheus.io/docs/prometheus/2.14/configuration/alerting_rules/.
      customRules: ""
      # CustomScrapingConfigs can be used to inject custom scraping rules. This must be a
      # YAML-formatted string containing an array of scrape configurations as documented
      # on https://prometheus.io/docs/prometheus/2.14/configuration/configuration/#scrape_config.
      customScrapingConfigs: ""
      # DisableDefaultRules disables the recording and alerting rules.
      disableDefaultRules: false
      # DisableDefaultScrapingConfigs disables the default scraping targets.
      disableDefaultScrapingConfigs: false
      # ScrapeAnnotationPrefix (if set) is used to make the in-cluster Prometheus scrape pods
      # inside the user clusters.
      scrapeAnnotationPrefix: monitoring.kubermatic.io
    # NodePortRange is the port range for customer clusters - this must match the NodePort
    # range of the seed cluster.
    nodePortRange: 30000-32767
    # OverwriteRegistry specifies a custom Docker registry which will be used for all images
    # used for user clusters (user cluster control plane + addons). This also applies to
    # the KubermaticDockerRepository and DNATControllerDockerRepository fields.
    overwriteRegistry: ""
  # Versions configures the available and default Kubernetes versions and updates.
  versions:
    # Default is the default version to offer users.
    default: v1.21.12
    # ProviderIncompatibilities lists all the Kubernetes version incompatibilities
    providerIncompatibilities:
      - # Condition is the cluster or datacenter condition that must be met to block a specific version
        condition: nonAMD64WithCanalAndIPVS
        # Operation is the operation triggering the compatibility check (CREATE or UPDATE)
        operation: UPGRADE
        # Provider to which to apply the compatibility check.
        # Empty string matches all providers
        provider: ""
        # Version is the Kubernetes version that must be checked. Wildcards are allowed, e.g. "1.22.*".
        version: 1.23.*
    # Updates is a list of available and automatic upgrades.
    # All 'to' versions must be configured in the version list for this orchestrator.
    # Each update may optionally be configured to be 'automatic: true', in which case the
    # controlplane of all clusters whose version matches the 'from' directive will get
    # updated to the 'to' version. If automatic is enabled, the 'to' version must be a
    # version and not a version range.
    # Also, updates may set 'automaticNodeUpdate: true', in which case Nodes will get
    # updates as well. 'automaticNodeUpdate: true' implies 'automatic: true' as well,
    # because Nodes may not have a newer version than the controlplane.
    updates:
      - # Automatic controls whether this update is executed automatically
        # for the control plane of all matching user clusters.
        automatic: true
        # Automatic controls whether this update is executed automatically
        # for the worker nodes of all matching user clusters.
        automaticNodeUpdate: false
        # From is the version from which an update is allowed. Wildcards are allowed, e.g. "1.18.*".
        from: 1.19.*
        # To is the version to which an update is allowed.
        # Must be a valid version if `automatic` is set to true, e.g. "1.20.13".
        # Can be a wildcard otherwise, e.g. "1.20.*".
        to: 1.20.13
      - from: 1.20.*
        to: 1.20.*
      - automatic: true
        from: '>= 1.20.0, < 1.20.13'
        to: 1.20.13
      - from: 1.20.*
        to: 1.21.*
      - from: 1.21.*
        to: 1.21.*
      - automatic: true
        from: '>= 1.21.0, < 1.21.8'
        to: 1.21.8
      - from: 1.21.*
        to: 1.22.*
      - from: 1.22.*
        to: 1.22.*
      - automatic: true
        from: '>= 1.22.0, < 1.22.5'
        to: 1.22.5
      - from: 1.22.*
        to: 1.23.*
      - from: 1.23.*
        to: 1.23.*
    # Versions lists the available versions.
    versions:
      - v1.20.13
      - v1.20.14
      - v1.21.8
      - v1.21.12
      - v1.22.5
      - v1.22.9
      - v1.23.6
  # VerticalPodAutoscaler configures the Kubernetes VPA integration.
  verticalPodAutoscaler:
    admissionController:
      # DockerRepository is the repository containing the component's image.
      dockerRepository: gcr.io/google_containers/vpa-admission-controller
      # Resources describes the requested and maximum allowed CPU/memory usage.
      resources:
        # Limits describes the maximum amount of compute resources allowed.
        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
        limits:
          cpu: 200m
          memory: 128Mi
        # Requests describes the minimum amount of compute resources required.
        # If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
        # otherwise to an implementation-defined value.
        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
        requests:
          cpu: 50m
          memory: 32Mi
    recommender:
      # DockerRepository is the repository containing the component's image.
      dockerRepository: gcr.io/google_containers/vpa-recommender
      # Resources describes the requested and maximum allowed CPU/memory usage.
      resources:
        # Limits describes the maximum amount of compute resources allowed.
        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
        limits:
          cpu: 200m
          memory: 3Gi
        # Requests describes the minimum amount of compute resources required.
        # If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
        # otherwise to an implementation-defined value.
        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
        requests:
          cpu: 50m
          memory: 512Mi
    updater:
      # DockerRepository is the repository containing the component's image.
      dockerRepository: gcr.io/google_containers/vpa-updater
      # Resources describes the requested and maximum allowed CPU/memory usage.
      resources:
        # Limits describes the maximum amount of compute resources allowed.
        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
        limits:
          cpu: 200m
          memory: 128Mi
        # Requests describes the minimum amount of compute resources required.
        # If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
        # otherwise to an implementation-defined value.
        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
        requests:
          cpu: 50m
          memory: 32Mi