diff --git a/ephemeral/instance.tf b/ephemeral/instance.tf index 517c41d2..06d2dfcc 100644 --- a/ephemeral/instance.tf +++ b/ephemeral/instance.tf @@ -31,3 +31,15 @@ resource "digitalocean_droplet" "cp" { local.prefix, ] } + +# mayastor volume +resource "digitalocean_volume" "mayastor" { + region = "fra1" + name = "rnd-${local.prefix}-spectrum-cp" + size = "100" +} + +resource "digitalocean_volume_attachment" "mayastor" { + droplet_id = digitalocean_droplet.cp[0].id + volume_id = digitalocean_volume.mayastor.id +} diff --git a/ephemeral/main.tf b/ephemeral/main.tf index 2eea5a7a..81a49daf 100644 --- a/ephemeral/main.tf +++ b/ephemeral/main.tf @@ -22,7 +22,8 @@ module "talos" { templatefile("${path.root}/patches/registry.yml", { docker_username = data.vault_generic_secret.docker.data.username, docker_password = data.vault_generic_secret.docker.data.password - }) + },), + file("${path.root}/patches/mayastor.yml") ] }, ] @@ -45,5 +46,6 @@ module "spectrum" { DOMAIN = "${local.prefix}.fluence.dev" PREFIX = local.prefix LOADBALANCER_IP = digitalocean_droplet.cp[0].ipv4_address + DO_DISK = digitalocean_volume.mayastor.name } } diff --git a/ephemeral/patches/mayastor.yml b/ephemeral/patches/mayastor.yml new file mode 100644 index 00000000..ccd9ce30 --- /dev/null +++ b/ephemeral/patches/mayastor.yml @@ -0,0 +1,27 @@ +machine: + kubelet: + extraArgs: + node-labels: openebs.io/engine=mayastor + max-pods: 500 + extraMounts: + - destination: /var/local # Destination is the absolute path where the mount will be placed in the container. + type: bind # Type specifies the mount kind. + source: /var/local # Source specifies the source path of the mount. + options: + - bind + - rshared + - rw + sysctls: + vm.nr_hugepages: "1024" + nodeLabels: + openebs.io/engine: "mayastor" +cluster: + apiServer: + admissionControl: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1beta1 + kind: PodSecurityConfiguration + exemptions: + namespaces: + - openebs diff --git a/flux/clusters/default/kustomization.yml b/flux/clusters/default/kustomization.yml index 42e85979..971a009d 100644 --- a/flux/clusters/default/kustomization.yml +++ b/flux/clusters/default/kustomization.yml @@ -4,3 +4,4 @@ resources: - ../../core/local-path-provisioner - ../../core/metrics-server - ../../core/kubelet-serving-cert-approver + - ../../core/openebs diff --git a/flux/clusters/ephemeral/kustomization.yml b/flux/clusters/ephemeral/kustomization.yml index d63b5510..b38614eb 100644 --- a/flux/clusters/ephemeral/kustomization.yml +++ b/flux/clusters/ephemeral/kustomization.yml @@ -11,7 +11,9 @@ resources: - ../../core/kubelet-serving-cert-approver - ../../components/kubevirt - ../../components/monitoring + - openebs-mayastor.yml - hubble-ingress.yml - grafana-ingress.yml # - cilium-l2.yml - lightmare.yml + - openebs-do-pool.yml diff --git a/flux/clusters/ephemeral/openebs-do-pool.yml b/flux/clusters/ephemeral/openebs-do-pool.yml new file mode 100644 index 00000000..eb4e56b4 --- /dev/null +++ b/flux/clusters/ephemeral/openebs-do-pool.yml @@ -0,0 +1,25 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: openebs-do-pool + namespace: flux-system +spec: + dependsOn: + - name: openebs + interval: 1m + path: "./flux/components/openebs-do-pool" + prune: true + sourceRef: + kind: GitRepository + name: spectrum + namespace: flux-system + patches: + - patch: |- + - op: replace + path: /spec/disks + value: ["aio:///dev/disk/by-id/scsi-0DO_Volume_${DO_DISK}"] + target: + kind: DiskPool + name: ebs-pool + namespace: openebs diff --git a/flux/clusters/ephemeral/openebs-mayastor.yml b/flux/clusters/ephemeral/openebs-mayastor.yml new file mode 100644 index 00000000..7197deb7 --- /dev/null +++ b/flux/clusters/ephemeral/openebs-mayastor.yml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: openebs + namespace: flux-system +spec: + interval: 1m + path: "./flux/core/openebs" + prune: true + sourceRef: + kind: GitRepository + name: spectrum + namespace: flux-system + \ No newline at end of file diff --git a/flux/components/openebs-do-pool/kustomization.yaml b/flux/components/openebs-do-pool/kustomization.yaml new file mode 100644 index 00000000..f81cecbb --- /dev/null +++ b/flux/components/openebs-do-pool/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./manifests.yaml diff --git a/flux/components/openebs-do-pool/manifests.yaml b/flux/components/openebs-do-pool/manifests.yaml new file mode 100644 index 00000000..65f87355 --- /dev/null +++ b/flux/components/openebs-do-pool/manifests.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: mayastor +parameters: + protocol: nvmf + repl: "1" +provisioner: io.openebs.csi-mayastor +--- +apiVersion: "openebs.io/v1beta2" +kind: DiskPool +metadata: + name: ebs-pool + namespace: openebs +spec: + node: cp-0 + disks: ["aio:///dev/disk/by-id/placeholder"] diff --git a/flux/components/openebs-pool/kustomization.yaml b/flux/components/openebs-pool/kustomization.yaml new file mode 100644 index 00000000..f81cecbb --- /dev/null +++ b/flux/components/openebs-pool/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./manifests.yaml diff --git a/flux/components/openebs-pool/manifests.yaml b/flux/components/openebs-pool/manifests.yaml new file mode 100644 index 00000000..e5e3e6d7 --- /dev/null +++ b/flux/components/openebs-pool/manifests.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: spectrum-pool +parameters: + protocol: nvmf + repl: "1" +provisioner: io.openebs.csi-mayastor diff --git a/flux/core/openebs/kustomization.yml b/flux/core/openebs/kustomization.yml new file mode 100644 index 00000000..d9d63633 --- /dev/null +++ b/flux/core/openebs/kustomization.yml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - repository.yml + - release.yml diff --git a/flux/core/openebs/release.yml b/flux/core/openebs/release.yml new file mode 100644 index 00000000..b79e6c9d --- /dev/null +++ b/flux/core/openebs/release.yml @@ -0,0 +1,35 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: openebs + namespace: kube-system +spec: + targetNamespace: openebs + install: + createNamespace: true + interval: 5m + chart: + spec: + chart: openebs + version: 4.1.3 + sourceRef: + name: openebs + kind: HelmRepository + namespace: flux-system + values: + mayastor: + csi: + node: + initContainers: + enabled: false + etcd: + replicaCount: 1 + engines: + replicated: + mayastor: + enabled: true + local: + lvm: + enabled: false + zfs: + enabled: false diff --git a/flux/core/openebs/repository.yml b/flux/core/openebs/repository.yml new file mode 100644 index 00000000..772bcdf8 --- /dev/null +++ b/flux/core/openebs/repository.yml @@ -0,0 +1,9 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: openebs + namespace: flux-system +spec: + interval: 5m + timeout: 3m + url: https://openebs.github.io/openebs