Well it works

.gitignore

@@ -0,0 +1,6 @@
+ceph_key*
+.terraform*
+*tfstate*
+variables.tf
+inventory
+data/

README.md

@@ -0,0 +1,25 @@
+ceph-on-lxd-by-terraform
+====
+
+I dont know how to name it correctly.
+
+Эти манифесты создадут виртуалки под CEPH, проект, по одному диску для каждого OSD и отдельную сеть с NAT'ом. Также на каждую ноду будет разложен ssh ключ и /etc/hosts.
+
+Сначала надо прогнать терраформ, затем запустить ансибл плейбук. В папке data/ должны лежать ключи и инвентарь для ансибла
+
+Перед запуском надо:
+
+- Сгенерировать ssh ключ в данной репе под именем `ceph_key` и `ceph_key.pub`, положить ключи в data/
+- Проверить что на компьютере есть lxd клиент и он направлен на нужный remote (или можно использовать локалхост)
+- Конечно же `terraform init`
+
+# TLDR:
+
+- `cd data/`
+- `ssh-keygen -f ceph_key`
+- `cd ../terraform`
+- `terraform init`
+- `terraform plan`
+- `terraform apply`
+- `cd ../data`
+- `ansible-playbook -i inventory ../ansible/playbook.yml`

ansible/hosts.j2

@@ -0,0 +1,3 @@
+{% for host in hostvars %}
+{{ hostvars[host]['ansible_facts']['enp5s0']['ipv4']['address'] }} {{ host }}
+{% endfor %}

ansible/playbook.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  tasks:
+    - name: Replace /etc/hosts
+      template:
+        src: hosts.j2
+        dest: /etc/hosts
+

terraform/ceph-disks.tf

@@ -0,0 +1,11 @@
+resource "lxd_volume" "ceph-osd-vol" {
+    pool = "default"
+    project = lxd_project.cephproject.name
+
+    name = format("ceph-osd-%d", count.index + 1)
+    count = 4
+    content_type = "block"
+    config = {
+        size = "10GiB"
+    }
+}

terraform/ceph-gen-inventory.tf

@@ -0,0 +1,12 @@
+# generate inventory file for Ansible
+resource "local_file" "ansible_inventory" {
+  content = templatefile(
+    "hosts.j2",
+    {
+      cephmons = {for key, value in lxd_instance.ceph-mon : value.name => value.ipv4_address }
+      cephosds = {for key, value in lxd_instance.ceph-osd : value.name => value.ipv4_address }
+      jumphost = var.ansible_jumphost
+    }
+  )
+  filename = "../data/inventory"
+}

terraform/ceph-net.tf

@@ -0,0 +1,12 @@
+resource "lxd_network" "cephnet" {
+    # project = lxd_project.cephproject.name
+    name = "cephnet"
+    type = "bridge"
+
+    config = {
+        "ipv4.address" = "10.99.99.1/24"
+        "ipv4.nat"     = "true"
+        "ipv6.address" = "fd42:474b:622d:259d::1/64"
+        "ipv6.nat"     = "true"
+    }
+}

terraform/ceph-vm.tf

@@ -0,0 +1,92 @@
+### ceph-mon
+resource "lxd_instance" "ceph-mon" {
+  name = format("ceph-mon-%d", count.index + 1)
+  count = 4
+
+  type = "virtual-machine"
+  image = "ubuntu:jammy"
+  project = lxd_project.cephproject.name
+  profiles  = ["default", "${lxd_profile.cephprofile.name}"]
+  
+  config = {
+    "boot.autostart" = false
+  }
+
+ device {
+    name = "eth0"
+    type = "nic"
+
+    properties = {
+      nictype = "bridged"
+      parent  = "${lxd_network.cephnet.name}"
+      "ipv4.address" = format("10.99.99.%d", count.index + 10)
+    }
+  }
+
+  limits = {
+    cpu = 2
+    memory = "4GiB"
+  }
+
+  execs = {
+    "shell_cmd" = {
+      command = ["/bin/sh", "-c", "echo $PUB_KEY | tee /root/.ssh/id_ed25519.pub /root/.ssh/authorized_keys ; echo \"$PRIV_KEY\"  > /root/.ssh/id_ed25519 ; chmod 600 /root/.ssh/*"]
+
+      environment = {
+        "PUB_KEY" = file("../data/ceph_key.pub")
+        "PRIV_KEY" = file("../data/ceph_key")
+      }
+    }
+  }
+}
+
+### ceph-osd
+resource "lxd_instance" "ceph-osd" {
+  name = format("ceph-osd-%d", count.index + 1)
+  count = 4
+
+  type = "virtual-machine"
+  image = "ubuntu:jammy"
+  project = lxd_project.cephproject.name
+  profiles  = ["default", "${lxd_profile.cephprofile.name}"]
+  
+  config = {
+    "boot.autostart" = false
+  }
+
+  device {
+    name = "eth0"
+    type = "nic"
+
+    properties = {
+      nictype = "bridged"
+      parent  = "${lxd_network.cephnet.name}"
+      "ipv4.address" = format("10.99.99.%d", count.index + 20)
+    }
+  }
+
+  device {
+    name = format("ceph-osd-%d", count.index + 1)
+    type = "disk"
+    properties = {
+      source = format("ceph-osd-%d", count.index + 1)
+      pool   = "default"
+    }
+  }
+
+  limits = {
+    cpu = 2
+    memory = "4GiB"
+  }
+  execs = {
+    "shell_cmd" = {
+      command = ["/bin/sh", "-c", "echo $PUB_KEY | tee /root/.ssh/id_ed25519.pub /root/.ssh/authorized_keys ; echo -e $PRIV_KEY > /root/.ssh/id_ed25519 ; chmod 600 /root/.ssh/*"]
+
+      environment = {
+        "PUB_KEY" = file("../data/ceph_key.pub")
+        "PRIV_KEY" = file("../data/ceph_key")
+      }
+    }
+  }
+
+}

terraform/hosts.j2

@@ -0,0 +1,9 @@
+[ceph-mon]
+%{ for name, ip in cephmons ~}
+${name} ansible_ssh_host=${ip} ansible_ssh_common_args='-J ${jumphost} -oStrictHostKeyChecking=no -i ../data/ceph_key' ansible_ssh_user='root'
+%{ endfor ~}
+
+[ceph-osd]
+%{ for name, ip in cephosds ~}
+${name} ansible_ssh_host=${ip} ansible_ssh_common_args='-J ${jumphost} -oStrictHostKeyChecking=no -i ../data/ceph_key' ansible_ssh_user='root'
+%{ endfor ~}

terraform/init.tf

@@ -0,0 +1,43 @@
+terraform {
+  required_providers {
+    lxd = {
+      source = "terraform-lxd/lxd"
+    }
+  }
+}
+
+resource "lxd_project" "cephproject" {
+  name        = "ceph"
+  description = "Terraform provider example project"
+  config = {
+    "features.storage.volumes" = false
+    "features.images"          = false
+    "features.profiles"        = false
+    "features.storage.buckets" = false
+    "features.networks"        = true
+  }
+}
+
+resource "lxd_profile" "cephprofile" {
+  name = "cephprofile"
+
+  device {
+    name = "eth0"
+    type = "nic"
+
+    properties = {
+      nictype = "bridged"
+      parent  = "${lxd_network.cephnet.name}"
+    }
+  }
+
+  device {
+    type = "disk"
+    name = "root"
+
+    properties = {
+      pool = "default"
+      path = "/"
+    }
+  }
+}

terraform/terraform.tfvars

@@ -0,0 +1,2 @@
+ansible_jumphost = "selded"
+ceph_subnet = "10.99.99"