mass update

- Upgrading to Terraform from 1.1.8 to 1.3.5
- Upgrading to Kubernetes from 1.23.5 to 1.25.4
- Using Q35 machine for workers
- Using UEFI for boot
- Refactored XSLT

.drone.yml

@@ -17,7 +17,7 @@ trigger:
 
 steps:
 - name: terraform init
-  image: hashicorp/terraform:1.1.8
+  image: hashicorp/terraform:1.3.5
   commands:
     - echo "$${CI_SSH_KEY}" | base64 -d > id_rsa
     - chmod 600 id_rsa
@@ -32,9 +32,9 @@ steps:
       from_secret: terraform-aws-secret-access-key
 
 - name: terraform plan
-  image: hashicorp/terraform:1.1.8
+  image: hashicorp/terraform:1.3.5
   commands:
-    - terraform plan -var-file ci.tfvars -out .tfplan
+    - terraform plan $([[ $${DRONE_BUILD_EVENT} = cron ]] && echo "-detailed-exitcode") -var-file ci.tfvars -out .tfplan
   environment:
     AWS_ACCESS_KEY_ID:
       from_secret: terraform-aws-key-id
@@ -53,21 +53,14 @@ steps:
     - main
     event:
     - push
-  image: hashicorp/terraform:1.1.8
+  image: hashicorp/terraform:1.3.5
   commands:
-    - terraform apply -var-file ci.tfvars .tfplan
+    - terraform apply .tfplan
   environment:
     AWS_ACCESS_KEY_ID:
       from_secret: terraform-aws-key-id
     AWS_SECRET_ACCESS_KEY:
       from_secret: terraform-aws-secret-access-key
-    KUBE_TOKEN:
-      from_secret: lawndale-k8s-ci-token
-    TF_VAR_lawndale_dns_key_secret:
-      from_secret: lawndale-dns-key-secret
-    TF_VAR_lawndale_dns_key_algorithm:
-      from_secret: lawndale-dns-key-algorithm
-
 ---
 kind: pipeline
 type: kubernetes
@@ -82,16 +75,16 @@ trigger:
 
 steps:
 - name: format and generate docs
-  image: hashicorp/terraform:1.1.8
+  image: hashicorp/terraform:1.3.5
   commands:
     - apk add bash wget 
     - wget -q https://github.com/terraform-docs/terraform-docs/releases/download/v0.16.0/terraform-docs-v0.16.0-linux-amd64.tar.gz -O - | tar -xz terraform-docs -C /usr/local/bin
     - terraform fmt
     - terraform-docs .
-    - git diff-index --exit-code --name-status HEAD
+    - git diff --exit-code
 
 ---
 kind: signature
-hmac: 05ee4e191f657d959752ba19c45bb980530c49140d75dc8eef5602c2e4e80fcc
+hmac: 95f8db197163e884f2eee4b14af136b9ea1e0f88f626079b4a3b38b43b91c6a8
 
 ...

.terraform.lock.hcl

@@ -24,24 +24,24 @@ provider "registry.terraform.io/cloudflare/cloudflare" {
 }
 
 provider "registry.terraform.io/dmacvicar/libvirt" {
-  version     = "0.6.14"
+  version     = "0.7.0"
-  constraints = "0.6.14, ~> 0.6.14"
+  constraints = ">= 0.6.14, 0.7.0"
   hashes = [
-    "h1:JyRYY5LmhQKvolpmwcfiWWlFJMMfvQKqP3PRWT0I2JU=",
+    "h1:1RiomFBEdgi6GvqaAf16mW6bRqrxAV0P47HKKwETH3E=",
-    "zh:0450e09888e0399722d6714467d5f0a61d2ab6505cd4c66423d58dd98404da80",
+    "zh:1161bfcac075d5790c9b36145811d95241622636b859222f995888471ba04efa",
-    "zh:263c80ca9743bcc699983803b85cac19f833663478b644c2b3000a6f3e1b5070",
+    "zh:317bca5edd36e2497e0ac2ed07dd5e7f09cfd5561a96607cb53fe1af6b0632c0",
-    "zh:2a3eda5b0dc170afd4339910396e6087181dd0f37da0d83ee175fed2975a5a40",
+    "zh:4fa67e3baf6845148f2b4e617fb01c47f0971ce2d945efa805ac5c3820bb0ca6",
-    "zh:363b8385d3340688fe58c67ea1e798d99892e25ac0a38f3e3fd615968b829e3e",
+    "zh:6e17f5f24373e21c0ff463d36d9caa4f08528e13764c5d1d7eceb719dcef6a14",
-    "zh:517efa2132c6ff6a849abea324916884a2d8e9361197209c56da99d1419542a3",
+    "zh:84622e2aca8bc91d71d3596fcd1b298c5dfe572c8722ab98084495d26b5c5e7d",
-    "zh:5f1424da9a9c9aef6b5583861750ed958fff4f1f85e17a493b58aff05b5a731b",
+    "zh:8ce125d872b26ce9b71a729437eb8ab36944a86da3784edaab7368af43ca3858",
-    "zh:778bd0ea056ed7e918bcc3c05ad651504af86e6b53e6480daf20879e7d01e0a1",
+    "zh:8fc7eee76776d515c023d013c018a7b9816f0e840578af01bfaf58e49f020c03",
-    "zh:8576f08eff1596d96072e6eb0c29febbbe316cc26537a949be76c71659bd3b63",
+    "zh:a4d6fccc0188746be35488396c431e4b313cd1221df408871c710d3a7382b02e",
-    "zh:a1f5bbadad4f809d4b96a332bda1b48787d08a8bf3bf23b40e68138fbaa727fb",
+    "zh:b575bb2d2f8987043aecbb22ac3bbf1e9c8b9da49b201b6b225baf2b4595dae4",
-    "zh:b24e1f6f1bd09acdfb87f76f76ee7adfa1af1e0798c8c0aeb20d2a5bf67d8a33",
+    "zh:b65b1733c29a09491912a98a829b19c9842af5971fbb358bc0e979b95bf33248",
-    "zh:b6359aab7499b6fab819c867901b32426eb8661f2279e12c0c07cbeadce119e1",
+    "zh:b8266ed7b4bce4791fee5433d102d89187974a273574d69f637cfdeb913462c2",
-    "zh:ca357e2424a41058571f4b437a5e440395755461dcc1041cbbb41ea23c29eab5",
+    "zh:bd0b842d6f694c6d558d3329a2c157dd9d84074d618d5ced891ef36798b1c97b",
-    "zh:ec57e6e3ee701522d2cfd57a8ae307e76bff4f4a4af36c0e10d4189fa8dd554d",
+    "zh:dacf0299c2c11d84bdaa2f614ca14aeac36ffba0f20dff5a63437a81a61f6867",
-    "zh:f46534893933d5b11f32fb0d55044ba84f4e69147955d0454a208c494bbb0882",
+    "zh:e8c92794a06df42c15ff071859e99c6e95e93dcb40797c4128d31d3a47a27923",
   ]
 }
 
@@ -126,21 +126,21 @@ provider "registry.terraform.io/hashicorp/kubernetes" {
 }
 
 provider "registry.terraform.io/hashicorp/random" {
-  version = "3.2.0"
+  version = "3.4.3"
   hashes = [
-    "h1:eeUh6cJ6wKLLuo4q9uQ0CA1Zvfqya4Wn1LecLCN8KKs=",
+    "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=",
-    "zh:2960977ce9a7d6a7d3e934e75ec5814735626f95c186ad95a9102344a1a38ac1",
+    "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752",
-    "zh:2fd012abfabe7076f3f2f402eeef4970e20574d20ffec57c162b02b6e848c32f",
+    "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b",
-    "zh:4cd3234671cf01c913023418b227eb78b0659f2cd2e0b387be1f0bb607d29889",
+    "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53",
-    "zh:52e695b4fa3fae735ffc901edff8183745f980923510a744db7616e8f10dc499",
     "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:848b4a294e5ba15192ee4bfd199c07f60a437d7572efcd2d89db036e1ebc0e6e",
+    "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3",
-    "zh:9d49aa432a05748a9527e95448cebee1238c87c97c7e8dec694bfd709683f9c7",
+    "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5",
-    "zh:b4ad4cf289d3f7408649b74b8639918833613f2a1f3cf51b51f4b2fdaa412dd2",
+    "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda",
-    "zh:c1544c4b416096fb8d8dbf84c4488584a2844a30dd533b957e9e9e60a165f24e",
+    "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6",
-    "zh:dc737d6b4591cad8c9a1d0b347e587e846d8d901789b29b4dd401b6cdf82c017",
+    "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1",
-    "zh:f5645fd39f749dbbf847cbdc87ba0dbd141143f12917a6a8904faf8a9b64111e",
+    "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d",
-    "zh:fdedf610e0d020878a8f1fedda8105e0c33a7e23c4792fca54460685552de308",
+    "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8",
+    "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93",
   ]
 }
 
@@ -163,20 +163,21 @@ provider "registry.terraform.io/hashicorp/template" {
 }
 
 provider "registry.terraform.io/hashicorp/time" {
-  version = "0.7.2"
+  version = "0.9.1"
   hashes = [
-    "h1:YYLAfhMFP5nhV2iZPslqsLkZN+6sZo7gMJW7pLcLfM8=",
+    "h1:NUv/YtEytDQncBQ2mTxnUZEy/rmDlPYmE9h2iokR0vk=",
-    "zh:0bbe0158c2a9e3f5be911b7e94477586110c51746bb13d102054f22754565bda",
+    "zh:00a1476ecf18c735cc08e27bfa835c33f8ac8fa6fa746b01cd3bcbad8ca84f7f",
-    "zh:3250af7fd49b8aaf2ccc895588af05197d886e38b727e3ba33bcbb8cc96ad34d",
+    "zh:3007f8fc4a4f8614c43e8ef1d4b0c773a5de1dcac50e701d8abc9fdc8fcb6bf5",
-    "zh:35e4de0437f4fa9c1ad69aaf8136413be2369ea607d78e04bb68dc66a6a520b8",
+    "zh:5f79d0730fdec8cb148b277de3f00485eff3e9cf1ff47fb715b1c969e5bbd9d4",
-    "zh:369756417a6272e79cad31eb2c82c202f6a4b6e4204a893f656644ba9e149fa2",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:390370f1179d89b33c3a0731691e772d5450a7d59fc66671ec625e201db74aa2",
+    "zh:8c8094689a2bed4bb597d24a418bbbf846e15507f08be447d0a5acea67c2265a",
-    "zh:3d12ac905259d225c685bc42e5507ed0fbdaa5a09c30dce7c1932d908df857f7",
+    "zh:a6d9206e95d5681229429b406bc7a9ba4b2d9b67470bda7df88fa161508ace57",
-    "zh:75f63e5e1c68e6c5bccba4568c3564e2774eb3a7a19189eb8e2b6e0d58c8f8cc",
+    "zh:aa299ec058f23ebe68976c7581017de50da6204883950de228ed9246f309e7f1",
-    "zh:7c22a2078a608e3e0278c4cbc9c483909062ebd1843bddaf8f176346c6d378b1",
+    "zh:b129f00f45fba1991db0aa954a6ba48d90f64a738629119bfb8e9a844b66e80b",
-    "zh:7cfb3c02f78f0060d59c757c4726ab45a962ce4a9cf4833beca704a1020785bd",
+    "zh:ef6cecf5f50cda971c1b215847938ced4cb4a30a18095509c068643b14030b00",
-    "zh:a0325917f47c28a2ed088dedcea0d9520d91b264e63cc667fe4336ac993c0c11",
+    "zh:f1f46a4f6c65886d2dd27b66d92632232adc64f92145bf8403fe64d5ffa5caea",
-    "zh:c181551d4c0a40b52e236f1755cc340aeca0fb5dcfd08b3b1c393a7667d2f327",
+    "zh:f79d6155cda7d559c60d74883a24879a01c4d5f6fd7e8d1e3250f3cd215fb904",
+    "zh:fd59fa73074805c3575f08cd627eef7acda14ab6dac2c135a66e7a38d262201c",
   ]
 }
 

README.md

@@ -69,7 +69,9 @@ Version:
 The following resources are used by this module:
 
 - [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
+- [helm_release.kube_state_metrics](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
 - [helm_release.metrics_server](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
+- [helm_release.traefik](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
 - [kubernetes_cluster_role.ci_cd](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) (resource)
 - [kubernetes_cluster_role.prometheus](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) (resource)
 - [kubernetes_cluster_role_binding.auto_approve_node_csrs](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) (resource)

bootstrap_token.tf

@@ -2,12 +2,18 @@ resource "random_password" "bootstrap_token_id" {
   special = false
   upper   = false
   length  = 6
+  lifecycle {
+    replace_triggered_by = [time_rotating.bootstrap_expiry_base]
+  }
 }
 
 resource "random_password" "bootstrap_token_secret" {
   special = false
   upper   = false
   length  = 16
+  lifecycle {
+    replace_triggered_by = [time_rotating.bootstrap_expiry_base]
+  }
 }
 
 resource "time_rotating" "bootstrap_expiry_base" {
@@ -34,6 +40,10 @@ resource "kubernetes_secret" "bootstrap_token" {
     "expiration" = timeadd(time_rotating.bootstrap_expiry_base.id, "${90 * 24}h")
   }
 
+  lifecycle {
+    replace_triggered_by = [time_rotating.bootstrap_expiry_base]
+  }
+
 }
 
 resource "kubernetes_cluster_role_binding" "auto_approve_node_csrs" {

ci.tf

@@ -100,4 +100,20 @@ resource "kubernetes_cluster_role" "ci_cd" {
     ]
     verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
   }
+
+  rule {
+    api_groups = ["traefik.containo.us"]
+    resources = [
+      "ingressroutes",
+      "ingressroutetcps",
+      "ingressrouteudps",
+      "middlewares",
+      "middlewaretcps",
+      "serverstransports",
+      "tlsoptions",
+      "tlsstores",
+      "traefikservices",
+    ]
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
+  }
 }

fixing-worker-domaindefs.xsl

@@ -0,0 +1,49 @@
+<xsl:stylesheet version="1.0"
+    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+    <xsl:output method="xml" indent="yes"/>
+    <xsl:strip-space elements="*" />
+
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()"/>
+        </xsl:copy>
+    </xsl:template>
+
+
+    <!-- Making the worker headless -->
+    <xsl:template match="/domain/devices/graphics" />
+    <xsl:template match="/domain/devices/video" />
+    <xsl:template match="/domain/devices/audio" />
+    <xsl:template match="/domain/devices/input[@type='mouse' or @type='keyboard']" />
+
+    <!-- SEE https://github.com/dmacvicar/terraform-provider-libvirt/issues/667 -->
+    <!-- Thanks dariush, https://gist.github.com/dariush/7405cbf62835e03d0b5c953d798a87cd -->
+    <!-- replace <target dev='hdd'...> with <target dev='sdd'...>  -->
+    <xsl:template match="/domain/devices/disk[@device='cdrom']/target/@dev">
+        <xsl:attribute name="dev">
+            <xsl:value-of select="'sdd'"/>
+        </xsl:attribute>
+    </xsl:template>
+
+    <!-- replace <target bus='ide'...> with <target bus='sata'...>  -->
+    <xsl:template match="/domain/devices/disk[@device='cdrom']/target/@bus">
+        <xsl:attribute name="bus">
+            <xsl:value-of select="'sata'"/>
+        </xsl:attribute>
+    </xsl:template>
+
+    <!-- replace <target bus='ide'...> with <target bus='sata'...>  -->
+    <xsl:template match="/domain/devices/disk[@device='disk' and target/@bus='scsi']">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|*[not(self::wwn) and not(self::target)]"/>
+            <target bus="sata">
+                <xsl:attribute name="dev"><xsl:value-of select="target/@dev" /></xsl:attribute>
+            </target> 
+        </xsl:copy>
+    </xsl:template>
+
+    <!-- replace <alias...> with nothing ie delete the <alias...> element  -->
+    <xsl:template match="/domain/devices/disk[@device='cdrom']/alias" />
+
+
+</xsl:stylesheet>

kube-state-metrics.tf

@@ -0,0 +1,35 @@
+resource "helm_release" "kube_state_metrics" {
+  name      = "kube-state-metrics"
+  namespace = "kube-system"
+
+  repository = "https://prometheus-community.github.io/helm-charts"
+  chart      = "kube-state-metrics"
+  version    = "4.13.0"
+
+  values = [
+    jsonencode({
+      rbac = {
+        create         = true
+        useClusterRole = true
+      }
+      serviceAccount = {
+        create = true
+      }
+      podSecurityPolicy = {
+        enabled = false
+      }
+    }),
+    jsonencode({
+      autosharding = {
+        enabled = false
+      }
+    }),
+    jsonencode({
+      customLabels = {
+        "prometheus.io/scrape" = "true"
+        "prometheus.io/port"   = "8080"
+        "prometheus.io/scheme" = "http"
+      }
+    }),
+  ]
+}

locals.tf

@@ -3,6 +3,7 @@ locals {
   kubernetes_server  = "https://nat.lawndale:6443"
   kubernetes_version = "1.23.5"
   cluster_dns        = "10.32.0.10"
+  traefik_namespace  = "traefik"
 
   bootstrap_token = "${random_password.bootstrap_token_id.result}.${random_password.bootstrap_token_secret.result}"
 

providers.tf

@@ -1,4 +1,5 @@
 terraform {
+  required_version = ">= 1.2.0"
   required_providers {
     aws = {
       source  = "hashicorp/aws"
@@ -27,7 +28,7 @@ terraform {
 
     libvirt = {
       source  = "dmacvicar/libvirt"
-      version = "0.6.14"
+      version = "0.7.0"
     }
 
     macaddress = {

traefik.tf

@@ -0,0 +1,32 @@
+resource "helm_release" "traefik" {
+  name             = "traefik"
+  namespace        = local.traefik_namespace
+  create_namespace = true
+
+  repository = "https://helm.traefik.io/traefik"
+  chart      = "traefik"
+
+  values = [
+    jsonencode({
+      rbac = {
+        enabled    = true
+        namespaced = false # traefik works for the whole cluster
+      }
+      podSecurityPolicy = {
+        enabled = false
+      }
+      deployment = {
+        enabled = false
+      }
+      daemonset = {
+        enabled = false
+      }
+      service = {
+        enabled = false
+      }
+      ingressroute = {
+        dashboard = { enabled = false }
+      }
+    })
+  ]
+}

worker.tf

@@ -1,7 +1,7 @@
 locals {
   pool_name       = "kubernetes-workers"
   worker_id_start = 80
-  worker_count    = 1
+  worker_count    = 2
 }
 
 resource "libvirt_pool" "kubernetes_workers" {
@@ -39,7 +39,7 @@ data "template_cloudinit_config" "worker" {
         }
       ]
       mounts = [
-        ["datastore", "/mnt/datastore", "9p", "trans=virtio"]
+        ["datastore", "/mnt/datastore", "9p", "trans=virtio,version=9p2000.L,rw,dirsync,cache=mmap"]
       ]
     })
   }
@@ -49,6 +49,8 @@ module "worker" {
   source = "git@git.thomasklein.me:thomasklein/terraform-modules//lawndale-vm"
 
   count   = local.worker_count
+  uefi    = true
+  machine = "q35"
 
   name = "k8s-worker-${count.index}"
   id   = local.worker_id_start + count.index
@@ -61,10 +63,10 @@ module "worker" {
 
   create_root_storage_pool    = false
   root_storage_pool           = local.pool_name
-  root_storage_volume_size_gb = 8
+  root_storage_volume_size_gb = 16
 
   base_image_pool   = "base-images"
-  base_image_volume = "ubuntu-jammy-20220516-k8s-v1.23.5-20220517"
+  base_image_volume = "ubuntu-jammy-20221127-k8s-v1.25.4-20221127"
 
   filesystems = [
     {
@@ -79,23 +81,6 @@ module "worker" {
 
   user_data = data.template_cloudinit_config.worker[count.index].rendered
 
-  ### Make the workers headless
-  xslt = <<EOM
-<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
-<xsl:output method="xml" indent="yes"/>
-<xsl:strip-space elements="*" />
 
-<xsl:template match="@*|node()">
+  xslt = file("${path.module}/fixing-worker-domaindefs.xsl")
- <xsl:copy>
-  <xsl:apply-templates select="@*|node()"/>
- </xsl:copy>
-</xsl:template>
-
-<xsl:template match="graphics" />
-<xsl:template match="video" />
-<xsl:template match="audio" />
-<xsl:template match="input[@type='mouse']|input[@type='keyboard']" />
-
-</xsl:stylesheet>
-EOM
 }