Update 'worker.tf'

From terraform plan --help
>  -detailed-exitcode  Return detailed exit codes when the command exits. This
>                      will change the meaning of exit codes to:
>                      0 - Succeeded, diff is empty (no changes)
>                      1 - Errored
>                      2 - Succeeded, there is a diff

.drone.yml

@@ -0,0 +1,90 @@
+---
+kind: pipeline
+type: kubernetes
+name: Terraform root module
+
+environment:
+  TF_IN_AUTOMATION: "1"
+  GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -i $${PWD}/id_rsa"
+  TF_VAR_lawndale_dns_host: "nat.lawndale"
+  TF_VAR_lawndale_libvirt_uri_transport: ssh
+  TF_VAR_lawndale_libvirt_uri_userhostport: "192.168.253.254:10022"
+
+trigger:
+  ref:
+    - refs/heads/main
+    - refs/pull/*/head
+
+steps:
+- name: terraform init
+  image: hashicorp/terraform:1.1.8
+  commands:
+    - echo "$${CI_SSH_KEY}" | base64 -d > id_rsa
+    - chmod 600 id_rsa
+    - echo 'lawndale_libvirt_uri_extra = {"sshauth"="privkey","keyfile"="'$${PWD}'/id_rsa","no_verify"="1"}' >> ci.tfvars
+    - terraform init
+  environment:
+    CI_SSH_KEY:
+      from_secret: ci-ssh-key
+    AWS_ACCESS_KEY_ID:
+      from_secret: terraform-aws-key-id
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: terraform-aws-secret-access-key
+
+- name: terraform plan
+  image: hashicorp/terraform:1.1.8
+  commands:
+    - terraform plan $([[ $${DRONE_BUILD_EVENT} = cron ]] && echo "-detailed-exitcode") -var-file ci.tfvars -out .tfplan
+  environment:
+    AWS_ACCESS_KEY_ID:
+      from_secret: terraform-aws-key-id
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: terraform-aws-secret-access-key
+    KUBE_TOKEN:
+      from_secret: lawndale-k8s-ci-token
+    TF_VAR_lawndale_dns_key_secret:
+      from_secret: lawndale-dns-key-secret
+    TF_VAR_lawndale_dns_key_algorithm:
+      from_secret: lawndale-dns-key-algorithm
+
+- name: terraform apply
+  when:
+    branch:
+    - main
+    event:
+    - push
+  image: hashicorp/terraform:1.1.8
+  commands:
+    - terraform apply .tfplan
+  environment:
+    AWS_ACCESS_KEY_ID:
+      from_secret: terraform-aws-key-id
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: terraform-aws-secret-access-key
+---
+kind: pipeline
+type: kubernetes
+name: Check docs and format
+
+environment:
+  TF_IN_AUTOMATION: "1"
+
+trigger:
+  ref:
+    - refs/pull/*/head
+
+steps:
+- name: format and generate docs
+  image: hashicorp/terraform:1.1.8
+  commands:
+    - apk add bash wget 
+    - wget -q https://github.com/terraform-docs/terraform-docs/releases/download/v0.16.0/terraform-docs-v0.16.0-linux-amd64.tar.gz -O - | tar -xz terraform-docs -C /usr/local/bin
+    - terraform fmt
+    - terraform-docs .
+    - git diff --exit-code
+
+---
+kind: signature
+hmac: 95f8db197163e884f2eee4b14af136b9ea1e0f88f626079b4a3b38b43b91c6a8
+
+...

.terraform-docs.yml

@@ -0,0 +1,5 @@
+formatter: markdown document
+
+output:
+  mode: inject
+  file: README.md

README.md

@@ -1,3 +1,11 @@
+# Terraform module for Lawndale Kubernetes environment
+[![Build Status](https://drone.thomasklein.me/api/badges/thomasklein/terraform-lawndale-k8s/status.svg)](https://drone.thomasklein.me/thomasklein/terraform-lawndale-k8s)
+
+This module manages the core Kubernetes cluster components running on Lawndale including workers. 
+
+
+
+<!-- BEGIN_TF_DOCS -->
 ## Requirements
 
 The following requirements are needed by this module:
@@ -61,7 +69,9 @@ Version:
 The following resources are used by this module:
 
 - [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
+- [helm_release.kube_state_metrics](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
 - [helm_release.metrics_server](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
+- [helm_release.traefik](https://registry.terraform.io/providers/hashicorp/helm/2.5.1/docs/resources/release) (resource)
 - [kubernetes_cluster_role.ci_cd](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) (resource)
 - [kubernetes_cluster_role.prometheus](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) (resource)
 - [kubernetes_cluster_role_binding.auto_approve_node_csrs](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) (resource)
@@ -85,29 +95,27 @@ The following resources are used by this module:
 
 The following input variables are required:
 
-### <a name="input_lawndale_dns_key_secret"></a> [lawndale\_dns\_key\_secret](#input\_lawndale\_dns\_key\_secret)
-
-Description: DNSSEC key to use sigining the NSUPDATE queries for Lawndale
-
-Type: `string`
-
-### <a name="input_lawndale_dns_port"></a> [lawndale\_dns\_port](#input\_lawndale\_dns\_port)
-
-Description: Port where the lawndale internal DNS server listens on
-
-Type: `number`
-
-## Optional Inputs
-
-The following input variables are optional (have default values):
-
 ### <a name="input_lawndale_dns_host"></a> [lawndale\_dns\_host](#input\_lawndale\_dns\_host)
 
 Description: Address to reach lawndale internal DNS server
 
 Type: `string`
 
-Default: `"lawndale-hyper"`
+### <a name="input_lawndale_dns_key_secret"></a> [lawndale\_dns\_key\_secret](#input\_lawndale\_dns\_key\_secret)
+
+Description: DNSSEC key to use sigining the NSUPDATE queries for Lawndale
+
+Type: `string`
+
+### <a name="input_lawndale_libvirt_uri_transport"></a> [lawndale\_libvirt\_uri\_transport](#input\_lawndale\_libvirt\_uri\_transport)
+
+Description: Transport protocol (could be ssh or tls) to dial Lawndale hypervisor
+
+Type: `string`
+
+## Optional Inputs
+
+The following input variables are optional (have default values):
 
 ### <a name="input_lawndale_dns_key_algorithm"></a> [lawndale\_dns\_key\_algorithm](#input\_lawndale\_dns\_key\_algorithm)
 
@@ -117,6 +125,14 @@ Type: `string`
 
 Default: `"hmac-sha256"`
 
+### <a name="input_lawndale_dns_port"></a> [lawndale\_dns\_port](#input\_lawndale\_dns\_port)
+
+Description: Port where the lawndale internal DNS server listens on
+
+Type: `number`
+
+Default: `53`
+
 ### <a name="input_lawndale_dns_transport"></a> [lawndale\_dns\_transport](#input\_lawndale\_dns\_transport)
 
 Description: Port where the lawndale internal DNS server listens on
@@ -125,6 +141,39 @@ Type: `string`
 
 Default: `"udp"`
 
+### <a name="input_lawndale_libvirt_uri"></a> [lawndale\_libvirt\_uri](#input\_lawndale\_libvirt\_uri)
+
+Description: Libvirt URI to use accessing Lawndale hypervisor
+
+Type: `string`
+
+Default: `null`
+
+### <a name="input_lawndale_libvirt_uri_extra"></a> [lawndale\_libvirt\_uri\_extra](#input\_lawndale\_libvirt\_uri\_extra)
+
+Description: Libvirt URI extra parameters [https://libvirt.org/uri.html#transport-configuration](See Libvirt transport configuration)
+
+Type: `map(any)`
+
+Default: `{}`
+
+### <a name="input_lawndale_libvirt_uri_path"></a> [lawndale\_libvirt\_uri\_path](#input\_lawndale\_libvirt\_uri\_path)
+
+Description: Libvirt URI path sent to the libvirt daemon
+
+Type: `string`
+
+Default: `"/system"`
+
+### <a name="input_lawndale_libvirt_uri_userhostport"></a> [lawndale\_libvirt\_uri\_userhostport](#input\_lawndale\_libvirt\_uri\_userhostport)
+
+Description: Libvirt URI username, hostname or ip address, and port to reach lawndale hypervisor, in the format of `[username@]<hostname>[:port]`
+
+Type: `string`
+
+Default: `"lawndale-hyper"`
+
 ## Outputs
 
 No outputs.
+<!-- END_TF_DOCS -->

backend.tf

@@ -1,10 +1,10 @@
 terraform {
-   backend "s3" {
+  backend "s3" {
-     bucket         = "thomasklein-global-states"
+    bucket         = "thomasklein-global-states"
-     key            = "envs/lawndale/k8s/main"
+    key            = "envs/lawndale/k8s/main"
-     region         = "eu-central-1"
+    region         = "eu-central-1"
-     encrypt        = true
+    encrypt        = true
-     kms_key_id     = "a767a3ce-c505-44c0-9f93-6f53678c69c7"
+    kms_key_id     = "a767a3ce-c505-44c0-9f93-6f53678c69c7"
-     dynamodb_table = "thomasklein-global-state-locks"
+    dynamodb_table = "thomasklein-global-state-locks"
-   }
+  }
 }

ci.tf

@@ -38,35 +38,23 @@ resource "kubernetes_cluster_role" "ci_cd" {
       "persistentvolumeclaims",
       "pods",
       "namespaces",
+      "nodes",
       "secrets",
+      "serviceaccounts",
       "services",
     ]
-    verbs = [
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
-      "create",
-      "delete",
-      "get",
-      "list",
-      "patch",
-      "update",
-      "watch",
-    ]
   }
 
   rule {
     api_groups = ["apps"]
     resources = [
+      "daemonsets",
       "deployments",
       "replicasets", # needed for 'helm upgrade --wait'
+      "statefulsets",
     ]
-    verbs = [
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
-      "create",
-      "delete",
-      "get",
-      "list",
-      "patch",
-      "update",
-      "watch",
-    ]
   }
 
   rule {
@@ -74,45 +62,58 @@ resource "kubernetes_cluster_role" "ci_cd" {
     resources = [
       "horizontalpodautoscalers"
     ]
-    verbs = [
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
-      "create",
-      "delete",
-      "get",
-      "list",
-      "patch",
-      "update",
-      "watch",
-    ]
   }
   rule {
     api_groups = ["networking.k8s.io"]
 
     resources = [
       "ingresses",
-    ]
-    verbs = [
-      "create",
-      "delete",
-      "get",
-      "list",
-      "patch",
-      "update",
-      "watch",
-    ]
-  }
-  rule {
-    api_groups = ["networking.k8s.io"]
-    resources = [
       "networkpolicies"
     ]
-    verbs = [
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
-      "create",
+  }
-      "delete",
+
-      "get",
+  rule {
-      "list",
+    api_groups = ["apiextensions.k8s.io"]
-      "patch",
+    resources = [
-      "update",
+      "customresourcedefinitions"
-      "watch",
     ]
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
+  }
+
+  rule {
+    api_groups = ["rbac.authorization.k8s.io"]
+    resources = [
+      "clusterrolebindings",
+      "clusterroles",
+      "rolebindings",
+      "roles",
+    ]
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
+  }
+
+  rule {
+    api_groups = ["policy"]
+    resources = [
+      "podsecuritypolicies",
+    ]
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
+  }
+
+  rule {
+    api_groups = ["traefik.containo.us"]
+    resources = [
+      "ingressroutes",
+      "ingressroutetcps",
+      "ingressrouteudps",
+      "middlewares",
+      "middlewaretcps",
+      "serverstransports",
+      "tlsoptions",
+      "tlsstores",
+      "traefikservices",
+    ]
+    verbs = ["create", "delete", "get", "list", "patch", "update", "watch"]
   }
 }

kube-state-metrics.tf

@@ -0,0 +1,35 @@
+resource "helm_release" "kube_state_metrics" {
+  name      = "kube-state-metrics"
+  namespace = "kube-system"
+
+  repository = "https://prometheus-community.github.io/helm-charts"
+  chart      = "kube-state-metrics"
+  version    = "4.13.0"
+
+  values = [
+    jsonencode({
+      rbac = {
+        create         = true
+        useClusterRole = true
+      }
+      serviceAccount = {
+        create = true
+      }
+      podSecurityPolicy = {
+        enabled = false
+      }
+    }),
+    jsonencode({
+      autosharding = {
+        enabled = false
+      }
+    }),
+    jsonencode({
+      customLabels = {
+        "prometheus.io/scrape" = "true"
+        "prometheus.io/port"   = "8080"
+        "prometheus.io/scheme" = "http"
+      }
+    }),
+  ]
+}

locals.tf

@@ -3,6 +3,7 @@ locals {
   kubernetes_server  = "https://nat.lawndale:6443"
   kubernetes_version = "1.23.5"
   cluster_dns        = "10.32.0.10"
+  traefik_namespace  = "traefik"
 
   bootstrap_token = "${random_password.bootstrap_token_id.result}.${random_password.bootstrap_token_secret.result}"
 
@@ -38,4 +39,13 @@ EehXHhbRmbtSZ7c4DrGeR2J0SZTyBQJfZczSGRvEiKyGNnyZlLVYKTTnV9b+aN3q
 Xw+ilWL3boYsSiqVN6SIUA==
 -----END CERTIFICATE-----
 EOM
+
+
+  lawndale_libvirt_uri_extra = join("&", [for k, v in var.lawndale_libvirt_uri_extra : format("%s=%s", urlencode(k), urlencode(v))])
+  lawndale_libvirt_uri = var.lawndale_libvirt_uri != null ? var.lawndale_libvirt_uri : format("qemu+%s://%s%s?%s",
+    var.lawndale_libvirt_uri_transport,
+    var.lawndale_libvirt_uri_userhostport,
+    var.lawndale_libvirt_uri_path,
+    local.lawndale_libvirt_uri_extra
+  )
 }

providers.tf

@@ -53,7 +53,7 @@ provider "aws" {
 
 provider "libvirt" {
   # no-pty ssh-rsa AAAAB3.....
-  uri = "qemu+ssh://lawndale-hyper.sch.bme.hu:10022/system?sshauth=privkey"
+  uri = local.lawndale_libvirt_uri
 }
 
 provider "dns" {

traefik.tf

@@ -0,0 +1,32 @@
+resource "helm_release" "traefik" {
+  name             = "traefik"
+  namespace        = local.traefik_namespace
+  create_namespace = true
+
+  repository = "https://helm.traefik.io/traefik"
+  chart      = "traefik"
+
+  values = [
+    jsonencode({
+      rbac = {
+        enabled    = true
+        namespaced = false # traefik works for the whole cluster
+      }
+      podSecurityPolicy = {
+        enabled = false
+      }
+      deployment = {
+        enabled = false
+      }
+      daemonset = {
+        enabled = false
+      }
+      service = {
+        enabled = false
+      }
+      ingressroute = {
+        dashboard = { enabled = false }
+      }
+    })
+  ]
+}

variables.tf

@@ -1,12 +1,12 @@
 variable "lawndale_dns_host" {
   type        = string
   description = "Address to reach lawndale internal DNS server"
-  default     = "lawndale-hyper"
 }
 
 variable "lawndale_dns_port" {
   type        = number
   description = "Port where the lawndale internal DNS server listens on"
+  default     = 53
 }
 
 variable "lawndale_dns_transport" {
@@ -31,3 +31,33 @@ variable "lawndale_dns_key_algorithm" {
   description = "DNSSEC key to use sigining the NSUPDATE queries for Lawndale"
   default     = "hmac-sha256"
 }
+
+variable "lawndale_libvirt_uri" {
+  type        = string
+  description = "Libvirt URI to use accessing Lawndale hypervisor"
+  default     = null
+  nullable    = true
+}
+
+variable "lawndale_libvirt_uri_transport" {
+  type        = string
+  description = "Transport protocol (could be ssh or tls) to dial Lawndale hypervisor"
+}
+
+variable "lawndale_libvirt_uri_userhostport" {
+  type        = string
+  description = "Libvirt URI username, hostname or ip address, and port to reach lawndale hypervisor, in the format of `[username@]<hostname>[:port]`"
+  default     = "lawndale-hyper"
+}
+
+variable "lawndale_libvirt_uri_path" {
+  type        = string
+  description = "Libvirt URI path sent to the libvirt daemon"
+  default     = "/system"
+}
+
+variable "lawndale_libvirt_uri_extra" {
+  type        = map(any)
+  description = "Libvirt URI extra parameters [https://libvirt.org/uri.html#transport-configuration](See Libvirt transport configuration)"
+  default     = {}
+}

worker.tf

@@ -1,7 +1,7 @@
 locals {
   pool_name       = "kubernetes-workers"
   worker_id_start = 80
-  worker_count    = 1
+  worker_count    = 2
 }
 
 resource "libvirt_pool" "kubernetes_workers" {