diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..7d79a5144bac1a4b96dfe53319c193c75969e9d4 --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,35 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc +.idea diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000000000000000000000000000000000000..492674abcdbd12157f66f383a9da5338fbfd5d5f --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,39 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + constraints = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} + +provider "registry.terraform.io/hashicorp/vsphere" { + version = "2.0.2" + constraints = "2.0.2" + hashes = [ + "h1:/9qlE9Ni7mhl3cTHLhmYg1AFuyBnXqie53Q5Ujor6oU=", + "zh:0b72856d2a89b118adf64dcfc75a0b0f7d8875e9872c637a9a0549d4a3dd9383", + "zh:2cf50f17464feeac60039f4d36656835895260798d4d9167a1ff2d4e1e464dce", + "zh:39d5f1c99bd4024d88b95892711f234afeb2b83700cdbd3df68a60c6b3bb0ac7", + "zh:551b68f91f5eefa4daee4d826d27d2bc10236e0642fdafd066604884b2a06816", + "zh:8454b5e7dbbf0300bbab1ecd73c12511cf35c0c493ad18e31742d58798e11026", + "zh:84a8233a60240e0020de513668a20d015a7fbc760ad6ab639d52e6ddabfa3376", + "zh:95574b7f092e77f9c629770b802d1635903f14d8d88d3828a96f58ca072876a5", + "zh:a262c779c112e4d6ac47152d6c6303308180dfd4d5929b980cf24707ca9218e9", + "zh:c7622cfa8bab6a6f122c0ab3d4767ecf0f52ccf32d64020aa4e9581f91da5586", + "zh:e01cfc79d21c7151c97047afdc79dfe7bf37c0063908829200480a03359e4c2e", + "zh:f64c3f283c31477886c3d16592426dbefa2cab55f2885cb3351cef094ad898a0", + ] +} diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000000000000000000000000000000000000..180c9975247929523f6df27a5a925a007ae763f7 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,62 @@ +# My projects Terraform repo + +It focuses on VMWare but you can find +other examples in the `./examples` folder. + +[Example documentation](https://docs.k8s-01.sch.bme.hu/tutorials/terraform#learn-by-doing-it) + +Each file contains on top what it should be used for. + +## Initialize + +We should store our terraform state in some remote +location, because this way others can work with +our project. +For that you should create your `login.sh` in the +`secret` folder by copying the `login.sh.example` file. +You will need an Acces token with `api` scope for your +repository which can be created under +`Settings` > `Access tokens`. +If you don't want to create a token for each project +just create one for your user and use that. + +After you created the `login.sh` file, just run the `init.sh` +file which configures the repository. + +`Infrastructure` should be enabled for the Repository! + +## Usage + +After the repo is initialized, just run `source secret/login.sh`. +This will load the environment variables for the Vsphere provider. + +### Example + +First time: + +```bash +cp secret/login.sh.example secret/login.sh +nano secret/login.sh +# ... +./init.sh +source secret/login.sh +terraform plan +``` + +After it was initialized: + +```bash +source secret/login.sh +terraform plan +``` + +## Cloud-init + +We can pass initialization script to our vms using +cloud-init. +In this repo you can find an example to it +under the `cloud-init` folder. +We can then base64 the file content and pass it as a +`vApp` property. + +https://cloudinit.readthedocs.io/en/latest/ diff --git a/terraform/cloud-init/example-vm.yaml b/terraform/cloud-init/example-vm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..445d5b90c00468108fd5c4b74ec3342d2e865e60 --- /dev/null +++ b/terraform/cloud-init/example-vm.yaml @@ -0,0 +1,55 @@ +#cloud-config +write_files: + - content: | + network: + version: 2 + renderer: networkd + ethernets: + wan: + optional: false + match: + macaddress: aa:aa:aa:aa:aa:aa + set-name: wan0 + dhcp4: true + + path: /etc/netplan/50-cloud-init.yaml + - content: | + Protocol 2 + Port 10022 + + HostKey /etc/ssh/ssh_host_rsa_key + HostKey /etc/ssh/ssh_host_ed25519_key + + KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com + + PermitRootLogin no + PubkeyAuthentication yes + PasswordAuthentication no + + MaxAuthTries 6 + + ChallengeResponseAuthentication no + KerberosAuthentication no + GSSAPIAuthentication no + UsePAM yes + AllowAgentForwarding yes + X11Forwarding yes + PrintMotd no + + AcceptEnv LANG LC_* + + Subsystem sftp /usr/lib/openssh/sftp-server + + AllowUsers ubuntu + path: /etc/ssh/sshd_config +runcmd: + - "netplan apply" + - "systemctl restart ssh" + # Fix slow sudo, because it tries to dns lookup the hostname first + - echo "127.0.1.1 $(hostname)" >> /etc/hosts + # Disable floppy + - echo "blacklist floppy" | sudo tee /etc/modprobe.d/blacklist-floppy.conf + - rmmod floppy || true + - update-initramfs -u \ No newline at end of file diff --git a/terraform/config.tf b/terraform/config.tf new file mode 100644 index 0000000000000000000000000000000000000000..3330a0fd75b0c817c00a7c412ea80f0d829ef29c --- /dev/null +++ b/terraform/config.tf @@ -0,0 +1,44 @@ +# Everything that is used for configuration + +terraform { + required_providers { + # List of providers + vsphere = { + source = "hashicorp/vsphere" + version = "2.0.2" + } + template = { + source = "hashicorp/template" + version = "2.2.0" + } + } + backend "http" { + # Store our State in the Gitlab Repo + } +} + +# Example: Vsphere provider config + +# Credentials are set by sourcing the secret/login.sh file +variable "vsphere_user" { + description = "Administrator user" + type = string + sensitive = false + default = "required" +} + +variable "vsphere_password" { + description = "Administrator password" + type = string + sensitive = true + default = "required" +} + +provider "vsphere" { + user = var.vsphere_user + password = var.vsphere_password + vsphere_server = "horizont.sch.bme.hu" + + # We use letsencrypt and that isn't verified somehow + allow_unverified_ssl = true +} diff --git a/terraform/data.tf b/terraform/data.tf new file mode 100644 index 0000000000000000000000000000000000000000..b7ac2b8ff5ea427da0f7f703b30caa77efa0570c --- /dev/null +++ b/terraform/data.tf @@ -0,0 +1,76 @@ +# Everythings that is a data source. + +data "vsphere_datacenter" "dc" { + name = "SCH" +} + +data "vsphere_compute_cluster" "SCH-Cluster-01" { + name = "SCH-Cluster-01" + datacenter_id = data.vsphere_datacenter.dc.id +} + +data "vsphere_resource_pool" "kubernetes" { + name = "Kubernetes" + datacenter_id = data.vsphere_datacenter.dc.id +} + +# * Templates + +data "vsphere_virtual_machine" "ubuntu2004-cloud-init" { + name = "ubuntu2004-cloud-init" + datacenter_id = data.vsphere_datacenter.dc.id +} + +# * Datastores + +## * Clusters +data "vsphere_datastore_cluster" "SCH-Cluster-FujiStorage" { + name = "SCH-Cluster-FujiStorage" + datacenter_id = data.vsphere_datacenter.dc.id +} + +## * Simple datastores +data "vsphere_datastore" "Memory-Blitzkrieg" { + name = "Memory-Blitzkrieg" + datacenter_id = data.vsphere_datacenter.dc.id +} + +data "vsphere_datastore" "N1_SSD-01" { + name = "N1_SSD-01" + datacenter_id = data.vsphere_datacenter.dc.id +} +data "vsphere_datastore" "N2_SSD-01" { + name = "N2_SSD-01" + datacenter_id = data.vsphere_datacenter.dc.id +} +data "vsphere_datastore" "N3_SSD-01" { + name = "N3_SSD-01" + datacenter_id = data.vsphere_datacenter.dc.id +} +data "vsphere_datastore" "N4_SSD-01" { + name = "N4_SSD-01" + datacenter_id = data.vsphere_datacenter.dc.id +} + +# * Network + +data "vsphere_network" "dvPG-151-Internal" { + name = "dvPG-151-Internal" + datacenter_id = data.vsphere_datacenter.dc.id +} +data "vsphere_network" "dvPG-208-Szerver" { + name = "dvPG-208-Szerver" + datacenter_id = data.vsphere_datacenter.dc.id +} +data "vsphere_network" "dvPG-502-K8S-Internal" { + name = "dvPG-502-K8S-Internal" + datacenter_id = data.vsphere_datacenter.dc.id +} +data "vsphere_network" "dvPG-503-K8S-Bgp-Nat" { + name = "dvPG-503-K8S-Bgp-Nat" + datacenter_id = data.vsphere_datacenter.dc.id +} +data "vsphere_network" "dvPG-504-K8S-Mgmt" { + name = "dvPG-504-K8S-Mgmt" + datacenter_id = data.vsphere_datacenter.dc.id +} diff --git a/terraform/init.sh b/terraform/init.sh new file mode 100755 index 0000000000000000000000000000000000000000..b5f614fe4b340ff4f877196470d6d770392c1520 --- /dev/null +++ b/terraform/init.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +BASEDIR=$(dirname "$0") + +if [ -d "$BASEDIR/.terraform" ]; then + echo ".terraform folder exists." + echo "Remove it before reinitializing terraform!" + exit 0 +fi + +if ! [ -f "$BASEDIR/secret/login.sh" ]; then + echo "Please create the login.sh file!" + exit 0 +fi + +source $BASEDIR/secret/login.sh + +terraform init \ + -backend-config="address=https://git.sch.bme.hu/api/v4/projects/${GITLAB_PROJECT_ID}/terraform/state/${GITLAB_STATE_NAME}" \ + -backend-config="lock_address=https://git.sch.bme.hu/api/v4/projects/${GITLAB_PROJECT_ID}/terraform/state/${GITLAB_STATE_NAME}/lock" \ + -backend-config="unlock_address=https://git.sch.bme.hu/api/v4/projects/${GITLAB_PROJECT_ID}/terraform/state/${GITLAB_STATE_NAME}/lock" \ + -backend-config="username=${GITLAB_USER}" \ + -backend-config="password=${GITLAB_REPO_TOKEN}" \ + -backend-config="lock_method=POST" \ + -backend-config="unlock_method=DELETE" \ + -backend-config="retry_wait_min=5" \ + -reconfigure diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000000000000000000000000000000000000..15e9d5be7ec3f33b84716d04729a99ae27d4c386 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,94 @@ +# Main configuration for our project. +# This example is quite overkill, this is an example that +# iterates over a list of vms and reuses the same resource object. + +# You don't have to use this file, but I recommend it. +# Larger example: https://git.sch.bme.hu/kszk/sysadmin/kubernetes/cluster-setup/-/tree/master/terraform + +locals { + vm_map = { + example-vm = { + num_cpus = 1 + memory = 1024 + template_uuid = data.vsphere_virtual_machine.ubuntu2004-cloud-init.id + datastore_cluster_id = data.vsphere_datastore_cluster.SCH-Cluster-FujiStorage.id + vapp_properties = { + public-keys = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPpH+TNAwcmxYc5cVctH04wUU83Pba6s/AkKXOnhDn+m rlacko@zen" + user-data = base64encode(file("${path.module}/cloud-init/example-vm.yaml")) + } + disk_map = { + 0 = { size = 16 } + } + network_interface_map = { + 01 = { + id = data.vsphere_network.dvPG-208-Szerver.id + mac_address = "aa:aa:aa:aa:aa:aa" + } + } + } + } +} + +resource "vsphere_virtual_machine" "vm" { + for_each = { for k, v in local.vm_map : k => v } + + name = each.key + resource_pool_id = data.vsphere_resource_pool.kubernetes.id + guest_id = try(each.value.guest_id, "ubuntu64Guest") + num_cpus = try(each.value.num_cpus, 1) + memory = try(each.value.memory, 1024) + firmware = try(each.value.firmware, "efi") + folder = "KSZK/sysadmin/k8s" + wait_for_guest_net_timeout = try(each.value.wait_for_guest_net_timeout, 0) + wait_for_guest_ip_timeout = try(each.value.wait_for_guest_ip_timeout, 0) + sync_time_with_host = true + sync_time_with_host_periodically = false + + datastore_cluster_id = try(each.value.datastore_cluster_id, null) + datastore_id = try(each.value.datastore_id, null) + + dynamic "clone" { + for_each = can(each.value.template_uuid) ? [1] : [] + content { + template_uuid = each.value.template_uuid + } + } + dynamic "vapp" { + for_each = can(each.value.template_uuid) ? [1] : [] + content { + properties = { + public-keys = each.value.vapp_properties.public-keys + hostname = try(each.value.vapp_properties.hostname, each.key) + instance-id = try(each.value.vapp_properties.instance-id, each.key) + password = try(each.value.vapp_properties.password, null) + user-data = try(each.value.vapp_properties.user-data, null) + } + } + } + + dynamic "network_interface" { + for_each = each.value.network_interface_map + content { + network_id = network_interface.value.id + use_static_mac = can(network_interface.value.mac_address) + mac_address = try(network_interface.value.mac_address, null) + } + } + + dynamic "disk" { + for_each = each.value.disk_map + content { + thin_provisioned = false + label = "disk${tostring(disk.key)}" + size = disk.value.size + unit_number = tonumber(disk.key) + } + } + + dynamic "cdrom" { + for_each = can(each.value.template_uuid) ? [1] : [] + content { + client_device = true + } + } +} diff --git a/terraform/secret/.gitignore b/terraform/secret/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..6b45583134f2f15bb3a087737ef911191f5e0a16 --- /dev/null +++ b/terraform/secret/.gitignore @@ -0,0 +1 @@ +login.sh \ No newline at end of file diff --git a/terraform/secret/login.sh.example b/terraform/secret/login.sh.example new file mode 100755 index 0000000000000000000000000000000000000000..7442c2be686eae035610ddf6e039987873e13f1b --- /dev/null +++ b/terraform/secret/login.sh.example @@ -0,0 +1,14 @@ +#!/bin/bash + +# Gitlab Project ID, this will be used as state store +export GITLAB_PROJECT_ID="0" +# If you want to rename the terraform state name, feel free. +export GITLAB_STATE_NAME="state" +# GitLab username +export GITLAB_USER="user" +# Access token with API Acces to the repo +export GITLAB_REPO_TOKEN="acces token" +# vSphere Username +export TF_VAR_vsphere_user="username" +# vSphere Password +export TF_VAR_vsphere_password="pw"