diff --git a/.gitattributes b/.gitattributes index f0396ee..e69de29 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +0,0 @@ -example/real_world_stuff/* filter=git-crypt diff=git-crypt diff --git a/README.md b/README.md index ae0c33b..676430a 100644 --- a/README.md +++ b/README.md @@ -18,15 +18,16 @@ To run tests simply run `nix-shell --run 'tree-sitter test'`. ## Compliance -The directory `example/real_world_stuff` contains a corpus of hcl files that I found with the github query `user:hashicorp language:HCL`. Just to be sure I encrypted them using `git-crypt`. +The directory `example/real_world_stuff` contains a corpus of hcl files that I found with the github query `language:HCL` for users `coreos` and `hashicorp` Given that some language features are still missing ( see TODO ) there are some expected parse errors: ```bash -nix-shell --run 'tree-sitter parse --quiet --stat example/real_world_stuff/*' +nix-shell --run 'tree-sitter parse --quiet --stat example/real_world_stuff/*/*' ... ... -Total parses: 886; successful parses: 801; failed parses: 85; success percentage: 90.41% +Total parses: 1130; successful parses: 1053; failed parses: 77; success percentage: 93.19% + ``` The aim is to build unit testcases from selected failure classes and slowly get to 100%. diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%config.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%config.tf new file mode 100644 index 0000000..5e850d1 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%config.tf @@ -0,0 +1,358 @@ +terraform { + required_version = ">= 0.10.7" +} + +provider "archive" { + version = "1.0.0" +} + +provider "external" { + version = "1.0.0" +} + +provider "ignition" { + version = "1.0.0" +} + +provider "local" { + version = "1.0.0" +} + +provider "null" { + version = "1.0.0" +} + +provider "random" { + version = "1.0.0" +} + +provider "template" { + version = "1.0.0" +} + +provider "tls" { + version = "1.0.1" +} + +variable "tectonic_config_version" { + description = < 0 ? + "${var.worker_azs[count.index]}" : + "${data.aws_availability_zones.azs.names[count.index]}" }", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "kubernetes.io/role/internal-elb", "", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +resource "aws_route_table_association" "worker_routing" { + count = "${var.external_vpc_id == "" ? var.worker_az_count : 0}" + route_table_id = "${aws_route_table.private_routes.*.id[count.index]}" + subnet_id = "${aws_subnet.worker_subnet.*.id[count.index]}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc-public.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc-public.tf new file mode 100644 index 0000000..8eb13ab --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc-public.tf @@ -0,0 +1,74 @@ +resource "aws_internet_gateway" "igw" { + count = "${local.external_vpc_mode ? 0 : 1}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}-igw", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +resource "aws_route_table" "default" { + count = "${var.external_vpc_id == "" ? 1 : 0}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}-public", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}", + ), var.extra_tags)}" +} + +resource "aws_main_route_table_association" "main_vpc_routes" { + count = "${local.external_vpc_mode ? 0 : 1}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + route_table_id = "${aws_route_table.default.id}" +} + +resource "aws_route" "igw_route" { + count = "${local.external_vpc_mode ? 0 : 1}" + destination_cidr_block = "0.0.0.0/0" + route_table_id = "${aws_route_table.default.id}" + gateway_id = "${aws_internet_gateway.igw.id}" +} + +resource "aws_subnet" "master_subnet" { + count = "${local.new_master_az_count}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + cidr_block = "${lookup(var.new_master_subnet_configs, + local.new_master_subnet_azs[count.index], + cidrsubnet(local.new_master_cidr_range, 3, count.index), + )}" + + availability_zone = "${local.new_master_subnet_azs[count.index]}" + + tags = "${merge(map( + "Name", "${var.cluster_name}-master-${local.new_master_subnet_azs[count.index]}", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +resource "aws_route_table_association" "route_net" { + count = "${local.new_master_az_count}" + route_table_id = "${aws_route_table.default.id}" + subnet_id = "${aws_subnet.master_subnet.*.id[count.index]}" +} + +resource "aws_eip" "nat_eip" { + count = "${min(local.new_master_az_count,local.new_worker_az_count)}" + vpc = true + + # Terraform does not declare an explicit dependency towards the internet gateway. + # this can cause the internet gateway to be deleted/detached before the EIPs. + # https://github.com/coreos/tectonic-installer/issues/1017#issuecomment-307780549 + depends_on = ["aws_internet_gateway.igw"] +} + +resource "aws_nat_gateway" "nat_gw" { + count = "${min(local.new_master_az_count,local.new_worker_az_count)}" + allocation_id = "${aws_eip.nat_eip.*.id[count.index]}" + subnet_id = "${aws_subnet.master_subnet.*.id[count.index]}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc-public.tf-192 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc-public.tf-192 new file mode 100644 index 0000000..8f5c0d0 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc-public.tf-192 @@ -0,0 +1,77 @@ +resource "aws_internet_gateway" "igw" { + count = "${var.external_vpc_id == "" ? 1 : 0}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}-igw", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +resource "aws_route_table" "default" { + count = "${var.external_vpc_id == "" ? 1 : 0}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}-public", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +resource "aws_main_route_table_association" "main_vpc_routes" { + count = "${var.external_vpc_id == "" ? 1 : 0}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + route_table_id = "${aws_route_table.default.id}" +} + +resource "aws_route" "igw_route" { + count = "${var.external_vpc_id == "" ? 1 : 0}" + destination_cidr_block = "0.0.0.0/0" + route_table_id = "${aws_route_table.default.id}" + gateway_id = "${aws_internet_gateway.igw.id}" +} + +resource "aws_subnet" "master_subnet" { + count = "${var.external_vpc_id == "" ? var.master_az_count : 0}" + + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + cidr_block = "${length(var.master_subnets) > 1 ? + "${element(var.master_subnets, count.index)}" : + "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index)}" + }" + + availability_zone = "${var.master_azs[count.index]}" + + tags = "${merge(map( + "Name", "${var.cluster_name}-master-${ "${length(var.master_azs)}" > 0 ? + "${var.master_azs[count.index]}" : + "${data.aws_availability_zones.azs.names[count.index]}" }", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +resource "aws_route_table_association" "route_net" { + count = "${var.external_vpc_id == "" ? var.master_az_count : 0}" + route_table_id = "${aws_route_table.default.id}" + subnet_id = "${aws_subnet.master_subnet.*.id[count.index]}" +} + +resource "aws_eip" "nat_eip" { + count = "${var.external_vpc_id == "" ? min(var.master_az_count, var.worker_az_count) : 0}" + vpc = true + + # Terraform does not declare an explicit dependency towards the internet gateway. + # this can cause the internet gateway to be deleted/detached before the EIPs. + # https://github.com/coreos/tectonic-installer/issues/1017#issuecomment-307780549 + depends_on = ["aws_internet_gateway.igw"] +} + +resource "aws_nat_gateway" "nat_gw" { + count = "${var.external_vpc_id == "" ? min(var.master_az_count, var.worker_az_count) : 0}" + allocation_id = "${aws_eip.nat_eip.*.id[count.index]}" + subnet_id = "${aws_subnet.master_subnet.*.id[count.index]}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc.tf new file mode 100644 index 0000000..7f112e3 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc.tf @@ -0,0 +1,17 @@ +locals { + new_worker_cidr_range = "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block,1,1)}" + new_master_cidr_range = "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block,1,0)}" +} + +resource "aws_vpc" "new_vpc" { + count = "${var.external_vpc_id == "" ? 1 : 0}" + cidr_block = "${var.cidr_block}" + enable_dns_hostnames = true + enable_dns_support = true + + tags = "${merge(map( + "Name", "${var.cluster_name}.${var.base_domain}", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc.tf-244 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc.tf-244 new file mode 100644 index 0000000..425a5db --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%vpc%vpc.tf-244 @@ -0,0 +1,30 @@ +data "aws_availability_zones" "azs" {} + +resource "aws_vpc" "new_vpc" { + count = "${var.external_vpc_id == "" ? 1 : 0}" + cidr_block = "${var.cidr_block}" + enable_dns_hostnames = true + enable_dns_support = true + + tags = "${merge(map( + "Name", "${var.cluster_name}.${var.base_domain}", + "kubernetes.io/cluster/${var.cluster_name}", "shared", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +data "aws_vpc" "cluster_vpc" { + # The join() hack is required because currently the ternary operator + # evaluates the expressions on both branches of the condition before + # returning a value. When providing and external VPC, the template VPC + # resource gets a count of zero which triggers an evaluation error. + # + # This is tracked upstream: https://github.com/hashicorp/hil/issues/50 + # + id = "${var.external_vpc_id == "" ? join(" ", aws_vpc.new_vpc.*.id) : var.external_vpc_id }" +} + +locals { + master_subnet_ids = ["${split(",", var.external_vpc_id == "" ? join(",", aws_subnet.master_subnet.*.id) : join(",", data.aws_subnet.external_master.*.id))}"] + worker_subnet_ids = ["${split(",", var.external_vpc_id == "" ? join(",", aws_subnet.worker_subnet.*.id) : join(",", data.aws_subnet.external_worker.*.id))}"] +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%ignition.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%ignition.tf new file mode 100644 index 0000000..aa405b2 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%ignition.tf @@ -0,0 +1,23 @@ +data "ignition_config" "main" { + files = ["${compact(list( + var.ign_installer_kubelet_env_id, + var.ign_installer_runtime_mappings_id, + var.ign_max_user_watches_id, + var.ign_nfs_config_id, + var.ign_ntp_dropin_id, + var.ign_profile_env_id, + var.ign_s3_puller_id, + var.ign_systemd_default_env_id, + ))}", + "${var.ign_ca_cert_id_list}", + ] + + systemd = [ + "${var.ign_docker_dropin_id}", + "${var.ign_iscsi_service_id}", + "${var.ign_k8s_node_bootstrap_service_id}", + "${var.ign_kubelet_service_id}", + "${var.ign_locksmithd_service_id}", + "${var.ign_update_ca_certificates_dropin_id}", + ] +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%ignition_s3.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%ignition_s3.tf new file mode 100644 index 0000000..9766c6b --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%ignition_s3.tf @@ -0,0 +1,21 @@ +resource "aws_s3_bucket_object" "ignition_worker" { + bucket = "${var.s3_bucket}" + key = "ignition_worker.json" + content = "${data.ignition_config.main.rendered}" + acl = "private" + + server_side_encryption = "AES256" + + tags = "${merge(map( + "Name", "${var.cluster_name}-ignition-worker", + "KubernetesCluster", "${var.cluster_name}", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + +data "ignition_config" "s3" { + replace { + source = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_worker.key)}" + verification = "sha512-${sha512(data.ignition_config.main.rendered)}" + } +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%variables.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%variables.tf new file mode 100644 index 0000000..9ef4076 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%variables.tf @@ -0,0 +1,95 @@ +variable "ssh_key" { + type = "string" +} + +variable "container_linux_channel" { + type = "string" +} + +variable "container_linux_version" { + type = "string" +} + +variable "cluster_id" { + type = "string" +} + +variable "cluster_name" { + type = "string" +} + +variable "ec2_type" { + type = "string" +} + +variable "ec2_ami" { + type = "string" + default = "" +} + +variable "instance_count" { + type = "string" +} + +variable "subnet_ids" { + type = "list" +} + +variable "sg_ids" { + type = "list" + description = "The security group IDs to be applied." +} + +variable "load_balancers" { + description = "List of ELBs to attach all worker instances to." + type = "list" + default = [] +} + +variable "extra_tags" { + description = "Extra AWS tags to be applied to created resources." + type = "map" + default = {} +} + +variable "autoscaling_group_extra_tags" { + description = "Extra AWS tags to be applied to created autoscaling group resources." + type = "list" + default = [] +} + +variable "root_volume_type" { + type = "string" + description = "The type of volume for the root block device." +} + +variable "root_volume_size" { + type = "string" + description = "The size of the volume in gigabytes for the root block device." +} + +variable "root_volume_iops" { + type = "string" + default = "100" + description = "The amount of provisioned IOPS for the root block device." +} + +variable "worker_iam_role" { + type = "string" + default = "" + description = "IAM role to use for the instance profiles of worker nodes." +} + +variable "base_domain" { + type = "string" + description = "Domain on which the ELB records will be created" +} + +variable "kubeconfig_content" { + type = "string" + default = "" +} + +variable "user_data_ign" { + type = "string" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%variables.tf-199 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%variables.tf-199 new file mode 100644 index 0000000..8863e4d --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%variables.tf-199 @@ -0,0 +1,93 @@ +variable "ssh_key" { + type = "string" +} + +variable "vpc_id" { + type = "string" +} + +variable "container_linux_channel" { + type = "string" +} + +variable "container_linux_version" { + type = "string" +} + +variable "cluster_id" { + type = "string" +} + +variable "cluster_name" { + type = "string" +} + +variable "ec2_type" { + type = "string" +} + +variable "ec2_ami" { + type = "string" + default = "" +} + +variable "instance_count" { + type = "string" +} + +variable "subnet_ids" { + type = "list" +} + +variable "sg_ids" { + type = "list" + description = "The security group IDs to be applied." +} + +variable "load_balancers" { + description = "List of ELBs to attach all worker instances to." + type = "list" + default = [] +} + +variable "extra_tags" { + description = "Extra AWS tags to be applied to created resources." + type = "map" + default = {} +} + +variable "autoscaling_group_extra_tags" { + description = "Extra AWS tags to be applied to created autoscaling group resources." + type = "list" + default = [] +} + +variable "root_volume_type" { + type = "string" + description = "The type of volume for the root block device." +} + +variable "root_volume_size" { + type = "string" + description = "The size of the volume in gigabytes for the root block device." +} + +variable "root_volume_iops" { + type = "string" + default = "100" + description = "The amount of provisioned IOPS for the root block device." +} + +variable "worker_iam_role" { + type = "string" + default = "" + description = "IAM role to use for the instance profiles of worker nodes." +} + +variable "ign_s3_puller_id" { + type = "string" +} + +variable "s3_bucket" { + type = "string" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%worker.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%worker.tf new file mode 100644 index 0000000..fa9540d --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%aws%worker-asg%worker.tf @@ -0,0 +1,175 @@ +locals { + ami_owner = "595879546273" + arn = "aws" +} + +data "aws_ami" "coreos_ami" { + filter { + name = "name" + values = ["CoreOS-${var.container_linux_channel}-${var.container_linux_version}-*"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "owner-id" + values = ["${local.ami_owner}"] + } +} + +resource "aws_launch_configuration" "worker_conf" { + instance_type = "${var.ec2_type}" + image_id = "${coalesce(var.ec2_ami, data.aws_ami.coreos_ami.image_id)}" + name_prefix = "${var.cluster_name}-worker-" + key_name = "${var.ssh_key}" + security_groups = ["${var.sg_ids}"] + iam_instance_profile = "${aws_iam_instance_profile.worker_profile.arn}" + user_data = "${var.user_data_ign}" + + lifecycle { + create_before_destroy = true + + # Ignore changes in the AMI which force recreation of the resource. This + # avoids accidental deletion of nodes whenever a new CoreOS Release comes + # out. + ignore_changes = ["image_id"] + } + + root_block_device { + volume_type = "${var.root_volume_type}" + volume_size = "${var.root_volume_size}" + iops = "${var.root_volume_type == "io1" ? var.root_volume_iops : 0}" + } +} + +resource "aws_autoscaling_group" "workers" { + name = "${var.cluster_name}-workers" + desired_capacity = "${var.instance_count}" + max_size = "${var.instance_count * 3}" + min_size = "${var.instance_count}" + launch_configuration = "${aws_launch_configuration.worker_conf.id}" + vpc_zone_identifier = ["${var.subnet_ids}"] + + tags = [ + { + key = "Name" + value = "${var.cluster_name}-worker" + propagate_at_launch = true + }, + { + key = "kubernetes.io/cluster/${var.cluster_name}" + value = "owned" + propagate_at_launch = true + }, + { + key = "tectonicClusterID" + value = "${var.cluster_id}" + propagate_at_launch = true + }, + "${var.autoscaling_group_extra_tags}", + ] + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_autoscaling_attachment" "workers" { + count = "${length(var.load_balancers)}" + + autoscaling_group_name = "${aws_autoscaling_group.workers.name}" + elb = "${var.load_balancers[count.index]}" +} + +resource "aws_iam_instance_profile" "worker_profile" { + name = "${var.cluster_name}-worker-profile" + + role = "${var.worker_iam_role == "" ? + join("|", aws_iam_role.worker_role.*.name) : + join("|", data.aws_iam_role.worker_role.*.name) + }" +} + +data "aws_iam_role" "worker_role" { + count = "${var.worker_iam_role == "" ? 0 : 1}" + name = "${var.worker_iam_role}" +} + +resource "aws_iam_role" "worker_role" { + count = "${var.worker_iam_role == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-role" + path = "/" + + assume_role_policy = < 0 ? 1 : 0}" + name = "${var.cluster_name}-etcd-out" + description = "${var.cluster_name} etcd - Outbound" + priority = 2000 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "*" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "etcd_ingress_ssh" { + count = "${var.external_nsg_master_id == "" && var.etcd_count > 0 ? 1 : 0}" + name = "${var.cluster_name}-etcd-in-ssh" + description = "${var.cluster_name} etcd - SSH" + priority = 400 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "22" + + # TODO: Reference subnet + source_address_prefix = "${var.ssh_network_internal}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "etcd_ingress_ssh_admin" { + count = "${var.external_nsg_master_id == "" && var.etcd_count > 0 ? 1 : 0}" + name = "${var.cluster_name}-etcd-in-ssh-external" + description = "${var.cluster_name} etcd - SSH external" + priority = 405 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "22" + + # TODO: Reference subnet + source_address_prefix = "${var.ssh_network_external}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "etcd_ingress_ssh_from_master" { + count = "${var.external_nsg_master_id == "" && var.etcd_count > 0 ? 1 : 0}" + name = "${var.cluster_name}-etcd-in-ssh-master" + description = "${var.cluster_name} etcd - SSH from master" + priority = 410 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "22" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "etcd_ingress_client_self" { + count = "${var.external_nsg_master_id == "" && var.etcd_count > 0 ? 1 : 0}" + name = "${var.cluster_name}-etcd-in-client-self" + description = "${var.cluster_name} etcd - etcd client" + priority = 415 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "2379" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "etcd_ingress_client_master" { + count = "${var.external_nsg_master_id == "" && var.etcd_count > 0 ? 1 : 0}" + name = "${var.cluster_name}-etcd-in-client-master" + description = "${var.cluster_name} etcd - etcd client from master" + priority = 420 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "2379" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "etcd_ingress_peer" { + count = "${var.external_nsg_master_id == "" && var.etcd_count > 0 ? 1 : 0}" + name = "${var.cluster_name}-etcd-in-peer" + description = "${var.cluster_name} etcd - etcd peer" + priority = 425 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "2380" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%nsg-master.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%nsg-master.tf new file mode 100644 index 0000000..f224a7e --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%nsg-master.tf @@ -0,0 +1,242 @@ +resource "azurerm_network_security_group" "master" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master" + location = "${var.location}" + resource_group_name = "${var.resource_group_name}" +} + +### LB rules +resource "azurerm_network_security_rule" "alb_probe" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-alb-probe" + description = "${var.cluster_name} master - Azure Load Balancer probe" + priority = 295 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "AzureLoadBalancer" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +# TODO: Fix NSG name and source +resource "azurerm_network_security_rule" "api_ingress_https" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-api-in-https" + description = "${var.cluster_name} Kubernetes API" + priority = 300 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "443" + + # TODO: Ternary on private implementation + source_address_prefix = "*" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "console_ingress_https" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-console-in-https" + description = "${var.cluster_name} Azure Load Balancer - Tectonic Console" + priority = 305 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "443" + + # TODO: Ternary on private implementation + source_address_prefix = "*" + destination_address_prefix = "AzureLoadBalancer" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "console_ingress_http" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-console-in-http" + description = "${var.cluster_name} Azure Load Balancer - Tectonic Identity" + priority = 310 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "80" + + # TODO: Ternary on private implementation + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "AzureLoadBalancer" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +### Master node rules + +resource "azurerm_network_security_rule" "master_egress" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-out" + description = "${var.cluster_name} master - Outbound" + priority = 2005 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "*" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "master_ingress_ssh" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-ssh" + description = "${var.cluster_name} master - SSH" + priority = 500 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "22" + + # TODO: Reference subnet + source_address_prefix = "${var.ssh_network_internal}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "master_ingress_ssh_admin" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-ssh-external" + description = "${var.cluster_name} master - SSH external" + priority = 505 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "22" + + # TODO: Reference subnet + source_address_prefix = "${var.ssh_network_external}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "master_ingress_flannel" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-udp-4789" + description = "${var.cluster_name} master - flannel" + priority = 510 + direction = "Inbound" + access = "Allow" + protocol = "UDP" + source_port_range = "*" + destination_port_range = "4789" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "master_ingress_node_exporter_from_master" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-tcp-9100-master" + description = "${var.cluster_name} master - Prometheus node exporter from master" + priority = 515 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "9100" + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "master_ingress_node_exporter_from_worker" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-tcp-9100-worker" + description = "${var.cluster_name} master - Prometheus node exporter from worker" + priority = 520 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "9100" + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +# TODO: Review NSG +resource "azurerm_network_security_rule" "master_ingress_k8s_nodeport_from_alb" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-any-30000-32767-alb" + description = "${var.cluster_name} master - Kubernetes NodePort range from Azure Load Balancer" + priority = 525 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "30000-32767" + + # TODO: Reference subnet + source_address_prefix = "AzureLoadBalancer" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +# TODO: Review NSG +resource "azurerm_network_security_rule" "master_ingress_k8s_nodeport" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-any-30000-32767" + description = "${var.cluster_name} master - Kubernetes NodePort range" + priority = 530 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "30000-32767" + + # TODO: Reference subnet + source_address_prefix = "*" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} + +resource "azurerm_network_security_rule" "master_ingress_kubelet_secure" { + count = "${var.external_nsg_master_id == "" ? 1 : 0}" + name = "${var.cluster_name}-master-in-tcp-10255-vnet" + description = "${var.cluster_name} master - kubelet" + priority = 535 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "10255" + + # TODO: CR on how open this should be + # TODO: Reference subnet + source_address_prefix = "VirtualNetwork" + + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.master.name}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%nsg-worker.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%nsg-worker.tf new file mode 100644 index 0000000..f007038 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%nsg-worker.tf @@ -0,0 +1,169 @@ +resource "azurerm_network_security_group" "worker" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker" + location = "${var.location}" + resource_group_name = "${var.resource_group_name}" +} + +resource "azurerm_network_security_rule" "worker_egress" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-out" + description = "${var.cluster_name} worker - Outbound" + priority = 2010 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "*" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +resource "azurerm_network_security_rule" "worker_ingress_ssh" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-ssh" + description = "${var.cluster_name} worker - SSH" + priority = 600 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "22" + + # TODO: Reference subnet + source_address_prefix = "${var.ssh_network_internal}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +resource "azurerm_network_security_rule" "worker_ingress_ssh_admin" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-ssh-external" + description = "${var.cluster_name} worker - SSH external" + priority = 605 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "22" + + # TODO: Reference subnet + source_address_prefix = "${var.ssh_network_external}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +# TODO: Determine if we need two rules for this +resource "azurerm_network_security_rule" "worker_ingress_k8s_nodeport" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-any-30000-32767" + description = "${var.cluster_name} worker - Kubernetes NodePort range" + priority = 610 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "30000-32767" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "*" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +resource "azurerm_network_security_rule" "worker_ingress_flannel" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-udp-4789" + description = "${var.cluster_name} worker - flannel" + priority = 615 + direction = "Inbound" + access = "Allow" + protocol = "UDP" + source_port_range = "*" + destination_port_range = "4789" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +resource "azurerm_network_security_rule" "worker_ingress_kubelet_secure" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-tcp-10255-vnet" + description = "${var.cluster_name} worker - kubelet" + priority = 620 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "10255" + + # TODO: CR on how open this should be + # TODO: Reference subnet + source_address_prefix = "VirtualNetwork" + + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +resource "azurerm_network_security_rule" "worker_ingress_node_exporter_from_worker" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-tcp-9100-vnet" + description = "${var.cluster_name} worker - Prometheus node exporter from worker" + priority = 625 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "9100" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +resource "azurerm_network_security_rule" "worker_ingress_node_exporter_from_master" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-tcp-9100-master" + description = "${var.cluster_name} worker - Prometheus node exporter from master" + priority = 630 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "9100" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} + +resource "azurerm_network_security_rule" "worker_ingress_heapster_from_master" { + count = "${var.external_nsg_worker_id == "" ? 1 : 0}" + name = "${var.cluster_name}-worker-in-tcp-4194-master" + description = "${var.cluster_name} worker - Heapster from master" + priority = 635 + direction = "Inbound" + access = "Allow" + protocol = "TCP" + source_port_range = "*" + destination_port_range = "4194" + + # TODO: Reference subnet + source_address_prefix = "${var.vnet_cidr_block}" + destination_address_prefix = "${var.vnet_cidr_block}" + resource_group_name = "${var.resource_group_name}" + network_security_group_name = "${azurerm_network_security_group.worker.name}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%outputs.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%outputs.tf new file mode 100644 index 0000000..d7eca83 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%outputs.tf @@ -0,0 +1,84 @@ +locals { + # A regular expression that parses a Azure subnet id to extract subnet name. + const_id_to_subnet_name_regex = "/^/subscriptions/[-\\w]+/resourceGroups/[-\\w]+/providers/Microsoft.Network/virtualNetworks/[.\\w]+/subnets/([.\\w-]+)$/" + + # A regular expression that parses Azure resource IDs into component identifiers + const_id_to_group_name_regex = "/^/subscriptions/[-\\w]+/resourceGroups/([\\w()-\\.]+)/providers/[.\\w]+/[.\\w]+/([.\\w-]+)$/" +} + +output "vnet_id" { + value = "${var.external_vnet_id == "" ? element(concat(azurerm_virtual_network.tectonic_vnet.*.name, list("")), 0) : replace(var.external_vnet_id, local.const_id_to_group_name_regex, "$2")}" +} + +output "master_subnet" { + value = "${var.external_master_subnet_id == "" ? element(concat(azurerm_subnet.master_subnet.*.id, list("")), 0) : var.external_master_subnet_id}" +} + +output "worker_subnet" { + value = "${var.external_worker_subnet_id == "" ? element(concat(azurerm_subnet.worker_subnet.*.id, list("")), 0) : var.external_worker_subnet_id}" +} + +output "worker_subnet_name" { + value = "${var.external_worker_subnet_id == "" ? element(concat(azurerm_subnet.worker_subnet.*.name, list("")), 0) : replace(var.external_worker_subnet_id, local.const_id_to_subnet_name_regex, "$1")}" +} + +output "vnet_resource_group" { + value = "${var.external_vnet_id == "" ? "" : replace(var.external_vnet_id, local.const_id_to_group_name_regex, "$1")}" +} + +# TODO: Allow user to provide their own network +output "etcd_cidr" { + value = "${element(concat(azurerm_subnet.master_subnet.*.address_prefix, list("")), 0)}" +} + +output "master_cidr" { + value = "${element(concat(azurerm_subnet.master_subnet.*.address_prefix, list("")), 0)}" +} + +output "worker_cidr" { + value = "${element(concat(azurerm_subnet.worker_subnet.*.address_prefix, list("")), 0)}" +} + +output "worker_nsg_name" { + value = "${var.external_nsg_worker_id == "" ? element(concat(azurerm_network_security_group.worker.*.name, list("")), 0) : var.external_nsg_worker_id}" +} + +output "etcd_network_interface_ids" { + value = ["${azurerm_network_interface.etcd_nic.*.id}"] +} + +output "etcd_endpoints" { + value = "${azurerm_network_interface.etcd_nic.*.private_ip_address}" +} + +output "master_network_interface_ids" { + value = ["${azurerm_network_interface.tectonic_master.*.id}"] +} + +output "worker_network_interface_ids" { + value = ["${azurerm_network_interface.tectonic_worker.*.id}"] +} + +output "master_private_ip_addresses" { + value = ["${azurerm_network_interface.tectonic_master.*.private_ip_address}"] +} + +output "worker_private_ip_addresses" { + value = ["${azurerm_network_interface.tectonic_worker.*.private_ip_address}"] +} + +output "api_ip_addresses" { + value = ["${split("|", var.private_cluster ? join("|", azurerm_network_interface.tectonic_master.*.private_ip_address) : join("|", azurerm_public_ip.api_ip.*.ip_address))}"] +} + +output "console_ip_addresses" { + value = ["${split("|", var.private_cluster ? join("|", azurerm_network_interface.tectonic_worker.*.private_ip_address) : join("|", azurerm_public_ip.console_ip.*.ip_address))}"] +} + +output "ingress_fqdn" { + value = "${var.base_domain == "" ? element(concat(azurerm_public_ip.console_ip.*.fqdn, list("")), 0) : "${var.cluster_name}.${var.base_domain}${var.private_cluster ? ":32000" : ""}"}" +} + +output "api_fqdn" { + value = "${var.base_domain == "" ? element(concat(azurerm_public_ip.api_ip.*.fqdn, list("")), 0) : "${var.cluster_name}-api.${var.base_domain}"}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%variables.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%variables.tf new file mode 100644 index 0000000..df7c218 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%variables.tf @@ -0,0 +1,96 @@ +variable "cluster_name" { + type = "string" +} + +variable "cluster_id" { + type = "string" +} + +// The base DNS domain of the cluster. +// Example: `azure.dev.coreos.systems` +variable "base_domain" { + type = "string" +} + +variable "resource_group_name" { + type = "string" +} + +variable "vnet_cidr_block" { + type = "string" +} + +variable "location" { + type = "string" +} + +variable "external_vnet_id" { + type = "string" + default = "" +} + +variable "external_master_subnet_id" { + type = "string" + default = "" +} + +variable "external_worker_subnet_id" { + type = "string" + default = "" +} + +variable "external_nsg_master_id" { + type = "string" + default = "" +} + +variable "external_nsg_worker_id" { + type = "string" + default = "" +} + +variable "etcd_cidr" { + type = "string" + default = "" +} + +variable "etcd_count" { + type = "string" + default = "" +} + +variable "master_cidr" { + type = "string" + default = "" +} + +variable "worker_cidr" { + type = "string" + default = "" +} + +variable "ssh_network_internal" { + type = "string" + default = "" +} + +variable "ssh_network_external" { + type = "string" + default = "" +} + +variable "master_count" { + type = "string" +} + +variable "worker_count" { + type = "string" +} + +variable "extra_tags" { + type = "map" +} + +variable "private_cluster" { + default = false +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%vnet-subnets.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%vnet-subnets.tf new file mode 100644 index 0000000..6604b92 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%vnet%vnet-subnets.tf @@ -0,0 +1,30 @@ +resource "azurerm_virtual_network" "tectonic_vnet" { + count = "${var.external_vnet_id == "" ? 1 : 0 }" + name = "${var.cluster_name}" + resource_group_name = "${var.resource_group_name}" + address_space = ["${var.vnet_cidr_block}"] + location = "${var.location}" + + tags = "${merge(map( + "Name", "${var.cluster_name}_vnet", + "tectonicClusterID", "${var.cluster_id}"), + var.extra_tags)}" +} + +resource "azurerm_subnet" "master_subnet" { + count = "${var.external_master_subnet_id == "" ? 1 : 0}" + name = "${var.cluster_name}_master_subnet" + resource_group_name = "${var.external_vnet_id == "" ? var.resource_group_name : replace(var.external_vnet_id, "${var.const_id_to_group_name_regex}", "$1")}" + virtual_network_name = "${var.external_vnet_id == "" ? join("",azurerm_virtual_network.tectonic_vnet.*.name) : replace(var.external_vnet_id, "${var.const_id_to_group_name_regex}", "$2")}" + address_prefix = "${cidrsubnet(var.vnet_cidr_block, 4, 0)}" + network_security_group_id = "${var.external_nsg_master_id == "" ? azurerm_network_security_group.master.id : var.external_nsg_master_id}" +} + +resource "azurerm_subnet" "worker_subnet" { + count = "${var.external_worker_subnet_id == "" ? 1 : 0}" + name = "${var.cluster_name}_worker_subnet" + resource_group_name = "${var.external_vnet_id == "" ? var.resource_group_name : replace(var.external_vnet_id, "${var.const_id_to_group_name_regex}", "$1")}" + virtual_network_name = "${var.external_vnet_id == "" ? join("",azurerm_virtual_network.tectonic_vnet.*.name) : replace(var.external_vnet_id, "${var.const_id_to_group_name_regex}", "$2") }" + address_prefix = "${cidrsubnet(var.vnet_cidr_block, 4, 1)}" + network_security_group_id = "${var.external_nsg_worker_id == "" ? azurerm_network_security_group.worker.id : var.external_nsg_worker_id}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%ignition-worker.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%ignition-worker.tf new file mode 100644 index 0000000..d388e45 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%ignition-worker.tf @@ -0,0 +1,58 @@ +data "ignition_config" "worker" { + files = ["${compact(list( + data.ignition_file.cloud-provider-config.id, + data.ignition_file.kubeconfig.id, + var.ign_azure_udev_rules_id, + var.ign_installer_kubelet_env_id, + var.ign_installer_runtime_mappings_id, + var.ign_max_user_watches_id, + var.ign_nfs_config_id, + var.ign_ntp_dropin_id, + var.ign_profile_env_id, + var.ign_systemd_default_env_id, + ))}", + "${var.ign_ca_cert_id_list}", + ] + + systemd = [ + "${var.ign_docker_dropin_id}", + "${var.ign_iscsi_service_id}", + "${var.ign_k8s_node_bootstrap_service_id}", + "${var.ign_kubelet_service_id}", + "${var.ign_locksmithd_service_id}", + "${var.ign_tx_off_service_id}", + "${var.ign_update_ca_certificates_dropin_id}", + ] + + users = [ + "${data.ignition_user.core.id}", + ] +} + +data "ignition_file" "kubeconfig" { + filesystem = "root" + path = "/etc/kubernetes/kubeconfig" + mode = 0644 + + content { + content = "${var.kubeconfig_content}" + } +} + +data "ignition_file" "cloud-provider-config" { + filesystem = "root" + path = "/etc/kubernetes/cloud/config" + mode = 0600 + + content { + content = "${var.cloud_provider_config}" + } +} + +data "ignition_user" "core" { + name = "core" + + ssh_authorized_keys = [ + "${file(var.public_ssh_key)}", + ] +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%output.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%output.tf new file mode 100644 index 0000000..eff294c --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%output.tf @@ -0,0 +1,3 @@ +output "availability_set_name" { + value = "${azurerm_availability_set.tectonic_workers.name}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%variables.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%variables.tf new file mode 100644 index 0000000..02c1895 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%variables.tf @@ -0,0 +1,92 @@ +variable "container_linux_channel" { + type = "string" +} + +variable "container_linux_version" { + type = "string" +} + +variable "cloud_provider" { + type = "string" + default = "azure" +} + +variable "cloud_provider_config" { + description = "Content of cloud provider config" + type = "string" +} + +variable "cluster_id" { + type = "string" +} + +variable "cluster_name" { + type = "string" + description = "The name of the cluster." +} + +variable "extra_tags" { + type = "map" +} + +variable "ign_azure_udev_rules_id" { + type = "string" +} + +variable "ign_tx_off_service_id" { + type = "string" +} + +variable "kubeconfig_content" { + type = "string" + default = "" +} + +variable "location" { + type = "string" + description = "Location is the Azure Location (East US, West US, etc)" +} + +variable "network_interface_ids" { + type = "list" + description = "List of NICs to use for master VMs" +} + +variable "public_ssh_key" { + type = "string" +} + +variable "resource_group_name" { + type = "string" +} + +variable "storage_id" { + type = "string" +} + +variable "storage_type" { + type = "string" + description = "Storage account type" +} + +variable "root_volume_size" { + type = "string" +} + +variable "tectonic_kube_dns_service_ip" { + type = "string" +} + +variable "vm_size" { + type = "string" + description = "VM Size name" +} + +variable "worker_count" { + type = "string" + description = "Count of worker nodes to be created." +} + +variable "fault_domains" { + type = "string" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%workers.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%workers.tf new file mode 100644 index 0000000..a0ed9f7 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%azure%worker-as%workers.tf @@ -0,0 +1,68 @@ +resource "azurerm_availability_set" "tectonic_workers" { + name = "${var.cluster_name}-workers" + location = "${var.location}" + resource_group_name = "${var.resource_group_name}" + managed = true + platform_fault_domain_count = "${var.fault_domains}" + + tags = "${merge(map( + "Name", "${var.cluster_name}-workers", + "tectonicClusterID", "${var.cluster_id}"), + var.extra_tags)}" +} + +resource "azurerm_virtual_machine" "tectonic_worker" { + count = "${var.worker_count}" + name = "${var.cluster_name}-worker-${count.index}" + location = "${var.location}" + resource_group_name = "${var.resource_group_name}" + network_interface_ids = ["${var.network_interface_ids[count.index]}"] + vm_size = "${var.vm_size}" + availability_set_id = "${azurerm_availability_set.tectonic_workers.id}" + + delete_os_disk_on_termination = true + + storage_image_reference { + publisher = "CoreOS" + offer = "CoreOS" + sku = "${var.container_linux_channel}" + version = "${var.container_linux_version}" + } + + storage_os_disk { + name = "worker-${count.index}-os-${var.storage_id}" + managed_disk_type = "${var.storage_type}" + create_option = "FromImage" + caching = "ReadWrite" + os_type = "linux" + disk_size_gb = "${var.root_volume_size}" + } + + os_profile { + computer_name = "${var.cluster_name}-worker-${count.index}" + admin_username = "core" + admin_password = "" + custom_data = "${base64encode("${data.ignition_config.worker.rendered}")}" + } + + os_profile_linux_config { + disable_password_authentication = true + + ssh_keys { + path = "/home/core/.ssh/authorized_keys" + key_data = "${file(var.public_ssh_key)}" + } + } + + tags = "${merge(map( + "Name", "${var.cluster_name}-worker-${count.index}", + "tectonicClusterID", "${var.cluster_id}"), + var.extra_tags)}" + + lifecycle { + ignore_changes = [ + "storage_os_disk", + "storage_data_disk", + ] + } +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%assets.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%assets.tf new file mode 100644 index 0000000..f9bad2d --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%assets.tf @@ -0,0 +1,85 @@ +# kubeconfig (/auth/kubeconfig) +data "template_file" "kubeconfig" { + template = "${file("${path.module}/resources/kubeconfig")}" + + vars { + root_ca_cert = "${base64encode(var.root_ca_cert_pem)}" + admin_cert = "${base64encode(var.admin_cert_pem)}" + admin_key = "${base64encode(var.admin_key_pem)}" + server = "${var.kube_apiserver_url}" + cluster_name = "${var.cluster_name}" + } +} + +data "ignition_file" "kubeconfig" { + filesystem = "root" + path = "/opt/tectonic/auth/kubeconfig" + mode = "0600" + + content { + content = "${data.template_file.kubeconfig.rendered}" + } +} + +# kubeconfig-kubelet +data "template_file" "kubeconfig-kubelet" { + template = "${file("${path.module}/resources/kubeconfig-kubelet")}" + + vars { + root_ca_cert = "${base64encode(var.root_ca_cert_pem)}" + client_cert = "${base64encode(var.kubelet_cert_pem)}" + client_key = "${base64encode(var.kubelet_key_pem)}" + server = "${var.kube_apiserver_url}" + cluster_name = "${var.cluster_name}" + } +} + +data "ignition_file" "kubeconfig-kubelet" { + filesystem = "root" + path = "/opt/tectonic/auth/kubeconfig-kubelet" + mode = "0600" + + content { + content = "${data.template_file.kubeconfig-kubelet.rendered}" + } +} + +# bootkube.sh +data "template_file" "bootkube_sh" { + template = "${file("${path.module}/resources/bootkube.sh")}" + + vars { + bootkube_image = "${var.container_images["bootkube"]}" + kube_core_renderer_image = "${var.container_images["kube_core_renderer"]}" + tnc_operator_image = "${var.container_images["tnc_operator"]}" + etcd_cert_signer_image = "${var.container_images["etcd_cert_signer"]}" + etcdctl_image = "${var.container_images["etcd"]}" + etcd_cluster = "${join(",", data.template_file.initial_cluster.*.rendered)}" + } +} + +data "ignition_file" "bootkube_sh" { + filesystem = "root" + path = "/opt/tectonic/bootkube.sh" + mode = "0755" + + content { + content = "${data.template_file.bootkube_sh.rendered}" + } +} + +# bootkube.service (available as output variable) +data "template_file" "bootkube_service" { + template = "${file("${path.module}/resources/bootkube.service")}" +} + +data "ignition_systemd_unit" "bootkube_service" { + name = "bootkube.service" + enabled = false + content = "${data.template_file.bootkube_service.rendered}" +} + +data "template_file" "initial_cluster" { + count = "${length(var.etcd_endpoints)}" + template = "https://${var.etcd_endpoints[count.index]}:2379" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%assets.tf-288 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%assets.tf-288 new file mode 100644 index 0000000..00ec5c6 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%assets.tf-288 @@ -0,0 +1,162 @@ +# Self-hosted manifests (resources/generated/manifests/) +resource "template_dir" "bootkube" { + source_dir = "${path.module}/resources/manifests" + destination_dir = "./generated/manifests" + + vars { + hyperkube_image = "${var.container_images["hyperkube"]}" + pod_checkpointer_image = "${var.container_images["pod_checkpointer"]}" + kubedns_image = "${var.container_images["kubedns"]}" + kubednsmasq_image = "${var.container_images["kubednsmasq"]}" + kubedns_sidecar_image = "${var.container_images["kubedns_sidecar"]}" + + # Choose the etcd endpoints to use. + # 1. If self-hosted etcd is enabled, then use + # var.etcd_service_ip. + # 2. Else if no etcd TLS certificates are provided, i.e. we bootstrap etcd + # nodes ourselves (using http), then use insecure http var.etcd_endpoints. + # 3. Else (if etcd TLS certific are provided), then use the secure https + # var.etcd_endpoints. + etcd_servers = "${ + var.self_hosted_etcd != "" + ? format("https://%s:2379", cidrhost(var.service_cidr, 15)) + : var.etcd_tls_enabled + ? join(",", formatlist("https://%s:2379", var.etcd_endpoints)) + : join(",", formatlist("http://%s:2379", var.etcd_endpoints)) + }" + + etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" + bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}" + + cloud_provider = "${var.cloud_provider}" + cloud_provider_config = "${var.cloud_provider_config}" + cloud_provider_config_flag = "${var.cloud_provider_config != "" ? "- --cloud-config=/etc/kubernetes/cloud/config" : "# no cloud provider config given"}" + + cluster_cidr = "${var.cluster_cidr}" + service_cidr = "${var.service_cidr}" + kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" + advertise_address = "${var.advertise_address}" + + anonymous_auth = "${var.anonymous_auth}" + oidc_issuer_url = "${var.oidc_issuer_url}" + oidc_client_id = "${var.oidc_client_id}" + oidc_username_claim = "${var.oidc_username_claim}" + oidc_groups_claim = "${var.oidc_groups_claim}" + oidc_ca_cert = "${base64encode(var.oidc_ca_cert)}" + + kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}" + apiserver_key = "${base64encode(var.apiserver_key_pem)}" + apiserver_cert = "${base64encode(var.apiserver_cert_pem)}" + serviceaccount_pub = "${base64encode(tls_private_key.service_account.public_key_pem)}" + serviceaccount_key = "${base64encode(tls_private_key.service_account.private_key_pem)}" + + etcd_ca_flag = "${var.etcd_ca_cert_pem != "" ? "- --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt" : "# no etcd-client-ca.crt given" }" + etcd_cert_flag = "${var.etcd_client_cert_pem != "" ? "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt" : "# no etcd-client.crt given" }" + etcd_key_flag = "${var.etcd_client_key_pem != "" ? "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key" : "# no etcd-client.key given" }" + + etcd_ca_cert = "${base64encode(var.etcd_ca_cert_pem)}" + etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}" + etcd_client_key = "${base64encode(var.etcd_client_key_pem)}" + + kubernetes_version = "${replace(var.versions["kubernetes"], "+", "-")}" + + master_count = "${var.master_count}" + node_monitor_grace_period = "${var.node_monitor_grace_period}" + pod_eviction_timeout = "${var.pod_eviction_timeout}" + + cloud_provider_profile = "${var.cloud_provider != "" ? "${var.cloud_provider}" : "metal"}" + cloud_config_path = "${var.cloud_config_path}" + } +} + +# Self-hosted bootstrapping manifests (resources/generated/manifests-bootstrap/) +resource "template_dir" "bootkube_bootstrap" { + source_dir = "${path.module}/resources/bootstrap-manifests" + destination_dir = "./generated/bootstrap-manifests" + + vars { + hyperkube_image = "${var.container_images["hyperkube"]}" + etcd_image = "${var.container_images["etcd"]}" + + # Choose the etcd endpoints to use. + # 1. If self-hosted etcd mode is enabled, then use + # var.etcd_service_ip. + # 2. Else if no etcd TLS certificates are provided, i.e. we bootstrap etcd + # nodes ourselves (using http), then use insecure http var.etcd_endpoints. + # 3. Else (if etcd TLS certific are provided), then use the secure https + # var.etcd_endpoints. + etcd_servers = "${ + var.self_hosted_etcd != "" + ? format("https://%s:2379,https://127.0.0.1:12379", cidrhost(var.service_cidr, 15)) + : var.etcd_tls_enabled + ? join(",", formatlist("https://%s:2379", var.etcd_endpoints)) + : join(",", formatlist("http://%s:2379", var.etcd_endpoints)) + }" + + etcd_ca_flag = "${var.etcd_ca_cert_pem != "" ? "- --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt" : "# no etcd-client-ca.crt given" }" + etcd_cert_flag = "${var.etcd_client_cert_pem != "" ? "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt" : "# no etcd-client.crt given" }" + etcd_key_flag = "${var.etcd_client_key_pem != "" ? "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key" : "# no etcd-client.key given" }" + + cloud_provider = "${var.cloud_provider}" + cloud_provider_config = "${var.cloud_provider_config}" + cloud_provider_config_flag = "${var.cloud_provider_config != "" ? "- --cloud-config=/etc/kubernetes/cloud/config" : "# no cloud provider config given"}" + + advertise_address = "${var.advertise_address}" + cluster_cidr = "${var.cluster_cidr}" + service_cidr = "${var.service_cidr}" + } +} + +# kubeconfig (resources/generated/auth/kubeconfig) +data "template_file" "kubeconfig" { + template = "${file("${path.module}/resources/kubeconfig")}" + + vars { + kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}" + kubelet_cert = "${base64encode(var.kubelet_cert_pem)}" + kubelet_key = "${base64encode(var.kubelet_key_pem)}" + server = "${var.kube_apiserver_url}" + cluster_name = "${var.cluster_name}" + } +} + +resource "local_file" "kubeconfig" { + content = "${data.template_file.kubeconfig.rendered}" + filename = "./generated/auth/kubeconfig" +} + +# bootkube.sh (resources/generated/bootkube.sh) +data "template_file" "bootkube_sh" { + template = "${file("${path.module}/resources/bootkube.sh")}" + + vars { + bootkube_image = "${var.container_images["bootkube"]}" + } +} + +resource "local_file" "bootkube_sh" { + content = "${data.template_file.bootkube_sh.rendered}" + filename = "./generated/bootkube.sh" +} + +# bootkube.service (available as output variable) +data "template_file" "bootkube_service" { + template = "${file("${path.module}/resources/bootkube.service")}" +} + +data "ignition_systemd_unit" "bootkube_service" { + name = "bootkube.service" + enabled = false + content = "${data.template_file.bootkube_service.rendered}" +} + +# bootkube.path (available as output variable) +data "template_file" "bootkube_path_unit" { + template = "${file("${path.module}/resources/bootkube.path")}" +} + +data "ignition_systemd_unit" "bootkube_path_unit" { + name = "bootkube.path" + enabled = true + content = "${data.template_file.bootkube_path_unit.rendered}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%outputs.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%outputs.tf new file mode 100644 index 0000000..582e029 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%outputs.tf @@ -0,0 +1,32 @@ +output "kubeconfig-kubelet" { + value = "${data.template_file.kubeconfig-kubelet.rendered}" +} + +output "systemd_service_id" { + value = "${data.ignition_systemd_unit.bootkube_service.id}" +} + +output "kube_dns_service_ip" { + value = "${cidrhost(var.service_cidr, 10)}" +} + +output "kubeconfig_rendered" { + value = "${data.template_file.kubeconfig.rendered}" +} + +output "kubeconfig-kubelet_rendered" { + value = "${data.template_file.kubeconfig-kubelet.rendered}" +} + +output "ignition_file_id_list" { + value = ["${flatten(list( + list( + data.ignition_file.bootkube_sh.id, + data.ignition_file.kubeconfig.id, + data.ignition_file.kubeconfig-kubelet.id, + data.ignition_file.service_account_key.id, + data.ignition_file.service_account_crt.id, + ), + data.ignition_file.manifest_file_list.*.id, + ))}"] +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%outputs.tf-37 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%outputs.tf-37 new file mode 100644 index 0000000..e6e0166 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%outputs.tf-37 @@ -0,0 +1,55 @@ +# This output is meant to be used to inject a dependency on the generated +# assets. As of Terraform v0.9, it is difficult to make a module depend on +# another module (no depends_on, no triggers), or to make a data source +# depend on a module (no depends_on, no triggers, generally no dummy variable). +# +# For instance, using the 'archive_file' data source against the generated +# assets, which is a common use-case, is tricky. There is no mechanism for +# defining explicit dependencies and the only available variables are for the +# source, destination and archive type, leaving little opportunities for us to +# inject a dependency. Thanks to the property described below, this output can +# be used as part of the output filename, in order to enforce the creation of +# the archive after the assets have been properly generated. +# +# Both localfile and template_dir providers compute their IDs by hashing +# the content of the resources on disk. Because this output is computed from the +# combination of all the resources' IDs, it can't be guessed and can only be +# interpolated once the assets have all been created. +output "id" { + value = "${sha1(" + ${local_file.kubeconfig.id} + ${local_file.bootkube_sh.id} + ${template_dir.bootkube.id} ${template_dir.bootkube_bootstrap.id} + ${join(" ", + template_dir.etcd_manifests.*.id, + template_dir.etcd_bootstrap_manifests.*.id, + local_file.etcd_bootstrap_service.*.id, + local_file.migrate_etcd_cluster.*.id, + local_file.migrate_etcd_cluster_pv_backup.*.id, + )} + ")}" +} + +output "kubeconfig" { + value = "${data.template_file.kubeconfig.rendered}" +} + +output "systemd_service_rendered" { + value = "${data.template_file.bootkube_service.rendered}" +} + +output "systemd_service_id" { + value = "${data.ignition_systemd_unit.bootkube_service.id}" +} + +output "systemd_path_unit_rendered" { + value = "${data.template_file.bootkube_path_unit.rendered}" +} + +output "systemd_path_unit_id" { + value = "${data.ignition_systemd_unit.bootkube_path_unit.id}" +} + +output "kube_dns_service_ip" { + value = "${cidrhost(var.service_cidr, 10)}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%self-hosted-etcd.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%self-hosted-etcd.tf new file mode 100644 index 0000000..4eaef96 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%self-hosted-etcd.tf @@ -0,0 +1,80 @@ +resource "template_dir" "etcd_manifests" { + count = "${var.self_hosted_etcd != "" ? 1 : 0}" + source_dir = "${path.module}/resources/self-hosted-etcd/manifests" + destination_dir = "./generated/etcd/manifests" + + vars { + etcd_operator_image = "${var.container_images["etcd_operator"]}" + etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" + kenc_image = "${var.container_images["kenc"]}" + + etcd_ca_cert = "${base64encode(var.etcd_ca_cert_pem)}" + + etcd_server_cert = "${base64encode(var.etcd_server_cert_pem)}" + etcd_server_key = "${base64encode(var.etcd_server_key_pem)}" + + etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}" + etcd_client_key = "${base64encode(var.etcd_client_key_pem)}" + + etcd_peer_cert = "${base64encode(var.etcd_peer_cert_pem)}" + etcd_peer_key = "${base64encode(var.etcd_peer_key_pem)}" + } +} + +resource "template_dir" "etcd_bootstrap_manifests" { + count = "${var.self_hosted_etcd != "" ? 1 : 0}" + source_dir = "${path.module}/resources/self-hosted-etcd/bootstrap-manifests" + destination_dir = "./generated/etcd/bootstrap-manifests" + + vars { + etcd_image = "${var.container_images["etcd"]}" + etcd_version = "${var.versions["etcd"]}" + bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}" + } +} + +data "template_file" "etcd_bootstrap_service" { + template = "${file("${path.module}/resources/self-hosted-etcd/bootstrap-etcd-service.json")}" + + vars { + bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}" + } +} + +resource "local_file" "etcd_bootstrap_service" { + count = "${var.self_hosted_etcd != "" ? 1 : 0}" + content = "${data.template_file.etcd_bootstrap_service.rendered}" + filename = "./generated/etcd/bootstrap-etcd-service.json" +} + +data "template_file" "migrate_etcd_cluster" { + template = "${file("${path.module}/resources/self-hosted-etcd/migrate-etcd-cluster.json")}" + + vars { + etcd_version = "${var.versions["etcd"]}" + bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}" + } +} + +resource "local_file" "migrate_etcd_cluster" { + count = "${var.self_hosted_etcd == "enabled" ? 1 : 0}" + content = "${data.template_file.migrate_etcd_cluster.rendered}" + filename = "./generated/etcd/migrate-etcd-cluster.json" +} + +data "template_file" "migrate_etcd_cluster_pv_backup" { + template = "${file("${path.module}/resources/self-hosted-etcd/migrate-etcd-cluster-pv-backup.json")}" + + vars { + etcd_version = "${var.versions["etcd"]}" + bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}" + etcd_backup_size = "${var.etcd_backup_size}" + etcd_backup_storage_class = "${var.etcd_backup_storage_class}" + } +} + +resource "local_file" "migrate_etcd_cluster_pv_backup" { + count = "${var.self_hosted_etcd == "pv_backup" ? 1 : 0}" + content = "${data.template_file.migrate_etcd_cluster_pv_backup.rendered}" + filename = "./generated/etcd/migrate-etcd-cluster.json" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%service-account.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%service-account.tf new file mode 100644 index 0000000..0c4e571 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%service-account.tf @@ -0,0 +1,35 @@ +# Kubernete's Service Account (resources/generated/tls/{service-account.key,service-account.pub}) +resource "tls_private_key" "service_account" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "local_file" "service_account_key" { + content = "${tls_private_key.service_account.private_key_pem}" + filename = "./generated/tls/service-account.key" +} + +data "ignition_file" "service_account_key" { + filesystem = "root" + path = "/opt/tectonic/tls/service-account.key" + mode = "0644" + + content { + content = "${tls_private_key.service_account.private_key_pem}" + } +} + +resource "local_file" "service_account_crt" { + content = "${tls_private_key.service_account.public_key_pem}" + filename = "./generated/tls/service-account.pub" +} + +data "ignition_file" "service_account_crt" { + filesystem = "root" + path = "/opt/tectonic/tls/service-account.pub" + mode = "0644" + + content { + content = "${tls_private_key.service_account.public_key_pem}" + } +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%service-account.tf-289 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%service-account.tf-289 new file mode 100644 index 0000000..00fbaf1 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%service-account.tf-289 @@ -0,0 +1,15 @@ +# Kubernete's Service Account (resources/generated/tls/{service-account.key,service-account.pub}) +resource "tls_private_key" "service_account" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "local_file" "service_account_key" { + content = "${tls_private_key.service_account.private_key_pem}" + filename = "./generated/tls/service-account.key" +} + +resource "local_file" "service_account_crt" { + content = "${tls_private_key.service_account.public_key_pem}" + filename = "./generated/tls/service-account.pub" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%variables.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%variables.tf new file mode 100644 index 0000000..6aba396 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%variables.tf @@ -0,0 +1,147 @@ +variable "apiserver_cert_pem" { + type = "string" + description = "The API server certificate in PEM format." +} + +variable "apiserver_key_pem" { + type = "string" + description = "The API server key in PEM format." +} + +variable "openshift_apiserver_cert_pem" { + type = "string" + description = "The Openshift API server certificate in PEM format." +} + +variable "openshift_apiserver_key_pem" { + type = "string" + description = "The Openshift API server key in PEM format." +} + +variable "apiserver_proxy_cert_pem" { + type = "string" + description = "The API server proxy certificate in PEM format." +} + +variable "apiserver_proxy_key_pem" { + type = "string" + description = "The API server proxy key in PEM format." +} + +variable "cloud_provider_config" { + description = "Content of cloud provider config" + type = "string" + default = "" +} + +variable "cluster_name" { + type = "string" +} + +variable "container_images" { + description = "Container images to use" + type = "map" +} + +variable "etcd_ca_cert_pem" { + type = "string" + description = "The etcd CA certificate in PEM format." +} + +variable "etcd_client_cert_pem" { + type = "string" + description = "The etcd client certificate in PEM format." +} + +variable "etcd_client_key_pem" { + type = "string" + description = "The etcd client key in PEM format." +} + +variable "etcd_endpoints" { + description = "List of etcd endpoints to connect with (hostnames/IPs only)" + type = "list" +} + +variable "kube_apiserver_url" { + description = "URL used to reach kube-apiserver" + type = "string" +} + +variable "root_ca_cert_pem" { + type = "string" + description = "The Root CA in PEM format." +} + +variable "aggregator_ca_cert_pem" { + type = "string" + description = "The Aggregated API Server CA in PEM format." +} + +variable "aggregator_ca_key_pem" { + type = "string" + description = "The Aggregated API Server CA key in PEM format." +} + +variable "kube_ca_cert_pem" { + type = "string" + description = "The Kubernetes CA in PEM format." +} + +variable "kube_ca_key_pem" { + type = "string" + description = "The Kubernetes CA key in PEM format." +} + +variable "service_serving_ca_cert_pem" { + type = "string" + description = "The Service Serving CA in PEM format." +} + +variable "service_serving_ca_key_pem" { + type = "string" + description = "The Service Serving CA key in PEM format." +} + +variable "admin_cert_pem" { + type = "string" + description = "The admin certificate in PEM format." +} + +variable "admin_key_pem" { + type = "string" + description = "The admin key in PEM format." +} + +variable "kubelet_cert_pem" { + type = "string" + description = "The kubelet certificate in PEM format." +} + +variable "kubelet_key_pem" { + type = "string" + description = "The kubelet key in PEM format." +} + +variable "tnc_cert_pem" { + type = "string" +} + +variable "tnc_key_pem" { + type = "string" +} + +variable "oidc_ca_cert" { + type = "string" +} + +variable "service_cidr" { + description = "A CIDR notation IP range from which to assign service cluster IPs" + type = "string" +} + +variable "pull_secret_path" { + type = "string" + description = "Path on disk to your Tectonic pull secret. Obtain this from your Tectonic Account: https://account.coreos.com." + default = "/Users/coreos/Desktop/config.json" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%variables.tf-277 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%variables.tf-277 new file mode 100644 index 0000000..0a02d2e --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootkube%variables.tf-277 @@ -0,0 +1,179 @@ +variable "advertise_address" { + description = "The IP address on which to advertise the apiserver to members of the cluster" + type = "string" +} + +variable "anonymous_auth" { + description = "Enables anonymous requests to the secure port of the API server" + type = "string" +} + +variable "apiserver_cert_pem" { + type = "string" + description = "The API server certificate in PEM format." +} + +variable "apiserver_key_pem" { + type = "string" + description = "The API server key in PEM format." +} + +variable "cloud_provider" { + description = "The provider for cloud services (empty string for no provider)" + type = "string" +} + +variable "cloud_provider_config" { + description = "Content of cloud provider config" + type = "string" + default = "" +} + +variable "cluster_cidr" { + description = "A CIDR notation IP range from which to assign pod IPs" + type = "string" +} + +variable "cluster_name" { + type = "string" +} + +variable "container_images" { + description = "Container images to use" + type = "map" +} + +variable "etcd_tls_enabled" { + default = true +} + +variable "etcd_ca_cert_pem" { + type = "string" + description = "The etcd CA certificate in PEM format." +} + +variable "etcd_client_cert_pem" { + type = "string" + description = "The etcd client certificate in PEM format." +} + +variable "etcd_client_key_pem" { + type = "string" + description = "The etcd client key in PEM format." +} + +variable "etcd_endpoints" { + description = "List of etcd endpoints to connect with (hostnames/IPs only)" + type = "list" +} + +variable "etcd_peer_cert_pem" { + type = "string" + description = "The etcd peer certificate in PEM format." +} + +variable "etcd_peer_key_pem" { + type = "string" + description = "The etcd peer key in PEM format." +} + +variable "etcd_server_cert_pem" { + type = "string" + description = "The etcd server certificate in PEM format." +} + +variable "etcd_server_key_pem" { + type = "string" + description = "The etcd server key in PEM format." +} + +variable "self_hosted_etcd" { + type = "string" + description = "See tectonic_self_hosted_etcd in config.tf" +} + +variable "kube_apiserver_url" { + description = "URL used to reach kube-apiserver" + type = "string" +} + +variable "kube_ca_cert_pem" { + type = "string" + description = "The Kubernetes CA in PEM format." +} + +variable "kubelet_cert_pem" { + type = "string" + description = "The kubelet certificate in PEM format." +} + +variable "kubelet_key_pem" { + type = "string" + description = "The kubelet key in PEM format." +} + +variable "master_count" { + description = "The number of the master nodes" + type = "string" +} + +variable "node_monitor_grace_period" { + description = "Amount of time which we allow running Node to be unresponsive before marking it unhealthy. Must be N times more than kubelet's nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet to post node status. N must be stricly > 1." + type = "string" + default = "40s" +} + +variable "oidc_ca_cert" { + type = "string" +} + +variable "oidc_client_id" { + description = "The client ID for the OpenID Connect client" + type = "string" +} + +variable "oidc_groups_claim" { + description = "The OpenID claim to use for specifying user groups (string or array of strings)" + type = "string" +} + +variable "oidc_issuer_url" { + description = "The URL of the OpenID issuer, only HTTPS scheme will be accepted" + type = "string" +} + +variable "oidc_username_claim" { + description = "The OpenID claim to use as the user name" + type = "string" +} + +variable "pod_eviction_timeout" { + description = "The grace period for deleting pods on failed nodes. The eviction process will start after node_monitor_grace_period + pod_eviction_timeout." + type = "string" + default = "5m" +} + +variable "cloud_config_path" { + description = "The path to the secret file that contains the cloud config contents. Either be empty ('') or ('/etc/kubernetes/cloud/config')." + type = "string" +} + +variable "etcd_backup_size" { + type = "string" + description = "The size of the PersistentVolume used to handle etcd backups" +} + +variable "etcd_backup_storage_class" { + type = "string" + description = "The name of the Kubernetes StorageClass that will be used to handle etcd backups" +} + +variable "service_cidr" { + description = "A CIDR notation IP range from which to assign service cluster IPs" + type = "string" +} + +variable "versions" { + description = "Container versions to use" + type = "map" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootstrap-ssh%main.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootstrap-ssh%main.tf new file mode 100644 index 0000000..00cc9a2 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootstrap-ssh%main.tf @@ -0,0 +1,28 @@ +resource "null_resource" "bootstrapper" { + triggers { + endpoint = "${var.bootstrapping_host}" + dependencies = "${join("", concat(flatten(var._dependencies)))}" + } + + connection { + host = "${var.bootstrapping_host}" + user = "core" + agent = true + } + + provisioner "file" { + when = "create" + source = "./generated" + destination = "$HOME/tectonic" + } + + provisioner "remote-exec" { + when = "create" + + inline = [ + "sudo mkdir -p /opt", + "sudo rm -rf /opt/tectonic", + "sudo mv /home/core/tectonic /opt/", + ] + } +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootstrap-ssh%variables.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootstrap-ssh%variables.tf new file mode 100644 index 0000000..ceb8fee --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%bootstrap-ssh%variables.tf @@ -0,0 +1,7 @@ +variable "bootstrapping_host" { + type = "string" +} + +variable "_dependencies" { + type = "list" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%main.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%main.tf new file mode 100644 index 0000000..b995987 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%main.tf @@ -0,0 +1,4 @@ +data "external" "version" { + count = "${var.release_version == "latest" ? 1 : 0}" + program = ["sh", "-c", "curl https://${var.release_channel}.release.core-os.net/amd64-usr/current/version.txt | sed -n 's/COREOS_VERSION=\\(.*\\)$/{\"version\": \"\\1\"}/p'"] +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%main.tf-236 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%main.tf-236 new file mode 100644 index 0000000..4e662cb --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%main.tf-236 @@ -0,0 +1,3 @@ +data "external" "version" { + program = ["sh", "-c", "curl https://${var.release_channel}.release.core-os.net/amd64-usr/current/version.txt | sed -n 's/COREOS_VERSION=\\(.*\\)$/{\"version\": \"\\1\"}/p'"] +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%outputs.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%outputs.tf new file mode 100644 index 0000000..8d9df4a --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%outputs.tf @@ -0,0 +1,13 @@ +locals { + // Create a map that matches the structure of the output of the external data source + // so we can avoid running the shell script and still parse the output consistently. + // Here, we jsonencode because ternaries can only operate on flat data types and + // Terraform `merge` and `element` do not play nicely with maps. + json = "${var.release_version == "latest" ? jsonencode(data.external.version.*.result) : jsonencode(map("version", var.release_version))}" +} + +output "version" { + // Parse out the version from the well-known JSON of format: + // {"version":""} + value = "${replace(local.json, "/.*\"version\":\"(.*)\".*/", "$1")}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%outputs.tf-204 b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%outputs.tf-204 new file mode 100644 index 0000000..7805bef --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%outputs.tf-204 @@ -0,0 +1,3 @@ +output "version" { + value = "${var.release_version == "latest" ? data.external.version.result["version"] : var.release_version}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%variables.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%variables.tf new file mode 100644 index 0000000..48a2493 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%modules%container_linux%variables.tf @@ -0,0 +1,19 @@ +variable "release_channel" { + type = "string" + + description = < 0 ? + join("|", keys(var.tectonic_aws_master_custom_subnets)) : + join("|", data.aws_availability_zones.azs.names) + )}" + worker_azs = "${ split("|", "${length(keys(var.tectonic_aws_worker_custom_subnets))}" > 0 ? + join("|", keys(var.tectonic_aws_worker_custom_subnets)) : + join("|", data.aws_availability_zones.azs.names) + )}" +} + +module "etcd" { + source = "../../modules/aws/etcd" + + base_domain = "${var.tectonic_base_domain}" + cluster_id = "${module.tectonic.cluster_id}" + cluster_name = "${var.tectonic_cluster_name}" + container_image = "${var.tectonic_container_images["etcd"]}" + container_linux_channel = "${var.tectonic_container_linux_channel}" + container_linux_version = "${module.container_linux.version}" + ec2_ami = "${var.tectonic_aws_ec2_ami_override}" + ec2_type = "${var.tectonic_aws_etcd_ec2_type}" + etcd_iam_role = "${var.tectonic_aws_etcd_iam_role_name}" + external_endpoints = "${compact(var.tectonic_etcd_servers)}" + extra_tags = "${var.tectonic_aws_extra_tags}" + ign_etcd_crt_id_list = "${module.ignition_masters.etcd_crt_id_list}" + ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}" + ign_ntp_dropin_id = "${length(var.tectonic_ntp_servers) > 0 ? module.ignition_masters.ntp_dropin_id : ""}" + ign_profile_env_id = "${module.ignition_masters.profile_env_id}" + ign_systemd_default_env_id = "${module.ignition_masters.systemd_default_env_id}" + instance_count = "${length(data.template_file.etcd_hostname_list.*.id)}" + root_volume_iops = "${var.tectonic_aws_etcd_root_volume_iops}" + root_volume_size = "${var.tectonic_aws_etcd_root_volume_size}" + root_volume_type = "${var.tectonic_aws_etcd_root_volume_type}" + s3_bucket = "${aws_s3_bucket.tectonic.bucket}" + sg_ids = "${concat(var.tectonic_aws_etcd_extra_sg_ids, list(module.vpc.etcd_sg_id))}" + ssh_key = "${var.tectonic_aws_ssh_key}" + subnets = "${module.vpc.worker_subnet_ids}" + tls_enabled = "${var.tectonic_etcd_tls_enabled}" +} + +module "ignition_masters" { + source = "../../modules/ignition" + + assets_location = "${aws_s3_bucket_object.tectonic_assets.bucket}/${aws_s3_bucket_object.tectonic_assets.key}" + base_domain = "${var.tectonic_base_domain}" + bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" + cloud_provider = "aws" + cluster_name = "${var.tectonic_cluster_name}" + container_images = "${var.tectonic_container_images}" + custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}" + etcd_advertise_name_list = "${data.template_file.etcd_hostname_list.*.rendered}" + etcd_ca_cert_pem = "${module.etcd_certs.etcd_ca_crt_pem}" + etcd_client_crt_pem = "${module.etcd_certs.etcd_client_crt_pem}" + etcd_client_key_pem = "${module.etcd_certs.etcd_client_key_pem}" + etcd_count = "${length(data.template_file.etcd_hostname_list.*.id)}" + etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}" + etcd_peer_crt_pem = "${module.etcd_certs.etcd_peer_crt_pem}" + etcd_peer_key_pem = "${module.etcd_certs.etcd_peer_key_pem}" + etcd_server_crt_pem = "${module.etcd_certs.etcd_server_crt_pem}" + etcd_server_key_pem = "${module.etcd_certs.etcd_server_key_pem}" + etcd_tls_enabled = "${var.tectonic_etcd_tls_enabled}" + http_proxy = "${var.tectonic_http_proxy_address}" + http_proxy_enabled = "${local.tectonic_http_proxy_enabled}" + https_proxy = "${var.tectonic_https_proxy_address}" + image_re = "${var.tectonic_image_re}" + ingress_ca_cert_pem = "${module.ingress_certs.ca_cert_pem}" + iscsi_enabled = "${var.tectonic_iscsi_enabled}" + kube_ca_cert_pem = "${module.kube_certs.ca_cert_pem}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubeconfig_fetch_cmd = "/opt/s3-puller.sh ${aws_s3_bucket_object.kubeconfig.bucket}/${aws_s3_bucket_object.kubeconfig.key} /etc/kubernetes/kubeconfig" + kubelet_cni_bin_dir = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }" + kubelet_debug_config = "${var.tectonic_kubelet_debug_config}" + kubelet_node_label = "node-role.kubernetes.io/master" + kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" + nfs_config_file = "${local._tectonic_nfs_config_file}" + no_proxy = "${var.tectonic_no_proxy}" + ntp_servers = "${var.tectonic_ntp_servers}" + proxy_exclusive_units = "${var.tectonic_proxy_exclusive_units}" + tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}" +} + +module "masters" { + source = "../../modules/aws/master-asg" + + assets_s3_location = "${aws_s3_bucket_object.tectonic_assets.bucket}/${aws_s3_bucket_object.tectonic_assets.key}" + autoscaling_group_extra_tags = "${var.tectonic_autoscaling_group_extra_tags}" + aws_lbs = "${module.vpc.aws_lbs}" + base_domain = "${var.tectonic_base_domain}" + cluster_id = "${module.tectonic.cluster_id}" + cluster_name = "${var.tectonic_cluster_name}" + container_images = "${var.tectonic_container_images}" + container_linux_channel = "${var.tectonic_container_linux_channel}" + container_linux_version = "${module.container_linux.version}" + ec2_ami = "${var.tectonic_aws_ec2_ami_override}" + ec2_type = "${var.tectonic_aws_master_ec2_type}" + extra_tags = "${var.tectonic_aws_extra_tags}" + ign_bootkube_path_unit_id = "${module.bootkube.systemd_path_unit_id}" + ign_bootkube_service_id = "${module.bootkube.systemd_service_id}" + ign_ca_cert_id_list = "${module.ignition_masters.ca_cert_id_list}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_init_assets_service_id = "${module.ignition_masters.init_assets_service_id}" + ign_installer_kubelet_env_id = "${module.ignition_masters.installer_kubelet_env_id}" + ign_installer_runtime_mappings_id = "${module.ignition_masters.installer_runtime_mappings_id}" + ign_iscsi_service_id = "${module.ignition_masters.iscsi_service_id}" + ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_nfs_config_id = "${var.tectonic_nfs_config_file != "" ? module.ignition_masters.nfs_config_id : ""}" + ign_ntp_dropin_id = "${length(var.tectonic_ntp_servers) > 0 ? module.ignition_masters.ntp_dropin_id : ""}" + ign_profile_env_id = "${module.ignition_masters.profile_env_id}" + ign_rm_assets_path_unit_id = "${module.ignition_masters.rm_assets_path_unit_id}" + ign_rm_assets_service_id = "${module.ignition_masters.rm_assets_service_id}" + ign_s3_puller_id = "${module.ignition_masters.s3_puller_id}" + ign_systemd_default_env_id = "${module.ignition_masters.systemd_default_env_id}" + ign_tectonic_path_unit_id = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}" + ign_tectonic_service_id = "${module.tectonic.systemd_service_id}" + ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}" + instance_count = "${var.tectonic_master_count}" + master_iam_role = "${var.tectonic_aws_master_iam_role_name}" + master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}" + private_endpoints = "${var.tectonic_aws_private_endpoints}" + public_endpoints = "${var.tectonic_aws_public_endpoints}" + root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}" + root_volume_size = "${var.tectonic_aws_master_root_volume_size}" + root_volume_type = "${var.tectonic_aws_master_root_volume_type}" + s3_bucket = "${aws_s3_bucket.tectonic.bucket}" + ssh_key = "${var.tectonic_aws_ssh_key}" + subnet_ids = "${module.vpc.master_subnet_ids}" +} + +module "ignition_workers" { + source = "../../modules/ignition" + + bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" + cloud_provider = "aws" + container_images = "${var.tectonic_container_images}" + custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}" + etcd_ca_cert_pem = "${module.etcd_certs.etcd_ca_crt_pem}" + http_proxy = "${var.tectonic_http_proxy_address}" + http_proxy_enabled = "${local.tectonic_http_proxy_enabled}" + https_proxy = "${var.tectonic_https_proxy_address}" + image_re = "${var.tectonic_image_re}" + ingress_ca_cert_pem = "${module.ingress_certs.ca_cert_pem}" + iscsi_enabled = "${var.tectonic_iscsi_enabled}" + kube_ca_cert_pem = "${module.kube_certs.ca_cert_pem}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubeconfig_fetch_cmd = "/opt/s3-puller.sh ${aws_s3_bucket_object.kubeconfig.bucket}/${aws_s3_bucket_object.kubeconfig.key} /etc/kubernetes/kubeconfig" + kubelet_cni_bin_dir = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }" + kubelet_debug_config = "${var.tectonic_kubelet_debug_config}" + kubelet_node_label = "node-role.kubernetes.io/node" + kubelet_node_taints = "" + nfs_config_file = "${local._tectonic_nfs_config_file}" + no_proxy = "${var.tectonic_no_proxy}" + ntp_servers = "${var.tectonic_ntp_servers}" + proxy_exclusive_units = "${var.tectonic_proxy_exclusive_units}" + tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}" +} + +module "workers" { + source = "../../modules/aws/worker-asg" + + autoscaling_group_extra_tags = "${var.tectonic_autoscaling_group_extra_tags}" + cluster_id = "${module.tectonic.cluster_id}" + cluster_name = "${var.tectonic_cluster_name}" + container_linux_channel = "${var.tectonic_container_linux_channel}" + container_linux_version = "${module.container_linux.version}" + ec2_ami = "${var.tectonic_aws_ec2_ami_override}" + ec2_type = "${var.tectonic_aws_worker_ec2_type}" + extra_tags = "${var.tectonic_aws_extra_tags}" + ign_ca_cert_id_list = "${module.ignition_masters.ca_cert_id_list}" + ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" + ign_installer_kubelet_env_id = "${module.ignition_workers.installer_kubelet_env_id}" + ign_installer_runtime_mappings_id = "${module.ignition_workers.installer_runtime_mappings_id}" + ign_iscsi_service_id = "${module.ignition_workers.iscsi_service_id}" + ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}" + ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" + ign_nfs_config_id = "${var.tectonic_nfs_config_file != "" ? module.ignition_workers.nfs_config_id : ""}" + ign_ntp_dropin_id = "${length(var.tectonic_ntp_servers) > 0 ? module.ignition_workers.ntp_dropin_id : ""}" + ign_profile_env_id = "${module.ignition_workers.profile_env_id}" + ign_s3_puller_id = "${module.ignition_workers.s3_puller_id}" + ign_systemd_default_env_id = "${module.ignition_workers.systemd_default_env_id}" + ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}" + instance_count = "${var.tectonic_worker_count}" + load_balancers = "${var.tectonic_aws_worker_load_balancers}" + root_volume_iops = "${var.tectonic_aws_worker_root_volume_iops}" + root_volume_size = "${var.tectonic_aws_worker_root_volume_size}" + root_volume_type = "${var.tectonic_aws_worker_root_volume_type}" + s3_bucket = "${aws_s3_bucket.tectonic.bucket}" + sg_ids = "${concat(var.tectonic_aws_worker_extra_sg_ids, list(module.vpc.worker_sg_id))}" + ssh_key = "${var.tectonic_aws_ssh_key}" + subnet_ids = "${module.vpc.worker_subnet_ids}" + vpc_id = "${module.vpc.vpc_id}" + worker_iam_role = "${var.tectonic_aws_worker_iam_role_name}" +} + +module "dns" { + source = "../../modules/dns/route53" + + api_external_elb_dns_name = "${module.vpc.aws_api_external_dns_name}" + api_external_elb_zone_id = "${module.vpc.aws_elb_api_external_zone_id}" + api_internal_elb_dns_name = "${module.vpc.aws_api_internal_dns_name}" + api_internal_elb_zone_id = "${module.vpc.aws_elb_api_internal_zone_id}" + api_ip_addresses = "${module.vpc.aws_lbs}" + base_domain = "${var.tectonic_base_domain}" + cluster_id = "${module.tectonic.cluster_id}" + cluster_name = "${var.tectonic_cluster_name}" + console_elb_dns_name = "${module.vpc.aws_console_dns_name}" + console_elb_zone_id = "${module.vpc.aws_elb_console_zone_id}" + custom_dns_name = "${var.tectonic_dns_name}" + elb_alias_enabled = true + etcd_count = "${length(data.template_file.etcd_hostname_list.*.id)}" + etcd_ip_addresses = "${module.etcd.ip_addresses}" + external_endpoints = ["${compact(var.tectonic_etcd_servers)}"] + master_count = "${var.tectonic_master_count}" + tectonic_external_private_zone = "${var.tectonic_aws_external_private_zone}" + tectonic_external_vpc_id = "${module.vpc.vpc_id}" + tectonic_extra_tags = "${var.tectonic_aws_extra_tags}" + tectonic_private_endpoints = "${var.tectonic_aws_private_endpoints}" + tectonic_public_endpoints = "${var.tectonic_aws_public_endpoints}" + tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%s3.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%s3.tf new file mode 100644 index 0000000..9fa6d7c --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%s3.tf @@ -0,0 +1,65 @@ +data "aws_region" "current" { + current = true +} + +resource "aws_s3_bucket" "tectonic" { + # Buckets must start with a lower case name and are limited to 63 characters, + # so we prepend the letter 'a' and use the md5 hex digest for the case of a long domain + # leaving 29 chars for the cluster name. + bucket = "${var.tectonic_aws_assets_s3_bucket_name == "" ? format("%s%s-%s", "a", var.tectonic_cluster_name, md5(format("%s-%s", data.aws_region.current.name , var.tectonic_base_domain))) : var.tectonic_aws_assets_s3_bucket_name }" + + acl = "private" + + tags = "${merge(map( + "Name", "${var.tectonic_cluster_name}-tectonic", + "KubernetesCluster", "${var.tectonic_cluster_name}", + "tectonicClusterID", "${module.tectonic.cluster_id}" + ), var.tectonic_aws_extra_tags)}" + + lifecycle { + ignore_changes = ["*"] + } +} + +# Bootkube / Tectonic assets +resource "aws_s3_bucket_object" "tectonic_assets" { + bucket = "${aws_s3_bucket.tectonic.bucket}" + key = "assets.zip" + source = "${data.archive_file.assets.output_path}" + acl = "private" + + # To be on par with the current Tectonic installer, we only do server-side + # encryption, using AES256. Eventually, we should start using KMS-based + # client-side encryption. + server_side_encryption = "AES256" + + tags = "${merge(map( + "Name", "${var.tectonic_cluster_name}-tectonic-assets", + "KubernetesCluster", "${var.tectonic_cluster_name}", + "tectonicClusterID", "${module.tectonic.cluster_id}" + ), var.tectonic_aws_extra_tags)}" + + lifecycle { + ignore_changes = ["*"] + } +} + +# kubeconfig +resource "aws_s3_bucket_object" "kubeconfig" { + bucket = "${aws_s3_bucket.tectonic.bucket}" + key = "kubeconfig" + content = "${module.bootkube.kubeconfig}" + acl = "private" + + # The current Tectonic installer stores bits of the kubeconfig in KMS. As we + # do not support KMS yet, we at least offload it to S3 for now. Eventually, + # we should consider using KMS-based client-side encryption, or uploading it + # to KMS. + server_side_encryption = "AES256" + + tags = "${merge(map( + "Name", "${var.tectonic_cluster_name}-kubeconfig", + "KubernetesCluster", "${var.tectonic_cluster_name}", + "tectonicClusterID", "${module.tectonic.cluster_id}" + ), var.tectonic_aws_extra_tags)}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%tectonic.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%tectonic.tf new file mode 100644 index 0000000..dd093f1 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%tectonic.tf @@ -0,0 +1,160 @@ +data "template_file" "etcd_hostname_list" { + count = "${var.tectonic_self_hosted_etcd != "" ? 0 : var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : length(data.aws_availability_zones.azs.names) == 5 ? 5 : 3}" + template = "${var.tectonic_cluster_name}-etcd-${count.index}.${var.tectonic_base_domain}" +} + +module "bootkube" { + source = "../../modules/bootkube" + cloud_provider = "aws" + + cluster_name = "${var.tectonic_cluster_name}" + + kube_apiserver_url = "https://${var.tectonic_aws_private_endpoints ? module.dns.api_internal_fqdn : module.dns.api_external_fqdn}:443" + oidc_issuer_url = "https://${var.tectonic_aws_private_endpoints ? module.dns.ingress_internal_fqdn : module.dns.ingress_external_fqdn}/identity" + + # Platform-independent variables wiring, do not modify. + container_images = "${var.tectonic_container_images}" + versions = "${var.tectonic_versions}" + self_hosted_etcd = "${var.tectonic_self_hosted_etcd}" + + service_cidr = "${var.tectonic_service_cidr}" + cluster_cidr = "${var.tectonic_cluster_cidr}" + + advertise_address = "0.0.0.0" + anonymous_auth = "false" + + oidc_username_claim = "email" + oidc_groups_claim = "groups" + oidc_client_id = "tectonic-kubectl" + oidc_ca_cert = "${module.ingress_certs.ca_cert_pem}" + + apiserver_cert_pem = "${module.kube_certs.apiserver_cert_pem}" + apiserver_key_pem = "${module.kube_certs.apiserver_key_pem}" + etcd_ca_cert_pem = "${module.etcd_certs.etcd_ca_crt_pem}" + etcd_client_cert_pem = "${module.etcd_certs.etcd_client_crt_pem}" + etcd_client_key_pem = "${module.etcd_certs.etcd_client_key_pem}" + etcd_peer_cert_pem = "${module.etcd_certs.etcd_peer_crt_pem}" + etcd_peer_key_pem = "${module.etcd_certs.etcd_peer_key_pem}" + etcd_server_cert_pem = "${module.etcd_certs.etcd_server_crt_pem}" + etcd_server_key_pem = "${module.etcd_certs.etcd_server_key_pem}" + kube_ca_cert_pem = "${module.kube_certs.ca_cert_pem}" + kubelet_cert_pem = "${module.kube_certs.kubelet_cert_pem}" + kubelet_key_pem = "${module.kube_certs.kubelet_key_pem}" + + etcd_backup_size = "${var.tectonic_etcd_backup_size}" + etcd_backup_storage_class = "${var.tectonic_etcd_backup_storage_class}" + etcd_endpoints = "${module.dns.etcd_endpoints}" + master_count = "${var.tectonic_master_count}" + + # The default behavior of Kubernetes's controller manager is to mark a node + # as Unhealthy after 40s without an update from the node's kubelet. However, + # AWS ELB's Route53 records have a fixed TTL of 60s. Therefore, when an ELB's + # node disappears (e.g. scaled down or crashed), kubelet might fail to report + # for a period of time that exceed the default grace period of 40s and the + # node might become Unhealthy. While the eviction process won't start until + # the pod_eviction_timeout is reached, 5min by default, certain operators + # might already have taken action. This is the case for the etcd operator as + # of v0.3.3, which removes the likely-healthy etcd pods from the the + # cluster, potentially leading to a loss-of-quorum as generally all kubelets + # are affected simultaneously. + # + # To cope with this issue, we increase the grace period, and reduce the + # pod eviction time-out accordingly so pods still get evicted after an total + # time of 340s after the first post-status failure. + # + # Ref: https://github.com/kubernetes/kubernetes/issues/41916 + # Ref: https://github.com/kubernetes-incubator/kube-aws/issues/598 + node_monitor_grace_period = "2m" + + pod_eviction_timeout = "220s" + + cloud_config_path = "" +} + +module "tectonic" { + source = "../../modules/tectonic" + platform = "aws" + + cluster_name = "${var.tectonic_cluster_name}" + + base_address = "${var.tectonic_aws_private_endpoints ? module.dns.ingress_internal_fqdn : module.dns.ingress_external_fqdn}" + kube_apiserver_url = "https://${var.tectonic_aws_private_endpoints ? module.dns.api_internal_fqdn : module.dns.api_external_fqdn}:443" + service_cidr = "${var.tectonic_service_cidr}" + + # Platform-independent variables wiring, do not modify. + container_images = "${var.tectonic_container_images}" + container_base_images = "${var.tectonic_container_base_images}" + versions = "${var.tectonic_versions}" + + license_path = "${var.tectonic_vanilla_k8s ? "/dev/null" : pathexpand(var.tectonic_license_path)}" + pull_secret_path = "${var.tectonic_vanilla_k8s ? "/dev/null" : pathexpand(var.tectonic_pull_secret_path)}" + + admin_email = "${var.tectonic_admin_email}" + admin_password = "${var.tectonic_admin_password}" + + update_channel = "${var.tectonic_update_channel}" + update_app_id = "${var.tectonic_update_app_id}" + update_server = "${var.tectonic_update_server}" + + ca_generated = "${var.tectonic_ca_cert == "" ? false : true}" + ca_cert = "${module.kube_certs.ca_cert_pem}" + + ingress_ca_cert_pem = "${module.ingress_certs.ca_cert_pem}" + ingress_cert_pem = "${module.ingress_certs.cert_pem}" + ingress_key_pem = "${module.ingress_certs.key_pem}" + + identity_client_cert_pem = "${module.identity_certs.client_cert_pem}" + identity_client_key_pem = "${module.identity_certs.client_key_pem}" + identity_server_cert_pem = "${module.identity_certs.server_cert_pem}" + identity_server_key_pem = "${module.identity_certs.server_key_pem}" + + console_client_id = "tectonic-console" + kubectl_client_id = "tectonic-kubectl" + ingress_kind = "NodePort" + self_hosted_etcd = "${var.tectonic_self_hosted_etcd}" + master_count = "${var.tectonic_master_count}" + stats_url = "${var.tectonic_stats_url}" + + image_re = "${var.tectonic_image_re}" +} + +module "flannel_vxlan" { + source = "../../modules/net/flannel_vxlan" + + cluster_cidr = "${var.tectonic_cluster_cidr}" + enabled = "${var.tectonic_networking == "flannel"}" + container_images = "${var.tectonic_container_images}" +} + +module "calico" { + source = "../../modules/net/calico" + + container_images = "${var.tectonic_container_images}" + cluster_cidr = "${var.tectonic_cluster_cidr}" + enabled = "${var.tectonic_networking == "calico"}" +} + +module "canal" { + source = "../../modules/net/canal" + + container_images = "${var.tectonic_container_images}" + cluster_cidr = "${var.tectonic_cluster_cidr}" + enabled = "${var.tectonic_networking == "canal"}" +} + +data "archive_file" "assets" { + type = "zip" + source_dir = "./generated/" + + # Because the archive_file provider is a data source, depends_on can't be + # used to guarantee that the tectonic/bootkube modules have generated + # all the assets on disk before trying to archive them. Instead, we use their + # ID outputs, that are only computed once the assets have actually been + # written to disk. We re-hash the IDs (or dedicated module outputs, like module.bootkube.content_hash) + # to make the filename shorter, since there is no security nor collision risk anyways. + # + # Additionally, data sources do not support managing any lifecycle whatsoever, + # and therefore, the archive is never deleted. To avoid cluttering the module + # folder, we write it in the Terraform managed hidden folder `.terraform`. + output_path = "./.terraform/generated_${sha1("${module.etcd_certs.id} ${module.tectonic.id} ${module.bootkube.id} ${module.flannel_vxlan.id} ${module.calico.id} ${module.canal.id}")}.zip" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%tls.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%tls.tf new file mode 100644 index 0000000..c33902e --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%tls.tf @@ -0,0 +1,40 @@ +module "kube_certs" { + source = "../../modules/tls/kube/self-signed" + + ca_cert_pem = "${var.tectonic_ca_cert}" + ca_key_alg = "${var.tectonic_ca_key_alg}" + ca_key_pem = "${var.tectonic_ca_key}" + kube_apiserver_url = "https://${var.tectonic_aws_private_endpoints ? module.dns.api_internal_fqdn : module.dns.api_external_fqdn}:443" + service_cidr = "${var.tectonic_service_cidr}" + validity_period = "${var.tectonic_tls_validity_period}" +} + +module "etcd_certs" { + source = "../../modules/tls/etcd/signed" + + etcd_ca_cert_path = "${var.tectonic_etcd_ca_cert_path}" + etcd_cert_dns_names = "${data.template_file.etcd_hostname_list.*.rendered}" + etcd_client_cert_path = "${var.tectonic_etcd_client_cert_path}" + etcd_client_key_path = "${var.tectonic_etcd_client_key_path}" + self_signed = "${var.tectonic_self_hosted_etcd != "" ? "true" : length(compact(var.tectonic_etcd_servers)) == 0 ? "true" : "false"}" + service_cidr = "${var.tectonic_service_cidr}" +} + +module "ingress_certs" { + source = "../../modules/tls/ingress/self-signed" + + base_address = "${var.tectonic_aws_private_endpoints ? module.dns.ingress_internal_fqdn : module.dns.ingress_external_fqdn}" + ca_cert_pem = "${module.kube_certs.ca_cert_pem}" + ca_key_alg = "${module.kube_certs.ca_key_alg}" + ca_key_pem = "${module.kube_certs.ca_key_pem}" + validity_period = "${var.tectonic_tls_validity_period}" +} + +module "identity_certs" { + source = "../../modules/tls/identity/self-signed" + + ca_cert_pem = "${module.kube_certs.ca_cert_pem}" + ca_key_alg = "${module.kube_certs.ca_key_alg}" + ca_key_pem = "${module.kube_certs.ca_key_pem}" + validity_period = "${var.tectonic_tls_validity_period}" +} diff --git a/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%variables.tf b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%variables.tf new file mode 100644 index 0000000..0dbec64 --- /dev/null +++ b/example/real_world_stuff/coreos/coreos%tectonic-installer%platforms%aws%variables.tf @@ -0,0 +1,347 @@ +variable "tectonic_aws_config_version" { + description = <,%,&,\,?,/' or control characters. +EOF + + default = {} +} + +variable "tectonic_azure_private_cluster" { + description = <