ansible: added option to provide accounts for genesis generation, terraform: added option to secure DigitalOcean servers, devops: added DNS name creation to tendermint terraform

ansible/README.md

@@ -190,7 +190,7 @@ The roles are self-sufficient under the `roles/` folder.
 * install-tendermint - install the tendermint application. It can install release packages or custom-compiled binaries.
 * install-basecoin - install the basecoin application. It can install release packages or custom-compiled binaries.
 * install-ethermint - install the ethermint application. It can install release packages or custom-compiled binaries.
-* cleanupconfig - delete all tendermint configuration and data.
+* cleanupdata - delete tendermint database.
 * config - configure the tendermint application
 * stop - stop an application. Requires the `service` parameter set.
 * start - start an application. Requires the `service` parameter set.

ansible/app_options_files/dev_money

@@ -0,0 +1,13 @@
+    "accounts": [{
+      "pub_key": {
+        "type": "ed25519",
+        "data": "619D3678599971ED29C7529DDD4DA537B97129893598A17C82E3AC9A8BA95279"
+      },
+      "coins": [
+        {
+          "denom": "mycoin",
+          "amount": 9007199254740992
+        }
+      ]
+    }]
+

ansible/app_options_files/public_testnet

@@ -0,0 +1,101 @@
+    "accounts": [
+      {
+        "name": "anton",
+        "address": "40CC622438D3E42148A1FFD3A27C07C100F8FA3D",
+        "pub_key": {
+          "type": "ed25519",
+          "data": "97BD389257763747488803DC686A8819C685936A3CD275D54EABEE51E0117EE6"
+        },
+        "coins": [
+          {
+            "denom": "anton",
+            "amount": 9007199254740992
+          },
+          {
+            "denom": "tank",
+            "amount": 99
+          }
+        ]
+      },
+      {
+        "name": "adrian",
+        "address": "98F28277FA8C512968BBDE443F5DB27AC743F814",
+        "pub_key": {
+          "type": "ed25519",
+          "data": "2E7EAB9E4C93D3657A63E063D6ABB851596BA97ED9F4EB9A2FC714043DC9685D"
+        },
+        "coins": [
+          {
+            "denom": "axi",
+            "amount": 1928936473812
+          },
+          {
+            "denom": "bear",
+            "amount": 42
+          }
+        ]
+      },
+      {
+        "name": "bucky",
+        "address": "2B24DEE2364762300168DF19B6C18BCE2D399EA2",
+        "pub_key": {
+          "type": "ed25519",
+          "data": "37BF9D6F8D66DC3FAECD330291CF08DF52E2F646B01F7E43BC0A746AED927A62"
+        },
+        "coins": [
+          {
+            "denom": "buckyball",
+            "amount": 20000000000
+          },
+          {
+            "denom": "bumblebee",
+            "amount": 100
+          },
+          {
+            "denom": "french",
+            "amount": 10
+          },
+          {
+            "denom": "frey",
+            "amount": 4838271727204
+          }
+        ]
+      },
+      {
+        "name": "phomer",
+        "address": "BAA67E0346AF48A05330149A3A72D9B974E9EBD2",
+        "pub_key": {
+          "type": "ed25519",
+          "data": "F5DF27FD625800D971A9E9B2607F0052480E49A12BE85E17336F773FE6FB9BC9"
+        },
+        "coins": [
+          {
+            "denom": "antipho",
+            "amount": 33
+          },
+          {
+            "denom": "pho",
+            "amount": 8367251830291
+          }
+        ]
+      },
+      {
+        "name": "shadow",
+        "address": "B140EFAAE6D5CA1C8E98814C557AF7112E3B9EAE",
+        "pub_key": {
+          "type": "ed25519",
+          "data": "F492282705DF29ACC3BB803D543B7BF98C8080FA28AE85B62B45827EA9DA8167"
+        },
+        "coins": [
+          {
+            "denom": "shadow",
+            "amount": 53712836452781
+          },
+          {
+            "denom": "unit",
+            "amount": 1
+          }
+        ]
+      }
+    ]
+

ansible/group_vars/all

@@ -57,14 +57,6 @@
 ##
 #basecoin_log_file: /var/log/basecoin.log
 
-###
-### Basecoin configuration
-###
-
-## basecoin_inprocess indicates if basecoin should run tendermint in-process
-##
-#basecoin_inprocess: false
-
 ###
 ### Development settings
 ###
@@ -73,9 +65,6 @@
 ##
 #testnet_name: testnet1
 
-## Used by the config role to add a lot of mycoins to wallet 0x1B1BE55F969F54064628A63B9559E7C21C925165.
-## Use /usr/share/basecoin/dev-keys/key.json to access them.
-## Tendermint data has to be cleared when setting or resetting this key.
-##
-#dev_money: false
+## app_options_file contains a path and filename which will be included in a generated genesis.json file on all nodes. The content will be dumped into the app_options dictionary.
+#app_options_file: "app_options_files/dev_money"
 

ansible/install-basecoin.yml

@@ -4,7 +4,7 @@
   roles:
     - install-tendermint
     - install-basecoin
-    - cleanupconfig
+    - cleanupdata
     - { role: config, testnet_name: "{{ lookup('env','TF_VAR_TESTNET_NAME') }}" }
     - { role: start, service: basecoin }
     - { role: start, service: tendermint }

ansible/install-ethermint.yml

@@ -4,7 +4,7 @@
   roles:
     - install-tendermint
     - install-ethermint
-    - cleanupconfig
+    - cleanupdata
     - { role: config, testnet_name: "{{ lookup('env','TF_VAR_TESTNET_NAME') }}" }
     - { role: start, service: ethermint }
     - { role: start, service: tendermint }

ansible/install-tendermint-core.yml

@@ -3,7 +3,7 @@
 - hosts: "{{ lookup('env','TF_VAR_TESTNET_NAME') }}"
   roles:
     - install-tendermint
-    - cleanupconfig
+    - cleanupdata
     - { role: config, testnet_name: "{{ lookup('env','TF_VAR_TESTNET_NAME') }}" }
     - { role: start, service: tendermint }
 

ansible/reset.yml

@@ -6,7 +6,7 @@
   roles:
     - { role: stop, service: tendermint }
     - stop
-    - { role: cleanupconfig, when: not reconfig|default(false)|bool }
+    - cleanupdata
     - { role: config, testnet_name: "{{ lookup('env','TF_VAR_TESTNET_NAME') }}" }
     - start
     - { role: start, service: tendermint }

ansible/roles/cleanupconfig/tasks/main.yml

@@ -1,10 +0,0 @@
----
-
-- name: clear tendermint configuration folder
-  file: "path={{tendermint_home}}/{{item}} state=absent"
-  with_items:
-  - config.toml
-  - genesis.json
-  - priv_validator.json
-  - data
-

ansible/roles/cleanupdata/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+
+- name: clear tendermint database
+  file: "path={{tendermint_home}}/{{item}} state=absent"
+  with_items:
+  - data
+

ansible/roles/config/defaults/main.yml

@@ -4,5 +4,6 @@ tendermint_group: tendermint
 tendermint_home: /etc/tendermint
 
 #tendermint_genesis_file="<undefined>"
+#app_options_file="<undefined>"
 testnet_name: testnet1
-dev_money: false
+

ansible/roles/config/templates/genesis.json.j2

@@ -15,19 +15,8 @@
   ],
   "app_hash":"",
   "app_options": {
-{% if dev_money|bool %}
-    "accounts": [{
-      "pub_key": {
-        "type": "ed25519",
-        "data": "619D3678599971ED29C7529DDD4DA537B97129893598A17C82E3AC9A8BA95279"
-      },
-      "coins": [
-        {
-          "denom": "mycoin",
-          "amount": 9007199254740992
-        }
-      ]
-    }]
+{% if app_options_file is defined %}
+{% include app_options_file %}
 {% endif %}
   }
 }

ansible/roles/install-basecoin/defaults/main.yml

@@ -10,5 +10,3 @@ basecoin_group: tendermint
 basecoin_home: /etc/tendermint
 basecoin_log_file: /var/log/basecoin.log
 
-basecoin_inprocess: false
-

ansible/roles/install-basecoin/templates/basecoin.conf.j2

@@ -6,7 +6,7 @@ stop on runlevel [!2345]
 
 script
   # Use su to become tendermint user non-interactively on old Upstart versions (see http://superuser.com/a/234541/76168)
-  exec su -s /bin/sh -c 'BCHOME={{basecoin_home}} exec "$0" "$@" >> {{ basecoin_log_file }} 2>&1' {{basecoin_user}} -- /usr/local/bin/basecoin start{{basecoin_inprocess|bool | ternary('',' --without-tendermint')}}
+  exec su -s /bin/sh -c 'BCHOME={{basecoin_home}} exec "$0" "$@" >> {{ basecoin_log_file }} 2>&1' {{basecoin_user}} -- /usr/local/bin/basecoin start --without-tendermint
 end script
 
 respawn

ansible/roles/install-basecoin/templates/basecoin.systemd.j2

@@ -9,7 +9,7 @@ Restart=on-failure
 User={{ basecoin_user }}
 Group={{ basecoin_group }}
 PermissionsStartOnly=true
-ExecStart=/usr/local/bin/basecoin start{{basecoin_inprocess|bool | ternary('',' --without-tendermint')}}
+ExecStart=/usr/local/bin/basecoin start --without-tendermint
 ExecReload=/bin/kill -HUP $MAINPID
 KillSignal=SIGINT
 

ansible/roles/install-tendermint/defaults/main.yml

@@ -10,5 +10,3 @@ tendermint_group: tendermint
 tendermint_home: /etc/tendermint
 tendermint_log_file: /var/log/tendermint.log
 
-basecoin_inprocess: false
-

ansible/roles/install-tendermint/templates/tendermint.conf.j2

@@ -1,8 +1,8 @@
 # Tendermint (Upstart unit)
 description "Tendermint"
 
-{{basecoin_inprocess|bool | ternary('','start on (runlevel [2345] and started basecoin)')}}
-{{basecoin_inprocess|bool | ternary('','stop on (runlevel [!2345] and stopping basecoin)')}}
+start on (runlevel [2345] and started basecoin)
+stop on (runlevel [!2345] and stopping basecoin)
 
 script
   # Use su to become tendermint user non-interactively on old Upstart versions (see http://superuser.com/a/234541/76168)

ansible/roles/install-tendermint/templates/tendermint.systemd.j2

@@ -1,6 +1,6 @@
 [Unit]
 Description=Tendermint
-Requires=network-online.target{{basecoin_inprocess|bool | ternary('',' basecoin.target')}}
+Requires=network-online.target basecoin.target
 After=network-online.target
 
 [Service]

ansible/roles/start/defaults/main.yml

@@ -1,3 +0,0 @@
----
-basecoin_inprocess: false
-

ansible/roles/stop/defaults/main.yml

@@ -1,3 +0,0 @@
----
-basecoin_inprocess: false
-

create-digitalocean-testnet.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# This is an example set of commands that uses Terraform and Ansible to create a testnet on Digital Ocean.
+# This is an example set of commands that uses Terraform and Ansible to create a basecoin testnet on Digital Ocean.
 
 # Prerequisites: terraform, ansible, DigitalOcean API token, ssh-agent running with the same SSH keys added that are set up during terraform
 # Optional: GOPATH if you build the app yourself
@@ -27,6 +27,8 @@ SERVERS=2
 cd terraform-digitalocean
 terraform init
 terraform env new "$TF_VAR_TESTNET_NAME"
+#The next step copies additional terraform rules that only apply in the Tendermint network.
+#cp -r ../devops/terraform-tendermint/* .
 terraform apply -var servers=$SERVERS -var DO_API_TOKEN="$DO_API_TOKEN"
 cd ..
 
@@ -45,13 +47,13 @@ fi
 #Note that SSH Agent needs to be running with SSH keys added or ansible-playbook requires the --private-key option.
 cd ansible
 python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null
-ansible-playbook -i inventory/digital_ocean.py install.yml $ANSIBLE_ADDITIONAL_VARS
+ansible-playbook -i inventory/digital_ocean.py install-basecoin.yml -u root -e app_options_file=ansible/app_options_files/dev_money $ANSIBLE_ADDITIONAL_VARS
 cd ..
 
 ###
 # Start application
 ###
 cd ansible
-ansible-playbook -i inventory/digital_ocean.py start.yml
+ansible-playbook -i inventory/digital_ocean.py start-basecoin.yml
 cd ..
 

devops/terraform-tendermint/cluster/networking.tf

@@ -0,0 +1,31 @@
+#resource "digitalocean_floating_ip" "cluster" {
+#  droplet_id = "${element(digitalocean_droplet.cluster.*.id,count.index)}"
+#  region     = "${element(digitalocean_droplet.cluster.*.region,count.index)}"
+#  count      = "${var.servers}"
+#}
+
+provider "aws" {
+}
+
+data "aws_route53_zone" "cluster" {
+  name         = "testnets.interblock.io."
+}
+
+resource "aws_route53_record" "cluster-node" {
+  zone_id = "${data.aws_route53_zone.cluster.zone_id}"
+  name    = "${var.name}-node${count.index}"
+  type    = "A"
+  ttl     = "300"
+  records = ["${element(digitalocean_droplet.cluster.*.ipv4_address,count.index)}"]
+  count      = "${var.servers}"
+}
+
+resource "aws_route53_record" "cluster-regions" {
+  zone_id = "${data.aws_route53_zone.cluster.zone_id}"
+  name    = "${var.name}-${element(digitalocean_droplet.cluster.*.region,count.index)}"
+  type    = "CNAME"
+  ttl     = "300"
+  records = ["${element(aws_route53_record.cluster-node.*.name,count.index)}.${data.aws_route53_zone.cluster.name}"]
+  count      = "${var.servers}"
+}
+

devops/terraform-tendermint/networking-output.tf

@@ -0,0 +1,4 @@
+#output "floating_ips" {
+#  value = "${module.cluster.floating_ips}"
+#}
+

terraform-digitalocean/.gitignore

@@ -0,0 +1,3 @@
+cluster/networking.tf
+networking-output.tf
+

terraform-digitalocean/README.md

@@ -6,7 +6,8 @@ This is a generic [Terraform](https://www.terraform.io/) configuration that sets
 
 * Install [HashiCorp Terraform](https://www.terraform.io) on a linux machine.
 * Create a [DigitalOcean API token](https://cloud.digitalocean.com/settings/api/tokens) with read and write capability.
-* Set an SSH key at the [DigitalOcean security page](https://cloud.digitalocean.com/settings/security). {Here](https://www.digitalocean.com/community/tutorials/how-to-use-ssh-keys-with-digitalocean-droplets)'s a tutorial.
+* Create a private/public key pair for SSH. This is needed to log onto your droplets as well as by Ansible to connect for configuration changes.
+* Set up the public SSH key at the [DigitalOcean security page](https://cloud.digitalocean.com/settings/security). [Here](https://www.digitalocean.com/community/tutorials/how-to-use-ssh-keys-with-digitalocean-droplets)'s a tutorial.
 * Find out your SSH key ID at DigitalOcean by querying the below command on your linux box:
 
 ```
@@ -53,10 +54,10 @@ terraform-apply
 ## Security
 
 DigitalOcean uses the root user by default on its droplets. This is fine as long as SSH keys are used. However some people still would like to disable root and use an alternative user to connect to the droplets - then `sudo` from there.
-There is a way to do this with Terraform but it requires SSH agent running on the machine where terraform is run, with one of the SSH keys of the droplets added to the agent. (This will be neede for ansible too, so it's worth setting it up here. Check out the [ansible](https://github.com/tendermint/tools/tree/master/ansible) page for more information.)
-After setting up the SSH key, uncomment the `provider` block under `cluster/main.tf` and run `terraform apply` to create your droplets. Terraform will create a user called `ec2-user` and move the SSH keys over, this way disabling SSH login for root. It also adds the `ec2-user` to the sudoers file, so after logging in as ec2-user you can `sudo` to `root`.
+Terraform can do this but it requires SSH agent running on the machine where terraform is run, with one of the SSH keys of the droplets added to the agent. (This will be neede for ansible too, so it's worth setting it up here. Check out the [ansible](https://github.com/tendermint/tools/tree/master/ansible) page for more information.)
+After setting up the SSH key, run `terraform apply` with `-var noroot=true` to create your droplets. Terraform will create a user called `ec2-user` and move the SSH keys over, this way disabling SSH login for root. It also adds the `ec2-user` to the sudoers file, so after logging in as ec2-user you can `sudo` to `root`.
 
-DigitalOcean announced firewalls but the current version of Terraform (0.9.6 as of this writing) does not support it yet. Fortunately it is quite easy to set it up through the web interface (and not that bad through the [RESTful API](https://developers.digitalocean.com/documentation/v2/#firewalls) either). When adding droplets to a firewall rule, you can add tags, so it's enough to define the testnet name. It is not necessary to add the nodes one-by-one. Also, the firewall rule "remembers" the testnet name tag so if you change the servers but keep the name, the firewall rules will still apply.
+DigitalOcean announced firewalls but the current version of Terraform (0.9.8 as of this writing) does not support it yet. Fortunately it is quite easy to set it up through the web interface (and not that bad through the [RESTful API](https://developers.digitalocean.com/documentation/v2/#firewalls) either). When adding droplets to a firewall rule, you can add tags. All droplets in a testnet are tagged with the testnet name so it's enough to define the testnet name in the firewall rule. It is not necessary to add the nodes one-by-one. Also, the firewall rule "remembers" the testnet name tag so if you change the servers but keep the name, the firewall rules will still apply.
 
 # What's next
 

terraform-digitalocean/cluster/main.tf

@@ -3,11 +3,9 @@ resource "digitalocean_tag" "cluster" {
 }
 
 resource "digitalocean_droplet" "cluster" {
-  # set the image and instance type
   name = "${var.name}-node${count.index}"
   image = "${var.image_id}"
   size = "${var.instance_size}"
-  # the `element` function handles modulo
   region = "${element(var.regions, count.index)}"
   ssh_keys = "${var.key_ids}"
   count = "${var.servers}"
@@ -21,23 +19,5 @@ resource "digitalocean_droplet" "cluster" {
     timeout = "30s"
   }
 
-#  #Additional SSH security: add ec2-user and remove root user credentials. You need to have ssh-agent running with your key for this to work.
-#  provisioner "remote-exec" {
-#    inline = [
-#      "useradd -m -s /bin/bash ec2-user",
-#      "echo 'ec2-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ec2-user",
-#      "cp -r /root/.ssh /home/ec2-user/.ssh",
-#      "chown -R ec2-user.ec2-user /home/ec2-user/.ssh",
-#      "chmod -R 700 /home/ec2-user/.ssh",
-#      "rm -rf /root/.ssh"
-#    ]
-#  }
-
 }
 
-#resource "digitalocean_floating_ip" "cluster" {
-#  droplet_id = "${element(digitalocean_droplet.cluster.*.id,count.index)}"
-#  region     = "${element(digitalocean_droplet.cluster.*.region,count.index)}"
-#  count      = "${var.servers}"
-#}
-

terraform-digitalocean/cluster/security.tf

@@ -0,0 +1,17 @@
+resource "null_resource" "cluster" {
+  count = "${ var.noroot ? var.servers : 0 }"
+  connection {
+    host = "${element(digitalocean_droplet.cluster.*.ipv4_address,count.index)}"
+  }
+  provisioner "remote-exec" {
+    inline = [
+      "useradd -m -s /bin/bash ec2-user",
+      "echo 'ec2-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ec2-user",
+      "cp -r /root/.ssh /home/ec2-user/.ssh",
+      "chown -R ec2-user.ec2-user /home/ec2-user/.ssh",
+      "chmod -R 700 /home/ec2-user/.ssh",
+      "rm -rf /root/.ssh"
+    ]
+  }
+}
+

terraform-digitalocean/cluster/variables.tf

@@ -28,3 +28,8 @@ variable "servers" {
   default     = 4
 }
 
+variable "noroot" {
+  description = "Set this variable to true, if you want SSH keys set for ec2-user instead of root."
+  default     = false
+}
+

terraform-digitalocean/main.tf

@@ -11,8 +11,16 @@ variable "TESTNET_NAME" {
 
 variable "ssh_keys" {
   description = "SSH keys provided in DigitalOcean to be used on the nodes"
-  # curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/account/keys"
-  default = ["9495227"]
+  # curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DO_API_TOKEN" "https://api.digitalocean.com/v2/account/keys"
+  default = [
+             "6259615",
+             "7658963",
+             "7668263",
+             "7668264",
+             "8036767",
+             "8163311",
+             "9495227"
+            ]
 }
 
 variable "servers" {
@@ -20,16 +28,21 @@ variable "servers" {
   default = "4"
 }
 
+variable "noroot" {
+  description = "Set this variable to true, if you want SSH keys set for ec2-user instead of root."
+  default     = false
+}
+
 provider "digitalocean" {
   token = "${var.DO_API_TOKEN}"
 }
 
-
 module "cluster" {
   source           = "./cluster"
   name             = "${var.TESTNET_NAME}"
   key_ids          = "${var.ssh_keys}"
   servers          = "${var.servers}"
+  noroot           = "${var.noroot}"
 }