--- apiVersion: v1 kind: Service metadata: annotations: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" name: basecoin labels: app: basecoin spec: ports: - port: 46656 name: p2p - port: 46657 name: rpc clusterIP: None selector: app: tm --- apiVersion: v1 kind: ConfigMap metadata: name: tm-config data: seeds: "tm-0,tm-1,tm-2,tm-3" validators: "tm-0,tm-1,tm-2,tm-3" validator.power: "10" genesis.json: |- { "genesis_time": "2016-02-05T06:02:31.526Z", "chain_id": "chain-tTH4mi", "validators": [], "app_hash": "" } pub_key_nginx.conf: |- server { listen 80 default_server; listen [::]:80 default_server ipv6only=on; location /pub_key.json { root /usr/share/nginx/; } location /app_pub_key.json { root /usr/share/nginx/; } } --- apiVersion: v1 kind: ConfigMap metadata: name: app-config data: genesis.json: |- { "chain_id": "chain-tTH4mi", "app_options": { "accounts": [ { "pub_key": "tm-0", "coins": [ { "denom": "mycoin", "amount": 1000000000 } ] }, { "pub_key": "tm-1", "coins": [ { "denom": "mycoin", "amount": 1000000000 } ] }, { "pub_key": "tm-2", "coins": [ { "denom": "mycoin", "amount": 1000000000 } ] }, { "pub_key": "tm-3", "coins": [ { "denom": "mycoin", "amount": 1000000000 } ] } ] } } --- apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: name: tm-budget spec: selector: matchLabels: app: tm minAvailable: 2 --- apiVersion: apps/v1beta1 kind: StatefulSet metadata: name: tm spec: serviceName: basecoin replicas: 4 template: metadata: labels: app: tm annotations: pod.beta.kubernetes.io/init-containers: '[{ "name": "tm-gen-validator", "image": "tendermint/tendermint:0.10.0", "imagePullPolicy": "IfNotPresent", "command": ["bash", "-c", " set -ex\n if [ ! -f /tendermint/priv_validator.json ]; then\n tendermint gen_validator > /tendermint/priv_validator.json\n # pub_key.json will be served by pub-key container\n cat /tendermint/priv_validator.json | jq \".pub_key\" > /tendermint/pub_key.json\n fi\n "], "volumeMounts": [ {"name": "tmdir", "mountPath": "/tendermint"} ] }, { "name": "app-gen-key", "image": "tendermint/basecoin:0.5.1", "imagePullPolicy": "IfNotPresent", "command": ["bash", "-c", " set -ex\n if [ ! -f /app/key.json ]; then\n basecoin key new > /app/key.json\n # pub_key.json will be served by app-pub-key container\n cat /app/key.json | jq \".pub_key\" > /app/pub_key.json\n fi\n "], "volumeMounts": [ {"name": "appdir", "mountPath": "/app"} ] }]' spec: containers: - name: tm imagePullPolicy: IfNotPresent image: tendermint/tendermint:0.10.0 ports: - containerPort: 46656 name: p2p - containerPort: 46657 name: rpc env: - name: SEEDS valueFrom: configMapKeyRef: name: tm-config key: seeds - name: VALIDATOR_POWER valueFrom: configMapKeyRef: name: tm-config key: validator.power - name: VALIDATORS valueFrom: configMapKeyRef: name: tm-config key: validators - name: TMHOME value: /tendermint command: - bash - "-c" - | set -ex # copy template cp /etc/tendermint/genesis.json /tendermint/genesis.json # fill genesis file with validators IFS=',' read -ra VALS_ARR <<< "$VALIDATORS" fqdn_suffix=$(hostname -f | sed 's#[^.]*\.\(\)#\1#') for v in "${VALS_ARR[@]}"; do # wait until validator generates priv/pub key pair set +e curl -s --fail "http://$v.$fqdn_suffix/pub_key.json" > /dev/null ERR=$? while [ "$ERR" != 0 ]; do sleep 5 curl -s --fail "http://$v.$fqdn_suffix/pub_key.json" > /dev/null ERR=$? done set -e # add validator to genesis file along with its pub_key curl -s "http://$v.$fqdn_suffix/pub_key.json" | jq ". as \$k | {pub_key: \$k, amount: $VALIDATOR_POWER, name: \"$v\"}" > pub_validator.json cat /tendermint/genesis.json | jq ".validators |= .+ [$(cat pub_validator.json)]" > tmpgenesis && mv tmpgenesis /tendermint/genesis.json rm pub_validator.json done # construct seeds IFS=',' read -ra SEEDS_ARR <<< "$SEEDS" seeds=() for s in "${SEEDS_ARR[@]}"; do seeds+=("$s.$fqdn_suffix:46656") done seeds=$(IFS=','; echo "${seeds[*]}") tendermint node --p2p.seeds="$seeds" --moniker="`hostname`" --proxy_app="unix:///socks/app.sock" volumeMounts: - name: tmdir mountPath: /tendermint - mountPath: /etc/tendermint/genesis.json name: tmconfigdir subPath: genesis.json - name: socksdir mountPath: /socks - name: app imagePullPolicy: IfNotPresent image: tendermint/basecoin:0.5.1 env: - name: BCHOME value: /app workingDir: /app command: - bash - "-c" - | set -ex # replace "tm-N" with public keys in genesis file cp /etc/app/genesis.json genesis.json fqdn_suffix=$(hostname -f | sed 's#[^.]*\.\(\)#\1#') # for every "base/account" i=0 length=$(cat genesis.json | jq ".app_options.accounts | length") while [[ $i -lt $length ]]; do # extract pod name ("tm-0") pod=$(cat genesis.json | jq -r ".app_options.accounts[$i].pub_key") # wait until pod starts to serve its pub_key set +e curl -s --fail "http://$pod.$fqdn_suffix/app_pub_key.json" > /dev/null ERR=$? while [ "$ERR" != 0 ]; do sleep 5 curl -s --fail "http://$pod.$fqdn_suffix/app_pub_key.json" > /dev/null ERR=$? done set -e # get its pub_key curl -s "http://$pod.$fqdn_suffix/app_pub_key.json" | jq "." > k.json # replace pod name with it ("tm-0" => "{"type": ..., "data": ...}") cat genesis.json | jq ".app_options.accounts[$i].pub_key = $(cat k.json | jq '.')" > tmpgenesis && mv tmpgenesis genesis.json rm -f k.json i=$((i+1)) done rm -f /socks/app.sock # remove old socket basecoin start --address="unix:///socks/app.sock" --without-tendermint volumeMounts: - name: appdir mountPath: /app - mountPath: /etc/app/genesis.json name: appconfigdir subPath: genesis.json - name: socksdir mountPath: /socks - name: pub-key imagePullPolicy: IfNotPresent image: nginx:latest ports: - containerPort: 80 command: - bash - "-c" - | set -ex # fixes 403 Permission Denied (open() "/tendermint/pub_key.json" failed (13: Permission denied)) # => we cannot serve from /tendermint, so we copy the file mkdir -p /usr/share/nginx cp /tendermint/pub_key.json /usr/share/nginx/pub_key.json cp /app/pub_key.json /usr/share/nginx/app_pub_key.json nginx -g "daemon off;" volumeMounts: - name: tmdir mountPath: /tendermint - name: appdir mountPath: /app - mountPath: /etc/nginx/conf.d/pub_key.conf name: tmconfigdir subPath: pub_key_nginx.conf volumes: - name: tmconfigdir configMap: name: tm-config - name: appconfigdir configMap: name: app-config - name: socksdir emptyDir: {} volumeClaimTemplates: - metadata: name: tmdir annotations: volume.alpha.kubernetes.io/storage-class: anything spec: accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 2Gi - metadata: name: appdir annotations: volume.alpha.kubernetes.io/storage-class: anything spec: accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 12Mi