Updated instructions for local deployment (#1582)

List of changes:
1) Created a new script from "docker-compose_test.sh" that allows
   users to deploy, and later cleanup, the KeyTransparency service
   locally;
2) Updated the README's related section with the new script's usage;
3) Substituted the instruction `go get github.com/google/keytransparency/...`
   which fails due to missing (autogenerated ?) files, with the
   equivalent `git clone` version;
4) Fixed the key file existance check in script "docker-compose_test.sh";

Author: gmichelo

README.md

@@ -176,33 +176,45 @@ Prerequisites
    - Docker Engine 1.17.6+ `docker version -f '{{.Server.APIVersion}}'`
    - Docker Compose 1.11.0+ `docker-compose --version`
 
-```sh
-go get github.com/google/keytransparency/...
-cd $(go env GOPATH)/src/github.com/google/keytransparency
-
-# Generate Private Keys
-./scripts/gen_monitor_keys.sh -f
-pushd genfiles
-go run "$(go env GOROOT)/src/crypto/tls/generate_cert.go" --host localhost,127.0.0.1,::
-popd
+### Deploy the KeyTransparency service
+
+1. Run the deployment script
+   ```sh
+   # Download the latest version of keytransparency
+   git clone https://github.com/google/keytransparency.git
+   cd keytransparency
+
+   # Run the deployment script for local environment
+   ./scripts/deploy_local.sh deploy
+   ```
+
+2. Check Docker's running containers
+   ```sh
+   docker container ls
+   ```
+   You should see 8 new running containers:
+   - gcr.io/key-transparency/keytransparency-monitor
+   - gcr.io/key-transparency/keytransparency-sequencer
+   - gcr.io/trillian-opensource-ci/map_server
+   - gcr.io/trillian-opensource-ci/log_signer
+   - gcr.io/trillian-opensource-ci/log_server
+   - gcr.io/key-transparency/keytransparency-server
+   - gcr.io/trillian-opensource-ci/db_server
+   - prom/prometheus
+
+3. Watch it Run
+- [Proof for foo@bar.com](https://localhost/v1/directories/default/users/foo@bar.com)
+- [Server configuration info](https://localhost/v1/directories/default)
 
-# Build Docker Images
-export TRAVIS_COMMIT=$(git rev-parse HEAD)
-docker-compose build --parallel
 
-# Run
-docker-compose -f docker-compose.yml docker-compose.prod.yml up -d
+### Terminate the KeyTransparency service
 
-# Create directory
-docker run -t --network kt_attachable gcr.io/key-transparency/init:${TRAVIS_COMMIT} sequencer:8080 -- curl -k -X POST https://sequencer:8080/v1/directories -d'{"directory_id":"default","min_interval":"1s","max_interval":"60s"}'
+The script will remove all the containers and their networks.
+```sh
+# Run the script to undeploy
+./scripts/deploy_local.sh undeploy
 ```
 
-2. Watch it Run
-- [Proof for foo@bar.com](https://localhost/v1/directories/default/users/foo@bar.com)
-- [Server configuration info](https://localhost/v1/directories/default)
-
-3. [Integration test](scripts/docker-compose_test.sh) for Docker Compose
-
 ## Development and Testing
 Key Transparency and its [Trillian](https://github.com/google/trillian) backend
 use a [MySQL database](https://github.com/google/trillian/blob/master/README.md#mysql-setup),

scripts/deploy_local.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -e
+set -o pipefail
+
+usage() {
+  echo "$(basename $0) deploy | undeploy"
+  echo "  deploy: deploys the KeyTransparency server locally"
+  echo "  undeploy: undeploys the KeyTransparency server"
+}
+
+function deploy() {
+	# Generate server's keys
+	if [ ! -f genfiles/key.pem ]; then
+		./scripts/gen_monitor_keys.sh -f
+		cd genfiles
+		go run "$(go env GOROOT)/src/crypto/tls/generate_cert.go" --host localhost,127.0.0.1,::
+		cd -
+	fi
+
+	# Start a docker swarm if not part of it
+	case "$(docker info --format '{{.Swarm.LocalNodeState}}')" in
+	active)
+		echo "Node is already in swarm cluster";;
+	*)
+		docker swarm init;;
+	esac
+
+	# Build the service's image
+	export TRAVIS_COMMIT=$(git rev-parse HEAD)
+	docker-compose build --parallel
+
+	# Deploy the set of services
+	docker stack deploy -c docker-compose.yml -c docker-compose.prod.yml kt
+	./scripts/docker-stack-wait.sh -t 180 -n sequencer kt
+	docker run -t --network kt_attachable gcr.io/key-transparency/init:${TRAVIS_COMMIT} sequencer:8080 -- curl -k -X POST https://sequencer:8080/v1/directories -d'{"directory_id":"default","min_interval":"1s","max_interval":"60s"}'
+	./scripts/docker-stack-wait.sh -t 180 kt
+
+	wget -T 60 --spider --retry-connrefused --waitretry=1 http://localhost:8081/readyz
+	wget -T 60 -O /dev/null --no-check-certificate  \
+		--retry-connrefused --waitretry=1 \
+		--retry-on-http-error=405,404,503 \
+		https://localhost/v1/directories/default
+
+	PASSWORD="foobar"
+	go run ./cmd/keytransparency-client authorized-keys create-keyset --password=${PASSWORD}
+	go run ./cmd/keytransparency-client post foo@bar.com \
+		--insecure \
+		--data='dGVzdA==' \
+		--password=${PASSWORD} \
+		--kt-url=localhost:443 \
+		--verbose \
+		--timeout=2m \
+		--logtostderr
+}
+
+function undeploy() {
+	# Remove the stack "kt"
+	docker stack rm kt
+}
+
+# Start a docker swarm if not part of it
+case "$1" in
+deploy)
+	deploy;;
+undeploy)
+	undeploy;;
+*)
+	usage;;
+esac

scripts/docker-compose_test.sh

@@ -2,7 +2,7 @@
 set -ex
 set -o pipefail
 
-if [ ! -f genfiles/server.key ]; then
+if [ ! -f genfiles/key.pem ]; then
 	./scripts/gen_monitor_keys.sh -f
 	cd genfiles
 	go run "$(go env GOROOT)/src/crypto/tls/generate_cert.go" --host localhost,127.0.0.1,::