Merge pull request #6 from 8thlight/VDB-337-Dockerize-vDB

VDB-337 Dockerize vDB

.dockerignore

@@ -0,0 +1,16 @@
+.git
+.travis.yml
+.idea
+bin
+.gitignore
+integration_test
+LICENSE
+postgraphile
+.private_blockchain_password
+README.md
+scripts
+Supfile
+test_config
+.travis.yml
+vulcanizedb.log
+Dockerfile

Dockerfile

@@ -0,0 +1,24 @@
+FROM golang:alpine as builder
+RUN apk --update --no-cache add make git g++
+
+# Build statically linked vDB binary (wonky path because of Dep)
+RUN mkdir -p /go/src/github.com/vulcanize/vulcanizedb
+ADD . /go/src/github.com/vulcanize/vulcanizedb
+WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
+
+# Build migration tool
+RUN go get -u -d github.com/pressly/goose/cmd/goose
+WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose
+
+# Second stage
+FROM alpine
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments/staging.toml /app/environments/
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/startup_script.sh /app/
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/
+COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose
+
+WORKDIR /app
+CMD ["./startup_script.sh"]

cmd/root.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -78,6 +79,9 @@ func database(cmd *cobra.Command, args []string) {
 
 func init() {
 	cobra.OnInitialize(initConfig)
+	// When searching for env variables, replace dots in config keys with underscores
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
 	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")

dockerfiles/README.md

@@ -0,0 +1,32 @@
+S
+`Dockerfile` will build an alpine image containing:
+- vDB as a binary with runtime deps statically linked: `/app/vulcanizedb`
+- The migration tool goose: `/app/goose`
+- Two services for running `lightSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
+
+By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`.
+
+## To use the container:
+1. Setup a postgres database with superuser `vulcanize`
+2. Set the env variables `DATABASE_NAME`, `DATABASE_HOSTNAME`,
+  `DATABASE_PORT`, `DATABASE_USER` & `DATABASE_PASSWORD`
+3. Run the DB migrations:
+  * `./goose postgres "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable"
+e`
+4. Set `CLIENT_IPCPATH` to a node endpoint
+5. Set the contract variables:
+  * `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...`
+  * `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"`
+  * `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching)
+6. Start the `lightSync` and `continuousLogSync` services:
+  * `./vulcanizedb lightSync --config environments/staging.toml`
+  * `./vulcanizedb continuousLogSync --config environments/staging.toml`
+
+### Automated
+The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like:
+
+`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize m0ar/images:vDB`
+
+### Logging
+When running, vDB services log to `/app/vulcanizedb.log`.
+

dockerfiles/startup_script.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Runs the migrations and starts the lightSync and continuousLogSync services
+
+# Exit if the variable tests fail
+set -e
+
+# Check the database variables are set
+test $DATABASE_NAME
+test $DATABASE_HOSTNAME
+test $DATABASE_PORT
+test $DATABASE_USER
+test $DATABASE_PASSWORD
+
+# Construct the connection string for postgres
+CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+echo "Connecting with: $CONNECT_STRING"
+
+set +e
+
+# Run the DB migrations
+./goose postgres "$CONNECT_STRING" up
+if [ $? -eq 0 ]; then
+  # Fire up the services
+  ./vulcanizedb lightSync --config environments/staging.toml &
+  ./vulcanizedb continuousLogSync --config environments/staging.toml &
+else
+  echo "Could not run migrations. Are the database details correct?"
+fi
+wait

environments/staging.toml

@@ -1,8 +1,4 @@
 [database]
-    name = "vulcanize_public"
-    hostname = "localhost"
-    user = "vulcanize"
-    password = "vulcanize"
     port = 5432
 
 [client]

pkg/datastore/postgres/postgres.go

@@ -18,6 +18,7 @@ package postgres
 
 import (
 	"errors"
+	"github.com/sirupsen/logrus"
 
 	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq" //postgres driver
@@ -40,6 +41,7 @@ var (
 
 func NewDB(databaseConfig config.Database, node core.Node) (*DB, error) {
 	connectString := config.DbConnectionString(databaseConfig)
+	logrus.Info("Using connection string: ", connectString)
 	db, err := sqlx.Connect("postgres", connectString)
 	if err != nil {
 		return &DB{}, ErrDBConnectionFailed

pkg/transformers/shared/constants/external.go

@@ -6,35 +6,19 @@ import (
 	"github.com/spf13/viper"
 )
 
-var initialized = false
-
-func initConfig() {
-	if initialized {
-		return
-	}
-
-	if err := viper.ReadInConfig(); err == nil {
-		fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
-	} else {
-		panic(fmt.Sprintf("Could not find environment file: %v", err))
-	}
-	initialized = true
-}
-
 func getEnvironmentString(key string) string {
-	initConfig()
 	value := viper.GetString(key)
 	if value == "" {
 		panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key))
 	}
 	return value
 }
 
+// Returns an int from the environment, defaulting to 0 if it does not exist
 func getEnvironmentInt64(key string) int64 {
-	initConfig()
 	value := viper.GetInt64(key)
 	if value == -1 {
-		panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key))
+		return 0
 	}
 	return value
 }

postgraphile/.dockerignore

@@ -0,0 +1,7 @@
+package-lock.json
+yarn.lock
+node_modules
+Dockerfile
+README.md
+spec
+.dockerignore

postgraphile/Dockerfile

@@ -0,0 +1,9 @@
+FROM mhart/alpine-node:10
+RUN apk --update --no-cache add make g++ python findutils postgresql-dev
+
+WORKDIR /app
+COPY . /app
+run yarn install
+RUN ["./node_modules/typescript/bin/tsc"]
+EXPOSE 3000
+CMD ["node", "/app/build/dist/index.js"]

postgraphile/README.md

@@ -2,6 +2,18 @@
 
 This application utilizes Postgraphile to expose GraphQL endpoints for exposure of the varied data that VulcanizeDB tracks.
 
+## Docker use
+_Note: currently this image is ~500MB large (unpacked)_
+
+Build the docker image in this directory. Start the `GraphiQL` frontend by:
+* Setting the env variables for the database connection: `DATABASE_HOST`,
+  `DATABASE_NAME`, `DATABASE_USER`, `DATABASE_PASSWORD` (and optionally
+  `DATABASE_PORT` if running on non-standard port).
+  * The specified user needs to be `superuser` on the vulcanizeDB database
+* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=vulcanize_public -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize -d m0ar/images:postgraphile-alpine`)
+* GraphiQL is available at `:3000/graphiql`
+
+
 ## Building
 
 *This application assumes the use of the [Yarn package manager](https://yarnpkg.com/en/). The use of npm may produce unexpected results.*
@@ -16,4 +28,4 @@ Provide the built bundle to node as a runnable script: `node ./build/dist/vulcan
 
 ## Testing
 
-Tests are executed via Jasmine with a console reporter via the `yarn test` task.
+Tests are executed via Jasmine with a console reporter via the `yarn test` task.

postgraphile/package.json

@@ -22,22 +22,22 @@
   "homepage": "https://github.com/vulcanize/vulcanizedb",
   "dependencies": {
     "express-session": "1.15.6",
+    "graphql-subscriptions": "0.5.8",
     "lodash": "4.17.10",
     "passport": "0.4.0",
+    "pg": "6.4.2",
     "pg-native": "3.0.0",
     "postgraphile": "4.0.0-rc.4",
-    "graphql-subscriptions": "0.5.8",
     "subscriptions-transport-ws": "0.9.14",
-    "toml": "2.3.3",
-    "pg": "6.4.2"
+    "toml": "2.3.3"
   },
   "devDependencies": {
-    "@types/graphql": "^0.13.4",
     "@types/express": "4.16.0",
     "@types/express-session": "1.15.10",
+    "@types/graphql": "^0.13.4",
     "@types/jasmine": "2.8.8",
     "@types/lodash": "4.14.116",
-    "@types/node": "10.9.3",
+    "@types/node": "^10.12.21",
     "@types/passport": "0.4.6",
     "awesome-typescript-loader": "5.2.0",
     "jasmine": "3.2.0",

postgraphile/src/config/parse.ts

@@ -6,34 +6,49 @@ export const MISSING_PATH_MESSAGE = `No path to config toml file provided, `
   + `please check the value of ${CONFIG_PATH_KEY} in your environment`;
 
 export const MISSING_HOST_MESSAGE = 'No database host provided in config toml';
-export const MISSING_DATABASE_MESSAGE = 'No database name provided in config '
-  + 'toml';
+export const MISSING_USER_MESSAGE = 'No database user & password provided in config toml';
+export const MISSING_DATABASE_MESSAGE = 'No database name provided in config toml';
 
 export function parseConfig(
   readCallback: ReadFileSyncCallback,
   tomlParseCallback: TomlParseCallback,
   configPath?: string
 ): DatabaseConfig {
-  if (!configPath || configPath.length < 1) {
-    throw new Error(MISSING_PATH_MESSAGE);
+  let host = '';
+  let port = '';
+  let database = '';
+  let user = '';
+  let password = '';
+
+  if (configPath) {
+      const tomlContents = readCallback(`${configPath}`).toString();
+      const parsedToml = tomlParseCallback(tomlContents);
+
+      host = parsedToml['database']['hostname'];
+      port = parsedToml['database']['port'];
+      database = parsedToml['database']['name'];
+      user = parsedToml['database']['user'];
+      password = parsedToml['database']['password'];
   }
 
-  const tomlContents = readCallback(`${configPath}`).toString();
-  const parsedToml = tomlParseCallback(tomlContents);
+  // Overwrite config values with env. vars if such are set
+  host = process.env.DATABASE_HOST || host;
+  port = process.env.DATABASE_PORT || port;
+  database = process.env.DATABASE_NAME || database;
+  user = process.env.DATABASE_USER || user;
+  password = process.env.DATABASE_PASSWORD || password;
 
-  const host = parsedToml['database']['hostname'];
-  const port = parsedToml['database']['port'];
-  const database = parsedToml['database']['name'];
-  const user = parsedToml['database']['user'] || '';
-  const password = parsedToml['database']['password'] || '';
-
-  if (!host || host.length < 1) {
+  if (!host) {
     throw new Error(MISSING_HOST_MESSAGE);
   }
 
-  if (!database || database.length < 1) {
+  if (!database) {
     throw new Error(MISSING_DATABASE_MESSAGE);
   }
 
+  if (!user || !password) {
+    throw new Error(MISSING_USER_MESSAGE);
+  }
+
   return { host: `postgres://${user}:${password}@${host}:${port}`, database };
 }

postgraphile/yarn.lock

@@ -108,9 +108,10 @@
   version "10.9.4"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.4.tgz#0f4cb2dc7c1de6096055357f70179043c33e9897"
 
-"@types/node@10.9.3":
-  version "10.9.3"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.3.tgz#85f288502503ade0b3bfc049fe1777b05d0327d5"
+"@types/node@^10.12.21":
+  version "10.12.21"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.21.tgz#7e8a0c34cf29f4e17a36e9bd0ea72d45ba03908e"
+  integrity sha512-CBgLNk4o3XMnqMc0rhb6lc77IwShMEglz05deDcn2lQxyXEZivfwgYJu7SMha9V5XcrP6qZuevTHV/QrN2vjKQ==
 
 "@types/passport@0.4.6":
   version "0.4.6"