diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..189c28b00 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.travis.yml +.idea +bin +.gitignore +integration_test +LICENSE +postgraphile +.private_blockchain_password +README.md +scripts +Supfile +test_config +.travis.yml +vulcanizedb.log +Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..4c853c43f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM golang:alpine as builder +RUN apk --update --no-cache add make git g++ + +# Build statically linked vDB binary (wonky path because of Dep) +RUN mkdir -p /go/src/github.com/vulcanize/vulcanizedb +ADD . /go/src/github.com/vulcanize/vulcanizedb +WORKDIR /go/src/github.com/vulcanize/vulcanizedb +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' . + +# Build migration tool +RUN go get -u -d github.com/pressly/goose/cmd/goose +WORKDIR /go/src/github.com/pressly/goose/cmd/goose +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose + +# Second stage +FROM alpine +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments/staging.toml /app/environments/ +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/startup_script.sh /app/ +COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/ +COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose + +WORKDIR /app +CMD ["./startup_script.sh"] diff --git a/cmd/root.go b/cmd/root.go index c0d7e7ef4..6b72c593f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "os" + "strings" "time" "github.com/ethereum/go-ethereum/ethclient" @@ -78,6 +79,9 @@ func database(cmd *cobra.Command, args []string) { func init() { cobra.OnInitialize(initConfig) + // When searching for env variables, replace dots in config keys with underscores + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.AutomaticEnv() rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location") rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name") diff --git a/dockerfiles/README.md b/dockerfiles/README.md new file mode 100644 index 000000000..5bf3c61ae --- /dev/null +++ b/dockerfiles/README.md @@ -0,0 +1,32 @@ +S +`Dockerfile` will build an alpine image containing: +- vDB as a binary with runtime deps statically linked: `/app/vulcanizedb` +- The migration tool goose: `/app/goose` +- Two services for running `lightSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`. + +By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`. + +## To use the container: +1. Setup a postgres database with superuser `vulcanize` +2. Set the env variables `DATABASE_NAME`, `DATABASE_HOSTNAME`, + `DATABASE_PORT`, `DATABASE_USER` & `DATABASE_PASSWORD` +3. Run the DB migrations: + * `./goose postgres "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable" +e` +4. Set `CLIENT_IPCPATH` to a node endpoint +5. Set the contract variables: + * `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...` + * `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"` + * `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching) +6. Start the `lightSync` and `continuousLogSync` services: + * `./vulcanizedb lightSync --config environments/staging.toml` + * `./vulcanizedb continuousLogSync --config environments/staging.toml` + +### Automated +The steps above have been rolled into a script: `/app/startup_script.sh`, which just assumes the DB env variables have been set, and defaults the rest to Kovan according to `environments/staging.toml`. This can be called with something like: + +`docker run -d -e DATABASE_NAME=vulcanize_public -e DATABASE_HOSTNAME=localhost -e DATABASE_PORT=5432 -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize m0ar/images:vDB` + +### Logging +When running, vDB services log to `/app/vulcanizedb.log`. + diff --git a/dockerfiles/startup_script.sh b/dockerfiles/startup_script.sh new file mode 100755 index 000000000..8373715a8 --- /dev/null +++ b/dockerfiles/startup_script.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Runs the migrations and starts the lightSync and continuousLogSync services + +# Exit if the variable tests fail +set -e + +# Check the database variables are set +test $DATABASE_NAME +test $DATABASE_HOSTNAME +test $DATABASE_PORT +test $DATABASE_USER +test $DATABASE_PASSWORD + +# Construct the connection string for postgres +CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable +echo "Connecting with: $CONNECT_STRING" + +set +e + +# Run the DB migrations +./goose postgres "$CONNECT_STRING" up +if [ $? -eq 0 ]; then + # Fire up the services + ./vulcanizedb lightSync --config environments/staging.toml & + ./vulcanizedb continuousLogSync --config environments/staging.toml & +else + echo "Could not run migrations. Are the database details correct?" +fi +wait diff --git a/environments/staging.toml b/environments/staging.toml index 8954a4839..ad320f279 100644 --- a/environments/staging.toml +++ b/environments/staging.toml @@ -1,8 +1,4 @@ [database] - name = "vulcanize_public" - hostname = "localhost" - user = "vulcanize" - password = "vulcanize" port = 5432 [client] diff --git a/pkg/datastore/postgres/postgres.go b/pkg/datastore/postgres/postgres.go index 82a1b2acb..dc43e3514 100644 --- a/pkg/datastore/postgres/postgres.go +++ b/pkg/datastore/postgres/postgres.go @@ -18,6 +18,7 @@ package postgres import ( "errors" + "github.com/sirupsen/logrus" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" //postgres driver @@ -40,6 +41,7 @@ var ( func NewDB(databaseConfig config.Database, node core.Node) (*DB, error) { connectString := config.DbConnectionString(databaseConfig) + logrus.Info("Using connection string: ", connectString) db, err := sqlx.Connect("postgres", connectString) if err != nil { return &DB{}, ErrDBConnectionFailed diff --git a/pkg/transformers/shared/constants/external.go b/pkg/transformers/shared/constants/external.go index 154f56f8c..5b14d4353 100644 --- a/pkg/transformers/shared/constants/external.go +++ b/pkg/transformers/shared/constants/external.go @@ -6,23 +6,7 @@ import ( "github.com/spf13/viper" ) -var initialized = false - -func initConfig() { - if initialized { - return - } - - if err := viper.ReadInConfig(); err == nil { - fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) - } else { - panic(fmt.Sprintf("Could not find environment file: %v", err)) - } - initialized = true -} - func getEnvironmentString(key string) string { - initConfig() value := viper.GetString(key) if value == "" { panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key)) @@ -30,11 +14,11 @@ func getEnvironmentString(key string) string { return value } +// Returns an int from the environment, defaulting to 0 if it does not exist func getEnvironmentInt64(key string) int64 { - initConfig() value := viper.GetInt64(key) if value == -1 { - panic(fmt.Sprintf("No environment configuration variable set for key: \"%v\"", key)) + return 0 } return value } diff --git a/postgraphile/.dockerignore b/postgraphile/.dockerignore new file mode 100644 index 000000000..1e5df4d80 --- /dev/null +++ b/postgraphile/.dockerignore @@ -0,0 +1,7 @@ +package-lock.json +yarn.lock +node_modules +Dockerfile +README.md +spec +.dockerignore diff --git a/postgraphile/Dockerfile b/postgraphile/Dockerfile new file mode 100644 index 000000000..f2785e366 --- /dev/null +++ b/postgraphile/Dockerfile @@ -0,0 +1,9 @@ +FROM mhart/alpine-node:10 +RUN apk --update --no-cache add make g++ python findutils postgresql-dev + +WORKDIR /app +COPY . /app +run yarn install +RUN ["./node_modules/typescript/bin/tsc"] +EXPOSE 3000 +CMD ["node", "/app/build/dist/index.js"] diff --git a/postgraphile/README.md b/postgraphile/README.md index 684247dac..0bd5958c9 100644 --- a/postgraphile/README.md +++ b/postgraphile/README.md @@ -2,6 +2,18 @@ This application utilizes Postgraphile to expose GraphQL endpoints for exposure of the varied data that VulcanizeDB tracks. +## Docker use +_Note: currently this image is ~500MB large (unpacked)_ + +Build the docker image in this directory. Start the `GraphiQL` frontend by: +* Setting the env variables for the database connection: `DATABASE_HOST`, + `DATABASE_NAME`, `DATABASE_USER`, `DATABASE_PASSWORD` (and optionally + `DATABASE_PORT` if running on non-standard port). + * The specified user needs to be `superuser` on the vulcanizeDB database +* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=vulcanize_public -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize -d m0ar/images:postgraphile-alpine`) +* GraphiQL is available at `:3000/graphiql` + + ## Building *This application assumes the use of the [Yarn package manager](https://yarnpkg.com/en/). The use of npm may produce unexpected results.* @@ -16,4 +28,4 @@ Provide the built bundle to node as a runnable script: `node ./build/dist/vulcan ## Testing -Tests are executed via Jasmine with a console reporter via the `yarn test` task. \ No newline at end of file +Tests are executed via Jasmine with a console reporter via the `yarn test` task. diff --git a/postgraphile/package.json b/postgraphile/package.json index 9c3dc3fac..9809865a5 100644 --- a/postgraphile/package.json +++ b/postgraphile/package.json @@ -22,22 +22,22 @@ "homepage": "https://github.com/vulcanize/vulcanizedb", "dependencies": { "express-session": "1.15.6", + "graphql-subscriptions": "0.5.8", "lodash": "4.17.10", "passport": "0.4.0", + "pg": "6.4.2", "pg-native": "3.0.0", "postgraphile": "4.0.0-rc.4", - "graphql-subscriptions": "0.5.8", "subscriptions-transport-ws": "0.9.14", - "toml": "2.3.3", - "pg": "6.4.2" + "toml": "2.3.3" }, "devDependencies": { - "@types/graphql": "^0.13.4", "@types/express": "4.16.0", "@types/express-session": "1.15.10", + "@types/graphql": "^0.13.4", "@types/jasmine": "2.8.8", "@types/lodash": "4.14.116", - "@types/node": "10.9.3", + "@types/node": "^10.12.21", "@types/passport": "0.4.6", "awesome-typescript-loader": "5.2.0", "jasmine": "3.2.0", diff --git a/postgraphile/src/config/parse.ts b/postgraphile/src/config/parse.ts index 762ba18b8..513aa4fe7 100644 --- a/postgraphile/src/config/parse.ts +++ b/postgraphile/src/config/parse.ts @@ -6,34 +6,49 @@ export const MISSING_PATH_MESSAGE = `No path to config toml file provided, ` + `please check the value of ${CONFIG_PATH_KEY} in your environment`; export const MISSING_HOST_MESSAGE = 'No database host provided in config toml'; -export const MISSING_DATABASE_MESSAGE = 'No database name provided in config ' - + 'toml'; +export const MISSING_USER_MESSAGE = 'No database user & password provided in config toml'; +export const MISSING_DATABASE_MESSAGE = 'No database name provided in config toml'; export function parseConfig( readCallback: ReadFileSyncCallback, tomlParseCallback: TomlParseCallback, configPath?: string ): DatabaseConfig { - if (!configPath || configPath.length < 1) { - throw new Error(MISSING_PATH_MESSAGE); + let host = ''; + let port = ''; + let database = ''; + let user = ''; + let password = ''; + + if (configPath) { + const tomlContents = readCallback(`${configPath}`).toString(); + const parsedToml = tomlParseCallback(tomlContents); + + host = parsedToml['database']['hostname']; + port = parsedToml['database']['port']; + database = parsedToml['database']['name']; + user = parsedToml['database']['user']; + password = parsedToml['database']['password']; } - const tomlContents = readCallback(`${configPath}`).toString(); - const parsedToml = tomlParseCallback(tomlContents); + // Overwrite config values with env. vars if such are set + host = process.env.DATABASE_HOST || host; + port = process.env.DATABASE_PORT || port; + database = process.env.DATABASE_NAME || database; + user = process.env.DATABASE_USER || user; + password = process.env.DATABASE_PASSWORD || password; - const host = parsedToml['database']['hostname']; - const port = parsedToml['database']['port']; - const database = parsedToml['database']['name']; - const user = parsedToml['database']['user'] || ''; - const password = parsedToml['database']['password'] || ''; - - if (!host || host.length < 1) { + if (!host) { throw new Error(MISSING_HOST_MESSAGE); } - if (!database || database.length < 1) { + if (!database) { throw new Error(MISSING_DATABASE_MESSAGE); } + if (!user || !password) { + throw new Error(MISSING_USER_MESSAGE); + } + return { host: `postgres://${user}:${password}@${host}:${port}`, database }; } diff --git a/postgraphile/yarn.lock b/postgraphile/yarn.lock index 825c7c4e7..c0e913de3 100644 --- a/postgraphile/yarn.lock +++ b/postgraphile/yarn.lock @@ -108,9 +108,10 @@ version "10.9.4" resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.4.tgz#0f4cb2dc7c1de6096055357f70179043c33e9897" -"@types/node@10.9.3": - version "10.9.3" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.3.tgz#85f288502503ade0b3bfc049fe1777b05d0327d5" +"@types/node@^10.12.21": + version "10.12.21" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.21.tgz#7e8a0c34cf29f4e17a36e9bd0ea72d45ba03908e" + integrity sha512-CBgLNk4o3XMnqMc0rhb6lc77IwShMEglz05deDcn2lQxyXEZivfwgYJu7SMha9V5XcrP6qZuevTHV/QrN2vjKQ== "@types/passport@0.4.6": version "0.4.6"