diff --git a/docs/content/_data/config/pipes/snowflake/common.yaml b/docs/content/_data/config/pipes/snowflake/common.yaml index 716ad99..9fd9a8a 100644 --- a/docs/content/_data/config/pipes/snowflake/common.yaml +++ b/docs/content/_data/config/pipes/snowflake/common.yaml @@ -12,12 +12,5 @@ fields: description: Snowflake options to specify such as key = value pairs. These options are then passed as option to the Spark connector for Snowflake example: Options { Database = "example_database" Schema = "example_schema" - Table = "example_table" - Mode = "append" - User = "username" - Url = "account_address" - pem_private_key = "private_key" - sfWarehouse = "warehouse" - sfRole = "role" - column_mapping = "name" + ... } diff --git a/docs/content/_data/config/pipes/snowflake/streaming.yaml b/docs/content/_data/config/pipes/snowflake/streaming.yaml new file mode 100644 index 0000000..3b9c47d --- /dev/null +++ b/docs/content/_data/config/pipes/snowflake/streaming.yaml @@ -0,0 +1,21 @@ +output: + type: com.amadeus.dataio.pipes.snowflake.streaming.SnowflakeOutput + fields: + - name: Duration + mandatory: "No" + description: Sets the trigger for the stream query. Controls the trigger() Spark function. + example: Duration = "60 seconds" + - name: Timeout + mandatory: "Yes" + description: Controls the amount of time before returning from the streaming query, in hours. It can be a String or an Int. + example: Timeout = 24 + - name: Mode + mandatory: "Yes" + description: The Spark Structured Streaming output mode. + example: Mode = "complete" + default: append + - name: AddTimestampOnInsert + mandatory: "No" + description: Add a column named "timestamp" containing the current timestamp at the start of query evaluation + example: AddTimestampOnInsert = "true" + default: false diff --git a/docs/content/configuration/pipes/snowflake.md b/docs/content/configuration/pipes/snowflake.md new file mode 100644 index 0000000..42c5a55 --- /dev/null +++ b/docs/content/configuration/pipes/snowflake.md @@ -0,0 +1,10 @@ +--- +title: Snowflake +layout: default +grand_parent: Configuration +parent: Pipes +--- + +# Snowflake +{% include pipe_description.md pipe=site.data.config.pipes.snowflake %} +