From f5301b6e80f6ca4dd73a791e620b39264b139f74 Mon Sep 17 00:00:00 2001 From: Jon Udell Date: Wed, 13 Nov 2024 20:13:00 +0700 Subject: [PATCH] cleanup --- docs/faq/index.md | 10 - docs/flowpipe-hcl/functions.md | 243 --------- docs/flowpipe-hcl/index.md | 42 -- docs/flowpipe-hcl/locals.md | 38 -- docs/flowpipe-hcl/mod.md | 131 ----- docs/flowpipe-hcl/pipeline.md | 103 ---- docs/flowpipe-hcl/step/container.md | 213 -------- docs/flowpipe-hcl/step/email.md | 55 -- docs/flowpipe-hcl/step/function.md | 121 ----- docs/flowpipe-hcl/step/http.md | 73 --- docs/flowpipe-hcl/step/index.md | 324 ------------ docs/flowpipe-hcl/step/input.md | 498 ------------------ docs/flowpipe-hcl/step/message.md | 41 -- docs/flowpipe-hcl/step/pipeline.md | 39 -- docs/flowpipe-hcl/step/query.md | 282 ---------- docs/flowpipe-hcl/step/sleep.md | 55 -- docs/flowpipe-hcl/step/transform.md | 47 -- docs/flowpipe-hcl/trigger/email.md.todo | 29 - docs/flowpipe-hcl/trigger/http.md | 124 ----- docs/flowpipe-hcl/trigger/index.md | 27 - docs/flowpipe-hcl/trigger/query.md | 235 --------- docs/flowpipe-hcl/trigger/schedule.md | 78 --- docs/flowpipe-hcl/variable.md | 109 ---- .../cli/{integration.md => collect.md} | 0 docs/reference/cli/help.md | 8 +- docs/reference/cli/index.md | 19 +- docs/reference/cli/notifier.md | 42 -- docs/reference/cli/pipeline.md | 109 ---- docs/reference/cli/plugin.md | 100 ++++ docs/reference/cli/process.md | 79 --- docs/reference/cli/{mod.md => query.md} | 0 docs/reference/cli/server.md | 72 --- docs/reference/cli/trigger.md | 70 --- docs/reference/cli/variable.md | 63 --- docs/reference/cli/variable.md.todo | 36 -- docs/reference/env-vars/flowpipe_base_url.md | 25 - .../env-vars/flowpipe_config_path.md | 28 - docs/reference/env-vars/flowpipe_host.md | 22 - .../env-vars/flowpipe_install_dir.md | 23 - docs/reference/env-vars/flowpipe_listen.md | 23 - docs/reference/env-vars/flowpipe_log_level.md | 26 - .../flowpipe_max_concurrency_container.md | 22 - .../flowpipe_max_concurrency_function.md | 22 - .../env-vars/flowpipe_max_concurrency_http.md | 22 - .../flowpipe_max_concurrency_query.md | 22 - .../env-vars/flowpipe_memory_max_mb.md | 21 - .../env-vars/flowpipe_mod_location.md | 14 - docs/reference/env-vars/flowpipe_port.md | 15 - docs/reference/env-vars/flowpipe_telemetry.md | 24 - .../env-vars/flowpipe_update_check.md | 22 - docs/reference/env-vars/flowpipe_workspace.md | 29 - docs/reference/env-vars/foo | 16 + docs/reference/env-vars/index.md | 49 +- docs/reference/env-vars/pipes_install_dir.md | 23 - docs/run/plugins.md | 9 + docs/sidebar.json | 182 +------ 56 files changed, 142 insertions(+), 4012 deletions(-) delete mode 100644 docs/flowpipe-hcl/functions.md delete mode 100644 docs/flowpipe-hcl/index.md delete mode 100644 docs/flowpipe-hcl/locals.md delete mode 100644 docs/flowpipe-hcl/mod.md delete mode 100644 docs/flowpipe-hcl/pipeline.md delete mode 100644 docs/flowpipe-hcl/step/container.md delete mode 100644 docs/flowpipe-hcl/step/email.md delete mode 100644 docs/flowpipe-hcl/step/function.md delete mode 100644 docs/flowpipe-hcl/step/http.md delete mode 100644 docs/flowpipe-hcl/step/index.md delete mode 100644 docs/flowpipe-hcl/step/input.md delete mode 100644 docs/flowpipe-hcl/step/message.md delete mode 100644 docs/flowpipe-hcl/step/pipeline.md delete mode 100644 docs/flowpipe-hcl/step/query.md delete mode 100644 docs/flowpipe-hcl/step/sleep.md delete mode 100644 docs/flowpipe-hcl/step/transform.md delete mode 100644 docs/flowpipe-hcl/trigger/email.md.todo delete mode 100644 docs/flowpipe-hcl/trigger/http.md delete mode 100644 docs/flowpipe-hcl/trigger/index.md delete mode 100644 docs/flowpipe-hcl/trigger/query.md delete mode 100644 docs/flowpipe-hcl/trigger/schedule.md delete mode 100644 docs/flowpipe-hcl/variable.md rename docs/reference/cli/{integration.md => collect.md} (100%) delete mode 100644 docs/reference/cli/notifier.md delete mode 100644 docs/reference/cli/pipeline.md create mode 100644 docs/reference/cli/plugin.md delete mode 100644 docs/reference/cli/process.md rename docs/reference/cli/{mod.md => query.md} (100%) delete mode 100644 docs/reference/cli/server.md delete mode 100644 docs/reference/cli/trigger.md delete mode 100644 docs/reference/cli/variable.md delete mode 100644 docs/reference/cli/variable.md.todo delete mode 100644 docs/reference/env-vars/flowpipe_base_url.md delete mode 100644 docs/reference/env-vars/flowpipe_config_path.md delete mode 100644 docs/reference/env-vars/flowpipe_host.md delete mode 100644 docs/reference/env-vars/flowpipe_install_dir.md delete mode 100644 docs/reference/env-vars/flowpipe_listen.md delete mode 100644 docs/reference/env-vars/flowpipe_log_level.md delete mode 100644 docs/reference/env-vars/flowpipe_max_concurrency_container.md delete mode 100644 docs/reference/env-vars/flowpipe_max_concurrency_function.md delete mode 100644 docs/reference/env-vars/flowpipe_max_concurrency_http.md delete mode 100644 docs/reference/env-vars/flowpipe_max_concurrency_query.md delete mode 100644 docs/reference/env-vars/flowpipe_memory_max_mb.md delete mode 100644 docs/reference/env-vars/flowpipe_mod_location.md delete mode 100644 docs/reference/env-vars/flowpipe_port.md delete mode 100644 docs/reference/env-vars/flowpipe_telemetry.md delete mode 100644 docs/reference/env-vars/flowpipe_update_check.md delete mode 100644 docs/reference/env-vars/flowpipe_workspace.md create mode 100644 docs/reference/env-vars/foo delete mode 100644 docs/reference/env-vars/pipes_install_dir.md create mode 100644 docs/run/plugins.md diff --git a/docs/faq/index.md b/docs/faq/index.md index 14ce595..5339323 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -5,13 +5,3 @@ sidebar_label: FAQ # FAQ -## Where is the Dockerfile or container example? - -Flowpipe can be run in a containerized setup. We run it ourselves that way as part of [Turbot Pipes](https://turbot.com/pipes). However, we don't publish or support a container definition because: - -* The CLI is optimized for developer use on the command line. -* Everyone has specific goals and requirements for their containers. -* Container setup requires various mounts and access to configuration files. -* It's hard to support containers across many different environments. - -We welcome users to create and share open-source container definitions for Flowpipe. \ No newline at end of file diff --git a/docs/flowpipe-hcl/functions.md b/docs/flowpipe-hcl/functions.md deleted file mode 100644 index 66ecc1a..0000000 --- a/docs/flowpipe-hcl/functions.md +++ /dev/null @@ -1,243 +0,0 @@ ---- -title: functions -sidebar_label: functions ---- - -# functions - -Flowpipe supports the following Terraform-compatible HCL functions: - -## Numeric Functions - -| Function | Description | -|----------|-------------| -| [abs](https://www.terraform.io/docs/language/functions/abs.html) | Returns the absolute value of a number. | -| [ceil](https://www.terraform.io/docs/language/functions/ceil.html) | Returns the smallest integer value greater than or equal to a given number. | -| [floor](https://www.terraform.io/docs/language/functions/floor.html) | Returns the largest integer value less than or equal to a given number. | -| [log](https://www.terraform.io/docs/language/functions/log.html) | Returns the natural logarithm of a number. | -| [max](https://www.terraform.io/docs/language/functions/max.html) | Returns the maximum value from a list of numbers. | -| [min](https://www.terraform.io/docs/language/functions/min.html) | Returns the minimum value from a list of numbers. | -| [parseint](https://www.terraform.io/docs/language/functions/parseint.html) | Parses a string as an integer. | -| [pow](https://www.terraform.io/docs/language/functions/pow.html) | Returns the result of raising a number to the power of another. | -| [signum](https://www.terraform.io/docs/language/functions/signum.html) | Returns -1, 0, or 1 depending on the sign of a number. | -| [sum](https://www.terraform.io/docs/language/functions/sum.html) | Returns the sum of a list of numbers. | - -## String Functions - -| Function | Description | -|----------|-------------| -| [chomp](https://www.terraform.io/docs/language/functions/chomp.html) | Removes trailing newline characters from a string. | -| [endswith](https://www.terraform.io/docs/language/functions/endswith.html) | Checks if a string ends with a specified suffix. | -| [format](https://www.terraform.io/docs/language/functions/format.html) | Formats a string using printf-style formatting. | -| [formatlist](https://www.terraform.io/docs/language/functions/formatlist.html) | Formats a list of values into a string using printf-style formatting. | -| [indent](https://www.terraform.io/docs/language/functions/indent.html) | Indents a string by a specified number of spaces. | -| [join](https://www.terraform.io/docs/language/functions/join.html) | Joins elements of a list into a string with a delimiter. | -| [lower](https://www.terraform.io/docs/language/functions/lower.html) | Converts a string to lowercase. | -| [regex](https://www.terraform.io/docs/language/functions/regex.html) | Searches a string using a regular expression. | -| [regexall](https://www.terraform.io/docs/language/functions/regexall.html) | Searches a string using a regular expression and returns all matches. | -| [replace](https://www.terraform.io/docs/language/functions/replace.html) | Replaces occurrences of a substring with another string. | -| [split](https://www.terraform.io/docs/language/functions/split.html) | Splits a string into a list of substrings. | -| [startswith](https://www.terraform.io/docs/language/functions/startswith.html) | Checks if a string starts with a specified prefix. | -| [strcontains](https://www.terraform.io/docs/language/functions/strcontains.html) | Checks if a string contains another substring. | -| [strrev](https://www.terraform.io/docs/language/functions/strrev.html) | Reverses a string. | -| [substr](https://www.terraform.io/docs/language/functions/substr.html) | Returns a substring from a string. | -| [title](https://www.terraform.io/docs/language/functions/title.html) | Converts the first character of each word in a string to uppercase. | -| [trim](https://www.terraform.io/docs/language/functions/trim.html) | Trim whitespace from a string | -| [trimprefix](https://www.terraform.io/docs/language/functions/trimprefix.html) | Trim a prefix from a string | -| [trimspace](https://www.terraform.io/docs/language/functions/trimspace.html) | Trim leading and trailing whitespace | -| [trimsuffix](https://www.terraform.io/docs/language/functions/trimsuffix.html) | Trim a suffix from a string | -| [upper](https://www.terraform.io/docs/language/functions/upper.html) | Converts a string to uppercase. | - - -## Collection Functions - -| Function | Description | -|----------|-------------| -| [alltrue](https://www.terraform.io/docs/language/functions/alltrue.html) | Returns true if all elements in a list are true. | -| [anytrue](https://www.terraform.io/docs/language/functions/anytrue.html) | Returns true if any element in a list is true. | -| [chunklist](https://www.terraform.io/docs/language/functions/chunklist.html) | Splits a list into chunks of a specified size. | -| [coalesce](https://www.terraform.io/docs/language/functions/coalesce.html) | Returns the first non-null value in a list. | -| [coalescelist](https://www.terraform.io/docs/language/functions/coalescelist.html) | Returns the first non-empty list in a list of lists. | -| [compact](https://www.terraform.io/docs/language/functions/compact.html) | Removes null and empty string elements from a list. | -| [concat](https://www.terraform.io/docs/language/functions/concat.html) | Concatenates multiple lists into one list. | -| [contains](https://www.terraform.io/docs/language/functions/contains.html) | Checks if a value is in a list, set, or map. | -| [distinct](https://www.terraform.io/docs/language/functions/distinct.html) | Returns a new list with duplicate elements removed. | -| [element](https://www.terraform.io/docs/language/functions/element.html) | Returns the element at a specified index in a list. | -| [flatten](https://www.terraform.io/docs/language/functions/flatten.html) | Flattens a list of lists into a single list. | -| [index](https://www.terraform.io/docs/language/functions/index.html) | Returns the index of a value in a list. | -| [keys](https://www.terraform.io/docs/language/functions/keys.html) | Returns the keys of a map as a list. | -| [length](https://www.terraform.io/docs/language/functions/length.html) | Returns the length of a string, list, or map. | -| [list](https://www.terraform.io/docs/language/functions/list.html) | Creates a list from the arguments. | -| [lookup](https://www.terraform.io/docs/language/functions/lookup.html) | Looks up a value in a map using a key. | -| [map](https://www.terraform.io/docs/language/functions/map.html) | Creates a map from a list of key-value pairs. | -| [matchkeys](https://www.terraform.io/docs/language/functions/matchkeys.html) | Returns a map with only the specified keys. | -| [merge](https://www.terraform.io/docs/language/functions/merge.html) | Merges maps together. | -| [one](https://www.terraform.io/docs/language/functions/one.html) | Returns true if only one element in a list is true. | -| [range](https://www.terraform.io/docs/language/functions/range.html) | Generates a sequence of numbers. | -| [reverse](https://www.terraform.io/docs/language/functions/reverse.html) | Reverses the order of elements in a list. | -| [setintersection](https://www.terraform.io/docs/language/functions/setintersection.html) | Returns the intersection of two sets. | -| [setproduct](https://www.terraform.io/docs/language/functions/setproduct.html) | Returns the Cartesian product of two or more sets. | -| [setsubtract](https://www.terraform.io/docs/language/functions/setsubtract.html) | Returns the difference of two sets. | -| [setunion](https://www.terraform.io/docs/language/functions/setunion.html) | Returns the union of two or more sets. | -| [slice](https://www.terraform.io/docs/language/functions/slice.html) | Extracts a subsequence from a list. | -| [sort](https://www.terraform.io/docs/language/functions/sort.html) | Sorts a list of strings or numbers. | -| [sum](https://www.terraform.io/docs/language/functions/sum.html) | Returns the sum of a list of numbers. | -| [transpose](https://www.terraform.io/docs/language/functions/transpose.html) | Transpose a list of lists | -| [values](https://www.terraform.io/docs/language/functions/values.html) | Returns the values of a map as a list. | -| [zipmap](https://www.terraform.io/docs/language/functions/zipmap.html) | Creates a map from two lists. | - - - -## Encoding Functions - -| Function | Description | -|----------|-------------| -| [base64decode](https://www.terraform.io/docs/language/functions/base64decode.html) | Decodes a base64-encoded string. | -| [base64encode](https://www.terraform.io/docs/language/functions/base64encode.html) | Encodes a string to base64. | -| [base64gzip](https://www.terraform.io/docs/language/functions/base64gzip.html) | Compresses and then base64-encodes a string. | -| [csvdecode](https://www.terraform.io/docs/language/functions/csvdecode.html) | Decodes a CSV string into a list of maps. | -| [jsondecode](https://www.terraform.io/docs/language/functions/jsondecode.html) | Decodes a JSON string into a map or list. | -| [jsonencode](https://www.terraform.io/docs/language/functions/jsonencode.html) | Encodes a value as a JSON string. | -| [textdecodebase64](https://www.terraform.io/docs/language/functions/textdecodebase64.html) | Decodes a base64-encoded string. | -| [textencodebase64](https://www.terraform.io/docs/language/functions/textencodebase64.html) | Encodes a string to base64. | -| [urlencode](https://www.terraform.io/docs/language/functions/urlencode.html) | URL-encodes a string. | -| [yamldecode](https://www.terraform.io/docs/language/functions/yamldecode.html) | Decodes a YAML string into a map or list. | -| [yamlencode](https://www.terraform.io/docs/language/functions/yamlencode.html) | Encodes a value as a YAML string. | - - -## Filesystem Functions - -| Function | Description | -|----------|-------------| -| [abspath](https://www.terraform.io/docs/language/functions/abspath.html) | Returns the absolute path of a file or directory. | -| [basename](https://www.terraform.io/docs/language/functions/basename.html) | Returns the last element of a path. | -| [dirname](https://www.terraform.io/docs/language/functions/dirname.html) | Returns the directory part of a path. | -| [pathexpand](https://www.terraform.io/docs/language/functions/pathexpand.html) | Expands the tilde (~) in a file path. | -| [file](https://www.terraform.io/docs/language/functions/file.html) | Reads the contents of a file. | -| [fileexists](https://www.terraform.io/docs/language/functions/fileexists.html) | Returns true if a file exists. | -| [fileset](https://www.terraform.io/docs/language/functions/fileset.html) | Returns a set of files matching a glob pattern. | -| [filebase64](https://www.terraform.io/docs/language/functions/filebase64.html) | Reads the contents of a file and base64-encodes it. | - - - - -## Date and Time Functions - -| Function | Description | -|----------|-------------| -| [formatdate](https://www.terraform.io/docs/language/functions/formatdate.html) | Formats a timestamp into a string. | -| [timeadd](https://www.terraform.io/docs/language/functions/timeadd.html) | Adds a duration to a timestamp. | -| [timecmp](https://www.terraform.io/docs/language/functions/timecmp.html) | Compares two timestamps. | -| [timestamp](https://www.terraform.io/docs/language/functions/timestamp.html) | Returns the current timestamp. | - - -## Hash and Crypto Functions - -| Function | Description | -|----------|-------------| -| [base64sha256](https://www.terraform.io/docs/language/functions/base64sha256.html) | Computes the SHA-256 hash of a string and base64-encodes the result. | -| [base64sha512](https://www.terraform.io/docs/language/functions/base64sha512.html) | Computes the SHA-512 hash of a string and base64-encodes the result. | -| [bcrypt](https://www.terraform.io/docs/language/functions/bcrypt.html) | Hashes a string using the bcrypt algorithm. | -| [filebase64sha256](https://www.terraform.io/docs/language/functions/filebase64sha256.html) | Reads the contents of a file, calculates the SHA256 hash, and base64-encodes it. | -| [filebase64sha512](https://www.terraform.io/docs/language/functions/filebase64sha512.html) | Reads the contents of a file, calculates the SHA512 hash, and base64-encodes it. | -| [filemd5](https://www.terraform.io/docs/language/functions/filemd5.html) | Reads the contents of a file and calculates the MD5 hash. | -| [filesha1](https://www.terraform.io/docs/language/functions/filesha1.html) | Reads the contents of a file and calculates the SHA1 hash. | -| [filesha256](https://www.terraform.io/docs/language/functions/filesha256.html) | Reads the contents of a file and calculates the SHA256 hash. | -| [filesha512](https://www.terraform.io/docs/language/functions/filesha512.html) | Reads the contents of a file and calculates the SHA512 hash. | -| [md5](https://www.terraform.io/docs/language/functions/md5.html) | Computes the MD5 hash of a string. | -| [rsadecrypt](https://www.terraform.io/docs/language/functions/rsadecrypt.html) | Decrypts data using an RSA private key. | -| [sha1](https://www.terraform.io/docs/language/functions/sha1.html) | Computes the SHA-1 hash of a string. | -| [sha256](https://www.terraform.io/docs/language/functions/sha256.html) | Computes the SHA-256 hash of a string. | -| [sha512](https://www.terraform.io/docs/language/functions/sha512.html) | Computes the SHA-512 hash of a string. | -| [uuid](https://www.terraform.io/docs/language/functions/uuid.html) | Generates a random UUID. | -| [uuidv5](https://www.terraform.io/docs/language/functions/uuidv5.html) | Generates a UUID using a name and namespace. | - - -## IP Network Functions - -| Function | Description | -|----------|-------------| -| [cidrhost](https://www.terraform.io/docs/language/functions/cidrhost.html) | Calculates the IP address of a host in a CIDR block. | -| [cidrnetmask](https://www.terraform.io/docs/language/functions/cidrnetmask.html) | Calculates the network mask of a CIDR block. | -| [cidrsubnet](https://www.terraform.io/docs/language/functions/cidrsubnet.html) | Calculates a subnet address within a CIDR block. | -| [cidrsubnets](https://www.terraform.io/docs/language/functions/cidrsubnets.html) | Splits a CIDR block into multiple subnets. | - -## Type Conversion Functions - -| Function | Description | -|----------|-------------| -| [can](https://www.terraform.io/docs/language/functions/can.html) | Checks if a value has a specific attribute or method. | -| [nonsensitive](https://www.terraform.io/docs/language/functions/nonsensitive.html) | Marks a value as non-sensitive, allowing it to be displayed in logs. | -| [sensitive](https://www.terraform.io/docs/language/functions/sensitive.html) | Marks a value as sensitive to avoid exposing sensitive information. | -| [tobool](https://www.terraform.io/docs/language/functions/tobool.html) | Converts a value to a boolean. | -| [tolist](https://www.terraform.io/docs/language/functions/tolist.html) | Converts a value to a list. | -| [tomap](https://www.terraform.io/docs/language/functions/tomap.html) | Converts a value to a map. | -| [tonumber](https://www.terraform.io/docs/language/functions/tonumber.html) | Converts a string to a number. | -| [toset](https://www.terraform.io/docs/language/functions/toset.html) | Converts a list to a set. | -| [tostring](https://www.terraform.io/docs/language/functions/tostring.html) | Converts a value to a string. | -| [try](https://www.terraform.io/docs/language/functions/try.html) | Attempts to evaluate an expression and returns a default value on failure. | - -## Flowpipe Functions - -| Function | Description | -|----------|-------------| -| [env](#env) | Gets the value of an environment variable. -| [error_message](#error_message) | Given a reference to a step, returns a string containing the first error message. -| [is_error](#is_error) | Given a reference to a step, returns `true`` if there are any errors. - - -### env - -`env` returns the value of an environment variable. Be careful when using environment variables, as they are global variables. When writing mod, you should generally use [variables](/docs/flowpipe-hcl/variable) or [pipeline parameters](/docs/flowpipe-hcl/pipeline#parameters) for passing data into the mod, though you may use `env` to set a default value. - -```hcl -param "region" { - default = env("AWS_REGION") -} -``` - -### error_message - -Given a reference to a step, `error_message` will return a string containing the first error message, if any. If there are no errors, it will return an empty string. - -```hcl -step "http" "my_request" { - url = "https://myapi.local/subscribe" - method = "post" - body = jsonencode({ - name = param.subscriber - }) - - error { - ignore = true - } -} - -output "error_message" { - value = err_message(step.http.my_request) -} -``` - -### is_error -Given a reference to a step, `is_error` returns a boolean true if there are 1 or more errors, or false if there are no errors. - -```hcl -step "http" "my_request" { - url = "https://myapi.local/subscribe" - method = "post" - body = jsonencode({ - name = param.subscriber - }) - - error { - ignore = true - } -} - -step "email" "send_it" { - to = param.subscriber - subject = "You have been subscribed" - body = step.http.my_request.response_body - if = !is_error(step.http.my_request) -} -``` \ No newline at end of file diff --git a/docs/flowpipe-hcl/index.md b/docs/flowpipe-hcl/index.md deleted file mode 100644 index 1e42f7f..0000000 --- a/docs/flowpipe-hcl/index.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Flowpipe HCL -sidebar_label: Flowpipe HCL ---- - -# Flowpipe HCL - -## General - -| Type | Description -|-|- -| [locals](/docs/flowpipe-hcl/locals) | Locals are internal, module level variables. -| [mod](/docs/flowpipe-hcl/mod) | The mod block contains metadata, documentation, and dependency data for the mod. -| [pipeline](/docs/flowpipe-hcl/pipeline) | A pipeline is a set of steps that do work. -| [trigger](/docs/flowpipe-hcl/trigger) | Triggers are used to execute a pipeline when an event occurs. -| [variable](/docs/flowpipe-hcl/variable) | Variables are module-level objects that essentially act as parameters for a module. - - -## Step Primitives - -| Type | Description -|-------------------|---------------- -| [container](/docs/flowpipe-hcl/step/container) | Run a Docker container. -| [email](/docs/flowpipe-hcl/step/email) | Send an email. -| [function](/docs/flowpipe-hcl/step/function) | Run an AWS Lambda-compatible function. -| [http](/docs/flowpipe-hcl/step/http) | Make an HTTP request. -| [pipeline](/docs/flowpipe-hcl/step/pipeline) | Run another Flowpipe pipeline. -| [query](/docs/flowpipe-hcl/step/query) | Run a SQL query. -| [sleep](/docs/flowpipe-hcl/step/sleep) | Wait for a defined time period. -| [transform](/docs/flowpipe-hcl/step/transform) | Use HCL functions to transform data . - -## Trigger Types -| Type | Description -|-------------------|---------------- -| [http](/docs/flowpipe-hcl/trigger/http) | Create a webhook and initiate a pipeline when a request is made to the webhook. -| [query](/docs/flowpipe-hcl/trigger/query) | Runs a query on schedule or at regular intervals and executes pipelines when there are changes to the result set. -| [schedule](/docs/flowpipe-hcl/trigger/schedule)| Runs a pipeline on schedule or at regular intervals. - - \ No newline at end of file diff --git a/docs/flowpipe-hcl/locals.md b/docs/flowpipe-hcl/locals.md deleted file mode 100644 index c4778c4..0000000 --- a/docs/flowpipe-hcl/locals.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: locals -sidebar_label: locals ---- - -# locals -The `locals` block defines and sets one or more [local variables](/docs/build/mod-variables#local-variables), using standard HCL assignment syntax. The locals are scoped to the mod, and a mod may contain multiple `locals` blocks. Locals may reference other values in the mod, including other local values. - -You can reference local values as `local.`. - - -## Example Usage - -```hcl -locals { - pipes_api_version = "latest" - pipes_baseurl = "https://pipes.turbot.com" - pipes_cred_file = "~/.steampipe/internal/pipes.turbot.com.tptt" -} - -locals { - pipes_api_url = "${local.pipes_baseurl}/api/${local.pipes_api_version}" -} - -pipeline "list_my_workspaces" { - step "http" "list_workspaces" { - url = "${local.pipes_api_url}/actor/workspace" - request_headers = { - Authorization = "Bearer ${file(local.pipes_cred_file)}" - } - } - - output "workspaces" { - value = step.http.list_workspaces.response_body.items[*].handle - } -} -``` - diff --git a/docs/flowpipe-hcl/mod.md b/docs/flowpipe-hcl/mod.md deleted file mode 100644 index 09e2c95..0000000 --- a/docs/flowpipe-hcl/mod.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -title: mod -sidebar_label: mod ---- - - -# mod -Every mod must contain a `mod.fp` file with a single `mod` block. - -The `mod` block contains metadata for the mod (including metadata used in the hub site and social media), as well as [dependency data](#mod-dependencies). A mod author may edit the mod block directly, but Flowpipe will **also** edit the file, adding, removing, and modifying dependencies in the file when users add and remove mods via the [`flowpipe mod` commands](/docs/reference/cli/mod). For this reason, it is recommended that the `mod.fp` *only* contain a `mod` block; do not add other mod resources (`trigger`, `pipeline`, etc) to this file. - -The block label is the mod name. Mod names use lower_snake_case. They may contain lowercase chars, numbers, or underscores, and must start with a letter. - - -## Example - Library mod with Hub Metadata -```hcl -mod "aws" { - # hub metadata - title = "AWS Library" - description = "Run pipelines to supercharge your AWS workflows using Flowpipe." - color = "#FF9900" - documentation = file("./docs/index.md") - icon = "/images/flowpipe/build/turbot/aws.svg" - categories = ["aws", "library"] - - opengraph { - title = "AWS Library Mod for Flowpipe" - description = "Run pipelines to supercharge your AWS workflows using Flowpipe." - image = "/images/flowpipe/build/turbot/aws-social-graphic.png" - } - - require { - flowpipe { - min_version = "0.1.0" - } - } -} - -``` - -## Example - Composite Mod with Dependency - -```hcl -variable "database" { - type = connection.steampipe - description = "Steampipe database connection string." - default = connection.steampipe.default -} - -mod "deactivate_expired_aws_iam_access_keys" { - title = "Deactivate expired AWS IAM keys" - description = "Deactivates AWS IAM keys that have been active for a certain period of time." - database = var.database - - require { - flowpipe { - min_version = "0.1.0" - } - - mod "github.com/turbot/flowpipe-mod-aws" { - version = "v0.0.2-dev-samples.3" - args = { - region = var.aws_region - access_key_id = var.aws_access_key_id - secret_access_key = var.aws_secret_access_key - } - } - } - -} -``` - -## Argument Reference - -| Name | Type | Required? | Description -|-|-|-|- -| `categories` | List(String) | Optional | A list of labels, used to categorize mods (such as on the Flowpipe Hub). -| `color` | String | Optional | A hexadecimal RGB value to use as the color scheme for the mod on hub.flowpipe.io. -| `database` | String | Optional | The database to use as the default for [query steps](/docs/flowpipe-hcl/step/query) and [query triggers](/docs/flowpipe-hcl/trigger/query). The `database` may be a connection reference (`connection.steampipe.default`), a connection string (`postgres://steampipe@127.0.0.1:9193/steampipe`), or a Pipes workspace (`acme/anvils`). If not set, the default is the local Steampipe database instance. -| `description` | String | Optional | A string containing a short description. -| `documentation` | String (Markdown)| Optional | A markdown string containing a long form description, used as documentation for the mod on hub.flowpipe.io. -| `icon` | String | Optional | The URL of an icon to use for the mod on hub.flowpipe.io. -| `opengraph` | Block | Optional | Block of metadata for use in social media applications that support [Opengraph](#opengraph) metadata. -| `require` | Block | Optional | A block that specifies one or more [mod dependencies](#mod-dependencies). -| `tags` | Map | Optional | A map of key:value metadata for the mod, used to categorize, search, and filter. -| `title` | String | Optional | The display title of the mod. - - -#### opengraph -The `opengraph` block is an optional block of metadata for use in social media applications that support [Opengraph](https://ogp.me/) metadata. - -| Name | Type| Description -|-|-|- -| `description` | String | The Opengraph description (`og:description`) of the mod, for use in social media applications. -| `title` | String | The Opengraph display title (`og:title`) of the mod, for use in social media applications. - - - -#### Mod Dependencies -A mod may contain a `require` block to specify version dependencies for the Flowpipe CLI and other mods. While it is possible to edit this section manually, Flowpipe will also modify it (including reordering and removing comments) when you run a `flowpipe mod` command to install, update, or uninstall a mod. - -A mod may specify a dependency on the Flowpipe CLI. Flowpipe will evaluate the dependency when the mod is loaded and will error if the constraint is not met, but it will not install or upgrade the CLI. A `flowpipe` constraint specifies a *minimum version*, and does not support semver syntax: -```hcl -flowpipe { - min_version = "0.1.0" -} -``` - - - -A mod may specify dependencies on other mods. While you can manually edit the `mod` dependencies in the `mod.fp`, they are more commonly managed by Flowpipe when you install, update, or uninstall mods via the [flowpipe mod commands](/docs/reference/cli/mod). The `version` can be an exact version or a [semver](https://semver.org/) string: - -```hcl -require { - mod "github.com/turbot/flowpipe-mod-aws" { - version = "^0.10" - } - mod "github.com/turbot/flowpipe-mod-gcp" { - version = "*" - } -} -``` diff --git a/docs/flowpipe-hcl/pipeline.md b/docs/flowpipe-hcl/pipeline.md deleted file mode 100644 index c0cddfe..0000000 --- a/docs/flowpipe-hcl/pipeline.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: pipeline -sidebar_label: pipeline ---- - -# pipeline - -A `pipeline` is a sequence of steps to do work. Pipelines may define: -- [Steps](/docs/flowpipe-hcl/step) (`step`) to perform specific actions, such as running a query, making an http request, or running another pipeline. By default, steps will run in parallel but Flowpipe will automatically serialize steps based on implicit (HCL reference) or explicit (depends_on) dependencies. -- [Parameters](#parameters) (`param`) to allow you to pass arguments explicitly or from a `trigger`. -- [Outputs](#outputs) (`output`) to return data or status. - -You can [run a pipeline manually](/docs/reference/cli/pipeline) from the command line or define a [trigger](/docs/flowpipe-hcl/trigger) that will start the pipeline in response to an event when running [`flowpipe server`](/docs/run/server). - - -```hcl -pipeline "get_astronauts" { - param "url" { - type = string - default = "http://api.open-notify.org/astros" - } - - step "http" "whos_in_space" { - url = param.url - method = "get" - } - - output "people_in_space" { - value = step.http.whos_in_space.response_body.people[*].name - } -} -``` - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|---------|-------------|----------------- -| `description` | String | Optional | A description of the pipeline. -| `documentation` | String | Optional | A markdown string containing a long form description, used as documentation for the mod on hub.steampipe.io. -| `max_concurrency` | Number | Optional | The maximum number of instances of the pipeline that can be run at a time. By default, there is no limit. -| `param` | Block | Optional | A [param](#parameters) block that defines the parameters that can be passed into the pipeline. -| `output` | Block | Optional | one or more [output](#outputs) blocks to return values from the pipeline -| `step` | Block | Optional | one or more [steps](#steps) to run -| `tags` | Map | Optional | A map of key:value metadata for the benchmark, used to categorize, search, and filter. The structure is up to the mod author and varies by benchmark and provider. -| `title` | String | Optional | Display title for the pipeline. - - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|--------------- -| `errors` | List | List of [error objects](#errors-read-only) from the pipeline. -| `output` | map | map of [outputs](#outputs) defined and set by the pipeline. - ----- - -## Parameters - -One or more `param` blocks may optionally be used in a pipeline to define parameters that the pipeline accepts. - -```hcl -param "url" { - type = string - default = "http://api.open-notify.org/astros" -} -``` - -You can then use the parameter value in the pipeline with the `param` object: -```hcl -step "http" "whos_in_space" { - url = param.url - method = "get" -} -``` - -### Arguments - - -| Name | Type | Description -|---------------|---------|-------------------------- -| `default` | Any | A value to use if no argument is passed for this parameter when the query is run. -| `description` | String | A description of the parameter. -| `enum` | Set | A set of allowed values for the param; no other values are allowed. -| `tags` | Map | A map of key:value metadata for the benchmark, used to categorize, search, and filter. The structure is up to the mod author and varies by benchmark and provider. -| `type` | String | The data type of the parameter: a primitive type `string`, `number`, `bool`, `connection`, `connection.{type}`, `notifier`, or a collection type `list()`, `map()`, or `any` (default `any`). - ----- - -## Outputs - -Output values make information from your pipeline run available to the caller. Output values are similar to return values in programming languages. - -```hcl -output "people_in_space" { - value = step.http.whos_in_space.response_body.people[*].name -} -``` -### Arguments - -| Name | Type | Description -|---------------|---------|-------------------------- -| `description` | String | A description of the output. -| `value` | Any | The value to export. (required) diff --git a/docs/flowpipe-hcl/step/container.md b/docs/flowpipe-hcl/step/container.md deleted file mode 100644 index 425d3db..0000000 --- a/docs/flowpipe-hcl/step/container.md +++ /dev/null @@ -1,213 +0,0 @@ ---- -title: container -sidebar_label: container ---- - - -# container -A `container` step is used to run a Docker container. - -> Important! Flowpipe `container` steps require Docker to be running. Make sure you have the Docker daemon installed and running. - -Each container definition is a distinct instance of the step (a distinct container image). A looped step (via `for_each`) will create a container instance for each iteration. - - -```hcl -pipeline "another_pipe" { - - step "container" "aws_s3_ls" { - image = "public.ecr.aws/aws-cli/aws-cli" - cmd = ["s3", "ls"] - - env = { - AWS_ACCESS_KEY_ID = "AKIAFAKEKEY25WNAQKRC" - AWS_SECRET_ACCESS_KEY = "+bOnLljsFEZfakekeyJWQq7kkQaOsh5JbkTQH5YR" - } - } - - output "buckets" { - value = step.container.aws_s3_ls.stdout - } -} - -``` - - - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|-----------|-------------|----------------- -| `image` | String | Optional* | The docker image to use, eg `turbot/steampipe:latest`. [You must specify `image` or `source` but not both](#source-vs-image). -| `source` | String | Optional* | The path to a folder that contains the `dockerfile` or `containerfile` to build the container. [You must specify `image` or `source` but not both](#source-vs-image). -| `cmd` | List<String>| Optional | The cmd to use to start the container. -| `cpu_shares` | Number | Optional | CPU shares (relative weight) for the container. -| `entrypoint` | List<String> | Optional | Overwrite the default `ENTRYPOINT` of the image. The entrypoint allows you to configure a container to run an executable. -| `env` | Map of Strings | Optional | A map of string to set environment variables -| `memory` | Number | Optional | Amount of memory in MB your container can use at runtime. Defaults to `128`. -| `memory_reservation` | Number | Optional | Allows you to specify a soft limit smaller than `memory` which is activated when Docker detects contention or low memory on the host machine. If you use `memory-reservation`, it must be set lower than `memory` for it to take precedence. Because it is a soft limit, it does not guarantee that the container doesn't exceed the limit. -| `memory_swap` | Number | Optional | The amount of memory this container is allowed to swap to disk. If `memory` and `memory-swap` are set to the same value, this prevents containers from using any swap. This is because `memory-swap` is the amount of combined memory and swap that can be used, while `memory` is only the amount of physical memory that can be used. Set to `-1` to enable unlimited swap -| `memory_swappiness` | Number | Optional | Tune container memory swappiness (0 to 100). -|`read_only` | Boolean | Optional | If `true`, the container will be started as read-only. Defaults to `false`. -| `user` | String | Optional | User to run the container as (format: `[:]`) -| `workdir` | String | Optional | The working directory for commands to run in. - - - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). - - - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|----------------- -| `container_id` | String | The container id -| `exit_code` | Number | The exit code from running the container -| `lines` | List of objects | [Ordered list of all lines of output](#lines) (stdout & stderr) from the container -| `stdout` | String | `STDOUT` output from the container -| `stderr` | String | `STDERR` output from the container - -### lines - -The `lines` attribute provides a list of all lines of output from the container, in order. `lines` is a list of objects, where each object has a `stream` (`stdout` or `stdout`) and a `line` (the line of output), for example: - -```hcl -[ - { - stream = "stdout" - line = "This is the first" - }, - { - stream = "stdout" - line = "block of STDOUT" - }, - { - stream = "stderr" - line = "This is the first" - }, - { - stream = "stderr" - line = "block of STDERR" - }, - { - stream = "stdout" - line = "This is the second" - }, - { - stream = "stdout" - line = "block of STDOUT" - }, -] -``` - -Usage example: - - -```hcl - -pipeline "simple_pipe" { - - step "container" "hello" { - image = "hello-world" - } - - output "combined_lines" { - value = step.container.hello.lines[*].line - } - - output "stdout_lines" { - value = [for v in step.container.hello.lines[*] : v.line if v.stream == "stdout"] - } - - output "stderr_lines" { - value = [for v in step.container.hello.lines[*] : v.line if v.stream == "stderr"] - } - - output "combined_text" { - value = join("\n",step.container.hello.lines[*].line) - } - - output "stdout_text" { - value = join("\n",[for v in step.container.hello.lines[*] : v.line if v.stream == "stdout"]) - } - - output "stderr_text" { - value = join("\n",[for v in step.container.hello.lines[*] : v.line if v.stream == "stderr"]) - } -} -``` - - -## Source v/s Image -You must specify either an `image` or a `source` but not both. The `image` will be pulled via standard docker client commands - The image must be publicly accessible or you must `docker login` to access a private repo. You may instead specify a `source` in which case a custom image will be built before the step is run. The image will be updated when anything in the `source` directory changes, and it will be deleted if the step is deleted. - - -### Container Step using an Image - -When using `image`, the image is pulled from a registry via standard docker client commands: - - -```hcl -pipeline "another_pipe" { - - step "container" "aws_s3_ls" { - image = "public.ecr.aws/aws-cli/aws-cli" - cmd = ["s3", "ls"] - - env = { - AWS_ACCESS_KEY_ID = "AKIAFAKEKEY25WNAQKRC" - AWS_SECRET_ACCESS_KEY = "+bOnLljsFEZfakekeyJWQq7kkQaOsh5JbkTQH5YR" - } - } - - output "buckets" { - value = step.container.aws_s3_ls.stdout - } -} - -``` - - - -### Container Step using Source -When using `source`, the path is relative to the [mod location](/docs/run#mod-location): - -```hcl -pipeline "another_pipe" { - - step "container" "aws_s3_ls" { - source = "./my_aws" - cmd = ["s3", "ls"] - - env = { - AWS_ACCESS_KEY_ID = "AKIAFAKEKEY25WNAQKRC" - AWS_SECRET_ACCESS_KEY = "+bOnLljsFEZfakekeyJWQq7kkQaOsh5JbkTQH5YR" - } - } - - output "buckets" { - value = step.container.aws_s3_ls.stdout - } -} - -``` - - -## Labels -Docker labels are automatically added to images, containers, volumes, etc using the [standard format](https://docs.docker.com/config/labels-custom-metadata/), including the standard `org.opencontainers` labels as well as Flowpipe-specific labels prefixed with `io.flowpipe`. - -```json -//container -"Labels": { - "io.flowpipe.name": "test_suite_mod.pipeline.with_functions.function.hello_nodejs_step", - "io.flowpipe.type": "container", - "org.opencontainers.container.created": "2023-10-12T04:29:32Z", - "org.opencontainers.image.created": "2023-10-12T04:29:29Z" -} -``` diff --git a/docs/flowpipe-hcl/step/email.md b/docs/flowpipe-hcl/step/email.md deleted file mode 100644 index 39cb7d1..0000000 --- a/docs/flowpipe-hcl/step/email.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: email -sidebar_label: email ---- - -# email - -***The `email` step is deprecated and will be removed in a future version of Flowpipe. Use the [message step](/docs/flowpipe-hcl/step/message) to send messages.*** - -The `email` step is used to send an email message. - - -```hcl -pipeline "another_pipe" { - - step "email" "send_it" { - to = ["darin@kramerica.com"] - from = "elaine@jpetermancatalog.com" - smtp_username = "elaine@jpetermancatalog.com" - smtp_password = "54678fakeblksdf09" - host = "smtp.example.com" - port = 587 - subject = "Order Shipped" - content_type = "text/html" - body = <<-EOS -

Your Urban Sombrero is on its way!

- -

Combining the spirit of old Mexico with a little big city panache...

Thank you for shopping with J Peterman.

- EOS - } -} - -``` - - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|-------------|------------|----------------- -| `to` | List<String>| Required | A list of recipient email addresses to which -| `bcc` | List<String>| Optional | A list of recipient email addresses to which to send the message via bcc. -| `body` | String | Optional | The message body. -| `cc` | List<String>| Optional | A list of recipient email addresses to which to send the message via cc. -| `content_type` | String | Optional | The content type of the email body, e.g. `text/plain`, `text/html` to send the message. -| `from` | String | Optional | The email address of the sender. -| `host` | String | Optional | The DNS name of the SMTP host to send the mail through. -| `port` | Number | Optional | The TCP port on the SMTP host. -| `sender_name` | String | Optional | A friendly display name of the sender -| `smtp_password` | String | Optional | Password to log in to the SMTP server. -| `smtp_username` | String | Optional | Username to log in to the SMTP server. -| `subject` | String | Optional | The message subject. - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). diff --git a/docs/flowpipe-hcl/step/function.md b/docs/flowpipe-hcl/step/function.md deleted file mode 100644 index 6da7894..0000000 --- a/docs/flowpipe-hcl/step/function.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: function -sidebar_label: function ---- - -# function - -A `function` step is used to run an AWS Lambda-compatible function. - -> Important! Flowpipe `function` steps require Docker to be running. Make sure you have the Docker daemon installed and running. - - -Each step *definition* is a distinct instance of the step (a distinct container image); a looped step (via `foreach`) is ONE definition, but you cannot change the `runtime` or other container-specific settings in the loop. - -```hcl -pipeline "simple_pipe" { - param "event" {} - - step "function" "my_func_defaults" { - source = "./my-other-function" - event = param.event - } -} -``` - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|-----------|-------------|----------------- -| `env` | Map | Optional | A map of string to set environment variables -| `event` | Object | Optional | JSON-compatible [event](https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-concepts.html#gettingstarted-concepts-event) object that contains data for a Lambda function to process -| `source` | String | Required | Path to the function's deployment package within the local filesystem. The path is relative to the root of the mod (where the `mod.hcl`). -| `handler` | String | Optional | Function entry point in your code, in the format `filename.functionname`. If not specified default handler for the runtime is used (`index.handler` for nodejs, `lambda_function.lambda_handler` for python, etc.) -| `runtime` | String | Required | Identifier of the function's runtime. Valid Values are `nodejs:18`, `nodejs:20`, `python:3.10`. - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). - - - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|----------------- -| `response` | Object | The response returned by the function - - - - - - -## Names & Labels - -Container names are randomly generated using standard docker names (`elegant_sutherland`, etc). - -Image names are generated using the following format: `flowpipe/{mod}.pipeline.{pipeline name}.function.{step name}:{timestamp}`, e.g: `flowpipe/test_suite_mod.pipeline.with_functions.function.hello_nodejs_step:20231012T155317200` - -Docker labels are automatically added to images & containers using the [standard format](https://docs.docker.com/config/labels-custom-metadata/), including the standard `org.opencontainers` labels, as well as Flowpipe-specific labels prefixed with `io.flowpipe`. - -```json -//container -"Labels": { - "io.flowpipe.name": "test_suite_mod.pipeline.with_functions.function.hello_nodejs_step", - "io.flowpipe.runtime": "nodejs:18", - "io.flowpipe.type": "function", - "org.opencontainers.container.created": "2023-10-12T04:29:32Z", - "org.opencontainers.image.created": "2023-10-12T04:29:29Z" -} -``` - - -## More Examples - - -```hcl -pipeline "another_pipe" { - - step "function" "my_func" { - # Now - description = "this is my function" - source = "./my-function" - runtime = "nodejs:18" - handler = "my_file.my_handler" - timeout = 10000 - - event = jsondecode(jsonencode({ - "Records":[ - { - "messageId":"916f5e95-bbbb-4148-1234-2ac8e764f06c", - "receiptHandle":"AQEBmLuoGWtLtFFgvyCFdSPMJh2HKgHOIPWNUq22EOwCzGT8iILZm97CE6j4J6oR71ZpDr3sgxQcJyVZ+dmmvGl+fFftT9GCJqZYrjMGsR2Q6WsMd8ciI8bTtDXyvsk8ektd7UGfh4gxIZoFp7WUKVRcMEeBkubKd8T4/Io81D0l/AK7MxcEfCj40vWEsex1kkGmMRlBtdSeGyy7fJgUq2543AYWciiWtbSit8S0Y38xZPmsIFhoxP0egQRoJcW4aUgMi469Gj5+khizetybtgC8vux5NCg/IejxcCueXkQ7LKVF8kfRdqRSUYB6DsOrfakeZpK4wpXIarByNz0R2p7J88meYpj2IVULv/emXsSYaKG4rXnpbH4J9ijbLWckYLAd7wPDzCYri1ZSTgAz0kchsEw==", - "body":"{\n\"name\": \"bob\",\n\"tag\": \"hello\"\n}", - "attributes":{ - "ApproximateReceiveCount":"1", - "SentTimestamp":"1602046897707", - "SenderId":"AIDAR3FAKE4FCWXL56NUU", - "ApproximateFirstReceiveTimestamp":"1602046897712" - }, - "messageAttributes":{ - - }, - "md5OfBody":"98da683a47692b39c1d43bd4fa21ed89", - "eventSource":"aws:sqs", - "eventSourceARN":"arn:aws:sqs:ap-south-1:123456789012:documentation", - "awsRegion":"ap-south-1" - } - ] - })) - - env = { - foo = "bar" - up = "down" - } - } -} - -``` \ No newline at end of file diff --git a/docs/flowpipe-hcl/step/http.md b/docs/flowpipe-hcl/step/http.md deleted file mode 100644 index 03c0a9a..0000000 --- a/docs/flowpipe-hcl/step/http.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: http -sidebar_label: http ---- - -# http - -An `http` step is used to make an HTTP request. - - -```hcl -pipeline "get_astronauts" { - - step "http" "whos_in_space" { - url = "http://api.open-notify.org/astros" - method = "get" - } - - output "people_in_space" { - value = step.http.whos_in_space.response_body.people[*].name - } -} -``` - - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|---------|------------|----------------- -| `url` | String | Required | The URL for the request. Supported schemes are `http` and `https` -| `basic_auth` | Block | Optional | A `basic_auth` block to specify a username and password using [Basic Authentication](#basic-authentication) -| `ca_cert_pem` | String | Optional | Certificate data of the Certificate Authority (CA) in PEM (RFC 1421) format. -| `insecure` | Boolean | Optional | Disables verification of the server's certificate chain and hostname. Defaults to `false` -| `method` | Boolean | Optional | The HTTP Method for the request. Allowed methods are a subset of methods defined in RFC7231 namely, `get`,`head`, and `post`, `put`, `patch`, `delete`. The default is `post`. -| `request_body` | String | Optional | The request body as a string. -| `request_headers` | Map | Optional | A map of request header field names and values. - - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|----------------- -| `response_body` | String or object | The response body. If the `Content-Type` is `application/json` then Flowpipe will decode it and return it as an object, otherwise, it is returned as a string. -| `response_headers` | Map of String | A map of response header field names and values. Duplicate headers are concatenated according to RFC2616. -| `status_code` | Number | The HTTP response status code. - - -## Basic Authentication -To use basic authentication, include a `basic_auth` block specifying a `username` and `password`. This will create the appropriate `Authorization` header and add it to the request. - -```hcl -step "http" "list_issues" { - method = "get" - url = "${param.api_base_url}/rest/api/2/search?jql=project=${param.project_key}" - request_headers = { - Content-Type = "application/json" - } - - basic_auth { - username = param.user_email - password = param.token - } - -} -``` - -| Attribute | Type | Description -|------------|---------|----------------- -| `username` | String | The basic auth username -| `password` | String | The basic auth password - diff --git a/docs/flowpipe-hcl/step/index.md b/docs/flowpipe-hcl/step/index.md deleted file mode 100644 index 000348e..0000000 --- a/docs/flowpipe-hcl/step/index.md +++ /dev/null @@ -1,324 +0,0 @@ ---- -title: step -sidebar_label: step ---- - -# step - -A pipeline is composed of one or more steps. Each step has a type and a name, and the arguments are dependent on the type. The step types are sometimes referred to as "primitives". - -Each step type has its own distinct set of attributes, though there are some [common step arguments](#common-step-arguments) and [attributes](#common-step-attributes-read-only) that all steps implement. - -## Step Types - -| Type | Description -|-------------------|---------------- -| [container](/docs/flowpipe-hcl/step/container) | Run a Docker container. -| [email](/docs/flowpipe-hcl/step/email) | [DEPRECATED] Send an email. -| [function](/docs/flowpipe-hcl/step/function) | Run an AWS Lambda-compatible function. -| [http](/docs/flowpipe-hcl/step/http) | Make an HTTP request. -| [input](/docs/flowpipe-hcl/step/input) | Prompt a user for input -| [message](/docs/flowpipe-hcl/step/message) | Send a message to an integration. -| [pipeline](/docs/flowpipe-hcl/step/pipeline) | Run another Flowpipe pipeline. -| [query](/docs/flowpipe-hcl/step/query) | Run a SQL query. -| [sleep](/docs/flowpipe-hcl/step/sleep) | Wait for a defined time period. -| [transform](/docs/flowpipe-hcl/step/transform) | Use HCL functions to transform data . - - -## Common Step Arguments - -| Argument | Type | Optional? | Description -|-----------------|-----------|-------------|----------------- -| `depends_on` | List of Steps | Optional | A list of steps that this step [depends on](#depends_on). -| `description` | String | Optional | A description of the step. -| `error` | Block | Optional | An [error block](#error) to handle errors from the step. -| `for_each` | Map or List | Optional | A map or list used as a [step iterator](#for_each). A step instance will be created for each item in the map or list. -| `if` | Condition| Optional | An [if condition](#if) to evaluate to determine whether to run this step. -| `loop` | Block | Optional | A [loop block](#loop) to run the step in a sequential loop. -| `max_concurrency` | Number | Optional | The maximum number of instances of the step that can be run at a time. By default, there is no limit but note the step is also subject to the per-step-type limits ([FLOWPIPE_MAX_CONCURRENCY_CONTAINER](/docs/reference/env-vars/flowpipe_max_concurrency_container), [FLOWPIPE_MAX_CONCURRENCY_FUNCTION](/docs/reference/env-vars/flowpipe_max_concurrency_function), [FLOWPIPE_MAX_CONCURRENCY_HTTP](/docs/reference/env-vars/flowpipe_max_concurrency_http), [FLOWPIPE_MAX_CONCURRENCY_QUERY](/docs/reference/env-vars/flowpipe_max_concurrency_query), etc). -| `output` | Block | Optional | One or more [output blocks](#output) to return custom values from the step. -| `retry` | Block | Optional | A [retry block](#retry) to retry the step when an error occurs. -| `throw` | Block | Optional | One or more [throw blocks](#throw) to raise an error from the step. -| `timeout` | Number or String| Optional | [Amount of time](#timeout) this step has to run before an error is raised. -| `title` | String | Optional | A display title for the step. - -### depends_on - -The `depends_on` argument allows you to define explicit dependencies to make steps run in a specific order. Note that Flowpipe will create [implicit dependencies](/docs/build/write-pipelines/control-flow) based on references to other steps so you only need `depends_on` when you don't have an implicit reference. - -The `depends_on` argument accepts a list of steps that this step depends on: -```hcl -step "sleep" "sleep_10_seconds" { - depends_on = [ step.http.http_1 ] - duration = "10s" -} -``` - - -### for_each - -The `for_each` argument is used to [run a step in a parallel loop](/docs/build/write-pipelines/iteration#for_each). This argument accepts a map or list used as a step iterator. A step instance will be created for each item in the map or list: - -```hcl -step "http" "add_a_user" { - for_each = ["Jerry","Elaine", "Newman"] - url = "https://myapi.local/api/v1/user" - method = "post" - - request_body = jsonencode({ - user_name = "${each.value}" - }) -} -``` - - -### if - -The `if` argument accepts a [condition to evaluate to determine whether to run this step](/docs/build/write-pipelines/conditionals). The step will only run when `if` evaluates to true: - -```hcl -step "email" "send_it" { - if = step.pipeline.order.output.order_count > 0 - to = ["darin@kramerica.com"] - from = "elaine@jpetermancatalog.com" - host = "smtp.example.com" - subject = "Order Shipped" - body = "Your order has shipped" -} -``` - -### timeout - -Most steps accept a `timeout` argument which specifies the amount of time to wait for the step to complete before raising an error. - -The `timeout` argument may be an integer or a [Go duration string](https://pkg.go.dev/time#Duration). If the duration is an integer, it will be interpreted as the number of milliseconds: - - -```hcl -step "http" "whos_in_space" { - url = "http://api.open-notify.org/astros" - method = "get" - timeout = 5000 -} -``` - -You may instead pass a string that specifies the number and type of units. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. Note that the granularity varies by step type, and fractional amounts will be rounded up to the appropriate granularity. For instance, The `timeout` for a `container` has a granularity of 1 second, so if you set the timeout to `500ms` it will be rounded up to 1 second. - -```hcl -step "http" "whos_in_space" { - url = "http://api.open-notify.org/astros" - method = "get" - timeout = "5s" -} -``` - -You can even include multiple units: - -```hcl -step "http" "whos_in_space" { - url = "http://api.open-notify.org/astros" - method = "get" - timeout = "1m5s" -} -``` - - -### error - -By default, all errors are fatal and are not retried - When a step encounters an error, it causes the step the fail. A failed step results in a failed pipeline - Any step instances that are already running will complete (but will not be retried) but then the pipeline will stop with a failed status. - -The `error` block allows you to [ignore the error](/docs/build/write-pipelines/errors#error) and continue execution. You can then "handle" the error in subsequent steps. - -```hcl -step "http" "my_request" { - url = "https://myapi.local/subscribe" - method = "post" - body = jsonencode({ - name = param.subscriber - }) - - error { - ignore = true - } -} - -step "email" "send_it" { - to = param.subscriber - subject = "You have been subscribed" - body = step.http.my_request.response_body - if = !is_error(step.http.my_request) -} -``` - -#### Arguments - -| Argument | Type | Optional? | Description -|-----------------|-------------|------------|----------------- -| `if` | Boolean | Optional | A condition to evaluate to determine whether to evaluate this error block. The `error` block will only be evaluated when the `if` condition evaluates to `true`. -| `ignore` | Boolean | Optional | If `true`, the error will be ignored. - - - -### loop -The `loop` block will [run a step in a sequential loop](/docs/build/write-pipelines/iteration#loop), changing the arguments with each iteration. This is useful for handling HTTP pagination, for example. - -```hcl -step "http" "list_workspaces" { - url = "https://latestpipe.turbot.io/api/v1/org/latesttank/workspace/?limit=3" - method = "get" - - request_headers = { - Content-Type = "application/json" - Authorization = "Bearer ${param.pipes_token}" - } - - loop { - until = result.response_body.next_token == null - url = "https://latestpipe.turbot.io/api/v1/org/latesttank/workspace/?limit=3&next_token=${result.response_body.next_token}" - } - -} -``` - -The loop is evaluated last after the step instance has finished executing and all retries have been completed. You can use the special value `result` to evaluate the attributes of the completed step instance. `result` is essentially a self-reference to "this" step after it has run (e.g. the attributes are populated). - -You can also use the special attribute called `loop` inside the block. `loop` has a single attribute `index` which is the (zero-based) index of the loop count. - -#### Arguments - -| Argument | Type | Optional? | Description -|-----------------|-------------|------------|----------------- -| `until` | Boolean | Optional | A condition to evaluate to determine whether to run this step again. The step will loop until this condition is `true`. -| `{any argument}`| any | Optional | A step argument to override for the next iteration. When the step runs again due to the loop, it inherits all of the arguments from the step, but you can override them inside the loop block if desired, allowing you to pass data from the step execution into the next step execution - - -### retry - -The `retry` block allows you to [retry the step](/docs/build/write-pipelines/errors#retry) when an error occurs. - -```hcl -step "http" "my_request" { - url = "https://myapi.local/subscribe" - method = "post" - body = jsonencode({ - name = param.subscriber - }) - - retry { - max_attempts = 5 - strategy = "exponential" - min_interval = 100 - max_interval = 10000 - } -} -``` - -#### Arguments - -| Argument | Default | Optional? | Description -|-----------------|-------------|------------|----------------- -| `if` | `true` | Optional | A condition to evaluate to determine whether to retry the step. The step will be retried only if the `if` condition evaluates to `true`. -| `max_attempts` | `3` | Optional | Specifies the maximum number attempts to run the step. -| `strategy` | `constant` | Optional | The backoff strategy. One of `exponential`, `linear`, `constant`. -| `min_interval` | `1000` | Optional | The first interval between retries, in milliseconds. If the strategy is exponential or linear, subsequent intervals will be scaled based on this value. -| `max_interval` | `10000` | Optional | The maximum interval between retries, in milliseconds. - - -### throw - -You can also [explicitly raise an exception](/docs/build/write-pipelines/errors#throw) with a `throw` block to raise an error when the step would usually succeed. You can include as many `throw` blocks as you want. Thrown errors are not retried, though they can be ignored. - - -```hcl -step "http" "my_request" { - url = "https://myapi.local/subscribe" - method = "post" - body = jsonencode({ - name = param.subscriber - }) - - throw { - if = length(result.response_body.errors) > 1 - message = result.response_body.errors[0] - } - -} -``` - -#### Arguments - -| Argument | Type | Optional? | Description -|-----------------|-------------|------------|----------------- -| `message` | String | Required | The error string for the thrown error. -| `if` | Boolean | Optional | A condition to evaluate to determine whether to throw an error. The step will only raise an error when the `if` condition evaluates to `true`. - - -### output -You may include one or more [output](#output) blocks to arbitrary values in the `output` attribute of the step. - -```hcl - -pipeline "get_astros" { - - step "http" "whos_in_space" { - url = "http://api.open-notify.org/astros" - method = "get" - } - - step "transform" "parse_astros" { - value = step.http.whos_in_space.response_body.people - - output "astronauts" { - value = step.http.whos_in_space.response_body.people[*].name - } - - output "spacecraft" { - value = distinct(step.http.whos_in_space.response_body.people[*].craft) - } - } - - output "people_in_space" { - value = step.transform.parse_astros.output.astronauts - } - - output "ships_in_space" { - value = step.transform.parse_astros.output.spacecraft - } -} -``` - -#### Arguments - -| Argument | Type | Optional? | Description -|-----------------|---------|------------|----------------- -| `value` | Any | Required | The output value. - - - -## Common Step Attributes (Read-Only) -| Attribute | Type | Description -|-----------------|---------|----------------- -| `errors` | List | List of [errors](#errors-read-only) from the step -| `flowpipe` | Map | A map of [Flowpipe metadata](#flowpipe-read-only) about the step instance -| `output` | Map | A [map of the step outputs](#output-read-only) defined in [output blocks](#output) - -### errors (Read-Only) - -Each step has an [`errors` attribute](/docs/build/write-pipelines/errors#the-errors-attribute) that contains a list of errors that occurred. [Unhandled errors](/docs/build/write-pipelines/errors#handling-errors) will cause the pipeline run to fail and will be returned in the pipeline `errors` list. - -To simplify common error-handling cases, Flowpipe provides some helper functions: -- `is_error`: Given a reference to a step, `is_error` returns a boolean `true` if there are 1 or more errors, or false if there are no errors. `is_error(step.http.my_request)` is equivalent to `length(step.http.my_request.errors) > 0` -- `error_message`: Given a reference to a step, `error_message` will return a string containing the first error message, if any. If there are no errors, then it will return an empty string. This is useful for simple step primitives. - - - -### output (Read-Only) -You can access [custom step outputs](#output-block) using the `output` attribute of a step. The `output` attribute contains a map of outputs for the step, with an entry for each `output` block. - -### flowpipe (Read-Only) -The `flowpipe` attribute includes metadata about the step instance: - -| Attribute | Description -|-----------------|-------------------------- -| `finished_at` | Timestamp when the step instance finished executing -| `started_at` | Timestamp when the step instance started executing diff --git a/docs/flowpipe-hcl/step/input.md b/docs/flowpipe-hcl/step/input.md deleted file mode 100644 index 79b3f59..0000000 --- a/docs/flowpipe-hcl/step/input.md +++ /dev/null @@ -1,498 +0,0 @@ ---- -title: input -sidebar_label: input ---- - -# input - -Use the `input` step primitive to prompt for user input. When the pipeline reaches an `input` step, it will prompt the user for input and then pause and wait for a response. When a response is received, the pipeline will continue and the step will return the selected `value`. - -When you run a pipeline in [client-mode](/docs/run#operating-modes), notifications for `input` steps will only appear on the command line. - -In [server-mode](/docs/run/server), notifications for `input` steps are not sent to the console. Instead, they are sent to [integrations](/docs/reference/config-files/integration/) such as Slack or Email via a [notifier](/docs/reference/config-files/notifier), allowing you to create collaborative workflows that integrate with your preferred communication channels. - - -```hcl -pipeline "my_step" { - - step "input" "my_step" { - type = "button" - prompt = "Do you want to approve?" - - option "Approve" {} - option "Deny" {} - - notifier = notifier.infosec - } - - step "pipeline" "do_the_thing" { - pipeline = pipeline.something - if = step.input.my_step.value == "Approve" - } -} -``` - -An `input` step is limited to a single prompt and a single input element. It asks a single question and returns a single answer (`value`). - -You must select a [notifier](/docs/reference/config-files/notifier) that defines which [integration](/docs/reference/config-files/integration) will handle the interaction: Slack, email, or other. - -There are multiple [input types](#input-types), but they all share these traits: -- There can be one or more [options](#options). -- The options may either be specified in `option` blocks or as a list of `options`. The block format is typically preferred when the list of options is statically defined, and the list format is useful when generating the options dynamically. -- Each option must have a `value`. -- An option may have a `label` (display value). If no `label` is specified, the `value` is used as the label. -- An `option` may be `selected` by default. - - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|-----------|-------------|----------------- -| `notifier` | Notifier Reference | Required | The [notifier](/docs/reference/config-files/notifier) to send the request to send the message. -| `cc` | List<String> | Optional | The email addresses to send to. This only applies to notifiers that uses `email` integrations. -| `bcc` | List<String> | Optional | The email addresses to send to. This only applies to notifiers that uses `email` integrations. -| `channel` | String | Optional | The channel to send the request to. This only applies to `slack` integrations. -| `option` | Block | Optional | The available [options](#options) to present to the user as `option` blocks. You may either specify one or more `option` blocks or a single `options` list, but not both. -| `options` | List | Optional | The available [options](#options) to present to the user as a list of objects. You may either specify one or more `option` blocks or a single `options` list, but not both. -| `prompt` | String | Optional | The text to present to the user as a prompt -| `subject` | String | Optional | The email subject. This only applies to notifiers that uses `email` integrations. -| `to` | List<String> | Optional | The email addresses to send to. This only applies to `email` integrations. -| `type` | String | Required | The [input type](#input-types) to present to the user - - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). - - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|----------------- -| `value ` | String | The value that the user has selected. This may be a scalar value (when the type is `button`, `text`, `select`) or a list (when the type is `multiselect`). - ----- - -## Input Types - -| Type | Description -|---------------|--------------------- -| `button` | Click a button -| `text` | Enter a single line of text -| `select` | Select a single item from a dropdown list -| `multiselect` | Select one or more items from a dropdown list - - - ----- -### Button - Simple - -##### Console (Client-Mode) -![](/images/docs/flowpipe-hcl/input_button_simple_console.png) - - -##### HTTP -![](/images/docs/flowpipe-hcl/input_button_simple_http.png) - -##### Slack -![](/images/docs/flowpipe-hcl/input_button_simple_slack.png) - - -```hcl -pipeline "my_pipe" { - - step "input" "my_step" { - notifier = notifier.default - type = "button" - prompt = "Do you want to approve?" - - option "Approve" {} - option "Deny" {} - - } - - step "pipeline" "do_the_thing" { - pipeline = pipeline.something - if = step.input.my_step.value == "Approve" - } -} -``` - ----- - -### Button - With labels and values - -##### Console (Client-Mode) -![](/images/docs/flowpipe-hcl/input_button_simple_console.png) - -##### HTTP -![](/images/docs/flowpipe-hcl/input_button_simple_http.png) - -##### Slack -![](/images/docs/flowpipe-hcl/input_button_simple_slack.png) - - -```hcl -pipeline "my_pipe" { - - step "input" "my_step" { - notifier = notifier.default - type = "button" - prompt = "Do you want to approve?" - - option "approve_button" { - label = "Approve" - value = "approve_button_pressed" - } - - option "deny_button" { - label = "Deny" - value = "deny_button_pressed" - } - } - - step "pipeline" "do_the_thing" { - pipeline = pipeline.something - if = step.input.my_step.value == "approve_button_pressed" - } -} - -``` - ----- - -### Select - basic - -##### Console (Client-Mode) -![](/images/docs/flowpipe-hcl/input_select_simple_console.png) - - -##### HTTP -![](/images/docs/flowpipe-hcl/input_select_simple_http_open.png) - -##### Slack -![](/images/docs/flowpipe-hcl/input_select_simple_slack_open.png) - - -```hcl -pipeline "my_select" { - - step "input" "select_region" { - notifier = notifier.default - type = "select" - prompt = "Select a region:" - - option "us-east-1" {} - option "us-east-2" {} - option "us-west-1" {} - option "us-west-2" {} - - } - - step "pipeline" "list_buckets" { - pipeline = aws.pipeline.list_buckets - args = { - region = step.input.select_region.value - } - } -} -``` - ----- - - - -#### Select - with labels & default selection - -##### Console (Client-Mode) -![](/images/docs/flowpipe-hcl/input_select_labels_console.png) - -##### HTTP -![](/images/docs/flowpipe-hcl/input_select_labels_http_open.png) - -##### Slack -![](/images/docs/flowpipe-hcl/input_select_labels_slack_open.png) - - -```hcl -pipeline "my_select_labels" { - - step "input" "select_region" { - notifier = notifier.default - type = "select" - prompt = "Select a region:" - - option "us-east-1" { - label = "N. Virginia" - selected = true - } - option "us-east-2" { - label = "Ohio" - } - option "us-west-1" { - label = "N. California" - } - option "us-west-2" { - label = "Oregon" - } - } - - step "pipeline" "list_buckets" { - pipeline = aws.pipeline.list_buckets - args = { - region = step.input.select_region.value - } - } - -} -``` - ----- - - - -### Multiselect - basic - -##### Console (Client-Mode) -![](/images/docs/flowpipe-hcl/input_multiselect_simple_console.png) - - -##### HTTP -![](/images/docs/flowpipe-hcl/input_multiselect_simple_http_selected.png) - -##### Slack -![](/images/docs/flowpipe-hcl/input_multiselect_simple_slack_selected.png) - - -```hcl -pipeline "my_multi" { - - step "input" "select_regions" { - notifier = notifier.default - type = "multiselect" - prompt = "Select regions:" - - option "us-east-1" {} - option "us-east-2" {} - option "us-west-1" {} - option "us-west-2" {} - } - - step "pipeline" "list_buckets" { - pipeline = aws.pipeline.list_buckets - - for_each = step.input.select_regions.value - args = { - region = each.value - } - } - -} -``` - ----- - - -### Multiselect with labels & default selection - -##### Console (Client-Mode) -![](/images/docs/flowpipe-hcl/input_multiselect_labels_console_selected.png) - - -##### HTTP -![](/images/docs/flowpipe-hcl/input_multiselect_labels_http_selected.png) - -##### Slack -![](/images/docs/flowpipe-hcl/input_multiselect_labels_slack_selected.png) - - -```hcl -pipeline "my_multi_labels" { - - step "input" "select_regions" { - notifier = notifier.default - type = "multiselect" - prompt = "Select regions:" - - option "us-east-1" { - label = "N. Virginia" - selected = true - } - option "us-east-2" { - label = "Ohio" - selected = true - } - - option "us-west-1" { - label = "N. California" - } - option "us-west-2" { - label = "Oregon" - } - } - - step "pipeline" "list_buckets" { - pipeline = aws.pipeline.list_buckets - - for_each = step.input.select_regions.value - args = { - region = each.value - } - } - -} -``` - ----- - - -## Options - -The available options to present to the user are specified in either `option` blocks, or in the `options` list. You may either specify one or more `option` blocks or a single `options` list, but not both. - -If no arguments are passed to an `option`, then the block label is used as both the `label` (the text to display) and `value` (the value to return if this option is selected): - -```hcl - step "input" "select_region" { - notifier = notifier.default - type = "select" - prompt = "Select a region:" - - option "us-east-1" {} - option "us-east-2" {} - option "us-west-1" {} - option "us-west-2" {} - - } -``` - - -Each option may optionally specify a `label` (the text to display) and a `value` (the value to return if this option is selected). For some types, you may specify a boolean `selected` value to pre-select the item: - -```hcl - step "input" "select_region" { - notifier = notifier.default - type = "select" - prompt = "Select a region:" - - option "us-east-1" { - label = "N. Virginia" - selected = true - } - option "us-east-2" { - label = "Ohio" - } - option "us-west-1" { - label = "N. California" - } - option "us-west-2" { - label = "Oregon" - } - } -``` - -You may instead pass all options as a list in the `options` argument: - - -```hcl - step "input" "select_regions" { - notifier = notifier.default - type = "select" - prompt = "Select a region:" - - options = [ - { - value = "us-east-1" - label = "N. Virginia" - selected = true - }, - { - value = "us-east-2" - label = "Ohio" - selected = true - }, - { - value = "us-west-2" - label = "N. California" - }, - { - value = "us-west-2" - label = "Oregon" - } - ] - } -``` - - -The `options` list form is useful for building the options dynamically: - - -```hcl -pipeline "my_pipe" { - - step "query" "get_regions" { - sql = <<-EOQ - select - name as value - from - aws_region - where - opt_in_status <> 'not-opted-in' - order by - name - EOQ - } - - step "input" "select_regions" { - notifier = notifier.default - type = "select" - prompt = "Select a region:" - options = step.query.get_regions.rows - } -} -``` - -Additionally, when using `button` input type, you can colorize the buttons by setting the `style` attribute on an `option`: - -```hcl -pipeline "my_pipe" { - - step "input" "my_step" { - notifier = notifier.default - type = "button" - prompt = "Do you want to approve?" - - option "approve_button" { - label = "Approve" - value = "approve_button_pressed" - style = "ok" - } - - option "deny_button" { - label = "Deny" - value = "deny_button_pressed" - style = "alert" - } - } - - step "pipeline" "do_the_thing" { - pipeline = pipeline.something - if = step.input.my_step.value == "approve_button_pressed" - } -} -``` - - -### Arguments - -| Argument | Type | Optional? | Description -|-----------------|-----------|-------------|----------------- -| `label` | String | Optional | The text to display for the option. -| `value` | String | Optional | The value to return when the option is selected. -| `selected` | Boolean | Optional | Set to `true` to pre-select the option. -| `style` | String | Optional | Set to `ok`, `alert`, `info` (default) to colorize option when used as a `button`. - - diff --git a/docs/flowpipe-hcl/step/message.md b/docs/flowpipe-hcl/step/message.md deleted file mode 100644 index ffc6699..0000000 --- a/docs/flowpipe-hcl/step/message.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: message -sidebar_label: message ---- - -# message - -Use the `message` step to send a one-way notification message from your pipeline. - -When you run a pipeline in [client-mode](/docs/run#operating-modes), notifications for `message` steps will only appear on the command line. - -In [server-mode](/docs/run/server), notifications for `message` steps are not sent to the console. Instead, they are sent to [integrations](/docs/reference/config-files/integration/) such as Slack or Email via a [notifier](/docs/reference/config-files/notifier), allowing you to create collaborative workflows that integrate with your preferred communication channels. - - -```hcl -pipeline "send_message" { - step "message" "send_a_message" { - - notifier = notifier.default - channel = "#ops" - to = ["ops@bluth.com"] - subject = "Sample message" - text = "I'm a sample message." - } -} -``` - - -## Arguments -| Argument | Type | Optional? | Description -|-----------------|-----------|-------------|----------------- -| `text` | String | Required | The message text to send to the integration. -| `notifier` | Notifier Reference | Required | The [notifier](/docs/reference/config-files/notifier) to send the request to send the message. -| `cc` | List<String> | Optional | The email addresses to send to. This only applies to `email` integrations. -| `bcc` | List<String> | Optional | The email addresses to send to. This only applies to notifiers that uses `email` integrations. -| `channel` | String | Optional | The channel to send the request to. This only applies to `slack` integrations. -| `subject` | String | Optional | The email subject. This only applies to notifiers that uses `email` integrations. -| `to` | List<String> | Optional | The email addresses to send to. This only applies to `email` integrations. - - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). diff --git a/docs/flowpipe-hcl/step/pipeline.md b/docs/flowpipe-hcl/step/pipeline.md deleted file mode 100644 index 9193cdb..0000000 --- a/docs/flowpipe-hcl/step/pipeline.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: pipeline -sidebar_label: pipeline ---- - -# pipeline - -A pipeline step executes another Flowpipe pipeline from the same mod or its direct dependencies: - -```hcl -pipeline "my_pipeline" { - - step "pipeline" "post_message" { - pipeline = slack.pipeline.chat_post_message - args = { - token = "xoxp-YOUR_TOKEN_HERE" - channel = "#general" - message = "This is a test message" - } - } -} -``` - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|---------|------------|----------------- -| `pipeline` | Pipeline Reference | Required | A reference to a `pipeline` resource to run in this step. -| `args` | Map | Optional | A map of arguments to pass to the pipeline. - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). - - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|----------------- -| `errors` | List | List of [errors](/docs/flowpipe-hcl/pipeline#errors-attribute-read-only) from the child pipeline. -| `output` | Map | A map of the `output` elements from the child pipeline merged with any [custom step outputs](/docs/flowpipe-hcl/pipeline#output--attribute-read-only) defined in [output blocks](/docs/flowpipe-hcl/pipeline#output-block) diff --git a/docs/flowpipe-hcl/step/query.md b/docs/flowpipe-hcl/step/query.md deleted file mode 100644 index b744bf2..0000000 --- a/docs/flowpipe-hcl/step/query.md +++ /dev/null @@ -1,282 +0,0 @@ ---- -title: query -sidebar_label: query ---- - -# query - -A query step runs a database query against a database. [Postgres](#postgres-query), [MySQL](#mysql-query), [SQLite](#sqlite-query), and [DuckDB](#duckdb-query) databases are currently supported. - -```hcl -pipeline "enabled_regions" { - - step "query" "get_enabled_regions" { - database = "postgres://steampipe@localhost:9193/steampipe" - sql = <<-EOQ - select - name, - account_id, - opt_in_status - from - aws_region - where - opt_in_status <> 'not-opted-in' - EOQ - - } - - output "enabled_regions" { - value = step.query.get_enabled_regions.rows - } -} -``` - -## Arguments - -| Argument | Type | Optional? | Description -| -----------| ------ | --------- | ---------------------------------------------------- -| `sql` | String | Required | A SQL query string. -| `args` | List | Optional | A list of arguments to pass to the query. -| `database` | String | Optional | The database to query. This may be a connection reference (`connection.steampipe.default`), a connection string (`postgres://steampipe@127.0.0.1:9193/steampipe`), or a Pipes workspace (`acme/anvils`). If not set, the default set in the [mod `database`](/docs/flowpipe-hcl/mod) will be used. - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). - -## Attributes (Read-Only) - -| Attribute | Type | Description | -| --------- | --------------- | --------------------------------------------- | -| `rows` | List of objects | The query results, as an array of row objects | - -The `rows` attribute contains the query result as a list of objects with an item for each row, where the column name is the key and the column value is the value. - -For example: - -```hcl -pipeline "enabled_regions" { - - step "query" "get_enabled_regions" { - database = "postgres://steampipe@localhost:9193/steampipe" - sql = <<-EOQ - select - name, - account_id, - opt_in_status - from - aws_region - where - opt_in_status <> 'not-opted-in' - EOQ - - } - - output "enabled_regions" { - value = step.query.get_enabled_regions.rows - } -} -``` - -Results in: - -```json -{ - "enabled_regions": [ - { - "account_id": "123456789012", - "name": "us-east-2", - "opt_in_status": "opt-in-not-required" - }, - { - "account_id": "123456789012", - "name": "us-west-1", - "opt_in_status": "opt-in-not-required" - }, - { - "account_id": "123456789012", - "name": "ap-south-1", - "opt_in_status": "opt-in-not-required" - }, - { - "account_id": "123456789012", - "name": "us-west-2", - "opt_in_status": "opt-in-not-required" - }, - ... - ] -} -``` - -Since `rows` is a list, you can use standard HCL `for` expressions: - -```hcl -output "enabled_region_names" { - value = [ for v in step.query.get_enabled_regions.rows : v.name ] -} -``` - -or splats: - -```hcl -output "enabled_region_names" { - value = step.query.get_enabled_regions.rows[*].name -} -``` - -to extract data. - -```json -{ - "enabled_region_names": [ - "us-east-2", - "us-west-1", - "ap-south-1", - "us-west-2", - ... - ] -} -``` - -## More Examples - - -### Steampipe Query - -If no `database` is specified, then the default defined in the [mod `database`](/docs/flowpipe-hcl/mod) will be used. If that is not set, the local Steampipe instance will be used by default. - -You can also specify a [Steampipe connection](/docs/reference/config-files/connection/postgres) to connect to a Steampipe database: - - -```hcl -pipeline "instances_by_region" { - step "query" "get_instances_by_region" { - database = connection.steampipe.default - sql = "select region, count(*) from aws_ec2_instance group by region;" - } -} -``` - - -### Postgres Query - -You can use a [Postgres connection](/docs/reference/config-files/connection/postgres) to connect to a PostgreSQL database: - - -```hcl -pipeline "enabled_regions" { - step "query" "get_enabled_regions" { - database = connection.postgres.mydb - sql = "select name, account_id, opt_in_status from aws_region where opt_in_status <> 'not-opted-in'" - } -} -``` - -Alternatively, you can pass the connection string directly, with the standard URI syntax supported by `psql` and `pgcli`: - -```bash -postgresql://[user[:password]@][host][:port][/dbname][?param1=value1&...] -``` - - -```hcl -pipeline "enabled_regions" { - step "query" "get_enabled_regions" { - database = "postgres://steampipe@localhost:9193/steampipe" - sql = "select name, account_id, opt_in_status from aws_region where opt_in_status <> 'not-opted-in'" - } -} -``` - -### SQLite query - -You can use a [SQLite connection](/docs/reference/config-files/connection/sqlite) to connect to a SQLite database: - - -```hcl -pipeline "sqlite_query" { - step "query" "step_1" { - database = connection.sqlite.my_db - sql = "select * from my_sqlite_table;" - } -} -``` - -Alternatively, pass the connection string directly, in the format `sqlite:path/to/file`: - -```hcl -pipeline "sqlite_query" { - step "query" "step_1" { - database = "sqlite:./my_sqlite_db.db" - sql = "select * from my_sqlite_table;" - } -} -``` - -The path is relative to the [mod location](/docs/run#mod-location), and `//` is optional after the scheme, thus the following are equivalent: - -```hcl -database = "sqlite:./my_sqlite_db.db" -database = "sqlite://./my_sqlite_db.db" -database = "sqlite://my_sqlite_db.db" -``` - - -### DuckDB query - -You can use a [DuckDB connection](/docs/reference/config-files/connection/duckdb) to connect to a DuckDB database: - -```hcl -pipeline "duckdb_query" { - step "query" "step_1" { - database = connection.duckdb.my_ducks" - sql = "select * from my_duckdb_table;" - } -} -``` - -Or pass a connection string directly in the format `duckdb:path/to/file`: - -```hcl -pipeline "duckdb_query" { - step "query" "step_1" { - database = "duckdb:./my_ducks.db" - sql = "select * from my_duckdb_table;" - } -} -``` - -The path is relative to the [mod location](/docs/run#mod-location), and `//` is optional after the scheme, thus the following are equivalent: - -```hcl -database = "duckdb:./my_ducks.db" -database = "duckdb://./my_ducks.db" -database = "duckdb://my_ducks.db" -``` - - - -### MySQL Query - -You can use a [MySQL connection](/docs/reference/config-files/connection/mysql) to connect to a MySQL database: - -```hcl -pipeline "mysql_query" { - step "query" "step_1" { - database = connection.mysql.default - sql = "select host, user from user;" - } -} -``` - -Or pass a connection string directly. The MySQL connection string supports the syntax of the [GO SQL driver for MySQL](https://github.com/go-sql-driver/mysql?tab=readme-ov-file#examples): - -```bash -mysql://[user[:password]@]network-location[:port][/dbname][?param1=value1&...] -``` - -```hcl -pipeline "mysql_query" { - step "query" "step_1" { - database = "mysql://root:my_pass@tcp(localhost)/mysql" - sql = "select host, user from user;" - } -} -``` \ No newline at end of file diff --git a/docs/flowpipe-hcl/step/sleep.md b/docs/flowpipe-hcl/step/sleep.md deleted file mode 100644 index fc18352..0000000 --- a/docs/flowpipe-hcl/step/sleep.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: sleep -sidebar_label: sleep ---- - - -# sleep - -A `sleep` step can be used to wait for a defined amount of time. You will usually want to add an explicit dependency when using a `sleep` step. - -```hcl -step "sleep" "sleep_10_seconds" { - depends_on = [ step.http.http_1 ] - duration = "10s" -} -``` - - -## Arguments - - -| Argument | Type | Optional? | Description -|-----------------|---------|------------|----------------- -| `duration` | String or Number | Required | The amount of time to sleep as an integer or a [duration string](#duration-strings). If the duration is an integer, it is interpreted as the number of milliseconds. - - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). - - -## Duration Strings - -The `duration` argument may be an integer or a [Go duration string](https://pkg.go.dev/time#Duration). If the duration is an integer, it will be interpreted as the number of milliseconds: - - -```hcl - step "sleep" "sleep_100_milliseconds" { - duration = 100 -} -``` - -You may instead pass a string that specifies the number and type of units. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`: - -```hcl - step "sleep" "sleep_10_seconds" { - duration = "10s" -} -``` - -You can even include multiple units: - -```hcl - step "sleep" "sleep_2_hours_and_45_minutes" { - duration = "2h45m" -} -``` \ No newline at end of file diff --git a/docs/flowpipe-hcl/step/transform.md b/docs/flowpipe-hcl/step/transform.md deleted file mode 100644 index 16daff8..0000000 --- a/docs/flowpipe-hcl/step/transform.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: transform -sidebar_label: transform ---- - - -# transform - -A transform step can be used to merge, filter, or reformat data using HCL functions. While you can transform the data "inline", it is often useful to: - - transform the data once and use the transformed data in multiple places - - transform the data through multiple distinct steps to makes it easier to troubleshoot / maintain - - perform conditional logic as part of the data transformation - -```hcl -pipeline "transform_example" { - - step "http" "whos_in_space" { - url = "http://api.open-notify.org/astros" - method = "get" - } - - step "transform" "get_spacecraft" { - value = distinct(step.http.whos_in_space.response_body.people[*].craft) - } - - step "transform" "get_astros" { - value = distinct(step.http.whos_in_space.response_body.people[*].name) - } - - output "astronauts" { - value = step.transform.get_astros.value - } - output "spacecraft" { - value = step.transform.get_spacecraft.value - } - -} -``` - -## Arguments - -| Argument | Type | Optional? | Description -|-----------------|---------|------------|----------------- -| `value` | Any | Required | The step value. Usually, this is the field where you will transform the data - - -This step also supports the [common step arguments](/docs/flowpipe-hcl/step#common-step-arguments) and [attributes](/docs/flowpipe-hcl/step#common-step-attributes-read-only). diff --git a/docs/flowpipe-hcl/trigger/email.md.todo b/docs/flowpipe-hcl/trigger/email.md.todo deleted file mode 100644 index 7eea6b0..0000000 --- a/docs/flowpipe-hcl/trigger/email.md.todo +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: email -sidebar_label: email ---- -# COMING SOON! -> This feature is not yet available for general release and is subject to change. - -
- -# email - - - -## Example Usage - -```hcl -``` - -##### Arguments - -| Name | Type | Required? | Description -|-|-|-|- -| `description` | String | Optional | A string containing a short description. -| `documentation` | String| Optional | A markdown string containing a long form description, used as documentation for the mod on hub.flowpipe.io. -| `enabled` | Boolean | Optional | Enable or disable the trigger. A disabled trigger will not fire, but it will retain its history and configuration. Default is `true`. -| `tags` | Map | Optional | A map of key:value metadata for the mod, used to categorize, search, and filter. -| `title` | String | Optional | The display title of the mod. - - diff --git a/docs/flowpipe-hcl/trigger/http.md b/docs/flowpipe-hcl/trigger/http.md deleted file mode 100644 index fbe54e1..0000000 --- a/docs/flowpipe-hcl/trigger/http.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -title: http -sidebar_label: http ---- - -# http - -An `http` trigger is used to create a webhook and initiate a pipeline when a request is made to the webhook. - -```hcl -trigger "http" "my_webhook" { - pipeline = pipeline.my_pipeline - args = { - event = self.request_body - } -} -``` - -It is common to pass in the `request_body` and/or `request_header` trigger attributes as arguments to the pipeline using `self` to reference the trigger instance. - - -## Arguments - - -| Argument | Type | Optional? | Description -|-----------------|---------|------------|----------------- -| `args` | Map | Optional | A map of arguments to pass to the pipeline. -| `description` | String | Optional | A string containing a short description of the step. -| `documentation` | String (Markdown)| Optional | A markdown string containing a long form description, used as documentation for the mod on hub.flowpipe.io. -| `enabled` | Boolean | Optional | Enable or disable the trigger. A disabled trigger will not fire, but it will retain its history and configuration. Default is `true`. -| `execution_mode` | String | Optional | Specifies whether the trigger should wait for the pipeline to complete and return its results in the response body (`synchronous`), or return immediately. The default is `asynchronous`, -| `method` | [method](#http-methods) block | Optional | A supported HTTP [http-methods](#method) block. The block label must be one of `get`, `post`. Note that if no `post` method block appears, only `post` is supported, and the top-level `pipeline`, `args`, and `execution_mode` apply. -| `pipeline` | Pipeline Reference | Optional | A reference to a `pipeline` resource to start when this trigger runs. -| `tags` | Map | Optional | A map of key:value metadata for the mod, used to categorize, search, and filter. -| `title` | String | Optional | Display title for the step. - - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|----------------- -| `request_body` | String | The request body as a string. -| `request_headers` | Map | A map of request header field names and values. -| `status_code` | Number | The HTTP response status code. -| `url` | String | The webhook URL. - - - -## HTTP Methods - -By default, the http trigger will only fire on `POST` events. You can use `method` blocks to trigger different pipelines for different HTTP methods. The block label must be the method name (`get` or `post`). - -Note that if no method blocks appear, only `post` is supported, and the top-level `pipeline`, `args`, and `execution_mode` apply. - -```hcl -trigger "http" "my_webhook" { - - method "post" { - pipeline = pipeline.my_pipeline - - args = { - event = self.request_body - } - } - - method "get" { - execution_mode = "synchronous" - pipeline = pipeline.confirm_setup - - args = { - headers = self.request_headers - } - } - -} -``` - -| Argument | Type | Optional? | Description -|-----------------|---------|------------|----------------- -| `pipeline` | Pipeline Reference | Required | A reference to a `pipeline` resource to start when this trigger runs. -| `args` | Map | Optional | A map of arguments to pass to the pipeline. -| `execution_mode`| String | Optional | Specifies whether the trigger should wait for the pipeline to complete and return its results in the response body (`synchronous`), or return immediately (`asynchronous`). The default is `asynchronous`, - - - -## Webhook Endpoint URL - -Flowpipe creates an endpoint on the flowpipe server for each `http` trigger. The HTTP webhook does not support any authentication mechanism, but it does have a URL with randomness to make it unguessable. The webhook URL path is: `/api/latest/hook/{trigger HCL label}/{random string}`, eg `/api/latest/hook/my_webhook/21ifp8truzi8y2r29jdl0qi7qt` - -The webhook URL will remain consistent across restarts. The {random string} is generated using the trigger name (the block label of the `trigger` ) and a global salt value. - - Because the URL contains the trigger name, changing the trigger name will generate a new URL. - - The salt value is stored in `~/.flowpipe/internal/salt` - - If the file is missing or empty, flowpipe will randomly generate a new salt and write it there. If you want to change *all* of your webhook URLs, remove the salt value. - - - -## Webhook Response - -Flowpipe will return metadata in the response headers to identify the trigger process execution ID and the pipeline process execution ID: -```bash -Flowpipe-Execution-Id: exec_cl92cgjjtoj4uv7etkq0 -Flowpipe-Pipeline-Execution-Id: pexec_cl92cgjjtoj4uv7etkqg -``` - - -If the HTTP trigger is configured to run a pipeline synchronously, it will return the outputs of the pipeline as top-level fields in the response. The response body will be formatted as JSON. - ```json - { - "my_string_outout": "foo", - "my_list_output": [ - "one", - "two", - "three" - ], - "my_object_output": { - "color": "blue", - "number": 34 - } - } - ``` - - - - diff --git a/docs/flowpipe-hcl/trigger/index.md b/docs/flowpipe-hcl/trigger/index.md deleted file mode 100644 index 0063281..0000000 --- a/docs/flowpipe-hcl/trigger/index.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: trigger -sidebar_label: trigger ---- - - -# trigger - -Triggers are used to execute a pipeline when an event occurs. They are defined in the mod and are based on a schedule, webhook, or other event. - -```hcl -trigger "http" "my_webhook" { - pipeline = pipeline.my_pipeline - args = { - event = self.request_body - } -} -``` - - -## Trigger Types - -| Type | Description -|-------------------|---------------- -| [http](/docs/flowpipe-hcl/trigger/http) | Create a webhook and initiate a pipeline when a request is made to the webhook. -| [query](/docs/flowpipe-hcl/trigger/query) | Runs a query on schedule or at regular intervals and executes pipelines when there are changes to the result set. -| [schedule](/docs/flowpipe-hcl/trigger/schedule)| Runs a pipeline on schedule or at regular intervals. diff --git a/docs/flowpipe-hcl/trigger/query.md b/docs/flowpipe-hcl/trigger/query.md deleted file mode 100644 index f96514b..0000000 --- a/docs/flowpipe-hcl/trigger/query.md +++ /dev/null @@ -1,235 +0,0 @@ ---- -title: query -sidebar_label: query ---- - -# query - -The query trigger will execute an SQL query on a schedule and can pass row changes as input to the defined pipeline. The query trigger supports the same database engines as the [query step](/docs/flowpipe-hcl/step/query). - -```hcl -trigger "query" "expired_access_keys" { - database = "postgres://steampipe@localhost:9193/steampipe" - primary_key = "access_key_id" - schedule = "daily" - - sql = <> 'connection_name' as connection - from - aws_iam_access_key - where - create_date < now() - interval '90 days'; - EOQ - - - capture "insert" { - pipeline = pipeline.delete_expired_access_keys - - args = { - access_keys = self.inserted_rows - } - } -} - -``` - -You may define a [capture block](#capture) for each type of change that you wish to handle. The block label must match the CRUD operation name that you wish to handle (`insert`, `update`, or `delete`). When the query runs, if there are any new, updated, or missing rows it will trigger a pipeline run for the relevant `capture` operations. The `inserted_rows`, `updated_rows`, and `deleted_rows` attributes will contain the details about which rows were added, updated, or deleted since the last query run, and you will usually want to pass them as arguments to the pipeline. You must define at least one `capture` block in a query trigger. - -Flowpipe saves the data from the query trigger in a SQLite database so that it can determine which items are new, changed, or deleted. - -The first time a query trigger runs, all items are considered new. The pipeline defined in the `insert` capture will be run (if it is defined), and Flowpipe will store each row's `primary_key` and a hash of the full row data for comparison on subsequent trigger executions. - -On subsequent query trigger runs: - -- Any rows with a new `primary_key` are considered inserts. The pipeline defined in the `insert` capture will be run (if it is defined). Flowpipe will store each row's `primary_key` and a hash of the full row data for comparison on subsequent trigger executions. -- Any rows with an existing `primary_key` whose hashed row data does not match what is stored are considered updates. The pipeline defined in the `update` capture will be run (if it is defined). Flowpipe will save the new row data and hash, overwriting the previous row data and hash for that key. -- If a `primary_key` that was previously stored is no longer in the result set, then the row is determined to be deleted. The pipeline defined in the `delete` capture will be run (if it is defined). Flowpipe will delete the item with that `primary_key`; if an item with that key is returned in a future trigger run, it will be considered an insert. - -## Arguments - -| Argument | Type | Optional? | Description -|---------------|---------|-----------|----------------- -| `sql` | String | Required | A SQL query string. -| `description` | String | Optional | A description of the trigger. -| `database` | String | Optional | The database to query. This may be a connection reference (`connection.steampipe.default`), a connection string (`postgres://steampipe@127.0.0.1:9193/steampipe`), or a Pipes workspace (`acme/anvils`). If not set, the default set in the [mod `database`](/docs/flowpipe-hcl/mod) will be used. -| `enabled` | Boolean | Optional | Enable or disable the trigger. A disabled trigger will not fire, but it will retain its history and configuration. Default is `true`. -| `param` | Block | Optional | A [param](#parameters) block that defines the parameters that can be passed into the trigger. -| `primary_key` | String | Optional | Primary key to use for update vs insert detection. If no primary key is defined, a hash of the row will be used as the key. -| `schedule` | String | Optional | [Schedule](/docs/flowpipe-hcl/trigger/schedule#more-examples) to run the query. This may be a named interval (`hourly`, `daily`, `weekly`, `5m`, `10m`, `15m`, `30m`, `60m`, `1h`, `2h`, `4h`, `6h`, `12h`, `24h`) or a custom schedule in cron syntax. The default is `15m` (every 15 minutes). -| `title` | String | Optional | Display title for the trigger. - -## Attributes (Read-Only) - -| Attribute | Type | Description -|-----------------|---------|----------------- -| `deleted_rows` | List | A list of rows that were deleted since the last time the trigger ran. `deleted_rows` does not return all the data for the deleted row, only its primary key. -| `inserted_rows` | List | A list of rows that were inserted since the last time the trigger ran. -| `updated_rows` | List | A list of rows that were updated since the last time the trigger ran. `updated_rows` contains the *new* row data (after it was updated). - -## Parameters - -One or more `param` blocks may optionally be used in a trigger to define parameters that the trigger accepts. - -```hcl -param "url" { - type = string - default = "http://api.open-notify.org/astros" -} -``` - -### Arguments - - -| Name | Type | Description -|---------------|---------|-------------------------- -| `default` | Any | A value to use if no argument is passed for this parameter when the query is run. -| `description` | String | A description of the parameter. -| `type` | String | The data type of the parameter: `string`, `number`, `bool`, `list`, `map`, `any` (default `any`). - ---- - -## capture - -You must define at least one `capture` block in a query trigger. - -You may define a `capture` block for each type of change that you wish to handle. The block label must match the CRUD operation name that you wish to handle (`insert`, `update`, or `delete`). When the query runs, if there are any new, updated, or missing rows it will trigger a pipeline run for the relevant `capture` operations. - -### Arguments - -| Argument | Type | Optional? | Description -|------------|--------------------|-----------|----------------- -| `pipeline` | Pipeline Reference | Required | A reference to a `pipeline` resource to start when this trigger runs. -| `args` | Map | Optional | A map of arguments to pass to the pipeline. - -## More Examples - -### Composite Key - -The query trigger does not explicitly support composite keys, however you can concatenate multiple columns to create one. - -```hcl -trigger "query" "dns_changes" { - database = "postgres://steampipe@localhost:9193/steampipe" - primary_key = "composite_key" - - sql = <` - - -## Example Usage -```hcl -variable "region" { - type = string - description = "The name of the Region." - default = "us-east-1" - enum = [ "us-east-1", "us-east-2", "us-west-1", "us-west-2" ] -} - -variable "conn" { - type = connection.aws - description = "AWS connection to connect with" - default = connection.aws.default -} - -pipeline "describe_vpcs" { - title = "Describe VPCs" - description = "Describes the specified VPCs or all VPCs." - - param "region" { - type = string - description = "The name of the region." - default = var.region - } - - param "conn" { - type = connection.aws - description = "AWS connection to connect with" - default = connection.aws.default - } - - param "vpc_ids" { - type = list(string) - description = "The VPC IDs." - optional = true - } - - step "container" "describe_vpcs" { - image = "amazon/aws-cli" - env = merge(param.conn.env, {AWS_REGION = param.region}) - - cmd = concat( - ["ec2", "describe-vpcs"], - try(length(param.vpc_ids), 0) > 0 ? concat(["--vpc-ids"], param.vpc_ids) : [] - ) - } - - output "stdout" { - description = "The standard output stream from the AWS CLI." - value = jsondecode(step.container.describe_vpcs.stdout) - } - -} -``` - -## Argument Reference -| Argument | Type | Optional? | Description -|-|-|-|- -| `default` | Any |Optional| A default value. If no value is passed, the user is not prompted and the default is used. -| `description` | String| Optional| A description of the variable. This text is included when the user is prompted for a variable's value. -| `enum` | Set | Optional| A set of allowed values for the variable; no other values are allowed. -| `tags` | Map | Optional | A map of key:value metadata for the benchmark, used to categorize, search, and filter. The structure is up to the mod author and varies by benchmark and provider. -| `type` | Type | Optional | The [variable type](#variable-types). This may be a primitive type `string`, `number`, `bool`, `connection`, `connection.{type}`, `notifier`, or a collection type `list()`, `map()`, or `any` (default `any`). - - - -## Variable Types -Variables may be simple types: -- `string` -- `number` -- `bool` -- `connection` -- a typed `connection`, eg `connection.aws`, `connection.github` , etc. -- `notifier` - - -Variables may also be collection types: -- `list()` -- `set()` -- `map()` -- `object({ = , ... })` -- `tuple([, ...])` - -The keyword `any` may be used to indicate that any type is acceptable - - -## Enum - -Flowpipe supports an `enum` argument to provide a set of allowed values; no other values are allowed. - -```hcl -variable "log_level" { - type = string - default = "info" - enum = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" ] -} -``` \ No newline at end of file diff --git a/docs/reference/cli/integration.md b/docs/reference/cli/collect.md similarity index 100% rename from docs/reference/cli/integration.md rename to docs/reference/cli/collect.md diff --git a/docs/reference/cli/help.md b/docs/reference/cli/help.md index bddd744..9aebaf4 100644 --- a/docs/reference/cli/help.md +++ b/docs/reference/cli/help.md @@ -1,14 +1,14 @@ --- -title: flowpipe help -sidebar_label: flowpipe help +title: tailpipe help +sidebar_label: tailpipe help --- -# flowpipe help +# tailpipe help Display help and usage information for any command in the application. ## Usage ```bash -flowpipe help [command] [flags] +tailpipe help [command] [flags] ``` ## Examples diff --git a/docs/reference/cli/index.md b/docs/reference/cli/index.md index c8cd132..09fe43a 100644 --- a/docs/reference/cli/index.md +++ b/docs/reference/cli/index.md @@ -1,24 +1,17 @@ --- -title: Flowpipe CLI -sidebar_label: Flowpipe CLI +title: Tailpipe CLI +sidebar_label: Tailpipe CLI --- -# Flowpipe CLI +# Tailpipe CLI ## Sub-Commands | Command | Description |-|- -| [flowpipe help](/docs/reference/cli/help) | Help about any command -| [flowpipe integration](/docs/reference/cli/integration) | List and view integrations -| [flowpipe mod](/docs/reference/cli/mod) | Flowpipe mod management -| [flowpipe notifier](/docs/reference/cli/notifier) | List and view notifiers -| [flowpipe pipeline](/docs/reference/cli/pipeline) | List, view, and run Flowpipe pipelines -| [flowpipe process](/docs/reference/cli/process) | List and view Flowpipe processes -| [flowpipe server](/docs/reference/cli/server) | Run Flowpipe server, including triggers and integrations -| [flowpipe trigger](/docs/reference/cli/trigger) | List and view Flowpipe triggers -| [flowpipe variable](/docs/reference/cli/variable) | List and view Flowpipe variables - +| [tailpipe help](/docs/reference/cli/help) | Help about any command +| [tailpipe collect](/docs/reference/cli/collect) | Collect from log sources +| [tailpipe query](/docs/reference/cli/query) | Query log sources - -## Flags - -| Flag | Applies to | Description -|-|-|- -| `--verbose` | `tail` | View detailed event information, including step arguments and attributes. - - -## Examples - -List processes: - -```bash -flowpipe process list -``` - -List processes for a server instance: - -```bash -flowpipe process list --host local -``` - -View process information: - -```bash -flowpipe process show exec_cl4l9ibjtoj9mk6ol0l0 -``` - -View process events: - -```bash -flowpipe process tail exec_cl4l9ibjtoj9mk6ol0l0 --host local -``` - -View detailed process events: - -```bash -flowpipe process tail exec_cl4l9ibjtoj9mk6ol0l0 --host local --verbose -``` - -List processes in JSON format: - -```bash -flowpipe process list --output json -``` - - \ No newline at end of file diff --git a/docs/reference/cli/mod.md b/docs/reference/cli/query.md similarity index 100% rename from docs/reference/cli/mod.md rename to docs/reference/cli/query.md diff --git a/docs/reference/cli/server.md b/docs/reference/cli/server.md deleted file mode 100644 index 474ad34..0000000 --- a/docs/reference/cli/server.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: flowpipe server -sidebar_label: flowpipe server ---- - - -# flowpipe server - -Run the Flowpipe server, including triggers, integrations, and the API. Flowpipe server runs in the foreground; Press `Ctrl-C` to exit. - - -## Usage -```bash -flowpipe server -``` - -## Flags - -| Flag | Description -|-|- -| `--base-url` | Set the [base URL](/docs/reference/env-vars/flowpipe_base_url) to use for [triggers](/docs/flowpipe-hcl/trigger) and [integrations](/docs/reference/config-files/integration). This is the base URL that Flowpipe advertises when it interacts with external systems to allow them to call back to Flowpipe. This may include the DNS or IP address of the system on which you are running Flowpipe, or it may be a reverse proxy such as [ngrok](https://ngrok.com/) (default `http://localhost:7103`). -| `--listen string` | Accept connections from `local` (localhost only) or `network` (all interfaces / IP addresses) (default `network`). -| `--port int` | Web server port (default `7103`). -| `--var string=string` | Specify the value of a variable. Multiple `--var` arguments may be passed. -| `--var-file string` | Specify a `.fpvar` file containing variable values. -| `--verbose` | View detailed event information, including all trigger executions. -| `--watch` | Watch mod files for changes when running `flowpipe server` (default `true`). - -## Examples - -Start Flowpipe in server mode: -```bash -flowpipe server -``` - - -Start Flowpipe in server mode and set the [base URL](/docs/reference/env-vars/flowpipe_base_url): -```bash -flowpipe server --base-url "https://84c5df474.ngrok-free.dev" -``` - -Start Flowpipe on port 7104 -```bash -flowpipe server --port 7104 -``` - -Start Flowpipe on `localhost` only -```bash -flowpipe server --listen local -``` - -Start Flowpipe using settings from a workspace -```bash -flowpipe server --workspace my_flowpipe_server -``` - -Start Flowpipe in server mode but turn off file watching: -```bash -flowpipe server --watch=false -``` - - \ No newline at end of file diff --git a/docs/reference/cli/trigger.md b/docs/reference/cli/trigger.md deleted file mode 100644 index 8c762af..0000000 --- a/docs/reference/cli/trigger.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: flowpipe trigger -sidebar_label: flowpipe trigger ---- - -# flowpipe trigger - -Manage Flowpipe triggers in the current mod and its direct dependents. - -## Usage - -```bash -flowpipe trigger [command] [flags] -``` - -## Sub-Commands - -| Command | Description -|-|- -| `list` | List triggers from the current mod. -| `run` | Run a trigger from the current mod or its direct dependents or from a Flowpipe server instance. -| `show` | List a trigger from the current mod. - -## Flags - -| Flag | Applies to | Description -|-|-|- -| `--arg string=string` | `run` | Specify the value of a trigger argument. Multiple `--arg` arguments may be passed. -| `--detach` | `run` | Start the trigger and return immediately. By default, `flowpipe trigger run` will run the trigger and wait for the results. You may only use `--detach` when running a trigger from a server instance (by specifying `--host`, for example). -| `--var string=string` | `run`| Specify the value of a variable. Multiple `--var` arguments may be passed. -| `--var-file strings`| `run`| Specify an .fpvar file containing variable values. -| `--verbose` | `run` | View detailed event information, including step arguments and attributes. - -## Examples - -List triggers: - -```bash -flowpipe trigger list -``` - -View trigger details: - -```bash -flowpipe trigger show my_trigger -``` - -List triggers in JSON format: - -```bash -flowpipe trigger list --output json -``` - -Run a trigger in the current mod: - -```bash -flowpipe trigger run schedule.my_trigger -``` - -Run a trigger with verbose output: - -```bash -flowpipe trigger run schedule.my_trigger --verbose -``` - -Run a trigger and pass parameters: - -```bash -flowpipe trigger run schedule.my_trigger --arg my_string_param="my name" --arg 'my_list_param=["Owner","Application","Environment"]' -``` diff --git a/docs/reference/cli/variable.md b/docs/reference/cli/variable.md deleted file mode 100644 index 160a5a1..0000000 --- a/docs/reference/cli/variable.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: flowpipe variable -sidebar_label: flowpipe variable ---- - -# flowpipe variable - -Manage Flowpipe variables in the current mod and its direct dependents. - -## Usage -```bash -flowpipe variable [command] -flowpipe variable show variable_name [args] -``` - - -## Sub-Commands - -| Command | Description -|-|- -| `list` | List variables from the current mod and its direct dependents. -| `show` | Show details of a variable from the current mod or its direct dependents. - - -## Examples - - -List variables: -```bash -flowpipe variable list -``` - - -List all variables in `JSON` format: -```bash -flowpipe variable list --output json -``` - -List variables using settings from a workspace: -```bash -flowpipe variable list --workspace my_workspace -``` - -Show details of a single variable in the current mod: -```bash -flowpipe variable show mandatory_tags -``` - - -Show details of a single variable in a direct dependency mod: -```bash -flowpipe variable show aws_tags.mandatory_tags -``` - -Show details of a variable in `JSON` format: -```bash -flowpipe variable show mandatory_tags --output json -``` - -Show details of a variable using settings from a workspace: -```bash -flowpipe variable show mandatory_tags -workspace my_workspace -``` diff --git a/docs/reference/cli/variable.md.todo b/docs/reference/cli/variable.md.todo deleted file mode 100644 index 77c9ff6..0000000 --- a/docs/reference/cli/variable.md.todo +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: flowpipe variable -sidebar_label: flowpipe variable ---- - -# flowpipe variable - -Manage Flowpipe variables in the current mod and its direct dependents. - - -## Usage -```bash -flowpipe variable [command] [flags] -``` - -## Sub-Commands - -| Command | Description -|-|- -| `list` | List variables for the the current mod and its direct dependents. - - -## Examples - -List variables: - -```bash -flowpipe variable list -``` - - -List variables in JSON format: - -```bash -flowpipe variable list --output json -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_base_url.md b/docs/reference/env-vars/flowpipe_base_url.md deleted file mode 100644 index f1d37c4..0000000 --- a/docs/reference/env-vars/flowpipe_base_url.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: FLOWPIPE_BASE_URL -sidebar_label: FLOWPIPE_BASE_URL ---- - -# FLOWPIPE_BASE_URL - -## Usage - -Sets the base URL to use for [triggers](/docs/flowpipe-hcl/trigger) and [integrations](/docs/reference/config-files/integration). This is the base URL that Flowpipe advertises when it interacts with external systems to allow them to call back to Flowpipe. - -The base URL defaults `localhost` on the [Flowpipe port](/docs/reference/env-vars/flowpipe_port), eg `http://localhost:7103`. This is sufficient for local testing of `http` triggers and integrations, but to allow external users and systems to interact with Flowpipe, you should set `FLOWPIPE_BASE_URL` to a URL that will route to your system. This may include the DNS or IP address of the system on which you are running Flowpipe, or it may be a reverse proxy such as [ngrok](https://ngrok.com/). - - -## Usage - -blah -```bash -export FLOWPIPE_BASE_URL=https://84c5df474.ngrok-free.dev -``` - -Reset the base URL to the default: -```bash -unset FLOWPIPE_BASE_URL -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_config_path.md b/docs/reference/env-vars/flowpipe_config_path.md deleted file mode 100644 index 27c739a..0000000 --- a/docs/reference/env-vars/flowpipe_config_path.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: FLOWPIPE_CONFIG_PATH -sidebar_label: FLOWPIPE_CONFIG_PATH ---- - -# FLOWPIPE_CONFIG_PATH - - Sets the search path for [configuration files](/docs/reference/config-files). `FLOWPIPE_CONFIG_PATH` accepts a colon-separated list of directories. - - All configuration files (`*.fpc`) will be loaded from each path, with decreasing precedence. The default is the mod location, followed by the `config` directory in the [FLOWPIPE_INSTALL_DIR](/docs/reference/env-vars/flowpipe_install_dir): `.:$FLOWPIPE_INSTALL_DIR/config`. This allows you to manage your [workspaces](/docs/reference/config-files/workspace) and [connections](/docs/reference/config-files/connection) centrally in the `~/.flowpipe/config` directory, but override them in the working directory/mod-location if desired. - - -## Usage - -Set the configuration search path to the current mod location, followed by `/flowpipe` -```bash -export FLOWPIPE_CONFIG_PATH=.:/flowpipe -``` - -Set the configuration search path to `~/.flowpipe/config` but don't include the mod location -```bash -export FLOWPIPE_CONFIG_PATH=~/.flowpipe/config -``` - -Reset the configuration search path to the default: -```bash -unset FLOWPIPE_CONFIG_PATH -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_host.md b/docs/reference/env-vars/flowpipe_host.md deleted file mode 100644 index 126052a..0000000 --- a/docs/reference/env-vars/flowpipe_host.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: FLOWPIPE_HOST -sidebar_label: FLOWPIPE_HOST ---- - -# FLOWPIPE_HOST -Sets the Flowpipe API host to connect to. By default, Flowpipe commands run in a [client-only context](/docs/run#operating-modes) using the mod from the current working directory or the `--mod-location` if specified. When `FLOWPIPE_HOST` is set (or `--host` is passed on the command line or in a workspace) Flowpipe will instead run the command against the specified server instance. - -You may specify the full host and port (e.g. `https://flowpipe.my-org.com:7103`), or use the keyword `local` to connect to the local server instance (`local` is a shortcut for `https://localhost:7103`). - - - -## Usage -Run Flowpipe commands against the local Flowpipe server instead of the current directory: -```bash -export FLOWPIPE_HOST=local -``` - -Run Flowpipe commands against a remote server instead of the current directory: -```bash -export FLOWPIPE_HOST=https://flowpipe.my-org.com:7103 -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_install_dir.md b/docs/reference/env-vars/flowpipe_install_dir.md deleted file mode 100644 index ede943a..0000000 --- a/docs/reference/env-vars/flowpipe_install_dir.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: FLOWPIPE_INSTALL_DIR -sidebar_label: FLOWPIPE_INSTALL_DIR ---- - -# FLOWPIPE_INSTALL_DIR - -Set the installation directory for Flowpipe. Internal Flowpipe files will be written to this path. The default is `~/.flowpipe`. By default, the [FLOWPIPE_CONFIG_PATH](/docs/reference/env-vars/flowpipe_config_path) is also relative to this installation directory. - - -## Usage - -Change the installation directory to `/flowpipe` : - -```bash -export FLOWPIPE_INSTALL_DIR=/flowpipe -``` - -Reset the installation directory to the default: - -```bash -unset FLOWPIPE_INSTALL_DIR -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_listen.md b/docs/reference/env-vars/flowpipe_listen.md deleted file mode 100644 index 7be53d6..0000000 --- a/docs/reference/env-vars/flowpipe_listen.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: FLOWPIPE_LISTEN -sidebar_label: FLOWPIPE_LISTEN ---- - -# FLOWPIPE_LISTEN -Specifies the IP addresses on which `flowpipe server` will listen for connections from clients. Currently supported values are `local` (localhost only) or `network` (all IP addresses). The default is `network`. This value can only be set when the server is started. - -## Usage - -When `flowpipe server` runs, listen only on localhost: - -```bash -export FLOWPIPE_LISTEN=local -``` - -When `flowpipe server` runs, listen on all IP addresses: - -```bash -export FLOWPIPE_LISTEN=local -# or -unset FLOWPIPE_LISTEN -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_log_level.md b/docs/reference/env-vars/flowpipe_log_level.md deleted file mode 100644 index 3535b32..0000000 --- a/docs/reference/env-vars/flowpipe_log_level.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: FLOWPIPE_LOG_LEVEL -sidebar_label: FLOWPIPE_LOG_LEVEL ---- -# FLOWPIPE_LOG_LEVEL -Sets the output logging level. Standard log levels are supported (`DEBUG`, `INFO`, `WARN`, `ERROR`). Logs are written to STDERR. By default, logging is off. - -Logs are written to STDERR. - -## Usage - -Turn on info level logging: -```bash -export FLOWPIPE_LOG_LEVEL=INFO -``` - -Turn off logging: - -```bash -unset FLOWPIPE_LOG_LEVEL -``` - -Redirect logs to a file: -```bash -FLOWPIPE_LOG_LEVEL=DEBUG flowpipe pipeline run my_pipeline 2> mylogs.txt -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_max_concurrency_container.md b/docs/reference/env-vars/flowpipe_max_concurrency_container.md deleted file mode 100644 index 0efc048..0000000 --- a/docs/reference/env-vars/flowpipe_max_concurrency_container.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: FLOWPIPE_MAX_CONCURRENCY_CONTAINER -sidebar_label: FLOWPIPE_MAX_CONCURRENCY_CONTAINER ---- -# FLOWPIPE_MAX_CONCURRENCY_CONTAINER - -Sets the maximum number of `container` step instances that can execute concurrently across all pipeline instances. When the limit is reached, `container` steps will be queued until enough `container` steps complete to bring the count under the limit. - -The minimum value is `1`, and the default is `25`. - -## Usage - - -Set the concurrency to 100: -```bash -export FLOWPIPE_MAX_CONCURRENCY_CONTAINER=100 -``` - -Reset to the default value: -```bash -unset FLOWPIPE_MAX_CONCURRENCY_CONTAINER -``` diff --git a/docs/reference/env-vars/flowpipe_max_concurrency_function.md b/docs/reference/env-vars/flowpipe_max_concurrency_function.md deleted file mode 100644 index f84712e..0000000 --- a/docs/reference/env-vars/flowpipe_max_concurrency_function.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: FLOWPIPE_MAX_CONCURRENCY_FUNCTION -sidebar_label: FLOWPIPE_MAX_CONCURRENCY_FUNCTION ---- -# FLOWPIPE_MAX_CONCURRENCY_FUNCTION - -Sets the maximum number of `function` step instances that can execute concurrently across all pipeline instances. When the limit is reached, `function` steps will be queued until enough `function` steps complete to bring the count under the limit. - -The minimum value is `1`, and the default is `50`. - -## Usage - - -Set the concurrency to 100: -```bash -export FLOWPIPE_MAX_CONCURRENCY_FUNCTION=100 -``` - -Reset to the default value: -```bash -unset FLOWPIPE_MAX_CONCURRENCY_FUNCTION -``` diff --git a/docs/reference/env-vars/flowpipe_max_concurrency_http.md b/docs/reference/env-vars/flowpipe_max_concurrency_http.md deleted file mode 100644 index b34cf08..0000000 --- a/docs/reference/env-vars/flowpipe_max_concurrency_http.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: FLOWPIPE_MAX_CONCURRENCY_HTTP -sidebar_label: FLOWPIPE_MAX_CONCURRENCY_HTTP ---- -# FLOWPIPE_MAX_CONCURRENCY_HTTP - -Sets the maximum number of `http` step instances that can execute concurrently across all pipeline instances. When the limit is reached, `http` steps will be queued until enough `http` steps complete to bring the count under the limit. - -The minimum value is `1`, and the default is `500`. - -## Usage - - -Set the concurrency to 100: -```bash -export FLOWPIPE_MAX_CONCURRENCY_HTTP=100 -``` - -Reset to the default value: -```bash -unset FLOWPIPE_MAX_CONCURRENCY_HTTP -``` diff --git a/docs/reference/env-vars/flowpipe_max_concurrency_query.md b/docs/reference/env-vars/flowpipe_max_concurrency_query.md deleted file mode 100644 index 8ff4fc2..0000000 --- a/docs/reference/env-vars/flowpipe_max_concurrency_query.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: FLOWPIPE_MAX_CONCURRENCY_QUERY -sidebar_label: FLOWPIPE_MAX_CONCURRENCY_QUERY ---- -# FLOWPIPE_MAX_CONCURRENCY_QUERY - -Sets the maximum number of `query` step instances that can execute concurrently across all pipeline instances. When the limit is reached, `query` steps will be queued until enough `query` steps complete to bring the count under the limit. - -The minimum value is `1`, and the default is `50`. - -## Usage - - -Set the concurrency to 100: -```bash -export FLOWPIPE_MAX_CONCURRENCY_QUERY=100 -``` - -Reset to the default value: -```bash -unset FLOWPIPE_MAX_CONCURRENCY_QUERY -``` diff --git a/docs/reference/env-vars/flowpipe_memory_max_mb.md b/docs/reference/env-vars/flowpipe_memory_max_mb.md deleted file mode 100644 index 6d9e2f4..0000000 --- a/docs/reference/env-vars/flowpipe_memory_max_mb.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: FLOWPIPE_MEMORY_MAX_MB -sidebar_label: FLOWPIPE_MEMORY_MAX_MB ---- - -# FLOWPIPE_MEMORY_MAX_MB -Set a soft memory limit for the `flowpipe` process. Flowpipe sets `GOMEMLIMIT` for the `flowpipe` process to the specified value. The Go runtime does not guarantee that the memory usage will not exceed the limit, but rather uses it as a target to optimize garbage collection. - -Set the `FLOWPIPE_MEMORY_MAX_MB` to `0` to disable the soft memory limit. - -## Usage - -Set the memory soft limit to 2GB: -```bash -export FLOWPIPE_MEMORY_MAX_MB=2048 -``` - -Disable the memory soft limit: -```bash -export FLOWPIPE_MEMORY_MAX_MB=0 -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_mod_location.md b/docs/reference/env-vars/flowpipe_mod_location.md deleted file mode 100644 index 37f2e41..0000000 --- a/docs/reference/env-vars/flowpipe_mod_location.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: FLOWPIPE_MOD_LOCATION -sidebar_label: FLOWPIPE_MOD_LOCATION ---- - -# FLOWPIPE_MOD_LOCATION -Sets the Flowpipe workspace directory. If not specified, the workspace directory will be set to the current working directory. - - -## Usage -Set your mod location: -```bash -export FLOWPIPE_MOD_LOCATION=~/my-steampipe-mods -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_port.md b/docs/reference/env-vars/flowpipe_port.md deleted file mode 100644 index 9a890e1..0000000 --- a/docs/reference/env-vars/flowpipe_port.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: FLOWPIPE_PORT -sidebar_label: FLOWPIPE_PORT ---- - -# FLOWPIPE_PORT -Specifies the TCP port on which `flowpipe server` will listen for connections from clients. The default is `7103`. This value can only be set when the server is started. - -## Usage - -When `flowpipe server` runs, listen on port 7777: - -```bash -export FLOWPIPE_PORT=7777 -``` diff --git a/docs/reference/env-vars/flowpipe_telemetry.md b/docs/reference/env-vars/flowpipe_telemetry.md deleted file mode 100644 index 89999a5..0000000 --- a/docs/reference/env-vars/flowpipe_telemetry.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: FLOWPIPE_TELEMETRY -sidebar_label: FLOWPIPE_TELEMETRY ---- - -# FLOWPIPE_TELEMETRY - -By default, Flowpipe collects usage information to help assess features, usage patterns, and bugs. This information helps us improve and optimize the Flowpipe experience. We do not collect any sensitive information such as secrets, environment variables, or file contents. We do not share your data with anyone. Current options are: -- `none`: do not collect or send any telemetry data. -- `info`: send basic information such as which mods are used, and how and when Flowpipe is started and stopped. - -## Usage - -Disable telemetry data: - -```bash -export FLOWPIPE_TELEMETRY=none -``` - -Enable telemetry data at `info` level (this is the default) - -```bash -export FLOWPIPE_TELEMETRY=info -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_update_check.md b/docs/reference/env-vars/flowpipe_update_check.md deleted file mode 100644 index bbaf418..0000000 --- a/docs/reference/env-vars/flowpipe_update_check.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: FLOWPIPE_UPDATE_CHECK -sidebar_label: FLOWPIPE_UPDATE_CHECK ---- - -# FLOWPIPE_UPDATE_CHECK -Automatic update checking informs you when a new Flowpipe version is available. The `FLOWPIPE_UPDATE_CHECK` environment variable allows you to enable or disable automatic update checking. Update checking is enabled by default. Set to `false` to disable update checking. - -## Usage -Disable update check: -```bash -export FLOWPIPE_UPDATE_CHECK=false -``` - -Enable update check: -```bash -export FLOWPIPE_UPDATE_CHECK=true -``` -or -```bash -unset FLOWPIPE_UPDATE_CHECK -``` \ No newline at end of file diff --git a/docs/reference/env-vars/flowpipe_workspace.md b/docs/reference/env-vars/flowpipe_workspace.md deleted file mode 100644 index 97afd19..0000000 --- a/docs/reference/env-vars/flowpipe_workspace.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: FLOWPIPE_WORKSPACE -sidebar_label: FLOWPIPE_WORKSPACE ---- - - -# FLOWPIPE_WORKSPACE - -Sets the Flowpipe [workspace](/docs/reference/config-files/workspace). - -A Flowpipe `workspace` is a "profile" that allows you to define a unified environment that the Flowpipe can interact with. - -To learn more, see **[Managing Workspaces →](/docs/run/workspaces)** - - - -## Usage -Use the `my_workspace` workspace: -```bash -export FLOWPIPE_WORKSPACE=my_workspace -``` - - \ No newline at end of file diff --git a/docs/reference/env-vars/foo b/docs/reference/env-vars/foo new file mode 100644 index 0000000..1786a6d --- /dev/null +++ b/docs/reference/env-vars/foo @@ -0,0 +1,16 @@ +--- +title: Environment Variables +sidebar_label: Environment Variables +--- + +# Environment Variables + +Tailpipe supports environment variables to allow you to change its default behavior. These are optional settings - You are not required to set any environment variables. + +Note that plugins may also support environment variables, but these are plugin-specific - refer to your plugin's documentation on the [Tailpipe Hub](https://hub.tailpipe.io/) for details. + +## Tailpipe Environment Variables + +| Command | Default | Description +|-|-|- +| [FLOWPIPE_CONFIG_PATH](/docs/reference/env-vars/flowpipe_config_path) | `.:$FLOWPIPE_INSTALL_DIR/config` | Set the search path for [configuration files](/docs/reference/config-files). \ No newline at end of file diff --git a/docs/reference/env-vars/index.md b/docs/reference/env-vars/index.md index 344865e..ae61cbe 100644 --- a/docs/reference/env-vars/index.md +++ b/docs/reference/env-vars/index.md @@ -5,53 +5,12 @@ sidebar_label: Environment Variables # Environment Variables -Flowpipe supports environment variables to allow you to change its default behavior. These are optional settings - You are not required to set any environment variables. +Tailpipe supports environment variables to allow you to change its default behavior. These are optional settings - You are not required to set any environment variables. -Note that plugins may also support environment variables, but these are plugin-specific - refer to your plugin's documentation on the [Flowpipe Hub](https://hub.flowpipe.io/) for details. +Note that plugins may also support environment variables, but these are plugin-specific - refer to your plugin's documentation on the [Tailpipe Hub](https://hub.tailpipe.io/) for details. -## Flowpipe Environment Variables +## Tailpipe Environment Variables | Command | Default | Description |-|-|- -| [FLOWPIPE_BASE_URL](/docs/reference/env-vars/flowpipe_base_url) | `http://localhost:7103` | Set the base URL to use for [triggers](/docs/flowpipe-hcl/trigger) and [integrations](/docs/reference/config-files/integration). This is the base URL that Flowpipe advertises when it interacts with external systems to allow them to call back to Flowpipe. -| [FLOWPIPE_CONFIG_PATH](/docs/reference/env-vars/flowpipe_config_path) | `.:$FLOWPIPE_INSTALL_DIR/config` | Set the search path for [configuration files](/docs/reference/config-files). `FLOWPIPE_CONFIG_PATH` accepts a colon-separated list of directories. -| [FLOWPIPE_HOST](/docs/reference/env-vars/flowpipe_host) | none | Set the remote Flowpipe API host to connect to. This allows you to run Flowpipe commands against a flowpipe host instead of the current working directory / mod location. -| [FLOWPIPE_INSTALL_DIR](/docs/reference/env-vars/flowpipe_install_dir) | `~/.flowpipe` | Set the installation directory for flowpipe. Internal flowpipe files will be written to this path. -| [FLOWPIPE_LISTEN](/docs/reference/env-vars/flowpipe_listen) | `network` | Specifies the IP addresses on which `flowpipe server` will listen for connections from clients. Currently supported values are `local` (localhost only) or `network` (all IP addresses). -| [FLOWPIPE_LOG_LEVEL](/docs/reference/env-vars/flowpipe_log_level) | off | Set the logging output level -| [FLOWPIPE_MAX_CONCURRENCY_CONTAINER](/docs/reference/env-vars/flowpipe_max_concurrency_container) | `25` | Set the maximum number of `container` step instances that can execute concurrently across all pipeline instances. -| [FLOWPIPE_MAX_CONCURRENCY_FUNCTION](/docs/reference/env-vars/flowpipe_max_concurrency_function) | `50` | Set the maximum number of `function` step instances that can execute concurrently across all pipeline instances. -| [FLOWPIPE_MAX_CONCURRENCY_HTTP](/docs/reference/env-vars/flowpipe_max_concurrency_http) | `500` | Set the maximum number of `http` step instances that can execute concurrently across all pipeline instances. -| [FLOWPIPE_MAX_CONCURRENCY_QUERY](/docs/reference/env-vars/flowpipe_max_concurrency_query) | `50` | Set the maximum number of `query` step instances that can execute concurrently across all pipeline instances. -| [FLOWPIPE_MEMORY_MAX_MB](/docs/reference/env-vars/flowpipe_memory_max_mb) | `1024` | Set a soft memory limit for the `flowpipe` process. -| [FLOWPIPE_MOD_LOCATION](/docs/reference/env-vars/flowpipe_mod_location) | current working directory | Set the workspace working directory -| [FLOWPIPE_PORT](/docs/reference/env-vars/flowpipe_port) | `7103` | Specifies the TCP port on which `flowpipe server` will listen for connections from clients. -| [FLOWPIPE_TELEMETRY](/docs/reference/env-vars/flowpipe_telemetry) | `info` | Set the level of telemetry data to collect and send -| [FLOWPIPE_UPDATE_CHECK](/docs/reference/env-vars/flowpipe_update_check)| `true` | Enable/disable automatic update checking -| [FLOWPIPE_WORKSPACE](/docs/reference/env-vars/flowpipe_workspace) | `default` | Set the Flowpipe workspace . This can be named workspace from `workspaces.fpc` or a remote Flowpipe Cloud workspace -| [PIPES_INSTALL_DIR](/docs/reference/env-vars/flowpipe_base_url) | `~/.pipes` | Set the installation directory for files used with [Turbot Pipes](https://turbot.com/pipes/docs), such as login tokens. - - - - - \ No newline at end of file +| [TAILPIPE_CONFIG_PATH](/docs/reference/env-vars/tailpipe_config_path) | `.:$TAILPIPE_INSTALL_DIR/config` | Set the search path for [configuration files](/docs/reference/config-files). \ No newline at end of file diff --git a/docs/reference/env-vars/pipes_install_dir.md b/docs/reference/env-vars/pipes_install_dir.md deleted file mode 100644 index ed81283..0000000 --- a/docs/reference/env-vars/pipes_install_dir.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: PIPES_INSTALL_DIR -sidebar_label: PIPES_INSTALL_DIR ---- - -# PIPES_INSTALL_DIR - -Set the installation directory for files used with [Turbot Pipes](https://turbot.com/pipes/docs), such as login tokens. The default is `~/.pipes`. - - -## Usage - -Change the Pipes installation directory to `/pipes` : - -```bash -export PIPES_INSTALL_DIR=/pipes -``` - -Reset the Pipes installation directory to the default: - -```bash -unset PIPES_INSTALL_DIR -``` \ No newline at end of file diff --git a/docs/run/plugins.md b/docs/run/plugins.md new file mode 100644 index 0000000..5e32aef --- /dev/null +++ b/docs/run/plugins.md @@ -0,0 +1,9 @@ +--- +title: Managing Plugins +sidebar_label: Plugins +--- + +Tailpipe provides an integrated, standardized SQL interface for querying various sources of log data. It relies on plugins to define and implement tables for those logs. This approach decouples the Tailpipe core from provider-specific implementations, providing flexibility and extensibility. + +# Installing Plugins + diff --git a/docs/sidebar.json b/docs/sidebar.json index 3e7e758..f26bfe2 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -11,77 +11,12 @@ "id": "run", "link": "run", "items": [ + "run/plugins", "run/collect", "run/query", "run/connections" ] }, - { - "type": "category", - "id": "working-with-mods", - "label": "Build Mods", - "link": "build", - "items": [ - { - "type": "category", - "id": "build/write-pipelines", - "label": "Learn Flowpipe", - "link": "build/write-pipelines", - "items": [ - "build/write-pipelines/control-flow", - "build/write-pipelines/inputs-outputs", - "build/write-pipelines/conditionals", - "build/write-pipelines/iteration", - "build/write-pipelines/errors" - ] - }, - "build/triggers", - "build/input", - "build/mod-variables", - "build/mod-dependencies" - ] - }, - - { - "type": "category", - "id": "flowpipe-hcl", - "link": "flowpipe-hcl", - "items": [ - "flowpipe-hcl/functions", - "flowpipe-hcl/locals", - "flowpipe-hcl/mod", - "flowpipe-hcl/pipeline", - { - "type": "category", - "id": "steps", - "link": "flowpipe-hcl/step", - "items": [ - "flowpipe-hcl/step/container", - "flowpipe-hcl/step/email", - "flowpipe-hcl/step/function", - "flowpipe-hcl/step/http", - "flowpipe-hcl/step/input", - "flowpipe-hcl/step/message", - "flowpipe-hcl/step/pipeline", - "flowpipe-hcl/step/query", - "flowpipe-hcl/step/sleep", - "flowpipe-hcl/step/transform" - ] - }, - { - "type": "category", - "id": "triggers", - "link": "flowpipe-hcl/trigger", - "items": [ - "flowpipe-hcl/trigger/http", - "flowpipe-hcl/trigger/query", - "flowpipe-hcl/trigger/schedule" - ] - }, - "flowpipe-hcl/variable" - ] - }, - { "type": "category", "id": "reference", @@ -94,44 +29,17 @@ "label": "Command Line Arguments", "link": "reference/cli", "items": [ - "reference/cli/help", - "reference/cli/integration", - "reference/cli/notifier", - "reference/cli/mod", - "reference/cli/pipeline", - "reference/cli/process", - "reference/cli/server", - "reference/cli/trigger", - "reference/cli/variable" + "reference/cli/help" ] }, - { "type": "category", "id": "env-vars", "label": "Environment Variables", "link": "reference/env-vars", "items": [ - "reference/env-vars/flowpipe_base_url", - "reference/env-vars/flowpipe_config_path", - "reference/env-vars/flowpipe_host", - "reference/env-vars/flowpipe_install_dir", - "reference/env-vars/flowpipe_listen", - "reference/env-vars/flowpipe_log_level", - "reference/env-vars/flowpipe_max_concurrency_container", - "reference/env-vars/flowpipe_max_concurrency_function", - "reference/env-vars/flowpipe_max_concurrency_http", - "reference/env-vars/flowpipe_max_concurrency_query", - "reference/env-vars/flowpipe_memory_max_mb", - "reference/env-vars/flowpipe_mod_location", - "reference/env-vars/flowpipe_port", - "reference/env-vars/flowpipe_telemetry", - "reference/env-vars/flowpipe_update_check", - "reference/env-vars/flowpipe_workspace", - "reference/env-vars/pipes_install_dir" ] }, - { "type": "category", "id": "config-files", @@ -143,44 +51,6 @@ "id": "connection", "link": "reference/config-files/connection", "items": [ - "reference/config-files/connection/abuseipdb", - "reference/config-files/connection/alicloud", - "reference/config-files/connection/aws", - "reference/config-files/connection/azure", - "reference/config-files/connection/bitbucket", - "reference/config-files/connection/clickup", - "reference/config-files/connection/datadog", - "reference/config-files/connection/discord", - "reference/config-files/connection/duckdb", - "reference/config-files/connection/freshdesk", - "reference/config-files/connection/gcp", - "reference/config-files/connection/github", - "reference/config-files/connection/gitlab", - "reference/config-files/connection/ip2locationio", - "reference/config-files/connection/ipstack", - "reference/config-files/connection/jira", - "reference/config-files/connection/jumpcloud", - "reference/config-files/connection/mastodon", - "reference/config-files/connection/mysql", - "reference/config-files/connection/okta", - "reference/config-files/connection/openai", - "reference/config-files/connection/opsgenie", - "reference/config-files/connection/pagerduty", - "reference/config-files/connection/pipes", - "reference/config-files/connection/postgres", - "reference/config-files/connection/sendgrid", - "reference/config-files/connection/servicenow", - "reference/config-files/connection/slack", - "reference/config-files/connection/sqlite", - "reference/config-files/connection/steampipe", - "reference/config-files/connection/teams", - "reference/config-files/connection/trello", - "reference/config-files/connection/turbot_guardrails", - "reference/config-files/connection/uptimerobot", - "reference/config-files/connection/urlscan", - "reference/config-files/connection/vault", - "reference/config-files/connection/virustotal", - "reference/config-files/connection/zendesk" ] }, { @@ -188,39 +58,6 @@ "id": "credential", "link": "reference/config-files/credential", "items": [ - "reference/config-files/credential/abuseipdb", - "reference/config-files/credential/alicloud", - "reference/config-files/credential/aws", - "reference/config-files/credential/azure", - "reference/config-files/credential/bitbucket", - "reference/config-files/credential/clickup", - "reference/config-files/credential/datadog", - "reference/config-files/credential/discord", - "reference/config-files/credential/freshdesk", - "reference/config-files/credential/gcp", - "reference/config-files/credential/github", - "reference/config-files/credential/gitlab", - "reference/config-files/credential/ip2locationio", - "reference/config-files/credential/ipstack", - "reference/config-files/credential/jira", - "reference/config-files/credential/jumpcloud", - "reference/config-files/credential/mastodon", - "reference/config-files/credential/okta", - "reference/config-files/credential/openai", - "reference/config-files/credential/opsgenie", - "reference/config-files/credential/pagerduty", - "reference/config-files/credential/pipes", - "reference/config-files/credential/sendgrid", - "reference/config-files/credential/servicenow", - "reference/config-files/credential/slack", - "reference/config-files/credential/teams", - "reference/config-files/credential/trello", - "reference/config-files/credential/turbot_guardrails", - "reference/config-files/credential/uptimerobot", - "reference/config-files/credential/urlscan", - "reference/config-files/credential/vault", - "reference/config-files/credential/virustotal", - "reference/config-files/credential/zendesk" ] }, { @@ -238,20 +75,7 @@ "items": [ "reference/config-files/credential_import/steampipe" ] - }, - { - "type": "category", - "id": "integration", - "link": "reference/config-files/integration", - "items": [ - "reference/config-files/integration/email", - "reference/config-files/integration/http", - "reference/config-files/integration/msteams", - "reference/config-files/integration/slack" - ] - }, - "reference/config-files/notifier", - "reference/config-files/workspace" + } ] } ]