diff --git a/docs/build/create-mod.md b/docs/build/create-mod.md new file mode 100644 index 0000000..b0cbab2 --- /dev/null +++ b/docs/build/create-mod.md @@ -0,0 +1,7 @@ +--- +title: Create a Mod +sidebar_label: Create a Mod +--- + +# Create a Mod + diff --git a/docs/build/write-dashboards.md b/docs/build/write-dashboards.md new file mode 100644 index 0000000..4352ecc --- /dev/null +++ b/docs/build/write-dashboards.md @@ -0,0 +1,7 @@ +--- +title: Write Dashboards +sidebar_label: Write Dashboards +--- + +# Write Dashboards + diff --git a/docs/build/write-detections.md b/docs/build/write-detections.md new file mode 100644 index 0000000..b5e46da --- /dev/null +++ b/docs/build/write-detections.md @@ -0,0 +1,7 @@ +--- +title: Write Detections +sidebar_label: Write Detections +--- + +# Write Detections + diff --git a/docs/build/write-mods/conditionals.md b/docs/build/write-mods/conditionals.md deleted file mode 100644 index 47e298f..0000000 --- a/docs/build/write-mods/conditionals.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: Conditional Execution -sidebar_label: Conditional Execution ---- -# Conditional Execution - -You can conditionally execute a step using the `if` argument: - -```hcl -pipeline "notify_unencrypted" { - step "query" "unencrypted_vols" { - database = "postgres://steampipe@localhost:9193/steampipe" - sql = "select arn from aws_ebs_volume where not encrypted" - } - - step "http" "notify_slack" { - if = length(step.query.unencrypted_vols.rows) > 0 - url = "https://hooks.slack.com/services/T04THIS54LQ/B04ISH1B2GM/vIakTJfqFAKET7M14g5H32w8" - method = "post" - - request_body = jsonencode({ - text = "Unencrypted Volumes: ${join(", ", step.query.unencrypted_vols.rows[*].arn)}" - }) - } -} -``` - -Flowpipe does not provide `else` or `case` arguments to `step`, but you can mimic the behavior by using different conditions in multiple steps: - -```hcl -pipeline "notify_unencrypted" { - step "query" "unencrypted_vols" { - database = "postgres://steampipe@localhost:9193/steampipe" - sql = "select arn from aws_ebs_volume where not encrypted" - } - - step "http" "notify_slack" { - if = length(step.query.unencrypted_vols.rows) > 0 - url = "https://hooks.slack.com/services/T04THIS54LQ/B04ISH1B2GM/vIakTJfqFAKET7M14g5H32w8" - method = "post" - - request_body = jsonencode({ - text = "There are ${length(step.query.unencrypted_vols.rows)} unencrypted volumes!" - }) - } - - step "http" "notify_slack_all_encrypted" { - if = length(step.query.unencrypted_vols.rows) == 0 - url = "https://hooks.slack.com/services/T04THIS54LQ/B04ISH1B2GM/vIakTJfqFAKET7M14g5H32w8" - method = "post" - - request_body = jsonencode({ - text = "All Volumes are encrypted!" - }) - } -} -``` - -In this instance, you could accomplish the same thing using [conditional expressions](https://developer.hashicorp.com/terraform/language/expressions/conditionals): - -```hcl -pipeline "notify_unencrypted" { - step "query" "unencrypted_vols" { - database = "postgres://steampipe@localhost:9193/steampipe" - sql = "select arn from aws_ebs_volume where not encrypted" - } - - step "http" "notify_slack" { - url = "https://hooks.slack.com/services/T04THIS54LQ/B04ISH1B2GM/vIakTJfqFAKET7M14g5H32w8" - method = "post" - - request_body = (length(step.query.unencrypted_vols.rows) > 0 - ? jsonencode({ text = "There are ${length(step.query.unencrypted_vols.rows)} unencrypted volumes!" }) - : jsonencode({ text = "All Volumes are encrypted!" }) - ) - } -} -``` - -or string [directives](https://developer.hashicorp.com/terraform/language/expressions/strings#directives): - -```hcl -pipeline "notify_unencrypted" { - step "query" "unencrypted_vols" { - database = "postgres://steampipe@localhost:9193/steampipe" - sql = "select arn from aws_ebs_volume where not encrypted" - } - - step "http" "notify_slack" { - url = "https://hooks.slack.com/services/T04THIS54LQ/B04ISH1B2GM/vIakTJfqFAKET7M14g5H32w8" - method = "post" - - request_body = jsonencode({ text = "%{if length(step.query.unencrypted_vols.rows) > 0}There are ${length(step.query.unencrypted_vols.rows)} unencrypted volumes! %{else} All Volumes are encrypted! %{endif}" }) - } -} -``` diff --git a/docs/build/write-mods/control-flow.md b/docs/build/write-mods/control-flow.md deleted file mode 100644 index 7bd209f..0000000 --- a/docs/build/write-mods/control-flow.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Control Flow -sidebar_label: Control Flow ---- - -# Control Flow - -By default, all steps are run in parallel: - -```hcl -pipeline "my_parallel_pipeline" { - step "http" "my_step_1" { - url = "https://example.com/my/webhook1" - } - - step "http" "my_step_2" { - url = "https://example.com/my/webhook1" - } -} -``` - -Flowpipe will define implicit dependencies based on references to other steps and run them in the required order: - -```hcl -pipeline "my_serial_pipeline" { - step "http" "my_step_1" { - url = "https://example.com/my/webhook1" - } - - step "http" "my_step_2" { - url = "https://example.com/my/webhook2" - body = step.http.my_step_1.response_body - } -} -``` - -You can also define explicit dependencies with `depends_on` to make them run in a specific order - -```hcl -pipeline "my_serial_pipeline" { - step "http" "my_step_1" { - url = "https://example.com/my/webhook1" - } - - step "http" "my_step_2" { - url = "https://example.com/my/webhook2" - depends_on = [step.http.my_step_1] - } -} -``` diff --git a/docs/build/write-mods/index.md b/docs/build/write-mods/index.md deleted file mode 100644 index db2ed2b..0000000 --- a/docs/build/write-mods/index.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -#id: learn -#title: Learn Flowpipe -#sidebar_label: Learn Flowpipe -#slug: / -title: Write Pipelines -sidebar_label: Write Pipelines ---- - -# Write Pipelines - -## Basics - -A pipeline is composed of one or more steps: - - -```hcl -pipeline "my_job" { - step "http" "my_http_step" { - url = "https://example.com/my/webhook" - } -} -``` - -Each step has a type and a name, and the arguments are dependent on the type. The step types are sometimes referred to as "primitives". The previous example included an `http` step, but there are others - `query`, `function`, `container`, etc. You can even run other pipelines using a `pipeline` step. - - -You can run a pipeline manually from the command line: -```bash -flowpipe pipeline run my_job -``` - -You can also define a trigger that will start the pipeline in response to an event. For example, you can run it on a schedule with an `interval` trigger: - -```hcl -trigger "schedule" "my_hourly_trigger" { - schedule = "hourly" - pipeline = pipeline.my_job -} - -pipeline "my_job" { - step "http" "my_http_step" { - url = "https://example.com/my/webhook" - } -} -``` - -Triggers, like steps, require 2 labels - a type and a name. There are multiple trigger types - `http`, `schedule`, `query`, etc. diff --git a/docs/build/write-mods/iteration.md b/docs/build/write-mods/iteration.md deleted file mode 100644 index bb8a569..0000000 --- a/docs/build/write-mods/iteration.md +++ /dev/null @@ -1,246 +0,0 @@ ---- -title: Iteration -sidebar_label: Iteration ---- - -# Iteration - -## for_each -You can run a step in a parallel loop using `for_each`: - - -```hcl -pipeline "send_keys_to_slack" { - param "access_keys" { - type = "map" - - default = { - AKIAXYSEVFAKE0123456 = { - user_name = "jenny" - account_id = "000008675309" - } - AKIAXYSEVFAKE0123456 = { - user_name = "janderson" - account_id = "000000090125" - } - } - } - - step "http" "send_to_slack" { - for_each = param.access_keys - url = var.slack_webhook_url - method = "post" - - request_body = jsonencode({ - text = "New AWS IAM access key: ${each.key} for user ${each.value.user_name} in ${each.value.account_id}" - }) - } -} -``` - - -Note that `for_each` accepts a *map* or a *list* of strings: - -```hcl -pipeline "add_users" { - param "users" { - type = "list" - default = ["jerry","Janis", "Jimi"] - } - - step "http" "add_a_user" { - for_each = param.users - url = "https://myapi.local/api/v1/user" - method = "post" - - request_body = jsonencode({ - user_name = "${each.value}" - }) - } -} -``` - -When using the list form, `each.key` will be the index of the current item in the list, and `each.value` is the actual list item. - -You may need to reference an "iterated" step in an `output` or another `step`. These step instances are identified by a map key (or set member) from the value provided to `for_each`: -- `step..` (e.g.`step.http.add_a_user`) refers to the block. Note that `depends_on` will use the block reference, and that the dependency means *all* instances of the step. You cannot depend on a specific instance of a step from a `for_each` loop. -- `step..[]` (e.g.`step.http.add_a_user["0"]`, `step.http.send_to_slack["AKIAXYSEVFAKE0123456"]`) refers to an individual step instance. - -For example, to wait until all the `add_a_user` steps have completed before sending a notification: -```hcl -pipeline "add_users" { - param "users" { - type = "list" - default = ["jerry","Janis", "Jimi"] - } - - step "http" "add_a_user" { - for_each = param.users - url = "https://myapi.local/api/v1/user" - method = "post" - - request_body = jsonencode({ - user_name = "${each.value}" - }) - } - - step "http" "notify_slack" { - depends_on = [step.http.add_a_user] - url = var.slack_webhook_url - - request_body = jsonencode({ - text = "New users created: ${join(",", param.users)}" - }) - } -} -``` - - The step instances for a `for_each` are represented by a map, you can chain steps together where there is a 1:1 relationship between them (see [Terraform resource chaining](https://developer.hashicorp.com/terraform/language/meta-arguments/for_each#chaining-for_each-between-resources)). For instance, we can modify the previous example to send a separate notification for each user: - -```hcl -pipeline "add_users" { - param "users" { - type = "list" - default = ["jerry","Janis", "Jimi"] - } - - step "http" "add_a_user" { - for_each = param.users - url = "https://myapi.local/api/v1/user" - method = "post" - - request_body = jsonencode({ - user_name = "${each.value}" - }) - } - - - step "http" "notify_slack" { - for_each = step.http.add_a_user - url = var.slack_webhook_url - - request_body = jsonencode({ - text = "New user created: ${jsondecode(each.value.request_body)["user_name"]}" - }) - } -} -``` - -In fact, you can use `for_each` and `if` in the same step (`for_each` is evaluated before `if`), so you could improve the example: - - -```hcl -pipeline "add_users" { - param "users" { - type = "list" - default = ["jerry","Janis", "Jimi"] - } - - step "http" "add_a_user" { - for_each = param.users - url = "https://myapi.local/api/v1/user" - method = "post" - - request_body = jsonencode({ - user_name = "${each.value}" - }) - } - - step "http" "notify_success" { - for_each = step.http.add_a_user - if = each.value.status_code >= 200 && each.value.status_code < 300 - url = var.slack_webhook_url - - request_body = jsonencode({ - text = "New user created: ${jsondecode(each.value.request_body)["user_name"]}" - }) - } - - step "http" "notify_failed" { - for_each = step.http.add_a_user - if = each.value.status_code < 200 || each.value.status_code >= 300 - url = var.slack_webhook_url - - request_body = jsonencode({ - text = "ERROR: New user creation failed: ${jsondecode(each.value.request_body)["user_name"]}" - }) - } -} -``` - - -## loop -You can also run a step in a sequential loop, changing the arguments with each iteration. This is useful for handling HTTP pagination, for example. To iterate the step, include a `loop` block. The `until` condition is used to break the loop - Flowpipe will run the step again unless the `until` condition is `true`. Once the `until` condition is true the loop is broken, the step completes, and execution continues at the next step. - -The loop is evaluated ***last*** after the step instance has been executed and all retries have been completed. You can use the special value `result` to evaluate the attributes of the completed step instance (you can use `result` in an error handling block like `throw`, `error`, and `retry` as well). `result` is essentially a self-reference to "this" step after it has run (e.g. the attributes are populated). - -When the step runs again due to the loop, it inherits all of the arguments from the step, but you can override them inside the `loop` block if desired, allowing you to pass data from the step execution into the next step execution. For instance, you may extract a pagination token from an HTTP response, and then override the step's `url` argument and pass that token in as a querystring parameter: - - -```hcl - step "http" "list_workspaces" { - url = "https://latestpipe.turbot.io/api/v1/org/latesttank/workspace/?limit=3" - method = "get" - - request_headers = { - Content-Type = "application/json" - Authorization = "Bearer ${param.pipes_token}" - } - - loop { - until = result.response_body.next_token == null - url = "https://latestpipe.turbot.io/api/v1/org/latesttank/workspace/?limit=3&next_token=${result.response_body.next_token}" - } - - } - -``` - -There is a special attribute called `loop` with a single attribute `index` which is the (zero-based) index of the loop count: - -```hcl -pipeline "simple_loop" { - - step "transform" "repeat" { - text = "iteration ${loop.index}" - - - loop { - until = loop.index >= 3 - } - } -} -``` \ No newline at end of file diff --git a/docs/reference/cli/collect.md b/docs/reference/cli/collect.md index fb6c602..f44f72f 100644 --- a/docs/reference/cli/collect.md +++ b/docs/reference/cli/collect.md @@ -1,11 +1,11 @@ --- -title: flowpipe integration -sidebar_label: flowpipe integration +title: tailpipe collect +sidebar_label: tailpipe collect --- -# flowpipe integration +# tailpipe collect -List and show Flowpipe [integrations](/docs/reference/config-files/integration). +Run a [collection](/docs/reference/config-files/integration). ## Usage diff --git a/docs/reference/cli/connect.md b/docs/reference/cli/connect.md new file mode 100644 index 0000000..56b2094 --- /dev/null +++ b/docs/reference/cli/connect.md @@ -0,0 +1,17 @@ +--- +title: tailpipe connect +sidebar_label: tailpipe connect +--- + +# tailpipe connect + +Return a connection string for a database, with a schema determined by the provided parameters. + + +## Usage +```bash +tailpipe connect [flags] +``` + +## Examples + diff --git a/docs/reference/cli/index.md b/docs/reference/cli/index.md index 09fe43a..690bdb3 100644 --- a/docs/reference/cli/index.md +++ b/docs/reference/cli/index.md @@ -11,16 +11,11 @@ sidebar_label: Tailpipe CLI |-|- | [tailpipe help](/docs/reference/cli/help) | Help about any command | [tailpipe collect](/docs/reference/cli/collect) | Collect from log sources +| [tailpipe connect](/docs/reference/cli/connect) | Return a connection string for a database +| [tailpipe plugin](/docs/reference/cli/plugin) | Tailpipe plugin management | [tailpipe query](/docs/reference/cli/query) | Query log sources - ## Global Flags diff --git a/docs/reference/index.md b/docs/reference/index.md index bcae12d..fb9eab5 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -1,9 +1,9 @@ --- -title: Flowpipe Reference +title: Tailpipe Reference sidebar_label: Reference --- -# Flowpipe Reference +# Tailpipe Reference When all else fails, read the manual... diff --git a/docs/run/workspaces.md b/docs/run/workspaces.md new file mode 100644 index 0000000..8740f06 --- /dev/null +++ b/docs/run/workspaces.md @@ -0,0 +1,11 @@ +--- +title: Managing Workspaces +sidebar_label: Workspaces +--- + +# Managing Workspaces + +A Tailpipe workspace is a profile that defines the environment in which Tailpipe operates. + + + diff --git a/docs/sidebar.json b/docs/sidebar.json index 4fce0ea..14e60a2 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -10,7 +10,7 @@ "label": "It's Just SQL!", "link": "sql/tailpipe-sql", "items": [] - } + } ] }, { @@ -33,7 +33,8 @@ "run/plugins", "run/collect", "run/query", - "run/connections" + "run/connections", + "run/workspaces" ] }, { @@ -41,11 +42,21 @@ "id": "build", "link": "build", "items": [ - "build/write-mods" + "build/create-mod", + "build/write-detections", + "build/write-dashboards", + "build/mod-variables", + "build/mod-dependencies" ] }, { "type": "category", + "id": "tailpipe-hcl", + "link": "tailpipe-hcl", + "items": [] + }, + { +"type": "category", "id": "reference", "label": "Reference", "link": "reference", @@ -56,7 +67,11 @@ "label": "Command Line Arguments", "link": "reference/cli", "items": [ - "reference/cli/help" + "reference/cli/help", + "reference/cli/connect", + "reference/cli/collect", + "reference/cli/plugin", + "reference/cli/query" ] }, { @@ -64,8 +79,7 @@ "id": "env-vars", "label": "Environment Variables", "link": "reference/env-vars", - "items": [ - ] + "items": [] }, { "type": "category", @@ -77,15 +91,13 @@ "type": "category", "id": "connection", "link": "reference/config-files/connection", - "items": [ - ] + "items": [] }, { "type": "category", "id": "credential", "link": "reference/config-files/credential", - "items": [ - ] + "items": [] }, { "type": "category", @@ -107,7 +119,6 @@ } ] }, - { "type": "category", "id": "faq", diff --git a/docs/tailpipe-hcl/index.md b/docs/tailpipe-hcl/index.md new file mode 100644 index 0000000..9c4d4b4 --- /dev/null +++ b/docs/tailpipe-hcl/index.md @@ -0,0 +1,8 @@ +--- +title: Tailpipe HCL +sidebar_label: Tailpipe HCL +--- + +# Tailpipe HCL + +