Saved segments spike

lib/plausible/helpers/list-traverse.ex

@@ -0,0 +1,56 @@
+defmodule Plausible.Helpers.ListTraverse do
+  @moduledoc """
+  This module contains utility functions for parsing and validating lists of values.
+  """
+
+  @doc """
+  Parses a list of values using a provided parser function.
+
+  ## Parameters
+
+    - `list`: A list of values to be parsed.
+    - `parser_function`: A function that takes a single value and returns either
+      `{:ok, result}` or `{:error, reason}`.
+
+  ## Returns
+
+    - `{:ok, parsed_list}` if all values are successfully parsed, where `parsed_list`
+      is a list containing the results of applying `parser_function` to each value.
+    - `{:error, reason}` if any value fails to parse, where `reason` is the error
+      returned by the first failing `parser_function` call.
+
+  ## Examples
+
+      iex> parse_list(["1", "2", "3"], &Integer.parse/1)
+      {:ok, [1, 2, 3]}
+
+      iex> parse_list(["1", "not_a_number", "3"], &Integer.parse/1)
+      {:error, :invalid}
+
+  """
+  @spec parse_list(list(), (any() -> {:ok, any()} | {:error, any()})) ::
+          {:ok, list()} | {:error, any()}
+  def parse_list(list, parser_function) do
+    Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->
+      case parser_function.(value) do
+        {:ok, result} -> {:cont, {:ok, results ++ [result]}}
+        {:error, _} = error -> {:halt, error}
+      end
+    end)
+  end
+
+  @doc """
+  Validates a list of values using a provided parser function.
+
+  Returns `:ok` if all values are valid, or `{:error, reason}` on first invalid value.
+  """
+  @spec validate_list(list(), (any() -> :ok | {:error, any()})) :: :ok | {:error, any()}
+  def validate_list(list, parser_function) do
+    Enum.reduce_while(list, :ok, fn value, :ok ->
+      case parser_function.(value) do
+        :ok -> {:cont, :ok}
+        {:error, _} = error -> {:halt, error}
+      end
+    end)
+  end
+end

lib/plausible/segment/schema.ex

@@ -0,0 +1,56 @@
+defmodule Plausible.Segment do
+  @moduledoc """
+  Schema for segments. Segments are saved filter combinations.
+  """
+  use Plausible
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @type t() :: %__MODULE__{}
+
+  @derive {Jason.Encoder,
+           only: [
+             :id,
+             :name,
+             :personal,
+             :segment_data,
+             :owner_id,
+             :inserted_at,
+             :updated_at
+           ]}
+
+  schema "segments" do
+    field :name, :string
+    field :segment_data, :map
+    field :personal, :boolean
+    belongs_to :owner, Plausible.Auth.User, foreign_key: :owner_id
+    belongs_to :site, Plausible.Site
+
+    timestamps()
+  end
+
+  def changeset(segment, attrs) do
+    segment
+    |> cast(attrs, [
+      :name,
+      :segment_data,
+      :site_id,
+      :personal,
+      :owner_id
+    ])
+    |> validate_required([:name, :segment_data, :site_id, :personal, :owner_id])
+    |> foreign_key_constraint(:site_id)
+    |> foreign_key_constraint(:owner_id)
+    |> validate_segment_data()
+  end
+
+  defp validate_segment_data(changeset) do
+    case get_field(changeset, :segment_data) do
+      %{"filters" => filters} when is_list(filters) ->
+        changeset
+
+      _ ->
+        add_error(changeset, :segment_data, "must contain property \"filters\" with array value")
+    end
+  end
+end

lib/plausible/segment/segments.md

@@ -0,0 +1,74 @@
+# Saved segments
+
+## Definitions
+
+| Term | Definition |
+|------|------------|
+| **Segment Owner** | Usually the user who authored the segment, but can change over time |
+| **Personal Segment** | A segment that has personal flag set as true and the user is the segment owner |
+| **Personal Segments of Other Users** | A segment that has personal flag set as true and the user is not the segment owner |
+| **Site Segment** | A segment that has personal flag set to false |
+| **Segment Contents** | A list of filters |
+
+## Capabilities
+
+| Capability | Public | Viewer | Admin | Owner | Super Admin |
+|------------|--------|--------|-------|-------|-------------|
+| Can view data filtered by any segment they know the ID of | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Can make API requests filtered by any segment they know the ID of |  | ✅ | ✅ | ✅ | ✅ |
+| Can create personal segments |  | ✅ | ✅ | ✅ | ✅ |
+| Can set personal segments to be site segments |  | ❓ | ✅ | ✅ | ✅ |
+| Can set site segments to be personal segments |  | ❓ | ✅ | ✅ | ✅ |
+| Can see list of personal segments |  | ✅ | ✅ | ✅ | ✅ |
+| Can see list of site segments | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Can see contents of personal segments |  | ✅ | ✅ | ✅ | ✅ |
+| Can see contents of site segments | ❓ | ❓ | ✅ | ✅ | ✅ |
+| Can edit personal segments |  | ✅ | ✅ | ✅ | ✅ |
+| Can edit site segments |  | ❓ | ✅ | ✅ | ✅ |
+| Can delete personal segments |  | ✅ | ✅ | ✅ | ✅ |
+| Can delete site segments |  | ❓ | ✅ | ✅ | ✅ |
+| Can list personal segments of other users [1] |  |  | ❓ | ❓ | ❓ |
+| Can see contents of personal segments of other users [1] |  |  | ❓ | ❓ | ❓ |
+| Can edit personal segments of other users [1] |  |  | ❓ | ❓ | ❓ |
+| Can delete personal segments of other users [1] |  |  | ❓ | ❓ | ❓ |
+
+### Notes
+
+__[1]__: maybe needed for elevated roles to be able to take control of segments of deleted / inactive / vacationing users, or to toggle whether such segments are site segments or not
+
+## Segment lifecycle
+
+| Action | Outcome |
+|--------|---------|
+| A user* selects filters that constitute the segment, chooses name, clicks save | Personal segment created (with user as segment owner) |
+| A user* views the contents of an existing segment, chooses name, clicks save as | Personal segment created (with user as segment owner) |
+| Segment owner* clicks edit segment, changes segment name or adds/removes/edits filters, clicks save | Segment updated |
+| Segment owner* or another user* toggles personal segment to be site segment | Segment elevated to be a site segment |
+| Segment owner* or another user* toggles site segment to be a personal segment | Segment de-elevated to be a personal segment |
+| Any user* except the segment owner opens the segment for editing and clicks save, with or without changes | Segment updated (with editor becoming the new segment owner) |
+| Segment owner* deletes segment | Segment deleted |
+| Any user* except the segment owner deletes segment | Segment deleted |
+| Site deleted | Segment autodeleted |
+| Segment owner is removed from site or deleted from Plausible | If personal segment, segment autodeleted; if site segment, nothing happens |
+| Any user* updates goal name, if site has any segments with "is goal ..." filters for that goal | Segment autoupdated |
+| Plausible engineer updates filters schema in backwards incompatible way | Segment autoupdated/automigrated |
+
+### Notes
+
+__*__: if the user has that particular capability
+
+## Schema
+
+| Field | Type | Constraints | Comment |
+|-------|------|-------------|---------|
+| :id | :bigint | null: false | |
+| :name | :string | null: false | |
+| :personal | :boolean | default: true, null: false | Needed to distinguish between segments we can clean up automatically and segments that need to remain stable over time |
+| :segment_data | :map | null: false | Contains the filters array at "filters" key |
+| :site_id | references(:sites) | on_delete: :delete_all, null: false | |
+| :owner_id | references(:users) | on_delete: :nothing, null: false | Used to display author info without repeating author name and email in the database |
+| timestamps() | | | Inserted_at, updated_at fields |
+
+## API
+
+[lib/plausible_web/router.ex](../../plausible_web/router.ex)

lib/plausible/site.ex

@@ -41,6 +41,7 @@ defmodule Plausible.Site do
     has_many :invitations, Plausible.Auth.Invitation
     has_many :goals, Plausible.Goal, preload_order: [desc: :id]
     has_many :revenue_goals, Plausible.Goal, where: [currency: {:not, nil}]
+    has_many :segments, Plausible.Segment, preload_order: [asc: :name]
     has_one :google_auth, GoogleAuth
     has_one :weekly_report, Plausible.Site.WeeklyReport
     has_one :monthly_report, Plausible.Site.MonthlyReport

lib/plausible/stats.ex

@@ -26,9 +26,9 @@ defmodule Plausible.Stats do
     |> QueryResult.from(site, optimized_query)
   end
 
-  def breakdown(site, query, metrics, pagination) do
+  def breakdown(site, query, metrics, pagination, opts \\ []) do
     include_sentry_replay_info()
-    Breakdown.breakdown(site, query, metrics, pagination)
+    Breakdown.breakdown(site, query, metrics, pagination, opts)
   end
 
   def aggregate(site, query, metrics) do

lib/plausible/stats/breakdown.ex

@@ -5,6 +5,7 @@ defmodule Plausible.Stats.Breakdown do
   Avoid adding new logic here - update QueryBuilder etc instead.
   """
 
+  use Plausible.Repo
   use Plausible.ClickhouseRepo
   use Plausible
   use Plausible.Stats.SQL.Fragments
@@ -20,6 +21,8 @@ defmodule Plausible.Stats.Breakdown do
         pagination,
         _opts \\ []
       ) do
+    get_available_segments = fn -> Repo.preload(site, :segments).segments end
+
     transformed_metrics = transform_metrics(metrics, dimension)
     transformed_order_by = transform_order_by(order_by || [], dimension)
 
@@ -37,7 +40,7 @@ defmodule Plausible.Stats.Breakdown do
         # Allow pageview and event metrics to be queried off of sessions table
         legacy_breakdown: true
       )
-      |> QueryOptimizer.optimize()
+      |> QueryOptimizer.optimize(get_available_segments: get_available_segments)
 
     q = SQL.QueryBuilder.build(query_with_metrics, site)
 

lib/plausible/stats/filters/filters.ex

@@ -3,7 +3,7 @@ defmodule Plausible.Stats.Filters do
   A module for parsing filters used in stat queries.
   """
 
-  alias Plausible.Stats.Filters.QueryParser
+  alias Plausible.Stats.Filters.{FiltersParser}
   alias Plausible.Stats.Filters.{LegacyDashboardFilterParser, StatsAPIFilterParser}
 
   @visit_props [
@@ -81,7 +81,7 @@ defmodule Plausible.Stats.Filters do
     do: LegacyDashboardFilterParser.parse_and_prefix(filters)
 
   def parse(filters) when is_list(filters) do
-    {:ok, parsed_filters} = QueryParser.parse_filters(filters)
+    {:ok, parsed_filters} = FiltersParser.parse_filters(filters)
     parsed_filters
   end
 
@@ -103,8 +103,8 @@ defmodule Plausible.Stats.Filters do
     |> Enum.map(fn {[_operator, dimension | _rest], _root, _depth} -> dimension end)
   end
 
-  def filtering_on_dimension?(query, dimension) do
-    dimension in dimensions_used_in_filters(query.filters)
+  def filtering_on_dimension?(filters, dimension) do
+    dimension in dimensions_used_in_filters(filters)
   end
 
   @doc """

lib/plausible/stats/filters/filters_parser.ex

@@ -0,0 +1,134 @@
+defmodule Plausible.Stats.Filters.FiltersParser do
+  @moduledoc """
+    FiltersParser is the module to verify that filters array is in the expected format.
+  """
+
+  alias Plausible.Stats.Filters
+  alias Plausible.Helpers.ListTraverse
+
+  @segment_filter_key "segment"
+  def segment_filter_key(), do: @segment_filter_key
+
+  @filter_entry_operators [
+    :is,
+    :is_not,
+    :matches,
+    :matches_not,
+    :matches_wildcard,
+    :matches_wildcard_not,
+    :contains,
+    :contains_not
+  ]
+
+  @filter_tree_operators [:not, :and, :or]
+
+  def parse_filters(filters) when is_list(filters) do
+    ListTraverse.parse_list(filters, &parse_filter/1)
+  end
+
+  def parse_filters(_invalid_metrics), do: {:error, "Invalid filters passed."}
+
+  defp parse_filter(filter) do
+    with {:ok, operator} <- parse_operator(filter),
+         {:ok, second} <- parse_filter_second(operator, filter),
+         {:ok, rest} <- parse_filter_rest(operator, filter) do
+      {:ok, [operator, second | rest]}
+    end
+  end
+
+  defp parse_operator(["is" | _rest]), do: {:ok, :is}
+  defp parse_operator(["is_not" | _rest]), do: {:ok, :is_not}
+  defp parse_operator(["matches" | _rest]), do: {:ok, :matches}
+  defp parse_operator(["matches_not" | _rest]), do: {:ok, :matches_not}
+  defp parse_operator(["matches_wildcard" | _rest]), do: {:ok, :matches_wildcard}
+  defp parse_operator(["matches_wildcard_not" | _rest]), do: {:ok, :matches_wildcard_not}
+  defp parse_operator(["contains" | _rest]), do: {:ok, :contains}
+  defp parse_operator(["contains_not" | _rest]), do: {:ok, :contains_not}
+  defp parse_operator(["not" | _rest]), do: {:ok, :not}
+  defp parse_operator(["and" | _rest]), do: {:ok, :and}
+  defp parse_operator(["or" | _rest]), do: {:ok, :or}
+  defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{i(filter)}'."}
+
+  def parse_filter_second(:not, [_, filter | _rest]), do: parse_filter(filter)
+
+  def parse_filter_second(operator, [_, filters | _rest]) when operator in [:and, :or],
+    do: parse_filters(filters)
+
+  def parse_filter_second(_operator, filter), do: parse_filter_key(filter)
+
+  defp parse_filter_key([_operator, filter_key | _rest] = filter) do
+    parse_filter_key_string(filter_key, "Invalid filter '#{i(filter)}")
+  end
+
+  defp parse_filter_key(filter), do: {:error, "Invalid filter '#{i(filter)}'."}
+
+  defp parse_filter_rest(operator, filter)
+       when operator in @filter_entry_operators,
+       do: parse_clauses_list(filter)
+
+  defp parse_filter_rest(operator, _filter)
+       when operator in @filter_tree_operators,
+       do: {:ok, []}
+
+  defp parse_clauses_list([operation, filter_key, list] = filter) when is_list(list) do
+    all_strings? = Enum.all?(list, &is_binary/1)
+    all_integers? = Enum.all?(list, &is_integer/1)
+
+    case {filter_key, all_strings?} do
+      {"visit:city", false} when all_integers? ->
+        {:ok, [list]}
+
+      {"visit:country", true} when operation in ["is", "is_not"] ->
+        if Enum.all?(list, &(String.length(&1) == 2)) do
+          {:ok, [list]}
+        else
+          {:error,
+           "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}
+        end
+
+      {@segment_filter_key, false} when all_integers? ->
+        {:ok, [list]}
+
+      {_, true} ->
+        {:ok, [list]}
+
+      _ ->
+        {:error, "Invalid filter '#{i(filter)}'."}
+    end
+  end
+
+  defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"}
+
+  def parse_filter_key_string(filter_key, error_message \\ "") do
+    case filter_key do
+      "event:props:" <> property_name ->
+        if String.length(property_name) > 0 do
+          {:ok, filter_key}
+        else
+          {:error, error_message}
+        end
+
+      "event:" <> key ->
+        if key in Filters.event_props() do
+          {:ok, filter_key}
+        else
+          {:error, error_message}
+        end
+
+      "visit:" <> key ->
+        if key in Filters.visit_props() do
+          {:ok, filter_key}
+        else
+          {:error, error_message}
+        end
+
+      @segment_filter_key ->
+        {:ok, filter_key}
+
+      _ ->
+        {:error, error_message}
+    end
+  end
+
+  defp i(value), do: inspect(value, charlists: :as_lists)
+end