Merge pull request #100 from EventStore/checkpoint-seeder

feat: introduce a configurable checkpoint seeder (none or chaser)
EventStore · Aug 26, 2024 · 71b0e01 · 71b0e01
2 parents aac63fc + aceedc2
commit 71b0e01
Show file tree

Hide file tree

Showing 22 changed files with 975 additions and 1,076 deletions.
diff --git a/docs/content/docs/Configuration/_index.md b/docs/content/docs/Configuration/_index.md
@@ -13,21 +13,28 @@ The settings file has the `replicator` root level, all settings are children to
 
 Available configuration options are:
 
-| Option | Description |
-| :----- | :---------- |
-| `replicator.reader.connectionString` | Connection string for the source cluster or instance |
-| `replicator.reader.protocol` | Reader protocol (`tcp` or `grpc`) |
-| `replicator.reader.pageSize` | Reader page size (only applicable for TCP protocol |
-| `replicator.sink.connectionString` | Connection string for the target cluster or instance |
-| `replicator.sink.protocol` | Writer protocol (`tcp` or `grpc`) |
-| `replicator.sink.partitionCount` | Number of [partitioned]({{% ref "writers" %}}) concurrent writers |
-| `replicator.sink.partitioner` | Custom JavaScript [partitioner]({{% ref "writers" %}}) |
-| `replicator.sink.bufferSize` | Size of the sink buffer, `1000` events by default |
-| `replicator.scavenge` | Enable real-time [scavenge]({{% ref "scavenge" %}}) |
-| `replicator.runContinuously` | Set to `false` if you want Replicator to stop when it reaches the end of `$all` stream. Default is `true`, so the replication continues until you stop it explicitly. |
-| `replicator.filters` | Add one or more of provided [filters]({{% ref "filters" %}}) |
-| `replicator.transform` | Configure the [event transformation]({{% ref "Transforms" %}}) |
-| `replicator.transform.bufferSize` | Size of the prepare buffer (filtering and transformations), `1000` events by default |
+| Option                                  | Description                                                                                                                                                           |
+|:----------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `replicator.reader.connectionString`    | Connection string for the source cluster or instance                                                                                                                  |
+| `replicator.reader.protocol`            | Reader protocol (`tcp` or `grpc`)                                                                                                                                     |
+| `replicator.reader.pageSize`            | Reader page size (only applicable for TCP protocol                                                                                                                    |
+| `replicator.sink.connectionString`      | Connection string for the target cluster or instance                                                                                                                  |
+| `replicator.sink.protocol`              | Writer protocol (`tcp` or `grpc`)                                                                                                                                     |
+| `replicator.sink.partitionCount`        | Number of [partitioned]({{% ref "writers" %}}) concurrent writers                                                                                                     |
+| `replicator.sink.partitioner`           | Custom JavaScript [partitioner]({{% ref "writers" %}})                                                                                                                |
+| `replicator.sink.bufferSize`            | Size of the sink buffer, `1000` events by default                                                                                                                     |
+| `replicator.scavenge`                   | Enable real-time [scavenge]({{% ref "scavenge" %}})                                                                                                                   |
+| `replicator.runContinuously`            | Set to `false` if you want Replicator to stop when it reaches the end of `$all` stream. Default is `true`, so the replication continues until you stop it explicitly. |
+| `replicator.filters`                    | Add one or more of provided [filters]({{% ref "filters" %}})                                                                                                          |
+| `replicator.transform`                  | Configure the [event transformation]({{% ref "Transforms" %}})                                                                                                        |
+| `replicator.transform.bufferSize`       | Size of the prepare buffer (filtering and transformations), `1000` events by default                                                                                  |
+| `replicator.checkpoint.type`            | Type of checkpoint store (`none`, `file` or `mongo`), `none` by default                                                                                               |
+| `replicator.checkpoint.path`            | The file path or connection string, empty by default                                                                                                                  |
+| `replicator.checkpoint.checkpointAfter` | The number of events that must be replicated before a checkpoint is stored, `1000` events by default                                                                  |
+| `replicator.checkpoint.database`        | The name of the Mongo database, `replicator` by default                                                                                                               |
+| `replicator.checkpoint.instanceId`      | The name of the replicator instance to isolate checkpoints with in the Mongo database, `default` by default                                                           |
+| `replicator.checkpoint.seeder.type`     | Type of checkpoint seeder to use (`none` or `chaser`), `none` by default                                                                                              |
+| `replicator.checkpoint.seeder.path`     | The file path of the `chaser.chk`, empty by default                                                                                                                   |
 
 ## Enable verbose logging
 

diff --git a/docs/content/docs/Features/Checkpoints/_index.md b/docs/content/docs/Features/Checkpoints/_index.md
@@ -36,6 +36,30 @@ Configuring a higher `checkpointAfter` improves write performance by ensuring Re
 
 Configure the `checkpointAfter` to align with your data consistency and performance requirements.
 
+### Checkpoint seeding
+
+Replicator supports checkpoint seeding, which allows you to start replication from a specific event number. This is optional and the default is to not seed.
+
+```yaml
+replicator:
+  checkpoint:
+    seeder: 
+      type: none
+```
+
+When the `type` of seeder is set to `chaser`, you can seed a checkpoint store from a `chaser.chk` file, like so:
+
+```yaml
+replicator:
+  checkpoint:
+    seeder: 
+      type: chaser
+      path: "path/to/chaser.chk"
+```
+
+This is useful when you want to start replication from the same event number as a backup's `chaser.chk`. It's not recommended to use this feature unless you are sure the `chaser.chk` file is immutable. This implies that the `chaser.chk` file of a running EventStoreDB node should not be used.
+Note that seeding will only happen if the checkpoint store has no corresponding stored checkpoint.
+
 ### Checkpoint stores
 
 Replicator supports storing checkpoints in different stores. Only one store can be configured per Replicator instance. 

diff --git a/docs/go.mod b/docs/go.mod
@@ -1,5 +1,5 @@
 module github.com/EventStore/replicator/docs
 
-go 1.22.0
+go 1.23
 
-require github.com/google/docsy v0.9.1 // indirect
+require github.com/google/docsy v0.10.0 // indirect
diff --git a/docs/go.sum b/docs/go.sum
@@ -2,6 +2,7 @@ github.com/FortAwesome/Font-Awesome v0.0.0-20210804190922-7d3d774145ac/go.mod h1
 github.com/FortAwesome/Font-Awesome v0.0.0-20220831210243-d3a7818c253f/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo=
 github.com/FortAwesome/Font-Awesome v0.0.0-20230327165841-0698449d50f2/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo=
 github.com/FortAwesome/Font-Awesome v0.0.0-20240108205627-a1232e345536/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo=
+github.com/FortAwesome/Font-Awesome v0.0.0-20240402185447-c0f460dca7f7/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo=
 github.com/google/docsy v0.2.0 h1:DN6wfyyp2rXsjdV1K3wioxOBTRvG6Gg48wLPDso2lc4=
 github.com/google/docsy v0.2.0/go.mod h1:shlabwAQakGX6qpXU6Iv/b/SilpHRd7d+xqtZQd3v+8=
 github.com/google/docsy v0.3.0 h1:wR0e0xJ6qIj/RPq2AP1Ufm44BCak9Sl72H2BWNAc/JA=
@@ -22,6 +23,8 @@ github.com/google/docsy v0.8.0 h1:RgHyKRTo8YwScMThrf01Ky2yCGpUS1hpkspwNv6szT4=
 github.com/google/docsy v0.8.0/go.mod h1:FqTNN2T7pWEGW8dc+v5hQ5VF29W5uaL00PQ1LdVw5F8=
 github.com/google/docsy v0.9.1 h1:+jqges1YCd+yHeuZ1BUvD8V8mEGVtPxULg5j/vaJ984=
 github.com/google/docsy v0.9.1/go.mod h1:saOqKEUOn07Bc0orM/JdIF3VkOanHta9LU5Y53bwN2U=
+github.com/google/docsy v0.10.0 h1:6tMDacPwAyRWNCfvsn/9qGOZDQ8b0aRzjRZvnZPY5dg=
+github.com/google/docsy v0.10.0/go.mod h1:c0nIAqmRTOuJ01F85U/wJPQtc3Zj9N58Kea9bOT2AJc=
 github.com/google/docsy/dependencies v0.2.0/go.mod h1:2zZxHF+2qvkyXhLZtsbnqMotxMukJXLaf8fAZER48oo=
 github.com/google/docsy/dependencies v0.3.0/go.mod h1:2zZxHF+2qvkyXhLZtsbnqMotxMukJXLaf8fAZER48oo=
 github.com/google/docsy/dependencies v0.4.0/go.mod h1:2zZxHF+2qvkyXhLZtsbnqMotxMukJXLaf8fAZER48oo=
@@ -33,3 +36,4 @@ github.com/google/docsy/dependencies v0.7.2/go.mod h1:gihhs5gmgeO+wuoay4FwOzob+j
 github.com/twbs/bootstrap v4.6.1+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0=
 github.com/twbs/bootstrap v4.6.2+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0=
 github.com/twbs/bootstrap v5.2.3+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0=
+github.com/twbs/bootstrap v5.3.3+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0=
diff --git a/docs/hugo.toml b/docs/hugo.toml
@@ -13,7 +13,7 @@ enableGitInfo = true
 # Language settings
 contentDir = "content"
 
-disableKinds = ["taxonomy", "taxonomyTerm"]
+disableKinds = ["taxonomy"]
 
 # Highlighting config
 pygmentsCodeFences = true
@@ -44,7 +44,7 @@ anchor = "smart"
 [services]
 [services.googleAnalytics]
 # Comment out the next line to disable GA tracking. Also disables the feature described in [params.ui.feedback].
-id = "UA-00000000-0"
+# id = "UA-00000000-0"
 
 # Language configuration
 
@@ -105,7 +105,7 @@ url_latest_version = "https://example.com"
 #gcs_engine_id = "d72aa9b2712488cc3"
 
 # Enable Algolia DocSearch
-algolia_docsearch = false
+# algolia_docsearch = false
 
 # Enable Lunr.js offline search
 offlineSearch = false
@@ -124,7 +124,7 @@ sidebar_search_disable = false
 #  Set to false if you don't want to display a logo (/assets/icons/logo.svg) in the top nav bar
 navbar_logo = false
 # Set to true to disable the About link in the site footer
-footer_about_disable = false
+footer_about_enable = true
 
 # Adds a H2 section titled "Feedback" to the bottom of each doc. The responses are sent to Google Analytics as events.
 # This feature depends on [services.googleAnalytics] and will be disabled if "services.googleAnalytics.id" is not set.