Configurable URL pipeline (#61)

* URL pipeline experiment start

* padding

* more thoughts on pipeline editor

* docs pipeline upgrade

* missing parameters for docs job

* try running unit tests

* debug pipeline

* pipeline config buttons

* what about this?

* submodule updates

* - config: save and load vector of steps
- save pipeline as it's changing

* loading pipeline

.github/workflows/build.yml

@@ -5,8 +5,11 @@ env:
   BUILD_TYPE: Release
   ARCH: x64
   VCPKG_CONFIG: Release
-  VCPKG_HASH: 8ee46dc72f75fc3d8d8dc90386bf7e84f2eb7450
+  VCPKG_HASH: 8e9dc3c690558456a667ede3555e01783c3fab9c
+  DOC_INSTANCE: wrs/bt
   DOC_ARTIFACT: webHelpBT2-all.zip
+  DOC_ALGOLIA_ARTIFACT: algolia-indexes-BT.zip
+  DOC_DOCKER_VERSION: 232.10275
 
 on: [push, workflow_dispatch]
 
@@ -57,9 +60,14 @@ jobs:
       run: cmake --build build --config ${{ env.VCPKG_CONFIG }}
       working-directory: . 
 
-    #- name: 🧪 unit tests
-    #  run: test.exe
-    #  working-directory: build/test/Release
+    - name: debug
+      run: ls -R
+      working-directory: build/test
+
+
+    - name: 🧪 unit tests
+      run: .\test.exe
+      working-directory: build/test/Release
 
     - name: 📦 pack
       run: cpack -C ${{ env.BUILD_TYPE }}
@@ -148,28 +156,27 @@ jobs:
   #     run: ./test
   #     working-directory: build/test
 
-  writerside-build:
+  wrs-build:
     runs-on: ubuntu-latest
     name: 'Build Docs'
-    env:
-      PRODUCT: Writerside/bt
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Build Writerside docs using Docker
-        uses: JetBrains/writerside-github-action@v3
+        uses: actions/checkout@v4
+      - name: Build docs
+        uses: JetBrains/writerside-github-action@v4
+        with:
+          instance: ${{ env.DOC_INSTANCE }}
+          artifact: ${{ env.DOC_INSTANCE }}
+          docker-version: ${{ env.DOC_DOCKER_VERSION }}
       - name: Upload documentation
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: docs
           path: artifacts/*.zip
-          retention-days: 7
+          retention-days: 1
 
-  writerside-deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    needs: writerside-build
+  wrs-deploy:
+    needs: wrs-build
     if: github.ref == 'refs/heads/master'
     runs-on: ubuntu-latest
     name: 'Deploy Docs to GitHub Pages'
@@ -179,9 +186,7 @@ jobs:
         with:
           name: docs
       - name: Unzip artifact
-        uses: montudor/action-zip@v1
-        with:
-          args: unzip -qq ${{ env.DOC_ARTIFACT }} -d dir
+        run: unzip -O UTF-8 -qq ${{ env.DOC_ARTIFACT }} -d dir
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact

CMakeLists.txt

@@ -16,6 +16,14 @@ add_compile_definitions(UNICODE) # required for WIN32 API to use correct version
 # see https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html
 set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
+# The cool "edit & continue" feature of MSVC is disabled by default in CMake.
+# https://learn.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=msvc-170#edit-and-continue-for-cmake-projects
+# enable globally: https://stackoverflow.com/a/76254117/80858
+if(MSVC AND CMAKE_BUILD_TYPE MATCHES Debug)
+    add_compile_options("/ZI")
+    add_link_options("/INCREMENTAL")
+endif()
+
 if(WIN32)
     add_subdirectory(grey)
     add_subdirectory(bt)

bt/app/config.cpp

@@ -21,6 +21,8 @@ namespace bt {
     const string PersistPopularityKey = "persist_popularity";
     const string ShowHiddenBrowsersKey = "browsers_show_hidden";
     const string UnshortEnabledKey = "unshort_enabled";
+    const string PipelineSectionName = "pipeline";
+    const string PipelineStepKeyName = "step";
 
     config::config() : cfg{config::get_data_file_path(FileName)} {
         migrate();
@@ -196,12 +198,12 @@ namespace bt {
         cfg.commit();
     }
 
-    bool config::get_unshort_enabled() {
-        return cfg.get_bool_value(UnshortEnabledKey, true);
+    std::vector<std::string> config::get_pipeline() {
+        return cfg.get_all_values(PipelineStepKeyName, PipelineSectionName);
     }
 
-    void config::set_unshort_enabled(bool enabled) {
-        cfg.set_bool_value(UnshortEnabledKey, enabled);
+    void config::set_pipeline(const std::vector<std::string>& steps) {
+        cfg.set_value(PipelineStepKeyName, steps, PipelineSectionName);
         cfg.commit();
     }
 

bt/app/config.h

@@ -54,8 +54,8 @@ namespace bt {
         bool get_show_hidden_browsers();
         void set_show_hidden_browsers(bool show);
 
-        bool get_unshort_enabled();
-        void set_unshort_enabled(bool enabled);
+        std::vector<std::string> get_pipeline();
+        void set_pipeline(const std::vector<std::string>& steps);
 
         // --- browser/instance
 

bt/app/config_window.cpp

@@ -80,7 +80,7 @@ namespace bt
             [this](repeater_bind_context<browser> ctx) {
             render(ctx.data, ctx.container);
         }, true);
-        rpt_browsers->on_item_clicked = [this](shared_ptr<container> c, shared_ptr<browser> b) {
+        rpt_browsers->on_item_clicked = [this](size_t idx, shared_ptr<container> c, shared_ptr<browser> b) {
             handle_selection(b);
         };
 
@@ -224,8 +224,7 @@ namespace bt
         mi_log_rule_hits->is_selected = log_rule_hits;
 
         mi_settings->add("", "-");
-        auto mi_cu = mi_settings->add("unshort", "Enable URL Un-Shortener", ICON_FA_SHIRT); 
-        mi_cu->is_selected = g_config.get_unshort_enabled();
+        mi_settings->add("pipeline_config", "Configure URL pipeline", ICON_FA_BOLT);
 
         // HELP
         auto mi_help = menu->items()->add("", "Help");
@@ -552,10 +551,8 @@ special keyword - %url% which is replaced by opening url.)";
         } else if(mi.id.starts_with("mi_ff_mode_")) {
             string name = mi.id.substr(11);
             update_firefox_mode(true, config::to_firefox_container_mode(name));
-        } else if(mi.id == "unshort") {
-            mi.is_selected = !mi.is_selected;
-            g_config.set_unshort_enabled(mi.is_selected);
-            g_pipeline.reconfigure();
+        } else if(mi.id == "pipeline_config") {
+            bt::ui::url_pipeline();
         }
     }
 
@@ -745,7 +742,7 @@ special keyword - %url% which is replaced by opening url.)";
                 lst_loc->items.push_back(list_item{"URL", ""});
                 lst_loc->items.push_back(list_item{"Title", ""});
                 lst_loc->items.push_back(list_item{"Process", ""});
-                lst_loc->selected_index = static_cast<size_t>(rule->loc);
+                lst_loc->set_selected_index(static_cast<int>(rule->loc));
 
                 // value
                 ctr->same_line();
@@ -785,7 +782,7 @@ special keyword - %url% which is replaced by opening url.)";
                 lst_scope->items.push_back(list_item{ICON_FA_GLOBE, "match anywhere"});
                 lst_scope->items.push_back(list_item{ICON_FA_LANDMARK_DOME, "match only in host name"});
                 lst_scope->items.push_back(list_item{ICON_FA_LINES_LEANING, "match only in path"});
-                lst_scope->selected_index = static_cast<size_t>(ctx.data->scope);
+                lst_scope->set_selected_index(static_cast<int>(ctx.data->scope));
                 g_scope->is_visible = &g_scope->tag_bool;
 
                 ctr->same_line();

bt/app/config_window.h

@@ -4,6 +4,7 @@
 #include "dash_window.h"
 #include "url_tester_window.h"
 #include "about_window.h"
+#include "url_pipeline_window.h"
 #include "config.h"
 
 namespace bt

bt/app/match_rule.cpp

@@ -17,7 +17,7 @@ namespace bt {
     match_rule::match_rule(const std::string& line) {
         string src = line;
         str::trim(src);
-        auto parts = split_line(src);
+        auto parts = str::split_pipe(src);
 
         for(string& p : parts) {
             if(p.empty()) continue;
@@ -111,7 +111,7 @@ namespace bt {
 
         parts.push_back(s);
 
-        return join_to_line(parts);
+        return str::join_with_pipe(parts);
     }
 
     std::string match_rule::to_string(match_scope s) {
@@ -193,56 +193,6 @@ namespace bt {
         }
     }
 
-    std::string match_rule::escape_pipe(const std::string& input) const {
-        std::string result;
-        for(char c : input) {
-            if(c == '|') {
-                result += "\\|";
-            } else {
-                result += c;
-            }
-        }
-        return result;
-    }
-
-    std::string match_rule::unescape_pipe(const std::string& input) const {
-        std::string result = input;
-        size_t position = 0;
-        while((position = result.find("\\|", position)) != std::string::npos) {
-            result.erase(position, 1);
-        }
-        return result;
-    }
-
-    std::string match_rule::join_to_line(const std::vector<std::string>& parts) const {
-        std::ostringstream joined;
-        for(auto it = parts.begin(); it != parts.end(); ++it) {
-            if(it != parts.begin()) {
-                joined << "|";
-            }
-            joined << escape_pipe(*it);
-        }
-        return joined.str();
-    }
-
-    std::vector<std::string> match_rule::split_line(const std::string& line) const {
-
-        // This function replaces all escaped pipes (\\|) with a non-printable character (\v),
-        // then splits the string by the pipe character (|). After splitting, it replaces the
-        // non-printable character back to the pipe character in each substring.
-        // Please note that this code assumes that the non - printable character \v is not used in
-        // the input string.If it is, you may need to choose a different character for temporary replacement.
-
-        std::string copy(line);
-        str::replace_all(copy, "\\|", "\v");
-        auto result = str::split(copy, "|");
-        for(auto& s : result) {
-            str::replace_all(s, "\v", "|");
-        }
-        return result;
-    }
-
-
     bool match_rule::is_match(const url_payload& up) const {
 
         if(value.empty()) return false;

bt/app/match_rule.h

@@ -46,11 +46,5 @@ namespace bt {
 
     private:
         bool contains(const std::string& input, const std::string& value) const;
-
-        std::string escape_pipe(const std::string& input) const;
-        std::string unescape_pipe(const std::string& input) const;
-
-        std::string join_to_line(const std::vector<std::string>& parts) const;
-        std::vector<std::string> split_line(const std::string& line) const;
     };
 }

bt/app/pick_window.cpp

@@ -156,7 +156,7 @@ namespace bt {
             lbl_spacer->padding_left = -10 * scale;
         }, true);
         rpt->bind(choices);
-        rpt->on_item_clicked = [this](shared_ptr<container> c, shared_ptr<browser_instance> bi) {
+        rpt->on_item_clicked = [this](size_t idx, shared_ptr<container> c, shared_ptr<browser_instance> bi) {
             launch(bi);
             if(persist_domain) {
                 string rule_value = str::get_domain_from_url(up.open_url);

bt/app/pipeline/o365.h

@@ -5,6 +5,8 @@
 namespace bt::pipeline {
     class o365 : public url_pipeline_step {
     public:
+        o365() : url_pipeline_step(url_pipeline_step_type::o365) {}
+
         // Inherited via url_pipeline_step
         void process(url_payload& up) override;
     };

bt/app/pipeline/replacer.cpp

@@ -0,0 +1,45 @@
+#include "replacer.h"
+#include "str.h"
+
+using namespace std;
+
+namespace bt::pipeline {
+
+    replacer::replacer(replacer_kind kind, const std::string& find, const std::string& replace) :
+        url_pipeline_step{url_pipeline_step_type::find_replace},
+        kind{kind}, find{find}, replace{replace} {
+    }
+
+    replacer::replacer(const std::string& rule) : url_pipeline_step(url_pipeline_step_type::find_replace) {
+        // parse rules - each string contains 3 parts separated by a pipe
+        // kind|match|replace
+
+        auto parts = str::split_pipe(rule);
+        //if(parts.size() != 3) {
+        //    continue;
+        //}
+
+        if(parts[0] == "rgx") {
+            kind = replacer_kind::regex;
+        } else {
+            kind = replacer_kind::find_replace;
+        }
+
+        if(parts.size() == 3) {
+            find = parts[1];
+            replace = parts[2];
+        }
+    }
+
+    void replacer::process(url_payload& up) {
+        string url = up.match_url.empty() ? up.url : up.match_url;
+
+        if(kind == replacer_kind::find_replace) {
+            size_t idx = url.find(find);
+            if(idx != string::npos) {
+                str::replace_all(url, find, replace);
+                up.match_url = up.open_url = url;
+            }
+        }
+    }
+}

bt/app/pipeline/replacer.h

@@ -0,0 +1,29 @@
+#pragma once
+#include <string>
+#include <vector>
+#include "../url_pipeline_step.h"
+
+namespace bt::pipeline {
+
+    enum class replacer_kind {
+        find_replace,
+        regex
+    };
+
+    class replacer : public url_pipeline_step {
+    public:
+
+        replacer_kind kind{replacer_kind::find_replace};
+        std::string find;
+        std::string replace;
+
+        replacer(replacer_kind kind, const std::string& find, const std::string& replace);
+        replacer(const std::string& rule);
+
+
+        // Inherited via url_pipeline_step
+        void process(url_payload& up) override;
+
+    private:
+    };
+}