Merge branch 'master' into rajan/vs2022-cmake

.github/workflows/asan.yml

@@ -21,7 +21,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, ubuntu-latest, macos-latest]
+        #os: [windows-latest, ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest] # Temporarily remove windows asan
         preset: [vcpkg-asan-debug, vcpkg-ubsan-debug]
         exclude:
           # UBSan not supported by MSVC on Windows

.github/workflows/build_windows_cmake.yml

@@ -25,7 +25,7 @@ jobs:
       CMAKE_BUILD_DIR: ${{ github.workspace }}/vw/build
       SOURCE_DIR: ${{ github.workspace }}/vw
       VCPKG_ROOT: ${{ github.workspace }}/vw/ext_libs/vcpkg
-      VCPKG_REF: 501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
+      VCPKG_REF: 53bef8994c541b6561884a8395ea35715ece75db
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/python_wheels.yml

@@ -284,7 +284,7 @@ jobs:
     runs-on: windows-2019
     env:
       VCPKG_ROOT: ${{ github.workspace }}\\vcpkg
-      VCPKG_REF: 501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
+      VCPKG_REF: 53bef8994c541b6561884a8395ea35715ece75db
       VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\vcpkg_binary_cache
     strategy:
       matrix:

.github/workflows/vendor_build.yml

@@ -48,7 +48,7 @@ jobs:
           -DVW_INSTALL=Off
           -DWARNINGS=On
           -DWARNING_AS_ERROR=On
-          -DUSE_LATEST_STD=On
+          -DVW_CXX_STANDARD=17
       - name: Build
         run: cmake --build build
       - name: Unit tests

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "(ctest) Launch",
+            "type": "cppdbg",
+            "cwd": "${workspaceFolder}",
+            "request": "launch",
+            "program": "${cmake.testProgram}",
+            "args": [ "${cmake.testArgs}" ]
+        }
+    ]
+}

CMakeLists.txt

@@ -60,10 +60,10 @@ if(VW_FEAT_LDA AND NOT BUILD_PYTHON)
   list(APPEND VCPKG_MANIFEST_FEATURES "lda")
 endif()
 
-option(BUILD_TESTING "Build tests" ON)
-if(BUILD_TESTING)
-  list(APPEND VCPKG_MANIFEST_FEATURES "tests")
-endif()
+#option(BUILD_TESTING "Build tests" ON)
+#if(BUILD_TESTING)
+#  list(APPEND VCPKG_MANIFEST_FEATURES "tests")
+#endif()
 
 option(BUILD_BENCHMARKS "Build benchmarks" OFF)
 if(BUILD_BENCHMARKS)
@@ -100,6 +100,31 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_VISIBILITY_INLINES_HIDDEN TRUE)
 set(CMAKE_CXX_VISIBILITY_PRESET "hidden")
 
+option(VW_USE_ASAN "Compile with AddressSanitizer" OFF)
+option(VW_USE_UBSAN "Compile with UndefinedBehaviorSanitizer" OFF)
+
+if(VW_USE_ASAN)
+  add_compile_definitions(VW_USE_ASAN)
+  if(MSVC)
+    add_compile_options(/fsanitize=address)
+    add_link_options(/InferASanLibs /incremental:no /debug)
+  else()
+    add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g3)
+    add_link_options(-fsanitize=address -fno-omit-frame-pointer -g3)
+  endif()
+endif()
+
+if(VW_USE_UBSAN)
+  add_compile_definitions(VW_USE_UBSAN)
+  if(MSVC)
+    message(FATAL_ERROR "UBSan not supported on MSVC")
+  else()
+    add_compile_options(-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer -g3)
+    add_link_options(-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer -g3)
+  endif()
+endif()
+
+
 include(VowpalWabbitUtils)
 
 if(MSVC)
@@ -152,33 +177,8 @@ option(VW_SSE2NEON_SYS_DEP "Override using the submodule for SSE2Neon dependency
 option(VW_BUILD_VW_C_WRAPPER "Enable building the c_wrapper project" ON)
 option(vw_BUILD_NET_CORE "Build .NET Core targets" OFF)
 option(vw_BUILD_NET_FRAMEWORK "Build .NET Framework targets" OFF)
-option(VW_USE_ASAN "Compile with AddressSanitizer" OFF)
-option(VW_USE_UBSAN "Compile with UndefinedBehaviorSanitizer" OFF)
 option(VW_BUILD_WASM "Add WASM target" OFF)
 
-if(VW_USE_ASAN)
-  add_compile_definitions(VW_USE_ASAN)
-  if(MSVC)
-    add_compile_options(/fsanitize=address /GS- /wd5072)
-    add_link_options(/InferASanLibs /incremental:no /debug)
-    # Workaround for MSVC ASan issue here: https://developercommunity.visualstudio.com/t/VS2022---Address-sanitizer-on-x86-Debug-/10116361
-    add_compile_definitions(_DISABLE_STRING_ANNOTATION)
-  else()
-    add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g3)
-    add_link_options(-fsanitize=address -fno-omit-frame-pointer -g3)
-  endif()
-endif()
-
-if(VW_USE_UBSAN)
-  add_compile_definitions(VW_USE_UBSAN)
-  if(MSVC)
-    message(FATAL_ERROR "UBSan not supported on MSVC")
-  else()
-    add_compile_options(-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer -g3)
-    add_link_options(-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer -g3)
-  endif()
-endif()
-
 if(VW_INSTALL AND NOT VW_ZLIB_SYS_DEP)
   message(WARNING "Installing with a vendored version of zlib is not recommended. Use VW_ZLIB_SYS_DEP to use a system dependency or specify VW_INSTALL=OFF to silence this warning.")
 endif()

CMakePresets.json

@@ -41,7 +41,7 @@
         },
         "VW_GTEST_SYS_DEP": {
           "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
         },
         "VW_EIGEN_SYS_DEP": {
           "type": "BOOL",

cmake/VowpalWabbitUtils.cmake

@@ -22,7 +22,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
       include(FetchContent)
       FetchContent_Declare(
         googletest
-        URL https://github.com/google/googletest/archive/refs/tags/release-1.11.0.zip
+        URL https://github.com/google/googletest/archive/refs/tags/v1.13.0.zip
       )
       # For Windows: Prevent overriding the parent project's compiler/linker settings
       set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)

ext_libs/ext_libs.cmake

@@ -38,7 +38,7 @@ if(RAPIDJSON_SYS_DEP)
   # Since EXACT is not specified, any version compatible with 1.1.0 is accepted (>= 1.1.0)
   find_package(RapidJSON 1.1.0 CONFIG REQUIRED)
   add_library(RapidJSON INTERFACE)
-  target_include_directories(RapidJSON INTERFACE ${RapidJSON_INCLUDE_DIRS})
+  target_include_directories(RapidJSON INTERFACE ${RapidJSON_INCLUDE_DIRS} ${RAPIDJSON_INCLUDE_DIRS})
 else()
   add_library(RapidJSON INTERFACE)
   target_include_directories(RapidJSON SYSTEM INTERFACE "${CMAKE_CURRENT_LIST_DIR}/rapidjson/include")
@@ -127,4 +127,4 @@ if(VW_FEAT_CB_GRAPH_FEEDBACK)
   target_include_directories(mlpack_ensmallen SYSTEM INTERFACE ${CMAKE_CURRENT_LIST_DIR}/armadillo-code/include)
 
   target_include_directories(mlpack_ensmallen SYSTEM INTERFACE ${CMAKE_CURRENT_LIST_DIR}/ensmallen/include)
-endif()
+endif()

python/docs/source/tutorials/DFtoVW_tutorial.ipynb

@@ -802,15 +802,17 @@
     "\n",
     "# Adding columns for easier visualization\n",
     "weights_df[\"feature_name\"] = weights_df.apply(\n",
-    "    lambda row: row.vw_feature_name.split(\"=\")[0]\n",
-    "    if row.is_cat\n",
-    "    else row.vw_feature_name,\n",
+    "    lambda row: (\n",
+    "        row.vw_feature_name.split(\"=\")[0] if row.is_cat else row.vw_feature_name\n",
+    "    ),\n",
     "    axis=1,\n",
     ")\n",
     "weights_df[\"feature_value\"] = weights_df.apply(\n",
-    "    lambda row: row.vw_feature_name.split(\"=\")[1].zfill(2)\n",
-    "    if row.is_cat\n",
-    "    else row.vw_feature_name,\n",
+    "    lambda row: (\n",
+    "        row.vw_feature_name.split(\"=\")[1].zfill(2)\n",
+    "        if row.is_cat\n",
+    "        else row.vw_feature_name\n",
+    "    ),\n",
     "    axis=1,\n",
     ")\n",
     "weights_df.sort_values([\"feature_name\", \"feature_value\"], inplace=True)"

python/docs/source/tutorials/cmd_first_steps.md

@@ -116,6 +116,6 @@ The model predicted a value of **0**. This result means our house will not need
 ## More to explore
 
 - See [Python tutorial](python_first_steps.ipynb) for a quick introduction to the basics of training and testing your model.
-- To learn more about how to approach a contextual bandits problem using  tVowpal Wabbit — including how to  work with different contextual bandits approaches, how to format data, and understand the results — see the [Contextual Bandit Reinforcement Learning Tutorial](python_Contextual_bandits_and_Vowpal_Wabbit.ipynb).
+- To learn more about how to approach a contextual bandits problem using Vowpal Wabbit — including how to work with different contextual bandits approaches, how to format data, and understand the results — see the [Contextual Bandit Reinforcement Learning Tutorial](python_Contextual_bandits_and_Vowpal_Wabbit.ipynb).
 - For more on the contextual bandits approach to reinforcement learning, including a content personalization scenario, see the [Contextual Bandit Simulation Tutorial](python_Simulating_a_news_personalization_scenario_using_Contextual_Bandits.ipynb).
 - See the [Linear Regression Tutorial](cmd_linear_regression.md) for a different look at the roof replacement problem and learn more about Vowpal Wabbit's format and understanding the results.

python/tests/confidence_sequence.py

@@ -189,6 +189,5 @@ def lblogwealth(self, *, t, sumXt, v, eta, s, alpha):
 
         return max(
             0,
-            (sumXt - sqrt(gamma1**2 * ll * v + gamma2**2 * ll**2) - gamma2 * ll)
-            / t,
+            (sumXt - sqrt(gamma1**2 * ll * v + gamma2**2 * ll**2) - gamma2 * ll) / t,
         )

python/tests/crminustwo.py

@@ -440,21 +440,23 @@ def intervaldiff(
                     candidates.append(
                         (
                             gstar,
-                            None
-                            if isclose(kappa, 0)
-                            else {
-                                "kappastar": kappa,
-                                "betastar": beta,
-                                "gammastar": gamma,
-                                "taustar": tau,
-                                "ufake": ufake,
-                                "wfake": wfake,
-                                "rfake": rex,
-                                "qfunc": lambda c, u, w, r, k=kappa, g=gamma, b=beta, t=tau, s=sign, num=n: -c
-                                * (b + g * u + t * w + s * (u - w) * r)
-                                / ((num + 1) * k),
-                                "mle": mle,
-                            },
+                            (
+                                None
+                                if isclose(kappa, 0)
+                                else {
+                                    "kappastar": kappa,
+                                    "betastar": beta,
+                                    "gammastar": gamma,
+                                    "taustar": tau,
+                                    "ufake": ufake,
+                                    "wfake": wfake,
+                                    "rfake": rex,
+                                    "qfunc": lambda c, u, w, r, k=kappa, g=gamma, b=beta, t=tau, s=sign, num=n: -c
+                                    * (b + g * u + t * w + s * (u - w) * r)
+                                    / ((num + 1) * k),
+                                    "mle": mle,
+                                }
+                            ),
                         )
                     )
 

python/vowpalwabbit/pyvw.py

@@ -532,9 +532,9 @@ def parse(
                     for ex in str_ex
                 ]
             ):
-                str_ex: List[
-                    Example
-                ] = str_ex  # pytype: disable=annotation-type-mismatch
+                str_ex: List[Example] = (
+                    str_ex  # pytype: disable=annotation-type-mismatch
+                )
                 return str_ex
 
         if not isinstance(str_ex, (list, str)):

python/vowpalwabbit/sklearn.py

@@ -738,12 +738,12 @@ class would be predicted.
             >>> model = VWMultiClassifier(oaa=3, loss_function='logistic')
             >>> _ = model.fit(X, y)
             >>> model.predict_proba(X)
-            array([[0.38924146, 0.30537927, 0.30537927],
-                   [0.40661219, 0.29669389, 0.29669389],
-                   [0.52335149, 0.23832427, 0.23832427],
-                   [0.52696788, 0.23651604, 0.23651604],
-                   [0.65430814, 0.17284594, 0.17284594],
-                   [0.61224216, 0.19387889, 0.19387889]])
+            array([[0.38926664, 0.30536669, 0.30536669],
+                   [0.40663728, 0.2966814 , 0.2966814 ],
+                   [0.52337217, 0.23831393, 0.23831393],
+                   [0.52698863, 0.23650573, 0.23650573],
+                   [0.6543135 , 0.17284323, 0.17284323],
+                   [0.61224902, 0.19387549, 0.19387549]])
         """
         return VW.predict(self, X=X)
 

test/core.vwtest.json

@@ -6021,7 +6021,7 @@
   {
     "id": 465,
     "desc": "cb_explore_adf with epsilon-greedy exploration using --sparse_weights and saving model",
-    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test.ldf --noconstant --sparse_weights -f standard_sparse_model.vw",
+    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test.ldf --noconstant --sparse_weights -f standard_sparse_model.vw -q::",
     "diff_files": {
       "stderr": "train-sets/ref/sparse_save_check.stderr",
       "stdout": "train-sets/ref/sparse_save_check.stdout"
@@ -6033,7 +6033,7 @@
   {
     "id": 466,
     "desc": "cb_explore_adf with epsilon-greedy exploration using --sparse_weights and loading model",
-    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test.ldf --noconstant --sparse_weights -i standard_sparse_model.vw",
+    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test.ldf --noconstant --sparse_weights -i standard_sparse_model.vw -q::",
     "diff_files": {
       "stderr": "train-sets/ref/sparse_load_check.stderr",
       "stdout": "train-sets/ref/sparse_load_check.stdout"
@@ -6045,5 +6045,62 @@
     "depends_on": [
       465
     ]
+  },
+  {
+    "id": 467,
+    "desc": "cb_explore_adf with epsilon-greedy exploration using --sparse_weights and saving model with random_weights",
+    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test.ldf --noconstant --sparse_weights -f standard_sparse_random_model.vw -q:: --random_weights",
+    "diff_files": {
+      "stderr": "train-sets/ref/sparse_save_check_random.stderr",
+      "stdout": "train-sets/ref/sparse_save_check_random.stdout"
+    },
+    "input_files": [
+      "train-sets/cb_test.ldf"
+    ]
+  },
+  {
+    "id": 468,
+    "desc": "cb_explore_adf with epsilon-greedy exploration using --sparse_weights and loading model with random_weights",
+    "vw_command": "--cb_explore_adf --epsilon 0.1 -d train-sets/cb_test.ldf --noconstant --sparse_weights -i standard_sparse_random_model.vw -q:: --random_weights",
+    "diff_files": {
+      "stderr": "train-sets/ref/sparse_load_check_random.stderr",
+      "stdout": "train-sets/ref/sparse_load_check_random.stdout"
+    },
+    "input_files": [
+      "train-sets/cb_test.ldf",
+      "standard_sparse_random_model.vw"
+    ],
+    "depends_on": [
+      467
+    ]
+  },
+  {
+    "id": 469,
+    "desc": "https://github.com/VowpalWabbit/vowpal_wabbit/issues/4669",
+    "vw_command": "--ccb_explore_adf --dsjson -d train-sets/issue4669.dsjson -f issue4669.model",
+    "diff_files": {
+      "stderr": "train-sets/ref/issue4669_train.stderr",
+      "stdout": "train-sets/ref/issue4669_train.stdout"
+    },
+    "input_files": [
+      "train-sets/issue4669.dsjson"
+    ]
+  },
+  {
+    "id": 470,
+    "desc": "https://github.com/VowpalWabbit/vowpal_wabbit/issues/4669",
+    "vw_command": "--ccb_explore_adf --dsjson --all_slots_loss --epsilon 0 -t -i issue4669.model -t -d train-sets/issue4669.dsjson -p issue4669_test_pred.txt",
+    "diff_files": {
+      "stderr": "train-sets/ref/issue4669_test.stderr",
+      "stdout": "train-sets/ref/issue4669_test.stdout",
+      "issue4669_test_pred.txt": "train-sets/ref/issue4669_test_pred.txt"
+    },
+    "input_files": [
+      "train-sets/issue4669.dsjson",
+      "issue4669.model"
+    ],
+    "depends_on": [
+      469
+    ]
   }
 ]