From 5932dec7ead5b13b3d264cdf762178e2c6cf10f0 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 1 May 2019 18:54:30 +0300 Subject: [PATCH 01/82] make set_config working with dicts --- polara/evaluation/pipelines.py | 45 +++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/polara/evaluation/pipelines.py b/polara/evaluation/pipelines.py index ad372d8..569e341 100644 --- a/polara/evaluation/pipelines.py +++ b/polara/evaluation/pipelines.py @@ -4,6 +4,14 @@ from functools import reduce from random import choice import pandas as pd +from collections import abc + + +def is_list_like(obj, allow_sets=False, allow_dict=False): + return (isinstance(obj, abc.Iterable) and + not isinstance(obj, (str, bytes)) and + not (allow_sets is False and isinstance(obj, abc.Set)) and + not (allow_dict is False and isinstance(obj, abc.Mapping))) def random_chooser(): @@ -45,13 +53,15 @@ def never_skip(config): return False return grid, param_names -def set_config(model, attributes, values): - for name, value in zip(attributes, values): +def set_config(model, config, convert_nan=True): + for name, value in config.items(): + if convert_nan: + value = value if value == value else None # convert NaN to None setattr(model, name, value) def evaluate_models(models, target_metric='precision', metric_type='all', **kwargs): - if not isinstance(models, (list, tuple)): + if not is_list_like(models, allow_sets=True): models = [models] model_scores = {} @@ -74,7 +84,7 @@ def find_optimal_svd_rank(model, ranks, target_metric, return_scores=False, evaluator = evaluator or evaluate_models model_verbose = model.verbose if config: - set_config(model, *zip(*config.items())) + set_config(model, config) model.rank = svd_rank = max(max(ranks), model.rank) if not model._is_ready: @@ -86,7 +96,7 @@ def find_optimal_svd_rank(model, ranks, target_metric, return_scores=False, res = {} try: - for rank in iterator(list(reversed(sorted(ranks)))): + for rank in iterator(sorted(ranks, key=lambda x: -x)): model.rank = rank res[rank] = evaluator(model, target_metric, **kwargs)[model.method] # prevent previous scores caching when assigning svd_rank @@ -112,7 +122,7 @@ def find_optimal_tucker_ranks(model, tucker_ranks, target_metric, return_scores= evaluator = evaluator or evaluate_models model_verbose = model.verbose if config: - set_config(model, *zip(*config.items())) + set_config(model, config) model.mlrank = tuple([max(mode_ranks) for mode_ranks in tucker_ranks]) @@ -155,21 +165,26 @@ def find_optimal_config(model, param_grid, param_names, target_metric, return_sc evaluator=None, iterator=lambda x: x, **kwargs): evaluator = evaluator or evaluate_models model_verbose = model.verbose + if init_config: - set_config(model, *zip(*init_config.items())) + if not is_list_like(init_config): + init_config = [init_config] + for config in init_config: + set_config(model, config) model.verbose = verbose grid_results = {} for params in iterator(param_grid): try: - set_config(model, param_names, params) + param_config = dict(zip(param_names, params)) + set_config(model, param_config) if not model._is_ready or force_build: model.build() grid_results[params] = evaluator(model, target_metric, **kwargs)[model.method] finally: if reset_config is not None: if isinstance(reset_config, dict): - set_config(model, *zip(*reset_config.items())) + set_config(model, reset_config) elif callable(reset_config): reset_config(model) else: @@ -179,9 +194,17 @@ def find_optimal_config(model, param_grid, param_names, target_metric, return_sc # workaround non-orderable configs (otherwise pandas raises error) scores = pd.Series(**dict(zip(('index', 'data'), (zip(*grid_results.items()))))) - best_config = scores.idxmax() + best_params = scores.idxmax() + try: + best_config = dict(zip(param_names, best_params)) + except TypeError: + best_config = {param_names: best_params} + if return_scores: - scores.index.names = param_names + try: + scores.index.names = param_names + except ValueError: # not list-like + scores.index.name = param_names scores.name = model.method return best_config, scores return best_config From e1a3831e3f88590f2301f7f41146b60c72de69f8 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 1 May 2019 18:56:07 +0300 Subject: [PATCH 02/82] add provisional implementation of LCE model --- polara/lib/optimize.py | 91 +++++++++++++++++++++++++++ polara/recommender/hybrid/models.py | 98 ++++++++++++++++++++++++++++- 2 files changed, 188 insertions(+), 1 deletion(-) diff --git a/polara/lib/optimize.py b/polara/lib/optimize.py index f9eb8ff..01d7492 100644 --- a/polara/lib/optimize.py +++ b/polara/lib/optimize.py @@ -1,6 +1,7 @@ from math import sqrt import numpy as np from numba import jit, njit, prange +from scipy import sparse from polara.tools.timing import track_time @@ -298,3 +299,93 @@ def kernelized_pmf_sgd(interactions, shape, nonzero_count, rank, adjustment_params=adjustment_params, seed=seed, verbose=verbose, iter_errors=iter_errors, iter_time=iter_time) + + +def trace(A, B): + if sparse.issparse(A): + return A.multiply(B).sum() + return (A * B).sum() + +def local_collective_embeddings(Xs, Xu, A, k=15, alpha=0.1, beta=0.05, + lamb=1, epsilon=0.0001, maxiter=15, + seed=None, verbose=True): + """ + Python Implementation of Local Collective Embeddings + + author : Abhishek Thakur, https://github.com/abhishekkrthakur/LCE + original : https://github.com/msaveski/LCE + adapted for Polara by: Evgeny Frolov + """ + n = Xs.shape[0] + v1 = Xs.shape[1] + v2 = Xu.shape[1] + + random = np.random if seed is None else np.random.RandomState(seed) + W = random.rand(n, k) + Hs = random.rand(k, v1) + Hu = random.rand(k, v2) + + D = sparse.dia_matrix((A.sum(axis=0), 0), A.shape) + + gamma = 1. - alpha + trXstXs = trace(Xs, Xs) + trXutXu = trace(Xu, Xu) + + WtW = W.T.dot(W) + WtXs = Xs.T.dot(W).T + WtXu = Xu.T.dot(W).T + WtWHs = WtW.dot(Hs) + WtWHu = WtW.dot(Hu) + DW = D.dot(W) + AW = A.dot(W) + + itNum = 1 + delta = 2.0 * epsilon + + ObjHist = [] + + while True: + + # update H + Hs_1 = np.divide( + (alpha * WtXs), np.maximum(alpha * WtWHs + lamb * Hs, 1e-10)) + Hs = np.multiply(Hs, Hs_1) + + Hu_1 = np.divide( + (gamma * WtXu), np.maximum(gamma * WtWHu + lamb * Hu, 1e-10)) + Hu = np.multiply(Hu, Hu_1) + + # update W + W_t1 = alpha * Xs.dot(Hs.T) + gamma * Xu.dot(Hu.T) + beta * AW + W_t2 = alpha * W.dot(Hs.dot(Hs.T)) + gamma * \ + W.dot(Hu.dot(Hu.T)) + beta * DW + lamb * W + W_t3 = np.divide(W_t1, np.maximum(W_t2, 1e-10)) + W = np.multiply(W, W_t3) + + # calculate objective function + WtW = W.T.dot(W) + WtXs = Xs.T.dot(W).T + WtXu = Xu.T.dot(W).T + WtWHs = WtW.dot(Hs) + WtWHu = WtW.dot(Hu) + DW = D.dot(W) + AW = A.dot(W) + + tr1 = alpha * (trXstXs - 2. * trace(Hs, WtXs) + trace(Hs, WtWHs)) + tr2 = gamma * (trXutXu - 2. * trace(Hu, WtXu) + trace(Hu, WtWHu)) + tr3 = beta * (trace(W, DW) - trace(W, AW)) + tr4 = lamb * (np.trace(WtW) + trace(Hs, Hs) + trace(Hu, Hu)) + + Obj = tr1 + tr2 + tr3 + tr4 + ObjHist.append(Obj) + + if itNum > 1: + delta = abs(ObjHist[-1] - ObjHist[-2]) + if verbose: + print("Iteration: ", itNum, "Objective: ", Obj, "Delta: ", delta) + if itNum > maxiter or delta < epsilon: + break + + itNum += 1 + + return W, Hu, Hs diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index 37560ae..e6dec48 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -2,7 +2,7 @@ import numpy as np from polara.recommender.models import RecommenderModel, ProbabilisticMF -from polara.lib.optimize import kernelized_pmf_sgd +from polara.lib.optimize import kernelized_pmf_sgd, local_collective_embeddings from polara.lib.sparse import sparse_dot from polara.tools.timing import track_time @@ -100,3 +100,99 @@ def build(self, *args, **kwargs): kernel_config = dict(kernel_update=self.kernel_update, sparse_kernel_format=self.sparse_kernel_format) super().build(kernel_matrices, *args, **kernel_config, **kwargs) + + +class LCEModel(RecommenderModel): + def __init__(self, *args, item_features=None, **kwargs): + super().__init__(*args, **kwargs) + self._rank = 10 + self.factors = {} + self.alpha = 0.1 + self.beta = 0.05 + self.max_neighbours = 10 + self.item_features = item_features + self.binary_features = True + self._item_data = None + self.feature_labels = None + self.seed = None + self.show_error = False + self.regularization = 1 + self.max_iterations = 15 + self.tolerance = 0.0001 + self.method = 'LCE' + self.data.subscribe(self.data.on_change_event, self._clean_metadata) + + def _clean_metadata(self): + self._item_data = None + self.feature_labels = None + + @property + def rank(self): + return self._rank + + @rank.setter + def rank(self, new_value): + if new_value != self._rank: + self._rank = new_value + self._is_ready = False + self._recommendations = None + + @property + def item_data(self): + if self.item_features is not None: + if self._item_data is None: + index_data = getattr(self.data.index, 'itemid') + + try: + item_index = index_data.training + except AttributeError: + item_index = index_data + + self._item_data = self.item_features.reindex(item_index.old.values, # make correct sorting + fill_value=[]) + else: + self._item_data = None + return self._item_data + + + def build(self): + # prepare input matrix for learning the model + Xs, lbls = stack_features(self.item_data, normalize=False) # item-features sparse matrix + Xu = self.get_training_matrix().T # item-user sparse matrix + + n_nbrs = min(self.max_neighbours, int(math.sqrt(Xs.shape[0]))) + A = construct_A(Xs, n_nbrs, binary=self.binary_features) + + with track_time(self.training_time, verbose=self.verbose, model=self.method): + W, Hu, Hs = local_collective_embeddings(Xs, Xu, A, + k=self.rank, + alpha=self.alpha, + beta=self.beta, + lamb=self.regularization, + epsilon=self.tolerance, + maxiter=self.max_iterations, + seed=self.seed, + verbose=self.show_error) + + userid = self.data.fields.userid + itemid = self.data.fields.itemid + self.factors[userid] = Hu.T + self.factors[itemid] = W + self.factors['item_features'] = Hs.T + self.feature_labels = lbls + + def get_recommendations(self): + if self.data.warm_start: + raise NotImplementedError + else: + return super().get_recommendations() + + def slice_recommendations(self, test_data, shape, start, stop, test_users=None): + userid = self.data.fields.userid + itemid = self.data.fields.itemid + slice_data = self._slice_test_data(test_data, start, stop) + + user_factors = self.factors[userid][test_users[start:stop], :] + item_factors = self.factors[itemid] + scores = user_factors.dot(item_factors.T) + return scores, slice_data From 16ffc6fe070c551fe9538bd9b074418a8bcdfcfd Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 2 May 2019 07:03:41 +0300 Subject: [PATCH 03/82] update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 08bb324..5d7d28e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ opts = dict(name="polara", description="Fast and flexible recommender system framework", keywords = "recommender system", - version = "0.6.4", + version = "0.6.4.dev", license="MIT", author="Evgeny Frolov", platforms=["any"], From bf437ee12796adec7ac43ed9ebae626460c61677 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 2 May 2019 07:07:51 +0300 Subject: [PATCH 04/82] add new tutorial on hyper-parameter tuning and comprehensive model evaluation --- ...ing_and_cross_validation_experiments.ipynb | 1729 +++++++++++++++++ 1 file changed, 1729 insertions(+) create mode 100644 examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb diff --git a/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb b/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb new file mode 100644 index 0000000..7e71c2b --- /dev/null +++ b/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb @@ -0,0 +1,1729 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial we will go through a full cycle of model tuning and evaluation with Polara. This will include 2 phases: grid-search for finding (almost) optimal values of hyper-parameters and verification of results via 5-fold cross-validation.\n", + "\n", + "
We will focus on performing a fair comparison of popular ALS-based matrix factorization (MF) model called Weighted Regularized Matrix Factorization (WRMF a.k.a. iALS) [Hu2008] with less popular model called PureSVD [Cremonesi2010] based on standard SVD.
\n", + "\n", + "We will use standard *Scipy*'s implementation for the latter and a great library called [*implicit*](https://github.com/benfred/implicit) for iALS. Both are wrapped by Polara and can be accessed via the corresponding classes. Due to its practicality the *implicit* library is often recommended to beginners and sometimes even serves as a default tool in production. On the other hand, there are some important yet often overlooked features, which make SVD-based models stand out. Ignoring them in my opinion leads to certain misconceptions and myths, not to say that it also overcomplicates things quite a bit.\n", + "\n", + "Note that by saying SVD I really mean *Singular Value Decomposition*, not just an arbitrary matrix factorization. In that sense, **methods like FunkSVD, SVD++, SVDFeature, etc., are not SVD-based at all**, even though historically they use SVD acronym in their names and are often referenced as if they are real substitutes for SVD. These methods utilize another optimization algorithm, typically based on stochastic gradient descent, and do not preserve the algebraic properties of SVD. This is really an important distinction, especially in the view of the following remarks:\n", + "\n", + "1. **SVD-based approach has a number of unique and beneficial properties**. To name a few, it produces stable and determenistic output with global guarantees. It admits the same prediction formula for both known and previously unseen users (as long as at least one user rating is known). It can take a hybrid form to include side information via the generalized formulation (see Chapter 6 of [my thesis](https://www.skoltech.ru/en/2018/09/phd-thesis-defense-evgeny-frolov)). Even without hybridization it can be quite successfully applied in the cold start regime (paper on this is coming). It requires minimal tuning and allows to compute and store a single latent feature matrix - either for users or for items - instead of computing and storing both of them. This luxury is not available in the majority of other matrix factorization approaches, definitely not in popular ones. \n", + "2. Computational complexity of truncated SVD **scales linearly with the number of known observations** and quadratically with the rank of decomposition (thanks to Lanczos procedure). There are [open source implementations](https://github.com/criteo/rsvd), allowing to handle in a distributed manner nearly *billion-scale problems* with its [efficient randomized version](https://medium.com/criteo-labs/sparkrsvd-open-sourced-by-criteo-for-large-scale-recommendation-engines-6695b649f519). \n", + "3. At least **in some cases PureSVD outperforms other more sophisticated matrix factorization methods** [Cremonesi2010].\n", + "4. Moreover, **PureSVD can be quite easily tuned to perform even better** [Nikolakopoulos2019].\n", + "\n", + "Despite that impresisve list, PureSVD technique (and especially its modifications) rarely gets into the list of baseline models to compare with. Hence, this tutorial also aims at performing a thorough assessment of the default choice of many practitioners to see whether it really provides the celebrated advantages over the simpler approach." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* [Hu 2008], Hu Y., Koren, Y. and Volinsky, C., 2008, December. *Collaborative Filtering for Implicit Feedback Datasets*. In ICDM (Vol. 8, pp. 263-272). [Link](http://yifanhu.net/PUB/cf.pdf). \n", + "* [Cremonesi2010] Cremonesi, P., Koren, Y. and Turrin, R., 2010, September. *Performance of recommender algorithms on top-n recommendation tasks*. In Proceedings of the fourth ACM conference on Recommender systems (pp. 39-46). ACM. [Link](https://www.researchgate.net/publication/221141030_Performance_of_recommender_algorithms_on_top-N_recommendation_tasks). \n", + "* [Nikolakopoulos2019] Nikolakopoulos, A.N., Kalantzis, V., Gallopoulos, E. and Garofalakis, J.D., 2019. *EigenRec: generalizing PureSVD for effective and efficient top-N recommendations*. Knowledge and Information Systems, 58(1), pp.59-81. [Link](https://arxiv.org/abs/1511.06033)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Downloading data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use the **[Movielens-10M](https://grouplens.org/datasets/movielens/)** dataset for our experiments. It's large enough to perform reliable evaluation; however, not that large that you would spend too many hours waiting for results. If you don't plan to play with the code, you may want to run all cells with a single command and leave the notebook running in the background, while reading the text. It takes around 1.5h to complete this notebook on a modern laptop." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that you'll need an internet connection in order to run the code below. It will automatically download data, store it in a temporary location, and convert into a `pandas` dataframe. \n", + "Alternatively, if you have already downloaded the dataset, you can use path to it as an input for the `get_movielens_data` function instead of `tmp_file`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import urllib\n", + "from polara import (get_movielens_data, # returns data in the pandas dataframe format\n", + " RecommenderData) # provides common interface to access data " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
useridmovieidratingtimestamp
011225.0838985046
111855.0838983525
212315.0838983392
312925.0838983421
413165.0838983392
\n", + "
" + ], + "text/plain": [ + " userid movieid rating timestamp\n", + "0 1 122 5.0 838985046\n", + "1 1 185 5.0 838983525\n", + "2 1 231 5.0 838983392\n", + "3 1 292 5.0 838983421\n", + "4 1 316 5.0 838983392" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url = 'http://files.grouplens.org/datasets/movielens/ml-10m.zip'\n", + "tmp_file, _ = urllib.request.urlretrieve(url) # this may take some time depending on your internet connection\n", + "\n", + "data = get_movielens_data(tmp_file, include_time=True)\n", + "data.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The resulting dataframe has a bit more than 10M ratings, as expected." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(10000054, 4)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Basic data stats:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "userid 69878\n", + "movieid 10677\n", + "rating 10\n", + "timestamp 7096905\n", + "dtype: int64" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.apply('nunique')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Preparing data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As always, you need to firstly define a data model that will provide a common interface for all recommendation algorithms used in experiments:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Fields(userid='userid', itemid='movieid', feedback='rating')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_model = RecommenderData(data, 'userid', 'movieid', 'rating', custom_order='timestamp', seed=0)\n", + "data_model.fields" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting `seed=0` ensures controllable randomization when sampling test data, which enhances reproducibility; `custom_order` allows to select observations for evaluation based on their timestamp, rather than on rating value (more on that later). " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's look at the default configuration of data model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'shuffle_data': False,\n", + " 'test_fold': 5,\n", + " 'test_ratio': 0.2,\n", + " 'permute_tops': False,\n", + " 'warm_start': True,\n", + " 'random_holdout': False,\n", + " 'negative_prediction': False,\n", + " 'test_sample': None,\n", + " 'holdout_size': 3}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_model.get_configuration()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default, `Polara` samples 20% of users and marks them for test (`test_ratio` attribute). These users would be excluded from the training dataset, if `warm_start` attribute remained set to `True` (strong generalization test). However, in the iALS case such setting would require running additional half-step optimization (folding-in) for each test user in order to obtain their latent representation. To avoid that we turn the \"warm start\" setting off and perform standard evaluation (weak generalization test). In that case test users are part of the training (except for the ratings that were held out for evaluation) and one can directly invoke scalar products of latent factors. Note that *SVD recommendations do not depend on this setting due to uniform projection formula applicable to both known and \"warm\" users*: $r = VV^\\top p$, where $r$ is a vector of predicted relevance scores, $p$ is a vector of *any* known preferences and $V$ is an item latent features matrix." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing data...\n", + "Done.\n", + "There are 9986078 events in the training and 13976 events in the holdout.\n" + ] + } + ], + "source": [ + "data_model.holdout_size = 1 # hold out 1 item from every test user\n", + "data_model.random_holdout = False # take items with the latest timstamp\n", + "data_model.warm_start = False # standard case\n", + "data_model.prepare()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `holdout_size` attribute controls how many user preferences will be used for evaluation. Current configuration instructs data model to holdout one item from every test user. The `random_holdout=False` setting along with `custom_order` input argumment of data model make sure that only the latest rated item is taken for evaluation, allowing to avoid \"recommendations from future\". All these items are available via `data_model.test.holdout`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# General configuration" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## technical settings" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sometimes due to the size of dataset evaluation make take considerably longer than actual training time. If that's the case, you may want to give Polara more resources to perform evaluation, which is mostly controlled by changing `memory_hard_limit` and `max_test_workers` settings from their defaults. The former defines how much memory is allowed to use when generating predictions for test users. Essentially, Polara avoids running an inefficient by-user-loop for that task and makes calculation in bulk, which allos to invoke linear algebra kernels and speed up calculations. This, however, generates dense $M \\times N$ matrix, where $M$ is the number of test users and $N$ is the number of all items seen during training. If this matrix doesn't fit into the memory constraint defined by `memory_hard_limit` (1Gb by default), calculations will be perfomed on a sequence of groups of $m\n", + "\n", + "100%\n", + "15/15\n", + "[00:57<00:04, 3.79s/it]" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 15/15 [00:57<00:04, 3.79s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# the model will be computed only once for the max value of rank\n", + "psvd_best_rank, psvd_rank_scores = find_optimal_svd_rank(psvd,\n", + " rank_grid,\n", + " target_metric,\n", + " config=init_config,\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that in this case the most of the time is spent on evaluation, rather than on model computation. The model was computed only once. You can verify it by calling `psvd.training_time` list attribute and seeing that it contains only one entry:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[9.0811659]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "psvd.training_time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's see how quality of recommendations changes with rank (number of latent features)." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8XOV97/HPV4sly5ssW17wbmwCNlATBIa0pEkoYEISwy2vYkoJaWndBUqTNmnh5iZNuG1vc9tXszQ0LQn7bQIpLYlLCiYJZGkCxjI2XnEQxmDZxguSV3mRpd/94xyZsZA0kg/jkezv+/Wa15zlOc88c2zNd855znlGEYGZmdnxKil2A8zMbGBzkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDIpK3YDToTRo0fH1KlTi90MM7MBZdmyZTsjojZfuVMiSKZOnUp9fX2xm3FKaW8PXn1zP6+/2ULtsAomVA+muqocScVumpn1kqTXelPulAgSK7ym/YdZsamZFa/vYvmmXby4aRd7Dh45pszg8lJOq67ktOrBTKgezGlHH5VMqB7MuBGVVJSVFukdmNnxcpBYnx060saaLXtY8fouVmxKHq83tQBQInjXuOFcde5pnDepmum1Q9i57xCbdx1ky64DRx/rtu5l575Db6u7dlhFGjSVnDbirbCZkAZOzZBBmY5qIoJDR9o5cLiNA63p43AbB3OmD7Sm84fbGFJRxvvPHMPooRXH/ZpmJzsHifUoInjtzRaWp0cbKzbtYu3WPbS2JaNGjxteyXmTq7lh7mTmTKrmnIkjqBrUu/9WB1vbeGN3EjCbdx1gS0fY7D7AS2/s5emXtnOwtf2YbSrKSnKOZioZOWQQh1rbuw6CoyHR/tb0kTb6OuC1BBdMqeHy2WO5YvY4JtVU9a0Cs5OcToVh5Ovq6sJ9JL2zq+Xw0aOMFekpquaWVgCqBpVyzoQRzJlczXmTqpkzaSTjRlQWrC0RQXNLa07QdDwOHp3f1dJKRXkJVYNKGVxeSmV5KYPT6cHlpVTmTA8elK4vL2Vwecmx891sv2X3ARav2cZTa97gpTf2AjBr/HCumD2OeWeP44yxQ93vYyctScsioi5vOQfJqau9PVi9ZTfLc05RvbpzP5B8Cz9jzDDmTKpmzuRq5kyqZuaYoZSVnrpXjG/cuZ+n1r7Bk6vf4IXXdwEwdVQVV8wex+Wzx3HepGpKShwqdvLoF0EiaR7wZaAU+EZE/G2n9RXAg8D5wJvAdRGxUdKFwN0dxYDPRcRjvamzKw6St7S3B8s37eLxlVv4r1Vb2bYn6acYM6zimNA4d2I1Qyt85rM72/cc5Km121i85g2efeVNjrQHY4ZVcNmsscw7exwXTR9F+SkcunZyKHqQSCoFfgFcBjQCS4HrI2JtTpk/As6NiD+QtAC4JiKuk1QFHI6II5LGAy8CpwGRr86unOpBEhG82Libx19MwmPL7oMMKivhfWfUcuU545g7bRTjR1T6FM1x2t3SytPrt7F49TZ+/IsdHGhtY3hlGZeeNZYrZo/lvWfU9rrfyKw/6W2QFPJ/94VAQ0RsSBv0MDAfyP3Qnw98Lp1+FPiqJEVES06ZSpIA6W2dRhIeqzfv4fGVW3h85VY27zpAean41TNq+dS8d/FrZ41lWGV5sZt5UhhRVc41503kmvMmcrC1jZ/8YgeL12zjhy9t47Hlm6ksL+GSmbXMmz2OS88aQ3XVoGI32ewdVcggmQBsyplvBOZ2VyY9+tgNjAJ2SpoL3AtMAW5M1/emzlNWRLB26x6+t3Ir31u1ldfebKGsRFwyczSfuOwMLps1lhGDHR6FVFleyuVpn0lrWztLX23iyTVv8NSabXx/7TZKS8RF02uSfpVZ4wp6sYLZiVLIIOnqPEnn82jdlomIJcBsSWcBD0h6opd1JhVLC4GFAJMnT+5tmweciGD9tr1JeKzcyoad+yktEe85fRS3vG8Gl88e62/ARVJeWsJ7ZozmPTNG87kPz2bl5t0sXvMGi9e8wWe/u4bPfncNM8YMZXJNFRNHDk4fVUyoTqaz3jPzTmk5fITNzQdo3HUgeW5OrqLb3NxCZXkpM8YMTR61yXPtsIp+0e58Dra2UVaiU/oCkndKIYOkEZiUMz8R2NJNmUZJZcAIoCm3QESsk7QfOLuXdXZsdzdph31dXd1Jd2nay9v28nh65NGwfR8lgotPH8XvvXc6V8weR80Qh0d/UlKi5GKGSdX8xbwzadi+lydXv8GLjbvZ3HyA+o1NXY4EkBswR4MmXTbqHQqa3Qda04BoSQMiJyx2HaBp/+FjypeX6uhNoi2H23jshc3sPfRW24dVlh0TLB2PiSOrKD2BV7W1trWzZdcBNjUdYFNzC5uaWtjUfIBNTS00Nrewc1/yvkYMLqdmyCBGViXPNUMGMXLIIGqqOj2n08Mqy3x1XieF7GwvI+kYvxTYTNIx/psRsSanzC3AOTmd7f8jIn5D0jRgU3o6awrwLHAusCtfnV05WTrbN+zYl4THyq2s37YXCeZOq+Gqc0/jyrPH+e7rAa7zB3pjOt2YfrDvPtB6TPnK8pKcgDn2aGbiyCpGD02+TDTtP5xzFHFs/ZubDxwTAh31JvUkofVWnYOZUF3FmGEVx3yQRgTb9x6iYfs+Grbv4+Xte9Pp/ceMXlBRVsL02qFvC5mpo6uOa2ic9vbkdY+GRE5gNDYfYOvuA7TnfLyVlSQBOKlmMJPSfdUWQfP+wzS1tCbP+w/T3HKYN/cf5vCR9i5ft7REjKwqZ2SnoBnVEUBDknWjhlQwY8xQBg8auMP+FP2qrbQRHwS+RHKp7r0R8deS7gTqI2KRpErgIeA8kiORBRGxQdKNwO1AK9AO3BkR3+muznztGMhBsnHnfr63aiuPr9zKuq17jt5lfdW547nynHGMGeZz7KeKPQdb3zpayAmYxl3J9K6WtweNEAda245ZPqyi7OhRTVeB8U6eUtvd0krDjr1HQ6Zh+z4aduyjsfnA0REGSkvE5JoqTu90BDNjzFBaj7Sn4XDsUUVjUwuNuw687cN+7PAKJo2sYlJNFZNGDmZiTVU6P5hxwyt7fRorIjjQ2sab+5Jg6QiYpv1p4LQcPiZ4mva30txymLb2Yz9Py0rE7AkjqJsykgumjuT8KTXUDhs4X/j6RZD0FwMlSCKC15taWLKhiedefZMlG5rYvOsAAOdPGclV54zng+eMdwetdWnvwdZjTk1tamoh4G2B0R8uuDhwuI0NO5NgeWX7Pl5OQ2bjm/uPDr/Tleqq8qPBMGlkVRoUg5lUkxxhVJYX79t/RLDn4JGjQbN9zyFWNu6i/rVmXty0i0Np6E0ZVUXdlBrqpo6kbspITq8d2m9PlTlIcvTXIIkINuzcz5INTSxJg+ONPQcBGDVkEBdOq+Gi6aP4tVljmVA9uMitNSu81rZ2Xm9qSQJmxz4qykqPBsXEkYMH7CXrh4+0s3rLbpZtbGbpxiaWvdbMm2nfU3VVOedPHsn5U0dywdQazpkwoqiBmMtBkqO/BElE0LB9H8+92sRzG97k+Veb2LE3OYc8emgFF02vYe70UVw0rYYZYzyGk9nJKiJ4ded+6l9rTsLltSY27EiGJxpUWsLZE4ZzwdQazp8ykvOnjGRUkfo/HSQ5ihUk7e3JpblLNrzJklebeP7VpqPfQsYNr2Tu9BrmThvF3Ok1TB89xMFhdgpr2n+YZa81U7+xifrXmlnVuJvDbcnpsOm1Q6ibMvLoKbFpJ+jzoj/c2X7KaWsP1m3dw3NpcCzd2HS0A3RC9WB+9V21XJQGx+SaKgeHmR1VM2QQl80ay2WzxgLJfS6rNu+mfmMSLk+t3ca36xuB5NT3nEnVVA4qpb09aGsP2iN5bguOLmuLSKZzntva6WJZp/Xpst5ykGS0busefvryDpZsaOL5jU3sTe8FmFxTxWVnjWXu9FHMnVbj37Awsz6pLC/lgqk1XDC1Bjid9vbglR37qH+tmfqNzazavIsjbUFJiSiVkucS3ppOn8tLSxhcKkokSks6nsmZ1tu26ahnZS/b6lNbGdz9k1f4m/96CYDpo4ccc6pq/Ah3jpvZwOZTWwUUEfz9U+u565lXuOqc8fzlh2cxZrgvyTWzU5ODpI/a24O/XLSGh557jQUXTOKvrznnhA77YGbW3zhI+qC1rZ1P/duLfGfFFn7/vdO5/coz3WFuZqc8B0kvHWxt49ZvvsAP1m3nU1e8iz963+kOETMzHCS9svdgK7/7QD3Pb2zif199NjdeNKXYTTIz6zccJHk07T/Mx+57njVb9vCl6+Ywf86EYjfJzKxfcZD04I3dB/mte5awqamFu288n0vPGlvsJpmZ9TsOkm5s3Lmf37pnCbtaWnngdy7koumjit0kM7N+yUHShXVb93DjPc/T1t7ON39vLudOrC52k8zM+i0HSSfLXmvmt+97nqpBZTy88GJmjBlW7CaZmfVrDpIcP315BwsfXMbY4RU8dPNcj49lZtYLDpLUE6u2ctvDyzm9digP3nyhf8LWzKyXHCTAt+s3cfu/r2TOpGru+9iFjKgamL/CZmZWDKd8kHzjpxv4q++t45KZo/mXG8+natApv0vMzPrklP3UjAi++P1f8JWnG7jy7HF8acEcKsr6x+8km5kNJKdkkLS3B3c+vpb7f76R36ibyN9ccw5lpSXFbpaZ2YB0ygXJkbZ2/vzRlfzH8s387q9M49NXneXBF83MMjilguRgaxt//K3lfH/tNj55+Rnc8v4ZDhEzs4xOmSDZd+gICx+s5+evvMmd82fz0YunFrtJZmYnhVMiSNragxu+sYTVm3fzxet+iWvOm1jsJpmZnTQK2sMsaZ6k9ZIaJN3exfoKSY+k65dImpouv0zSMkmr0ucP5Gzzo7TOFeljTL52vLJjH+u27uGff+t8h4iZ2TusYEckkkqBu4DLgEZgqaRFEbE2p9jNQHNEzJC0APgCcB2wE/hwRGyRdDawGMj9IZAbIqK+t21pbQvu/+0LeM/pozO+KzMz66yQRyQXAg0RsSEiDgMPA/M7lZkPPJBOPwpcKkkRsTwitqTL1wCVkiqOtyHTRw9xiJiZFUghg2QCsClnvpFjjyqOKRMRR4DdQOcf/vh1YHlEHMpZdl96Wusz6uayK0kLJdVLqt+3uynL+zAzsx4UMki6+oCPvpSRNJvkdNfv56y/ISLOAS5JHzd29eIRcXdE1EVEXW1tbZ8abmZmvVfIIGkEJuXMTwS2dFdGUhkwAmhK5ycCjwEfjYhXOjaIiM3p817gmySn0MzMrEgKGSRLgZmSpkkaBCwAFnUqswi4KZ2+Fng6IkJSNfA94I6I+FlHYUllkkan0+XAh4DVBXwPZmaWR8GCJO3zuJXkiqt1wLcjYo2kOyV9JC12DzBKUgPwp0DHJcK3AjOAz3S6zLcCWCxpJbAC2Ax8vVDvwczM8lNE526Lk09dXV3U1/f6amEzMwMkLYuIunzlPOStmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZllUtAgkTRP0npJDZJu72J9haRH0vVLJE1Nl18maZmkVenzB3K2OT9d3iDpK5JUyPdgZmY9K1iQSCoF7gKuBGYB10ua1anYzUBzRMwAvgh8IV2+E/hwRJwD3AQ8lLPN14CFwMz0Ma9Q78HMzPIr5BHJhUBDRGyIiMPAw8D8TmXmAw+k048Cl0pSRCyPiC3p8jVAZXr0Mh4YHhHPRkQADwJXF/A9mJlZHoUMkgnAppz5xnRZl2Ui4giwGxjVqcyvA8sj4lBavjFPnWZmdgKVFbDurvouoi9lJM0mOd11eR/q7Nh2IckpMCZPnpyvrWZmdpwKeUTSCEzKmZ8IbOmujKQyYATQlM5PBB4DPhoRr+SUn5inTgAi4u6IqIuIutra2oxvxczMulPIIFkKzJQ0TdIgYAGwqFOZRSSd6QDXAk9HREiqBr4H3BERP+soHBFbgb2SLkqv1voo8N0CvgczM8ujYEGS9nncCiwG1gHfjog1ku6U9JG02D3AKEkNwJ8CHZcI3wrMAD4jaUX6GJOu+0PgG0AD8ArwRKHeg5mZ5afk4qeTW11dXdTX1xe7GWZmA4qkZRFRl6+c72w3M7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZllkjdIJJVKGp0zP0jSQknrCts0MzMbCHoMEkkLgCZgpaQfS3o/sAG4ErjhBLTPzMz6ubI86/8XcH5ENEh6N/AssCAiHit808zMbCDId2rrcEQ0AETEC8CrDhEzM8uV74hkjKQ/zZkfmjsfEf9QmGaZmdlAkS9Ivg4M62HezMxOcT0GSUR8vrt1koa8880xM7OBpjeX/06QVCdpUDo/RtLfAC8XvHVmZtbv5bv89+PACuAfgeck3QSsAwYD5xe+eWZm1t/l6yNZCLwrIpokTQYagPdGxHOFb5qZmQ0E+U5tHYyIJoCIeB34RV9CRNI8SeslNUi6vYv1FZIeSdcvkTQ1XT5K0jOS9kn6aqdtfpTWuSJ9jOlte8zM7J2X74hkoqSv5MyPyZ2PiNu621BSKXAXcBnQCCyVtCgi1uYUuxlojogZ6V30XwCuAw4CnwHOTh+d3RAR9XnabmZmJ0C+IPlUp/llfaj7QqAhIjYASHoYmA/kBsl84HPp9KPAVyUpIvYD/y1pRh9ez8zMiiDf5b8PZKh7ArApZ74RmNtdmYg4Imk3MArYmafu+yS1Af8O/FVEROcCkhaS9PEwefLk43oDZmaWX49BImlRT+sj4iM9bd7VJsdRprMbImKzpGEkQXIj8GAXbbsbuBugrq4uX51mZnac8p3aupjkiOFbwBK6/uDvTiMwKWd+IrClmzKNksqAESSjDXcrIjanz3slfZPkFNrbgsTMzE6MfFdtjQP+J0mH95dJOs53RsSPI+LHebZdCsyUNC29mXEB0PkIZxFwUzp9LfB0V6epOkgq6/htFEnlwIeA1XnaYWZmBZSvj6QNeBJ4UlIFcD3wI0l3RsQ/5tn2iKRbgcVAKXBvRKyRdCdQHxGLgHuAhyQ1kByJLOjYXtJGYDgwSNLVwOXAa8DiNERKgR+QjP9lZmZFoh4OAJICSYBcRRIiU0mOIu7tOMU0ENTV1UV9va8WNjPrC0nLIqIuX7l8ne0PkJzWegL4fET4NJKZmR0jX2f7jcB+4AzgNuloX7uAiIjhBWybmZkNAPn6SPKODmxmZqc2B4WZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLpKBBImmepPWSGiTd3sX6CkmPpOuXSJqaLh8l6RlJ+yR9tdM250talW7zFUkq5HswM7OeFSxIJJUCdwFXArOA6yXN6lTsZqA5ImYAXwS+kC4/CHwG+GQXVX8NWAjMTB/z3vnWm5lZbxXyiORCoCEiNkTEYeBhYH6nMvOBB9LpR4FLJSki9kfEf5MEylGSxgPDI+LZiAjgQeDqAr4HMzPLo5BBMgHYlDPfmC7rskxEHAF2A6Py1NmYp04zMzuBChkkXfVdxHGUOa7ykhZKqpdUv2PHjh6qNDOzLAoZJI3ApJz5icCW7spIKgNGAE156pyYp04AIuLuiKiLiLra2to+Nt3MzHqrkEGyFJgpaZqkQcACYFGnMouAm9Lpa4Gn076PLkXEVmCvpIvSq7U+Cnz3nW+6mZn1VlmhKo6II5JuBRYDpcC9EbFG0p1AfUQsAu4BHpLUQHIksqBje0kbgeHAIElXA5dHxFrgD4H7gcHAE+nDzMyKRD0cAJw06urqor6+vtjNMDMbUCQti4i6fOV8Z7uZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wKGiSS5klaL6lB0u1drK+Q9Ei6fomkqTnr7kiXr5d0Rc7yjZJWSVohqb6Q7Tczs/zKClWxpFLgLuAyoBFYKmlRRKzNKXYz0BwRMyQtAL4AXCdpFrAAmA2cBvxA0hkR0ZZu9/6I2FmotpuZWe8V8ojkQqAhIjZExGHgYWB+pzLzgQfS6UeBSyUpXf5wRByKiFeBhrQ+MzPrZwoZJBOATTnzjemyLstExBFgNzAqz7YBPCVpmaSF3b24pIWS6iXV79ixI9MbMTOz7hUySNTFsuhlmZ62/eWIeDdwJXCLpPd29eIRcXdE1EVEXW1tbW/bbGZmfVTIIGkEJuXMTwS2dFdGUhkwAmjqaduI6HjeDjyGT3mZmRVVIYNkKTBT0jRJg0g6zxd1KrMIuCmdvhZ4OiIiXb4gvaprGjATeF7SEEnDACQNAS4HVhfwPZiZWR4Fu2orIo5IuhVYDJQC90bEGkl3AvURsQi4B3hIUgPJkciCdNs1kr4NrAWOALdERJukscBjSX88ZcA3I+LJQr0HMzPLT8kBwMmtrq4u6ut9y4mZWV9IWhYRdfnK+c52MzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLpKBBImmepPWSGiTd3sX6CkmPpOuXSJqas+6OdPl6SVf0tk4zMzuxChYkkkqBu4ArgVnA9ZJmdSp2M9AcETOALwJfSLedBSwAZgPzgH+SVNrLOs3M7AQq5BHJhUBDRGyIiMPAw8D8TmXmAw+k048Cl0pSuvzhiDgUEa8CDWl9vanTzMxOoEIGyQRgU858Y7qsyzIRcQTYDYzqYdve1GlmZidQWQHrVhfLopdlulveVfB1rjOpWFoILExn90la30073wmjgZ0FrL+QBnLbwe0vNre/uArd/im9KVTIIGkEJuXMTwS2dFOmUVIZMAJoyrNtvjoBiIi7gbuPt/F9Iak+IupOxGu90wZy28HtLza3v7j6S/sLeWprKTBT0jRJg0g6zxd1KrMIuCmdvhZ4OiIiXb4gvaprGjATeL6XdZqZ2QlUsCOSiDgi6VZgMVAK3BsRayTdCdRHxCLgHuAhSQ0kRyIL0m3XSPo2sBY4AtwSEW0AXdVZqPdgZmb5KTkAsCwkLUxPpQ04A7nt4PYXm9tfXP2l/Q4SMzPLxEOkmJlZJg6SPpA0SdIzktZJWiPpT9LlNZK+L+nl9Hlksdvak3SUgOWSHk/np6VD1LycDlkzqNht7I6kakmPSnop/Xe4eKDsf0mfSP/frJb0LUmV/X3fS7pX0nZJq3OWdbm/lfhKOnzRSknvLl7Lu23736X/d1ZKekxSdc66LodlKpau2p+z7pOSQtLodL6o+95B0jdHgD+LiLOAi4Bb0iFabgd+GBEzgR+m8/3ZnwDrcua/AHwxbX8zydA1/dWXgScj4kzgl0jeR7/f/5ImALcBdRFxNsnFIgvo//v+fpJhinJ1t7+vJLnCcibJPVxfO0Ft7M79vL3t3wfOjohzgV8Ad0D3wzKduKZ26X7e3n4kTQIuA17PWVzcfR8RfhznA/hu+g+6HhifLhsPrC9223po80SSP/4PAI+T3Py5EyhL118MLC52O7tp+3DgVdK+vZzl/X7/89aoDDUkV0s+DlwxEPY9MBVYnW9/A/8CXN9Vuf7S9k7rrgH+NZ2+A7gjZ91i4OL+tu/TZY+SfInaCIzuD/veRyTHSclIxecBS4CxEbEVIH0eU7yW5fUl4M+B9nR+FLArkiFqoH8POzMd2AHcl56a+4akIQyA/R8Rm4G/J/kWuZVkOKBlDJx9n6u7/T3QhjD6HeCJdHpAtF3SR4DNEfFip1VFbb+D5DhIGgr8O/DxiNhT7Pb0lqQPAdsjYlnu4i6K9tdL+cqAdwNfi4jzgP30w9NYXUn7EeYD04DTgCEkpyM666/7vjcGzP8lSZ8mOVX9rx2LuijWr9ouqQr4NPDZrlZ3seyEtd9B0keSyklC5F8j4j/SxdskjU/Xjwe2F6t9efwy8BFJG0lGTv4AyRFKtZIhaqCHYWf6gUagMSKWpPOPkgTLQNj/vwa8GhE7IqIV+A/gPQycfZ+ru/3dm2GRik7STcCHgBsiPQ/EwGj76SRfRF5M/4YnAi9IGkeR2+8g6QNJIrkbf11E/EPOqtyhXm4i6TvpdyLijoiYGBFTSToWn46IG4BnSIaogf7d/jeATZLelS66lGT0g4Gw/18HLpJUlf4/6mj7gNj3nXS3vxcBH02vILoI2N1xCqy/kDQP+AvgIxHRkrOqu2GZ+o2IWBURYyJiavo33Ai8O/27KO6+L3Zn0kB6AL9Ccri4EliRPj5I0s/wQ+Dl9Lmm2G3txXt5H/B4Oj2d5I+mAfg3oKLY7euh3XOA+vTf4DvAyIGy/4HPAy8Bq4GHgIr+vu+Bb5H06bSSfHDd3N3+Jjm9chfwCrCK5Aq1/tb2BpK+hI6/33/OKf/ptO3rgSv7477vtH4jb3W2F3Xf+852MzPLxKe2zMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkNhJSdL/kfQ+SVdL6tPd75Jq0xF5l0u6pNO6H0nq8TeyJX08vQv5uKTtfk836yok/UDSCknX9bHeqZJ+83jpISyNAAADkUlEQVTbZdYdB4mdrOaSjIP2q8BP+7jtpcBLEXFeRPR1W4CPA8cdJCT3+HQZJCTju5VHxJyIeKSP9U4F+hQkOXfdm3XLQWInlfT3JlYCFwDPAr8LfE3S28YnkjRF0g/T32/4oaTJkuYA/xf4YPqtf3APr/U1SfVKfmPk8+my20jG0npG0jPpssslPSvpBUn/lo7VhqSNkj6fLl8l6cx0MNA/AD6Rvv4lOa83Bvh/wJx03emSPitpqZLfOLk7vWseSTPSI5cX0/pPB/4WuCTd9hNKfg/lvvS1l0t6f7rtx9J2/ifwlKTxkn6Sbre681GaWdHvnPXDj3f6AVwI/CNQDvysh3L/CdyUTv8O8J10+mPAV7vZ5kekdw3z1h3dpenyc9P5jbx1x/Fo4CfAkHT+L4DP5pT743T6j4BvpNOfAz7Zzeu/j3REgtw2pNMPAR9Op5cA16TTlSRHSJ23/TPgvnT6TJJhXCrT99+Y8/7+DPh0znsdVux/Yz/618NHJHYyOo9k+IszScaz6s7FwDfT6YdIhsDpi9+Q9AKwnOQHkWZ1UeaidPnPJK0gGZtqSs76joE/l5Gceuqr96f9OatIBuGcLWkYMCEiHgOIiINx7LhSHX6F5H0TES8BrwFnpOu+HxFN6fRS4LclfQ44JyL2Hkc77STm85920khPS91PMvLpTpJv4Uo/wC+OiAN5quj1eEHpwH6fBC6IiGZJ95N8m39bUZIP5eu7qepQ+txGH/8eJVUC/0RyhLQp/aCvpOshxbusood1+zsmIuInkt4LXAU8JOnvIuLBvrTVTm4+IrGTRkSsiIg5JD+hOgt4Grgiko7prkLk5ySjIAPcAPx3H15uOMmH7W5JYzn2t0X2AsPS6eeAX5Y0A5LflJB0Bj3L3b4nHcG1M+13uRYgkt/IaZR0dfqaFelVZJ3r/QnJ+yZt02SSAQuPIWkKye/YfJ1k9Oui/ha79T8OEjupSKoFmiOiHTgzIno6tXUbySmblcCNJL9l3yuR/ELdcmANcC/ws5zVdwNPSHomInaQ9Dl8K32d50hOufXkP4FrOne2d9GGXcDXSUZ7/Q7JKagONwK3pa/5c2AcyYjJR9IO+E+QHM2UpqfFHgE+FhGHeLv3ASskLQd+HfhynvbbKcaj/5qZWSY+IjEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXy/wEPZfeUlgnphAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ax = psvd_rank_scores.plot(ylim=(0, None))\n", + "ax.set_xlabel('# of latent factors')\n", + "ax.set_ylabel(target_metric.upper());" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scaled PureSVD model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will employ a simple scaling trick over the rating matrix $R$ that was proposed by the authors of [EIGENREC model](https://arxiv.org/abs/1511.06033) [Nikolakopoulos2019]: $R \\rightarrow RD^{f-1},$\n", + "where $D$ is a diagonal scaling matrix with elements corresponding to the norm of the matrix columns (or square root of the number of nonzero elements in each column for the binary case). Parameter $f$ controls the effect of scaling and typically lies in the range [0, 1]. Finding the optimal value is an experimental task and will be performed via grid-search. We will use built-in support for such model in Polara." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Grid search" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.models import ScaledSVD\n", + "from polara.evaluation.pipelines import find_optimal_config # generic routine for grid-search" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we have to compute SVD model for every value of $f$. However, we can still avoid computing the model for each rank value by the virtue of rank truncation." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def fine_tune_scaledsvd(model, ranks, scale_params, target_metric, config=None):\n", + " rev_ranks = sorted(ranks, key=lambda x: -x) # descending order helps avoiding model recomputation\n", + " param_grid = [(s1, r) for s1 in scale_params for r in rev_ranks]\n", + " param_names = ('col_scaling', 'rank')\n", + " return find_optimal_config(model,\n", + " param_grid,\n", + " param_names,\n", + " target_metric,\n", + " init_config=config,\n", + " return_scores=True,\n", + " force_build=False, # avoid recomputing the model\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We already know an approximate range of values for the scaling factor. You may also want to play with other values, especially when working with a different dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "60/60\n", + "[04:17<00:03, 4.29s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 60/60 [04:17<00:03, 4.29s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ssvd = ScaledSVD(data_model) # create model\n", + "scaling = [0.2, 0.4, 0.6, 0.8]\n", + "\n", + "ssvd_best_config, ssvd_scores = fine_tune_scaledsvd(ssvd,\n", + " rank_grid,\n", + " scaling,\n", + " target_metric,\n", + " config=init_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that during this grid search the model was computed only `len(scaling)=4` number of times, other points were found via rank truncation. Let's see how quality changes with different values of scaling parameter $f$." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXd8VfXdx9/fO7MHGZAQICyVIajgQqtSFVctVmmlrtZRnlrtbrV9qtVa22rr83TpU2sfd4ciLtQ6akWfVlEBBwoOQMCEMLL3Te74Pn+ck+QmZEFyITHf9+t1Xue37/ece+/5nN8WVcUwDMMw9hbP/jbAMAzDGN6YkBiGYRgDwoTEMAzDGBAmJIZhGMaAMCExDMMwBoQJiWEYhjEgTEiMfYKIvCgil+1l3vEi0iAi3sG2a38gIltE5CTX/Z8i8r+DWPaNIlIhIjsGq0wj8YiIisiUfqQ7QURK94VNe4IJySDhPhya3QfeDhG5R0TS9rddw5H4By2Aqn6sqmmqGt2fdiUCVf25ql4GICLF7gPFtzdlicg44LvAdFUdMxj2ufY0ur/rbSLy34kSdBE5VkReEZFaEakSkZdF5HAROdq1Ib2bPG+KyJVx967BPXaKyJMicnIibDU6Y0IyuJypqmnAIcChwA/3sz3GyGICUKmqu/Y0Yx/iNdv9XZ8InAd8ZZDLR0QygCeB3wOjgLHAT4AWVV0JlALndMkzE5gO/C0uOMu1dTbwD+BREfnyntpr7BkmJAlAVXcAz+IICgAiEhSRW0TkY/dt6XYRSY6LXygib4lInYhsEpFT3fBCEVnuvqFtFJGvxOW5XkQeEpE/i0i9iLwjIgeIyA9FZJeIlIjIgrj0L7pNH6+4b21PiEiOiPzF/dxVIlIcl/4gEfmH+9kfiMgX4uLuEZHbROQp97NfE5HJcfEni8j77tvlrYDExU0WkRdEpNJthvmLiGS5cfcD44EnXBuv6vqm3o97slRE7nPtWicic+Pir3bfrOvdazqxu+/QvS/L3fvyuoj8VET+7cbtVnOQuKa73q6vm8+5XkT+7Hr/zz3XuNd+vHuNB8elzxen5pvXpZyTcB6chW7ee9zwz7r3oMa1cVpcni3u/VgLNPb1sFfV94F/ATPd/J2aY9zfxI2u+wQRKXXL3wHc7YZ/xv2d17i/w1lu9gPcz/ibqkZVtVlVn1PVtW78vcBFXUy6CHhKVSu7sXWHqv4WuB64WUS6fdaJyBEistr9nneKyH/3kK7teq5y/1vbReQsETldRD50v6f/jEsfFJHfiEiZe/xGRIJx8d93yygTkUu6fFavz4ohiaraMQgHsAU4yXUXAe8Av42L/w2wHOdtKx14AviFG3cEUAucjCPuY4GD3LiXgP8BknCEqRw40Y27HggBpwA+4D5gM/AjwI/z5rg5zoYXgY3AZCATWA98CJwUl/9uN20qUAJc7MYdBlQAM9z4e4Aq13Yf8BfgATcuF6gDFrl2fBuIAJe58VPcaw0CeTgP0N90dy9dfzGggG8P7snpgBf4BfCqG3ege02FceVO7uH7fABY6t6HmcA24N/d2RN3b/f4+lx7/9xLuf8D3Bzn/ybwRA82nwCUxvkPABpdW/zAVe73H4iz4y1gHJDcQ5kKTHHd04EdwKVd4+J+EzfG2RIBbnbvQzLOb2gXcKT73XzJtSEIZACVOIJxGpDdxY5xQBgY7/o9OLWUs3q6d274JDd8Wg/XtxK40HWnAUf1cm8jwI/p+G+VA3/F+T/PwPndTXLT3wC8CuS7v4FXgJ+6cacCO3F+V6luGfH3ubdnRafveKgc+92AT8rh/iEagHr3R/FPnGo2OG/jjcQ9tICjcR/ywB+BX3dT5jggCqTHhf0CuMd1Xw/8Iy7uTNcGr+tPd21ps+NF4Edx6f8LeLpL/rdc97nAv7rY80fgOtd9D/C/cXGnA++77otwH95x11+K+6Dt5jrPAt7sci+7FZJ+3pPn4+KmA82uewrOg+wkwN/Ld+nFeWgdFBf2c/opJHtyffQtJEfiiJ/H9a8GvtDD55xAZyG5Flga5/fgCOIJcXZc0sfvWnFeCqqBTcCNcbb0JSStQFJc/B9wH6ZxYR8Ax7vuaW4ZpTgP7eXA6Li0zwP/6bpPxnmx8fd079zwJDf8mB6u7/9wmtBy+7gPJwDN7P7fOjIuzRo6hG0TcHpc3CnAFtd9F3BTXNwBbfeSvp8Vnb7joXJY09bgcpaqpuN82QfhvJmD80aSAqxxq/Q1wDNuODgPx03dlFcIVKlqfVzYVpwaSxs749zNQIV2dEo3u+e0XtJ39belnQAc2Wava/P5QHwnbvzIoKa4vIU4Dz8A1PkHtPvd5pkH3CamOuDPdNyrvujPPelqV5KI+FR1I/AtnIf3LteGwm4+Iw9HtEriwrb2076BXl8nVPU1nAfL8SJyEM7DZnk/sxcSZ7eqxnCuKf5elXTN1A2HqWq2qk5W1WvccvpDuaqG4vwTgO92+U2Nc+1EVd9T1S+rahHO23ohztt5G/HNWxcCf1XVcB82tF1rlYicLx2d8U+74ZfiPMjfF6dp9zO9lFXZzX+rp/9Pp3vvugvj4nr6bfX1rBiSmJAkAFV9CefN6hY3qALnRzZDVbPcI1OdTkFwflSTdy+JMmCUdB6tMh7nrTLRlAAvxdmbpc7Iqcv7kXc7zgMCABGReD9ODUKBWaqaAVxAXB+KG9cTA7onqvpXVT0W56GmOE0vXSnHeSOOt3l8nLvRPafEhcULbF/X16N5PYTf65ZxIbCsy8O5N8pwrhPo9D3E36ve7nVfNNHzPeiu7BLgZ11+Uymq+je6ZnT6Y+7B7Y9xeQQYKyLzgbNxmmL74nM4tdAPVPUv7m84TVVPcz9ng6p+EacJ6mZgmYik9qPcvuh073F+P2Wuu9P/g86/rb6eFUMSE5LE8RvgZBE5xH2D+xPwaxHJBxCRsSJyipv2TuBiETlRRDxu3EGqWoLTtvoLEUlyOyYvxemPSDRPAgeIyIUi4nePw+M7a3vhKWCGiJztduB+g84PmXScJrgaERkLfL9L/p04bdu7MZB7IiIHisin3U7PEM4fdrchxe5b5yPA9SKSIiLTcdrz2+LLcR7GF4iI1+0sjX8R6Ov6eqIciLH7td+P80C8gP49PNtYCpzh/q78OEODW3Du32DwFnCeew9OBY7vI/2fgK+KyJHikCoiZ4hIujgDO74rIkXQPpT5izj9DACoaiOwDKfjfquqru7pg0RktIhcCVwH/LCnWpSIXCAieW58jRs8GMPM/wZcIyJ5IpKL07fSNqhiKfBlEZkuIimujUB7rbG3Z8WQxIQkQbgPm/tw2qkBrsbp6HzVbe54HqfzF1V9HadT+9c4ne4v0fE280Wc9t8y4FGcPop/7AP764EFwGL3s3fQ0XHaV94K4PPATTgdqFOBl+OS/ASn47UWR3Qe6VLEL3D+hDUi8r1uPmJv70nQtanCvZ584D97SHslTjPFDpw347u7xH8FRyAqcTpa4x/OfV1ft6hqE/Az4GX32o9yw0uBN3De8P/Vn7LcfB/giM/vca75TJwh6q39LaMPvumW2dbs+Vgf9qzGuW+34vS5bAS+7EbX4/QHvSYijTgC8i6O+MVzL85/oydBrXHzv4PTb/d5Vb2rF7NOBdaJSAPwW2DxHtT4euNGnP6sta4tb7hhqOrTOC+aL+Dcgxe65O3xWTFUEbcDxzCMXhBnLsJlbrPY/vj8u4AyVb1mf3y+YfTGXs2gNQxj3yHO3J6zcSa5GsaQw5q2DGMIIyI/xWni+ZWqbt7f9hhGd1jTlmEYhjEgrEZiGIZhDIgR0UeSm5urxcXF+9sMwzCMYcWaNWsqVLXPyZAjQkiKi4tZvbrHIeeGYRhGN4hIv1Z0sKYtwzAMY0CYkBiGYRgDYkQ0bRmGYSSKmMaobK6krLGMsoaOY2fTTgQh6AuS5E0iyZfUcXbd3ca54cneZCfeDfN7/DjLpQ09TEgMwzB6IRqLUt5c7ghEF7Eoayxje8N2WmOdV53JDGYyJmUMHvHQHGmmJdpCKBIiFA0RioTQvVgr0yMegt4gyb7k9nN2UjZ5yXnkJueSm5xLXkoeuUm55KbkkpecR2YwE0/3e3oNKiYkhmGMaCKxCDubdu4mEG3uHY07iGikU55RSaMYmzaWA7MP5NPjPk1BWgFj08ZSkFpAYVohqf6eFxBWVVpjrY6wREK0RFtojjQTioZoibS0i037Od4dl6Y50kxlcyXrK9dT3lxOc6R5t8/yiY+c5BxHZJLzyE3JbXfnJOd0EqGAN7DX99CExDCMIYeqUtdaR0O4gdZoa8cRc87hWJiWaEt7eCd/rJVwtMMfjoXbw1uiLYSj4XZ3eVM5O5t2EuuyOHB+cj6FaYUcnHcwp048lYJUVyjSCihILSDZt/c734oIQW+QoDdIZjBzoLeqnaZwE+XN5VQ0V1DeXE5lcyXlTY6/ormC7Y3bWVuxlupQdbc1osxgZnttpk1s+ktChcRdWvq3ODvO/a+q3tQlPoiziuccnFVUz1XVLSJyBHBHWzLgelV91M2zBWel0CgQUdW5GIYxbIhpjKpQFTubdrKzcWfnc9NOdjXtYmfjTkLRvV+E1yc+/F4/AW+AoCfY7g54AgS9jj/Vl8qE0RM61SbGpo1lTOqYAb2d7y9S/ClM8E9gQsaEXtNFYhGqQlXtAlPRXNFJcMqby3lr11tUNFf0+7MTtkSKiHhx9gM/GWfrzFXAF1V1fVyar+Fs/vNVEVkMfE5Vz3XX6G9V1YiIFABv4+yzHXGFZK67VHm/mDt3rto8EsNIPNFYlIrminZR6E4odjbtJBLr3FTk8/gYnTKa0SmjyU/Jbz9nBDMIeAKOCHgD+D1+gt5guzvgDezmD3gCeD3e/XQHPjmoKh6PZ01/XtYTWSM5Atioqh8BiMgDwEJgfVyahTjbnoKzYc2tIiLuvgxttO25bBjGfiYai1LWUMaWui1sqdvSPjqpTSwqmiuIaud9oYLeoCMSqaM5NP/Qdnf8eVTSqH3SKWz0nz0ZIZZIIRlL532JS3E2ruk2jVvbqAVygAoRORK4C2cTmwtV23u7FHhORBT4o6reQTeIyBJgCcD48eO7S2IYRjeoKtUt1Wyp3cLWuq1srtvM1tqtbKnbQkl9CeFYxzbpyb5kxqSOYXTKaI4qOKpdGNrCRqeMJjOYOWSHrRqDQyKFpLtfTteaRY9pVPU1nO1apwH3isjT7s5lx6hqmbsN5T9E5H1V/b/dCnEE5g5wmrYGciGG8UkkFAmxtW4rW+sckdhat5UttVvYXLeZ+tb69nQ+j4/x6eMpzijm+HHHMzFjIsWZxUzImEB2MNtEwkiokJTSeYP7IpytUbtLU+ru7Z0JVMUnUNX33K0zZwKrVbXMDd8lIo/iNKHtJiSGYTgd29sbt7O11q1ZuGKxpW4LOxp3dBq9k5+Sz8SMiZw+8XQmZEygOKOY4oxiCtIK8HlsgKfRM4n8dawCporIRGAbzt7f53VJsxz4ErASWAS8oKrq5ilxm7sm4OxXvEVEUgGPqta77gXADQm8BsMYsrR1bO9q2uWMdHJHPMX7tzdupyXa0p4n1Z9KcUYxh40+jAkZE5iYMZEJGc5InxR/yn68GmM4kzAhcUXgSuBZnOG/d6nqOhG5AadmsRy4E7hfRDbi1EQWu9mPBX4gImEgBnxNVStEZBLwqFuV9gF/VdVnEnUNhrG/aAw37i4MjZ39FaGK3eY/+MRHXkoe+Sn5TM2eyvFFx7c3Q03MnEhOUo41RRmDzojYIdGG/xpDCVVlZ9NONtVsoqS+pGPuRJxwNIYbd8uX7k8nPyW/09E2TDY/Nd9GPxmDjojs9+G/hpFQog0NtHzwAd6MDLyjRuHNzER8Q+cn3SYYG2s2sqlmU8dRu6mTUHjFS25yLqNTRjM5czJHFxy9u1Ck5FvTkzFkGTr/OsPoJ83vvEvN0gepffIptDlufSERR1Sysx1hyc7Gm52FL9t1j8rGFxfny85GUlIG3NSjquxo3MGm2k29CsaopFFMyZrCmZPOZErWFCZlTaI4o5hRSaNsAp0xrDEhMYYFscZGap98ipoHHyS0fj2SnEzGGaeT/ukT0VAzkapqotXVRKuriFRXE62qJlxSQmjtWiLV1RCJdFuuBAKdhMXbdrii48nIwJOUhASCSCBAlTZQ0rKDktadfNRcykdNJWxo3kqNNhL1ACLtgvHZyZ9lcuZkJmc5R3ZS9r69aYaxjzAhMYY0ofXrqX5wKXVPPEGsqYngAQcw+sfXknnmmXjT0/tVhqoSa2ggWlVFtLq6XWiiNdVEqqqIVte0x7VuKyVaVU2svr7H8rLdY1bXz/EInkAQT7AFCW5EgiVI8BU8gSC1SUnUBQN4AkEk2HYEEL8f8XjB6+l89ngQrwc83h7PeATxtqX19pzW50O8PsTvc9LFu30+xOe4xedz08a5fXHprJN++KEKkRCE6qClzj3Xdvhb6nuP6ycmJMaQI9bURN3TT1P94FJCa9ciwSAZp51G1rlfIPmQQ/b4gSYieNPTHeGZ0POCdi3RFl7b/horSlbw780vEqouJzUEud5MipPGMj5pDEWB0RT4cxjjzSZZfcRaWtCWVrSlhVhLqN2trS1OXKjFiWt10kXq6p04N1zDYTQWg2i089l1Dym83t0FJykJT0oKntRU59yXOzWlx7ih1L81ZGltguotUPWRc26udh747UJQB6HaOH89xK1E0COBNAhmQFKGc07JgeyJwBv9Msu+OWPIEPrgQ2oefJDa5cuJNTQQmDKZ0f/5n2Qu/CzezMFbbjue2pZa/q/0/1hRsoKXt71MU6SJZF8yx449lvlHzmde4TxyknMS8tn9oVuRiRMbjUbbRWf3NArRCBqNouFIJ7dGIxCJoJFoZ3ekLS7e3UO6SBhtaSXW1ESssZFYUxPhmpp2d6ypqXMfVh9IILCbwHjzcgmMLcJfVIS/aCz+sWMJFBXhSfkEDzxoaYDqzY5YtB+boXIT1Hed0y2dBSApA9ILIO/AuPB01525e9qgG99jH909/TLZhMTYr8RCIeqeeYaaBx6k+a23kECA9FNOIXvxuSQfdlhCmlNK6kt4seRFVpSs4I2dbxDVKHnJeZwx6Qzmj5vPEQVHEPQGB/1z9wbxeJymq/1tyF6i0Six5mZijU3Emho7znHio01NRBsbUVd82oWosZHWzVto/PfLaKjzkvLeUaPwFxURcMXF74pNoGgsvsJCPIEhvgx8qG53oaj6CKo2QcPOzmlT82DUJJh0vHNuO7KLITkbhkCTowmJsV9o2bSJ6gcfpPbx5cRqawkUF5N/9dVknrUQX/bgdkrHNMb6yvW88PELrChZwcaajQBMyZrCJTMvYf64+czInWHzLxKAeL1409LwpqXtdRmqSrSyknBpKa2l2whv20a4tJTwtlKa311H3T+eh3Bc840Ivvz8noVm9OjENqO19Uu0NEDtx3Ei8ZFTq6j6CJq67IKRNsYRhyknw6iJjjtnstO8lJSROFsHCZuQaOwzYi0t1D/3HNUPPkjz6jXg95Nx8slknXsuKUccPqi1j9ZoK6/veJ0VH6/gxZIX2dW8C494OCz/MOaPm8/8cfMZlzGu74KMIY9Go0R27eoQmtLSdrFp3baNyI4dzsO9DZ8P/+jR+MfkEZxQQNLkIpKKRxMsyEa0BcLNEG5yz/HuPQjrjoyxbm3CFYpRkztqFsG9F9pEYhMSjSFDy+bN1Cx9iNpHHyVaU4N//Hjyv/ddMj/3OXw5g9f/0Ft/xwnjTuC4sceRlZQ1aJ9nDA3E68VfUIC/oICUQw6Ghh1Q33FozTbCJR8TLiujdUcl4co6wjUbaN28mZq3fWjEqYmKRwlmhknK7jiCWRE8QT/4k8Gf0uWcDEkFu4f5UzqOzLEdYuHf++15hzomJEZCiLW0UP+P56l56CGaXnsNfD7STzyR7HO/QMpRRzlt/4PAtoZtrPh4BStKVrBm5xqiGiU3OZfTJ53O/HHzObLgyCHT32EMgHCzIwwNO6F+O9S7567+UM1uWcXjJ5A+hkDhaFIPnAbpY5wjbQzqT6O1vJHQ1p2ENpcR2lRK/YbN1HzkDn31eAhMmkjStOkkTZ9O0rRpJE2fhjdj6Dc37UtMSIxBpXndOmoffoTaJ58kVleHf+xY8r79bbLO/hy+vLy9KjMUCVFSX9Jp34y2oyrk7DowJWsKF8+8mPnj5jMzd6b1dww3YjGoLYHKjU4/QuVG56jb5ghINwKBx98hCjmTofhYSB/tjFpyhYL0AqdDuocXFwGC7tE2LlBViZSVEXrvPULr1xNa/x5Nr79O3RNPtOfzFxU5wjJ9WrvA7O3vu79oJOIMXGhqItbYhIaandFtGRl409MRvz+hn98b1kdiDJhoTY0z6/zhh2l57z1n5NWCBWSdczYpRx7Zr9pHNBalrLGsY6Ol2g7B2N64vdO+GbnJue37ZUzJmsJxRccxPsN2wRzyqEJTZYdItB+bnCNuuXsC6ZAzCbLGu4IQfxQ4Yb0IRCKIVFYSWu+Ky3vvEXpvPeGtH7fHe/Ny42ot00k66CDE620fCt3paOwaFjdkutv4JrSlpRfrQFJSnCWCMjLwZKTjzcjEm56OJzMDb3oG3swMR3Ta0rhh3vT0HpcK6m8fiQmJsVdoLEbTq69Ss+xh6p9/Hm1tJWn6dDIXnUPmGWd0O+9DValoruhUq2hzl9SXEIl1LGOS5k+jOKOYCZnOXhkT0ic47vQJpAWGZsek4dLaGFer2NRZNOJrFh6/0/GcM8WpUeRMcY+pkJY/JIa19kW0vp6W999vr7mE3nuPlk2b9mgyafukzq6HO3lTOoW3TepMxZMUJNbURLS2jmh9HbHaOqL19UTraju76+qJNTT0boTP5whMejqeTEeAvJkZFP3619bZbgw+4bIyah55lNpHHiFcVoYnM5OsL3yBrHPOJmnatPZ0LdEW3tj5Bm/seqN9v++tdVtpinSMaAl4AozPGM+kzEnMHzffEQ53k6VRSaNsSY6hTCzqzKzurnZRt61z2oyxjkDMPCdOLCZD1gTwDu9HkDc9nZTDDyfl8MPbw2KhEC0bNtDywQcg0kkcpKsgJCc7S9AkGI1EiNbXE6uvJ1pbR6y+jmidc8Tq6jrEqK6OaF090bo6wqWl/S7faiRGn8RaW2l4/nlqlj1M48qVAKQefTSZ55xN+kkn4QkGUVU+qv2Il7e9zCvbX2HNjjWEoiE84qEwtZAJmRM6CUVxRjGjU0bbqrdDndYmqNwAFRug/AOo+NA5KjdCtLUjXVKmU5PIndq5djFqEgRS95/9xoCw4b/GgAm9/z41yx6m7okniNbW4issIPdrXyPzc58jUDSWmlANz21fwSvbXuGVslfY2eTMyJ2YOZFzDjiHeYXzmDt6ru2jMRxorHAEovwDRzQqPoDyD50JdW2IxxnGmnsATDnJOedOdQQkZdSwaIoyEoMJidGJaF0dtU8+Se3DjxBatw7x+0k/+SQyzzmHwBFzeLdqPQ+UPcLKt1fybsW7KEp6IJ2jCo5iXuE85hXOozCtcH9fhtEdsZgjDOVuraLig46aRnNVRzpfMuROgXFHwKEXQN4BjmiMmgz+pP1nvzFkMSExnI7z11+n5uFHqH/uObSlheBBBzH6Rz+i4YRDeblpHS9vW8rrD32PhnADHvFwcO7BXD77cuaNnceMnBn4PPZT2m+ousuB1zqd2aFaaK5x3LWlHbWMyg3O0h1tpORA7oEw7Uxnkb9cVzAyx+3T0VDG8Mf+/SOY8I4d1D76KDWPPEq4pARPejqpZ32WLcdN5qWUj1m5/W98/MLNABSmFnLqxFOZVziPIwuOJCNgE7IGlWjYFYI4EQjVuO5uBKLd7ebRnkYJiTOENvcAZ9G/3KmOeOQeAKn7b1Vj45NFQoVERE4Ffgt4gf9V1Zu6xAeB+4A5QCVwrqpuEZEjgDvakgHXq+qj/SnT6B2NRGh46SVqlj5Ew7/+5SxFPmcmH35uJk8WVbCm5gkipRGSfckcPuZwzpt2HvMK51GcUWyjqAZCqLZjwb7KTc4qr5Wb3NnYtdDax/BMjx+SsyApyzmn5DhNTUmZHeHduVPzIWB9VEZiSZiQiIgXuA04GSgFVonIclVdH5fsUqBaVaeIyGLgZuBc4F1grqpGRKQAeFtEngC0H2Ua3dBaUkLNsoepfeQRIuXlePJy+OjMQ7ireCsfprwPvM80mcZFMy7imMJjOCT/EALeIb4U91CjpT5OJNwlwduEo+tqr20L+E08vnchSMp0/P5k68w2hiyJrJEcAWxU1Y8AROQBYCEQ/9BfCFzvupcBt4qIqGr88plJ0D6tuT9lGi5tw3arH3qIppWvgseDb94RvHreDG5PXUWzvsP8cfP58oSTOLrwaHKTc/e3yUOfloa4PSTiBKNyEzTu6pw2vcCpNRx0unPOmeycs4utlmB8okikkIwFSuL8pcCRPaVxax+1QA5QISJHAncBE4AL3fj+lAmAiCwBlgCMHz+yls9o2bTJWW338ced1XYLC/EuuYCHplSwrOZFAD4z+TNcPPNiJmVO2r/GDjXCzU5zU91251xb0rlJqmFH5/Rpox1xOGBBZ7EYNdHmTxgjhkQKSXf18K6zH3tMo6qvATNEZBpwr4g83c8ycfPfgdvPMnfu3E/8rMtYczN1zzxLzUMP0fzGG+2r7daccjh/DL7OC6UPEqwLsvigxVw0/SIK0gr2t8n7llgUGnY5W5XW74A691y/vbNwdLc4YGqeIw5TTuzYcKhNLILp+/5aDGOIkUghKQXidw4qArpuONyWplREfDgLcFbFJ1DV90SkEZjZzzJHFKH166l+6CHqnnjS2ee8uJi8732XTfPG81+lS3lt+02kB9L5yqyvcP608xmVNGp/mzy4qDoP/zYh6CQMOzqEo2EnaKxzXvE6NYqMgrjVY8dARqG7OGCh4x4GO9QZxv4kkUKyCpgqIhOBbcBi4LwuaZYDXwJWAouAF1RV3TwlbnPWBOBAYAtQ048yP/FEGxqoe/JJapY+RGj9eiQYJP2UBWQuWsRro+u48927eGfVO+Qm5/KdOd/h8wd8fvgvdBhpdfpG3RmMAAAgAElEQVQiyj9wj/edc9VHEGnePX1ytruceAHkz3DEok0c2sQiNQ9siRbDGDAJExJXBK4EnsUZqnuXqq4TkRuA1aq6HLgTuF9ENuLURBa72Y8FfiAiYSAGfE1VKwC6KzNR1zCUUFWa33qLmoeWUff002hzM8EDD2T0NdeQcsapPFf1Mne983M2vbeJorQirj3qWhZOWTj8NnUKN3fMti5/312q4wOnf6J9roQ7NyLvIJh0giMKGQUdwpE+5hO9G51hDDVs0cYhTqS6mrrly6lZtoyWDRuRlBQyzzidrM9/Hp02hcc2PsY96+5he+N2Dsg+gEtnXsqC4gVDf6Z5S72zVEe8WJS/D9Vbae/2Eq/TJ5F3oCMaeQc6R85UG/VkGPsAW7RxGBFtaCRc8jGtH5cQLi1xzm3+7dshGiVp1izG/PQGMk47ncZAlD+//yB/fuTrVIWqODT/UK456ho+NfZTQ2/SYFOVuxjg+3FNUh9CXdwS1d6AIw6Fh8HsL3YIx6hJ4BtmNSrD2AeoKqFwjJhbEYj/24s7Jqnro6DNL3FjljrC2vzSyd9fTEj2AapKZFe5Iw4lpe0i0VryMeGSUqJVncYX4M3Kwj9+PMmzZ5P52c+SfsoCkg48kIrmCn63/g6WfrCUhnADx449lssOvow5o+fspyvrhkgLfLwSNj4PG56H8vc64vwpztIcxcd0iEXugc68imG+L4Vh7A2RaIza5jC1zWFqmsPUNrnuplbHHx/mhtc2R6htbiUcHTqtSfbvHSRira2ES7ftXqMoLaG1pBQNxS2W5/HgLyjAP34c6SedhH9cEYFx4wmMH4d/3Di86Z2HlJbWl3LPqzfy6IZHiWiEBRMWcOnBl3LQqIP28VX2QPUW2PAP2PhP2Px/EG50lvSYMA9mfQFGz3SEwxYDNIYYsZgSiSmRWIxITIlGlXAsRjSmRKJOXDQWIxxVJyymRKJu2pgSjsbiwp1y2vKGYzHqQxFXGMLUNre2u2uawtQ1h6lvifRqX3rQR2aKn8xkP1kpfg4ck05mcoCsFD/pST58HiG+d6LN2RbWtkV1dz0Ybd0aHWl3z/vtm/t3H01IBkBr6TZ23ngjoQ8/ILJ9R6dvS5KTCRQV4R8/gdRjjsU/fpwjFuOK8BcWIoHulx9RVXY07mBz7WY2127mrV1v8dzW5xARFk5eyMUzL2ZCxoR9dYndE26GLS/Dxn84NY/KjU541gQ45IvOXhXFn4LgMB8pZgwbojGltjlMdVMr1Y2tVDc57pom193Y6sQ1hdvDapr2zVu93ytkJgfITPaRlRJgTEYSB45OJzPFT1ZceLtgJPvJSgmQkeTD592/L17f7mc6E5K9pPnttyn52hVoOEzaCccTGDfeqVmMH09g3Di8ubm99lc0R5qdPctrtziiUbeZLbVb2FK3hea44azpgXTOn3Y+F02/iNGpo/fFpe2OqiMWG593ah5bX3aWI/clOXMvDv+KIx45k209KGPAqCp1zRF21oeoaGihpl0UwlS5gtA1rC4U7vatG5wHeVZKgFEpzpv8pNw0slOdh3WSz4vPK/g8gtcj+L0evB7H7/N62sN383sFn8fj5mlL4+lUls/jIT3JR0rAO/T6LgcZE5K9oO6ZZym7+mp8+fmM++PtBCd1v8yIqrKraVe7SGyu3cyWOue8vXF7ezpBKEwrpDh9PHPGHMPEcJiJdeUU79pIbsV6ZGsprH4MMoucxf4yizqOjLHO8Fevf3AvsqXBaaba+Lxz1Gx1wnOmwpyLYepJMOEYG2Zr7BGhcJSddSF21rWwoy7ErroQO2pD7KxvYWdtiJ31IXbWhQiFY93mTwl4yU4JkJ3qJzslQFF2Ctkpjjs7xU92aqCTaGSnBkgdAQ/y/Y0JyR6gqlTdeSe7bvkvkg89lKLbbsU3ahShSMipXdR1FosttVtoinSsP5niS6E4s5jDRh9Gcdo4JuKjuLGGCRVbSNr+Dry7rGOuRHoBFB4K085ylhmvLXWOktegubqLZeLMnehOaDKLIKMIUnN7ry2owq73Opqrtq6EWBj8qc4+Fsd8w6l1ZBcP+n01hj+RaIyKhlZXJEKdxCLeX9sc3i1vkt/DmIwkRmckMbsoizGZSeSnBxmdkUROWoBRqQGyXWEI+mwC6VDE5pH0Ew2H2XHDDdQ8tIyM00+j4Be/YGXFan656pdsrt3c3qkFUJBawMTMiRRnFDMxcyIT0ydQHI2RX7EZ2f4WlL0BO96FaIuTITnbGfo69jBHPAoPcybY9URrI9RucxYUrNvmiozrry11wuJ3wgPwBiFzbIewZBY5fn+KW/P4p7OcCDgzwaec6AjH+KNsCO4wRVUJR5XWaIyWcJSWSIyWSIzWSIyWiOsPx2iNRmkJx9z4qBvvHuEoLdFYe3ynvJEYNU2t7Kh1mqBiXR4lXo+Qnx4kPyOJMRmOMHQcQcZkJJGfkURGks9qDEOU/s4jMSHpB9H6erZ985s0vrKSnK/+B6mXX8Z/v/Frln64lEmZkzi1+FRHODKLGZ82jpT6HVD2pnNsewO2v+2MZAIIpEPhIe7hikfWhMHtW1CFpsqOWkxdm8hs6/DXb+9YeyqYCZNPcIRj8omOwBhDltZIjIqGFsrrnWOXey5vCLWHlbvxPTUR7QlBn8c5/F4CXg9Bv4egz0vA5yEr2d9JFNpqFqMzg+SkBvF6TCCGMzYhcZBoLd1GyVf/g9YtWyn4+c/56JgJ/OjJz7OtYRtfmv4lvj71XILb34JNr0HZHxzxCNU6mX1JMOZgOPSCjtpGztTED4EVcZqyUnMdweqOaLhjd768gwa/j8XYI2IxpaY5HCcEcaLQSSyczufuyE7xk5ceJC89yJzx2eSlB8lI8pPkdx76jhi4IhAnCG3hTpi3XTgCPifMagtGX5iQ9EL8yKyCO/7AXYHXueeZ6yhMK+SuU+5ibjgGv5/jNFF5fJA/HWZ8rqN5Kn/a0H1Ae/3OelXDjEg0RmNLlIbWCE0tEULhGCLgEcHjcQYueMSZoetpCxdx0nicOb0e6SGNx3G3pWl7fjrNOh3NOZ2bhTo3G3VqGorLt3tzkuOvD0Uor2+hoqGl26GoSX4P+elJ5KUHmZyXxlGTctrFIi8t2O7OTQsS8NkcHWP/YELSA3XPPkfZVVfhy8sj/PvruLjkv9hYs5FFByzie3O/R2pM4fZjnWXIF93p1DyG4QimaExJZPNmVJWmliiNrRFHAFoiNLVGaGyJ0NASpbEl4sY58W3+9rj2eMffEhl4U82+xO+V9mag9iYin7e9BpCdEuDA0ekd4hAnEPkZSTbiyBgWmJB0IX5kVtIhs1lxxZH8/t2ryE7K5rYTb+O4ouOchE98y5nR/eWnYNwR+9Xm/lDV2MrGXQ0dR3kDm3Y1sK2mmyXY9wNej5Aa8JIa9HUcAS+jUlNIC/pIDXpJDXSOSw36SPJ7UVVi6nx3CsTi/DFVYjEnTNU9d00Tc9xtaZQOP0BSvBB0afppaxpKius3aBOLgM9jfQTGiMCEJA5nZNZPqXnoITwnHcePT6rlrY/u4tTiU/nRkT8iKynLSfjhs7Dmbjjmm866UUMEVaWsNtRJMDa5olHV2NqeLsnvYXJeGnOLszln1Fj8CZw9KwIpAZ8rBq4gBH2kumEpQS9pQR9Bn7XFG8ZwxYTExRmZ9S0aX3mFHeccw1UHrSYYSuZXx/2KUyee2pGwsQIev9JZP2r+j/aLreFojK2VTY5QlMeJRnkDTa3R9nRZKX6m5KWxYPpopuSnMTk/jSl5aYzNSsZjb8qGYQwSJiTEjczavIVnvjiZu4pf41MFn+In835CXkpeR0JVeOKbztauFz22T+ZXfLiznvVldZ2apLZWNnbqmC3ITGJKfhpfmDuOKflp7UdOasDe8g3DSDgjXkia166l5PKv0Rpq5L/OS2J9cTnXHX4d50w9Z/eH8Ft/hfefhAU3wugZCbUrHI1x45PruXelszSJ1yNMGJXC5Pw0Tp4+mil5ae21jLTgiP8aDcPYj4zoJ1Dds8+x7arvU5vm5brzwxRMn83Dx9xIUXrR7omrt8DTV8OEY+GoKxJqV2VDC1/7yxu8trmKS46ZyOIjxjEhJ8WWhzAMY0gyIoVEVam66y52/eoWNhX5uGWRl4s/9X0unH4hHumm4zkWhUcvd9yf+0NCJxS+u62W/7h/DRUNLfz63Nl87tBuRM0wDGMIMeKERMNhPr7+WpoefpyXpwkvXHggd376ZiZnTe4508pb4eNX4KzbEzqJ77E3t3H1w2vJSQ2w7KvzOLgoM2GfZRiGMViMKCGJ1tez7qtfxr9mPY/O85J15eXce8gS/J5eZp/veAf++VOY9lmYvTghdkWiMW5+5n3+9K/NHDFxFP9z/mHkptlCiYZhDA8SKiQicirwW8AL/K+q3tQlPgjcB8wBKoFzVXWLiJwM3AQEgFbg+6r6gpvnRaAAaJtJt0BVd/VlS92Wjay/9AJSt9fy0KJ8zv7WbczMndl7pnAIHvkPSBkFn/lNQjZtqm5s5et/e5N/b6zgS0dP4JrPTE/ovA7DMIzBJmFCIiJe4DbgZKAUWCUiy1V1fVyyS4FqVZ0iIouBm4FzgQrgTFUtE5GZwLNA/JK056tqv5fzba6v5r1FZ+ELR3n9qlO46oKbSPIl9Z1xxY2wax2cvwxSc/r7cf3mve11LLl/NTtrW/jlObP4wuHjBv0zDMMwEk0iX32PADaq6keq2go8ACzskmYhcK/rXgacKCKiqm+qqrs5BuuAJLf2slfox2W0+oE7fsGlX/5N/0Rk87/glVth7qUw9eS9/ege+fs72zn7f16hJRzjgf84ykTEMIxhSyKbtsYCJXH+UuDIntKoakREaoEcnBpJG+cAb6pqS1zY3SISBR4GbtRuVh0UkSXAEoCp6SnMeuTvZBb0s6M8VAuPXQ6jJsGCn/YvTz+JxpT//scH3LZiE4eNz+L2C+aQn9EPYTMMwxiiJFJIuutQ6PrA7zWNiMzAae5aEBd/vqpuE5F0HCG5EKefpXMhqncAdwDMnTNH+y0i4MwXqSuDS5+DQGr/8/VBbXOYbz/4Fi+8v4vFh4/jJwtn2NwQwzCGPYkUklIgvr2mCCjrIU2piPiATKAKQESKgEeBi1R1U1sGVd3mnutF5K84TWi7CUkn9qSTfP3j8Pbf4PiroajPjcH6zcZd9Sy5bw0fVzVx41kzOf/I8bZ8iWEYnwgS2UeyCpgqIhNFJAAsBpZ3SbMc+JLrXgS8oKoqIlnAU8APVfXltsQi4hORXNftBz4DvDtoFtfvcJaHLzwUjvv+oBX7j/U7Oeu2V6gLhfnrV47igqMmmIgYhvGJIWE1ErfP40qcEVde4C5VXSciNwCrVXU5cCdwv4hsxKmJtE3UuBKYAlwrIte6YQuARuBZV0S8wPPAnwbJYGdV33AznP2nQdnZMBZTfv/CRn79/IccPDaTP144h8Ks4bf5lWEYRm9IInfHGyrMnTtXV6/uY7Twqjvhqe/A6bfAEV8Z8Gc2tET4zoNv8dz6nZx96Fh+fvbBJPmtP8QwjOGDiKxR1T7b+EfUzPYeqdgIz10Dkz8Nh1824OK2VDTylftW81FFIz/+zHQuPqbYmrIMw/jEYkISjcCjS8AbgIW3DXj2+osf7OIbf3sTr0e475IjOGZK7iAZahiGMTQxIfnXf8G2NbDobsgo3OtiVJXbX/qIXz77PgeNyeCOC+cwblTKIBpqGIYxNBnZQrJtDbx0Mxz8BZh59l4X09Qa4fvL1vLU2u18ZlYBv1w0i5TAyL61hmGMHEbu0661yVmQMX0MnP6rvS6mpKqJJfev4f0ddVx96kF89fhJ1h9iGMaIYuQKyfPXQeUGuGg5JGftVRHPvLudHzzyDrGYcveXD+eEA/MH2UjDMIyhz8gUko3Pw+t3wFFfg0nH73H2ulCY65ev45E3tjFzbAa//+JhTMwdvKVUDMMwhhMjT0iaquCxKyDvIDjxx3uc/ZVNFXz/obXsqAvxjU9P4esnTrX9QwzDGNGMLCFRdSYdNlXC+UvB3/9Z5qFwlF89+wF3/nszE3NTWfbVozl0fHYCjTUMwxgejCwheWcZrHvUqYkUzO53tne31fLtB99iw64GLjxqAj88/SAblWUYhuEycp6GtaXw1Hdh3JFwzLf6lSUSjXH7S5v4zfMbyEkLcO8lR3D8AXkJNtQwDGN4MXKE5LHLQaPwuT+Cp+81rzZXNPKdpW/x5sc1nDm7kJ8unEFWSmAfGGoYhjG8GBlC0lgOmzfAmb+DURN7Taqq/Pm1j/n5U+/h9wq/XXwICw8Z22sewzCMkczIEJK6MjjgHDjsol6T7awLcdWytbz0YTmfmprLrxbNZkymbYNrGIbRGyNDSMQDn/1drwsyPrm2jGsee5dQOMoNC2dwoW0+ZRiG0S9GhpBkjYe07med1zaF+fHyd3n8rTJmj8viv78wm8l5afvYQMMwjOHLyBCSpMxug/+9oYLvPfQ25Q0tfPukA7hi/mR8NrnQMAxjjxgZQtKF5tYoNz/zPve8soXJeanccdE8ZhXt3XpbhmEYI50RJyRvl9Tw7aVv8VF5I1+eV8wPTjvItsA1DMMYACNGSMLRGLe+sJFbV2wkPz3Iny89kmOn2u6FhmEYAyWhHQIicqqIfCAiG0XkB93EB0XkQTf+NREpdsNPFpE1IvKOe/50XJ45bvhGEfmd9GNoVUskxqI/vMJv/7mBz84u5JlvHWciYhiGMUgkrEYiIl7gNuBkoBRYJSLLVXV9XLJLgWpVnSIii4GbgXOBCuBMVS0TkZnAs0DbrMA/AEuAV4G/A6cCT/dmy4Zd9Xirmvif8w/j9IMLBu8iDcMwjITWSI4ANqrqR6raCjwALOySZiFwr+teBpwoIqKqb6pqmRu+Dkhyay8FQIaqrlRVBe4DzurLkLSAj+e+dZyJiGEYRgJIpJCMBUri/KV01Cp2S6OqEaAWyOmS5hzgTVVtcdOX9lHmbhTnppKfYTPUDcMwEkEiO9u767vQPUkjIjNwmrsW7EGZbXmX4DSBMX78+L5sNQzDMPaSPmskIuIVkdw4f0BElojIe31kLQXGxfmLgLKe0oiID8gEqlx/EfAocJGqbopLX9RHmQCo6h2qOldV5+bl2dLvhmEYiaJXIXE7wKuAtSLykojMBz4CTgPO76PsVcBUEZkoIgFgMbC8S5rlwJdc9yLgBVVVEckCngJ+qKovtyVW1e1AvYgc5Y7Wugh4vD8XahiGYSSGvpq2rgHmqOpGETkMWAksVtVH+ypYVSMiciXOiCsvcJeqrhORG4DVqrocuBO4X0Q24gjWYjf7lcAU4FoRudYNW6Cqu4DLgXuAZJzRWr2O2DIMwzASiziDn3qIFHlDVQ+L87+vqgftE8sGkblz5+rq1av3txmGYRjDChFZo6pz+0rXV40kX0S+E+dPi/er6n/vrYGGYRjGJ4O+hORPQHovfsMwDGOE06uQqOpPeooTkdTBN8cwDMMYbvRn+O9YEZnrjrxCRPJF5OfAhoRbZxiGYQx5+hr++y3gLeD3wKsi8iXgPZwRU3MSb55hGIYx1Omrj2QJcKCqVonIeGAjcJyqvpp40wzDMIzhQF9NWyFVrQJQ1Y+BD01EDMMwjHj6qpEUicjv4vz58X5V/UZizDIMwzCGC30Jyfe7+NckyhDDMAxjeNLX8N97e4s3DMMwjF6FRES6LrLYCVX97OCaYxiGYQw3+mraOhpn46m/Aa/R/X4ghmEYxgimLyEZg7Pn+heB83CWdv+bqq5LtGGGYRjG8KDX4b+qGlXVZ1T1S8BROPNIXhSRr+8T6wzDMIwhT59b7YpIEDgDp1ZSDPwOeCSxZhmGYRjDhb462+8FZuJsHvUTVX13n1hlGIZhDBv6qpFcCDQCBwDfcHa3BZxOd1XVjATaZhiGYQwD+ppH0ufqwIZhGMbIxoTCMAzDGBAmJIZhGMaAMCExDMMwBkRChUREThWRD0Rko4j8oJv4oIg86Ma/JiLFbniOiKwQkQYRubVLnhfdMt9yj/xEXoNhGIbRO33OI9lbRMQL3IYzM74UWCUiy1V1fVyyS4FqVZ0iIouBm4FzgRBwLc7Q45ndFH++qq5OlO2GYRhG/0lkjeQIYKOqfqSqrcADwMIuaRYCbSsMLwNOFBFR1UZV/TeOoBiGYRhDmEQKyVicBR/bKHXDuk2jqhGgFsjpR9l3u81a10rc5JZ4RGSJiKwWkdXl5eV7br1hGIbRLxIpJN094HUv0nTlfFU9GPiUe1zYXSJVvUNV56rq3Ly8vD6NNQzDMPaORApJKTAuzl8ElPWURkR8QCZQ1VuhqrrNPdcDf8VpQjMMwzD2E4kUklXAVBGZKCIBYDHQdaOs5cCXXPci4AVV7bFGIiI+Ecl13X7gM4Ct/2UYhrEfSdioLVWNiMiVwLOAF7hLVdeJyA3AalVdDtwJ3C8iG3FqIovb8ovIFiADCIjIWcACYCvwrCsiXuB54E+JugbDMAyjb6SXCsAnhrlz5+rq1TZa2DAMY08QkTWqOrevdDaz3TAMwxgQJiSGYRjGgDAhMQzDMAaECYlhGIYxIExIDMMwjAFhQmIYhmEMCBMSwzAMY0CYkBiGYRgDwoTEMAzDGBAmJIZhGMaAMCExDMMwBkTCFm0c6oTDYUpLSwmFbBPG4UhSUhJFRUX4/f79bYphjHhGrJCUlpaSnp5OcXExPWyyaAxRVJXKykpKS0uZOHHi/jbHMEY8I7ZpKxQKkZOTYyIyDBERcnJyrDZpGEOEESskgInIMMa+O8MYOoxoITEMwzAGjgmJYRiGMSBMSIYJ119/PbfccktCP+OEE06gbSfJ008/nZqamkH/jDVr1nDwwQczZcoUvvGNb9DdDp1/+ctfmDVrFrNmzWLevHm8/fbbg26HYRiDhwmJ0S1///vfycrKGvRyL7/8cu644w42bNjAhg0beOaZZ3ZLM3HiRF566SXWrl3Ltddey5IlSwbdDsMwBo8RO/w3np88sY71ZXWDWub0wgyuO3NGr2nuu+8+brnlFkSEWbNmcf/997N161YuueQSysvLycvL4+6772b8+PF9ft7vfvc7br/9dnw+H9OnT+eBBx6goaGBr3/966xevRoR4brrruOcc87h8ssvZ9WqVTQ3N7No0SJ+8pOf7FZecXExq1evpqGhgdNOO41jjz2WV155hbFjx/L444+TnJzMqlWruPTSS0lNTeXYY4/l6aef5t133+3Rxu3bt1NXV8fRRx8NwEUXXcRjjz3Gaaed1indvHnz2t1HHXUUpaWlfV6/YRj7D6uR7CfWrVvHz372M1544QXefvttfvvb3wJw5ZVXctFFF7F27VrOP/98vvGNb/SrvJtuuok333yTtWvXcvvttwPw05/+lMzMTN555x3Wrl3Lpz/9aQB+9rOfsXr1atauXdv+5t8bGzZs4IorrmDdunVkZWXx8MMPA3DxxRdz++23s3LlSrxeb3v6srIyTj/99N3K2bZtG0VFRe3+oqIitm3b1utn33nnnbsJjWEYQ4uE1khE5FTgt4AX+F9VvalLfBC4D5gDVALnquoWEckBlgGHA/eo6pVxeeYA9wDJwN+Bb2p3De17QF81h0TwwgsvsGjRInJzcwEYNWoUACtXruSRRx4B4MILL+Sqq67qV3mzZs3i/PPP56yzzuKss84C4Pnnn+eBBx5oT5OdnQ3A0qVLueOOO4hEImzfvp3169cza9asHsueOHEihxxyCABz5sxhy5Yt1NTUUF9f3157OO+883jyyScBKCws5O9///tu5XT3NfU2jHfFihXceeed/Pvf/+7r8g3D2I8krEYiIl7gNuA0YDrwRRGZ3iXZpUC1qk4Bfg3c7IaHgGuB73VT9B+AJcBU9zh18K1PPKrar7kQ/Z0v8dRTT3HFFVewZs0a5syZQyQS6fYzNm/ezC233MI///lP1q5dyxlnnNHnxL5gMNju9nq97WXvKUVFRZ2aqUpLSyksLOw27dq1a7nssst4/PHHycnJ2ePPMgxj35HIpq0jgI2q+pGqtgIPAAu7pFkI3Ou6lwEnioioaqOq/htHUNoRkQIgQ1VXurWQ+4CzEngNCePEE09k6dKlVFZWAlBVVQU4/QNttYi//OUvHHvssX2WFYvFKCkpYf78+fzyl7+kpqaGhoYGFixYwK233tqerrq6mrq6OlJTU8nMzGTnzp08/fTTe2V/dnY26enpvPrqqwCdaj49UVBQ0J5HVbnvvvtYuLDrTwI+/vhjzj77bO6//34OOOCAvbLPMIx9RyKFZCxQEucvdcO6TaOqEaAW6O31c6xbTm9lAiAiS0RktYisLi8v30PTE8+MGTP40Y9+xPHHH8/s2bP5zne+Azid5nfffXd753tb30lvRKNRLrjgAg4++GAOPfRQvv3tb5OVlcU111xDdXU1M2fOZPbs2axYsYLZs2dz6KGHMmPGDC655BKOOeaYvb6GO++8kyVLlnD00UejqmRmZgI995EA/OEPf+Cyyy5jypQpTJ48ub3/4/bbb2/v27nhhhuorKzka1/7Gocccghz587daxsNw0g8MsDuhZ4LFvk8cIqqXub6LwSOUNWvx6VZ56Ypdf2b3DSVrv/LwNy2PhIRORz4haqe5Po/BVylqmf2ZsvcuXO1bX5EG++99x7Tpk0blGsdqTQ0NJCWlgY4nf3bt2/vl/ANFvYdGkZiEZE1qtrnm1wiO9tLgXFx/iKgrIc0pSLiAzKBqj7KLIrzd1emsY946qmn+MUvfkEkEmHChAncc889+9skwzD2A4kUklXAVBGZCGwDFgPndUmzHPgSsBJYBLzQ2wgsVd0uIvUichTwGnAR8PtEGD9UueKKK3j55Zc7hX3zm9/k4osv3ue2nHvuuZx77rn7/HMNwxhaJExIVDUiIlcCz+IM/71LVdeJyA3AalVdDtwJ3C8iG3Xs2isAAA/PSURBVHFqIovb8ovIFiADCIjIWcACVV0PXE7H8N+n3WPEcNttt+1vEwzDMDqR0Hkkqvp3nLke8WE/jnOHgM/3kLe4h/DVwMzBs9IwDMMYCDaz3TAMwxgQJiSGYRjGgDAhMQzDMAaECckwYSTtR9LGqlWr8Hq9LFu2bNDtMAxj8DAhMbplf+5HAs5s/auvvppTTjll0G0wDGNwsf1IAJ7+Aex4Z3DLHHMwnHZTr0lsP5Lu9yMB/r+9uw+uqs7vOP7+Dgjp0u4KoitrREJFIJHL41BwO0I3rRtETYto8RFZ0ao4YmdXkXVc1mXcMeNO3dXCtrgCFZ/a4tpGVOiK66DYxYAIAvJUExXYlfAgDytaQ7/945xkr+He5Iabk3MufF4zmdxzzu+efO8vufnmPOT75dFHH+Xyyy+npqam1dcuIvHSEUlM1I8kez+SnTt38vzzz3PLLbfk9NpFJF46IoFWjxyioH4kgUxl8u+8806qqqq+lJxEJLmUSGISRT+SFStWUF1dzezZs9m4cWOL/Uhqamro3r07N9xwQ5v7kRw5ciTSfiSrV69m0qSgyMGePXt46aWX6Ny5c1OCFJFk0amtmKgfSfZ+JLW1tdTV1VFXV8fEiROZO3eukohIgimRxET9SLL3IxGRwhJZP5IkUT+SaKgficiJLQn9SOQEp34kIgJKJAVH/UhEJGmUSAqM+pGISNLoYruIiORFiURERPKiRCIiInlRIhERkbwokRSIk60fyWuvvcaQIUMoKytjzJgx7R6HiLSfSO/aMrMK4GdAJ+AX7v5gs+1dgSeA4cBe4G/dvS7cNhO4ETgK3OHuy8L1dcChcH1DLv8s05qqt6rYvG9zvrv5kgE9BjBj5Ix23WdHylR0sT009iMZNWoUF198MUuXLj2mjPwnn3zCbbfdxtKlS+nduze7d++OJBYRaR+RHZGYWSdgDjAOKAWuMrPSZsNuBPa7+7nAw0BV+NxSYBJQBlQAc8P9NfoLdx/SHkkkTk888QSpVIrBgwdz3XXXAfDBBx9QXl5OKpWivLycDz/8MKd9PfLII5SWlpJKpZoKHh4+fJgpU6YwaNAgUqlUU/n3W2+9lREjRlBWVsasWbMy7q9Pnz7s2bOHuro6Bg4cyE033URZWRkXXXQRR44cAYIOhqlUitGjR3PXXXdx/vnntxhjej8SM2vqR9Lc008/zYQJE5r6sJxxxhk5zYGIxCPKI5KRwHZ3fx/AzJ4FKoFNaWMqgR+GjxcD/2hBudpK4Fl3/xyoNbPt4f7+O4pA4zhyaOxHsnLlSnr27NlUtLGxH8nkyZOZP38+d9xxR8Zfts09+OCD1NbW0rVr16ZTUun9SCAo2ghBP5IePXpw9OhRysvLWb9+fYtl5Ldt28YzzzzDY489xpVXXslzzz3Htddey5QpU5g3bx4XXHAB99xzT9P4Xbt2MXXq1GOOanLtR7J161a++OILxo4dy6FDh5g+fTrXX399q3MgIvGI8hrJWcBHacs7wnUZx7h7A3AAOK2V5zrwX2a2xsxuzvbFzexmM1ttZqvr6+vzeiFRaKkfydVXXw0E/UjeeOONnPbX2I/kySefpHPn4O+DV155hWnTpjWNSe9HMmzYMIYOHcrGjRvZtGlTxn02yrUfSaN8+5E0NDSwZs0aXnzxRZYtW8bs2bPZunVra1MgIjGJMpFkaqTR/DdJtjEtPfeb7j6M4JTZNDO7MNMXd/d57j7C3UecfvrpucbcYaLoRzJt2jTWrFnD8OHDaWhoaLEfyfLly1m/fj3jx49vcz+Sxn23Va79SIqLi6moqKBbt2707NmTCy+8kHXr1rX564lIx4gykewAzk5bLgZ2ZRtjZp2BrwH7Wnquuzd+3g08T3DKq+CoH0n2fiSVlZW8/vrrNDQ08Omnn7Jq1SpV+RVJsCgTSQ3Qz8xKzKwLwcXz6mZjqoHJ4eOJwKse/KlbDUwys65mVgL0A94ys25m9icAZtYNuAjYEOFriIz6kWTvRzJw4EAqKipIpVKMHDmSqVOntnohX0TiE2k/EjO7GPgpwe2/8939ATP7EbDa3avNrAhYBAwlOBKZlHZx/l7gO0ADcKe7v2xmfQmOQiC4UeBpd3+gtTjUjyQa6kcicmJLRD8Sd38JeKnZuh+kPf4MuCLLcx8AHmi27n1gcPtHKsdD/UhEBFRGvuCoH4mIJI0SSYFRPxIRSRrV2hIRkbwokYiISF6USEREJC9KJCIikhclkgJxMvUjOXDgAJdeeimDBw+mrKyMBQsWtHscItJ+dNcW8Lsf/5jP32vffiRdBw7gzO9/v1332ZHi7EcyZ84cSktLeeGFF6ivr6d///5cc801dOnSJZKYRCQ/OiKJkfqRZO5HYmYcOnQId+fw4cP06NGjqaKxiCSQu5/wH8OHD/fmNm3adMy6jrRhwwY/77zzvL6+3t3d9+7d6+7ul1xyiS9cuNDd3R9//HGvrKx0d/dZs2b5Qw89lHV/vXr18s8++8zd3ffv3+/u7nfffbdPnz69acy+ffu+9LUaGhp8zJgxvm7dOnd3HzNmjNfU1Li7+znnnOP19fVeW1vrnTp18rVr17q7+xVXXOGLFi1yd/eysjJfuXKlu7vPmDHDy8rK3N19586dPm7cuGNirKmp8fLy8qblFStW+Pjx448Zd/DgQR87dqyfeeaZ3q1bN1+yZEnG1xz391DkREdQzqrV37E6IomJ+pEEMpXJX7ZsGUOGDGHXrl2888473H777Rw8eLC1KRCRmCiRxMTVjyRrP5IFCxYwYcIEzIxzzz2XkpISNm9u32tYItJ+lEhion4k2fuR9O7dm+XLlwPw8ccfs2XLFvr27XtccYpI9JRIYqJ+JNn7kdx33328+eabDBo0iPLycqqqqppOAYpI8kTajyQp1I8kGupHInJiS0Q/EjmxqR+JiIASScFRPxIRSZqTOpHkeudUkqgfSeBkOCUrUihO2ovtRUVF7N27V7+QCpC7s3fvXoqKiuIORUQ4iY9IGv+nob6+Pu5Q5DgUFRVRXFwcdxgiwkmcSE455RRKSkriDkNEpOBFemrLzCrMbIuZbTezezJs72pm/xpuX2VmfdK2zQzXbzGzb+e6TxER6ViRJRIz6wTMAcYBpcBVZlbabNiNwH53Pxd4GKgKn1sKTALKgApgrpl1ynGfIiLSgaI8IhkJbHf39939f4Fngeb1MCqBfwkfLwbKLbiNqhJ41t0/d/daYHu4v1z2KSIiHSjKayRnAR+lLe8A/izbGHdvMLMDwGnh+t80e+5Z4ePW9gmAmd0M3BwuHjazLcfxGnLVE9gT4f6jVMixg+KPm+KPV9Txn5PLoCgTSaZ/0Gh+r222MdnWZzqCynj/rrvPA+a1FGB7MbPVuZQRSKJCjh0Uf9wUf7ySEn+Up7Z2AGenLRcDu7KNMbPOwNeAfS08N5d9iohIB4oykdQA/cysxMy6EFw8r242phqYHD6eCLwaduWqBiaFd3WVAP2At3Lcp4iIdKDITm2F1zxuB5YBnYD57r7RzH5E0L6xGngcWGRm2wmORCaFz91oZv8GbAIagGnufhQg0z6jeg1t0CGn0CJSyLGD4o+b4o9XIuI/KcrIi4hIdE7aWlsiItI+lEhERCQvSiRtYGZnm9mvzew9M9toZtPD9T3M7Fdmti383D3uWFsSVglYa2ZLwuWSsETNtrBkTZe4Y8zGzE41s8Vmtjn8PowulPk3s78Pf242mNkzZlaU9Lk3s/lmttvMNqStyzjfFngkLF+03syGxRd51tgfCn921pvZ82Z2atq2jGWZ4pIp/rRt3zMzN7Oe4XKsc69E0jYNwHfdfSAwCpgWlmi5B1ju7v2A5eFykk0H3ktbrgIeDuPfT1C6Jql+Bix19wHAYILXkfj5N7OzgDuAEe5+PsHNIpNI/twvJChTlC7bfI8juMOyH8E/A/+8g2LMZiHHxv4r4Hx3TwFbgZmQvSxTx4Wa0UKOjR8zOxv4K+DDtNXxzr276+M4P4D/DL+hW4Be4bpewJa4Y2sh5mKCN/+3gCUE//y5B+gcbh8NLIs7ziyxfxWoJbxJJG194uefP1Rx6EFwt+QS4NuFMPdAH2BDa/MN/DNwVaZxSYm92ba/AZ4KH88EZqZtWwaMTtrch+sWE/wRVQf0TMLc64jkOIWViocCq4Cvu/tvAcLPZ8QXWat+CtwN/F+4fBrwibs3hMvp5WiSpi9QDywIT839wsy6UQDz7+47gZ8Q/BX5W+AAsIbCmft02eY7U1mkJL+e7wAvh48LInYzuwzY6e7rmm2KNX4lkuNgZn8MPAfc6e4H444nV2Z2CbDb3dekr84wNKn3hHcGhgE/d/ehwO9J4GmsTMLrCJVACfANoBvB6Yjmkjr3uSiYnyUzu5fgVPVTjasyDEtU7Gb2FeBe4AeZNmdY12HxK5G0kZmdQpBEnnL3X4arPzazXuH2XsDuuOJrxTeBy8ysjqBy8rcIjlBODUvUQLLLzuwAdrj7qnB5MUFiKYT5/0ug1t3r3f0L4JfABRTO3KfLNt8FUcLIzCYDlwDXeHgeiMKI/U8J/hBZF76Hi4G3zexMYo5fiaQNzMwI/hv/PXf/h7RN6aVeJhNcO0kcd5/p7sXu3ofgwuKr7n4N8GuCEjWQ7Ph/B3xkZv3DVeUE1Q8KYf4/BEaZ2VfCn6PG2Ati7pvJNt/VwPXhHUSjgAONp8CSwswqgBnAZe7+adqmbGWZEsPd33X3M9y9T/ge3gEMC98X8c593BeTCukD+HOCw8X1wDvhx8UE1xmWA9vCzz3ijjWH1zIWWBI+7kvwptkO/DvQNe74Woh7CLA6/B78B9C9UOYfuB/YDGwAFgFdkz73wDME13S+IPjFdWO2+SY4vTIH+B/gXYI71JIW+3aCawmN799/Sht/bxj7FmBcEue+2fY6/nCxPda5V4kUERHJi05tiYhIXpRIREQkL0okIiKSFyUSERHJixKJiIjkRYlEJIHM7Idm9r244xDJhRKJSMTCfxLTe01OWPrhFomAmfUJ+6XMBd4GHjez1WE/kvvTxtWZ2f1m9raZvWtmAzLs6yYze9nM/qgjX4NIrpRIRKLTH3jCgwKT33X3EUAKGGNmqbRxe9x9GEEPiS+dzjKz24FLgb929yMdFLdImyiRiETnA3f/Tfj4SjN7G1hL0DypNG1cY/HPNQT9JxpdR1Ah+HJ3/zziWEWOmxKJSHR+D0ErY4IjjXIPOvO9CBSljWtMEkcJSuU32kCQWIojj1QkD0okItH7KkFSOWBmXydzH5JM1gJ/B1Sb2TeiCk4kX0okIhHzoJvdWmAjMB9Y2YbnvkFwNPOimfWMJkKR/Kj6r4iI5EVHJCIikhclEhERyYsSiYiI5EWJRERE8qJEIiIieVEiERGRvCiRiIhIXv4fpMZzyi2PXncAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "for cs in scaling:\n", + " cs_scores = ssvd_scores.xs(cs, level='col_scaling')\n", + " ax = cs_scores.plot(label=f'col_scaling: {cs}')\n", + "ax.set_title(f'Recommendations quality for {ssvd.method} model')\n", + "ax.set_ylim(0, None)\n", + "ax.set_ylabel(target_metric.upper())\n", + "ax.legend();" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "ssvd_rank_scores = ssvd_scores.xs(ssvd_best_config['col_scaling'], level='col_scaling')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The optimal set of hyper-parameters:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'col_scaling': 0.6, 'rank': 130}" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ssvd_best_config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## iALS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using *implicit* library in Polara is almost as simple as using SVD-based models. Make sure you have it installed in your python environment (follow instructions at https://github.com/benfred/implicit )." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "import os; os.environ[\"MKL_NUM_THREADS\"] = \"1\" # as required by implicit\n", + "import numpy as np\n", + "\n", + "from polara.recommender.external.implicit.ialswrapper import ImplicitALS\n", + "from polara.evaluation.pipelines import random_grid, set_config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### defining hyper-parameter grid" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hyper-parameter space in that case is much broader. We will start by adjusting all hyper-parameters expect the rank value and then, once an optimal config is found, we will perform full grid-search over the range of rank values defined by `rank_grid`." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "als_params = dict(alpha = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100],\n", + " epsilon = [0.01, 0.03, 0.1, 0.3, 1],\n", + " weight_func = [None, np.sign, np.sqrt, np.log2, np.log10],\n", + " regularization = [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],\n", + " rank = [40] # enforce rank value for quick exploration of other parameters\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to avoid too long computation time, grid-search is performed over 60 random points, which is enough to get within 5% of the optimum with 95% confidence [Bergstra]. The grid is generated with the built-in `random_grid` function." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "als_param_grid, als_param_names = random_grid(als_params, n=60)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### random grid search " + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "ials = ImplicitALS(data_model) # create model" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "60/60\n", + "[24:14<00:24, 24.23s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 60/60 [24:14<00:24, 24.23s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ials_best_config, ials_grid_scores = find_optimal_config(ials,\n", + " als_param_grid, # hyper-parameters grid\n", + " als_param_names, # hyper-parameters' names\n", + " target_metric,\n", + " init_config=init_config,\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### rank tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In contrast to the case of SVD-based algorithms, iALS requires recomputing the model for every new rank value, therefore in addition to the previous 60 times, the model will be computed `len(rank_grid)` more times for all rank values." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "15/15\n", + "[06:23<00:32, 25.56s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 15/15 [06:23<00:32, 25.56s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ials_best_rank, ials_rank_scores = find_optimal_config(ials,\n", + " rank_grid,\n", + " 'rank',\n", + " target_metric,\n", + " # configs are applied in the order they're provided\n", + " init_config=[init_config,\n", + " ials_best_config],\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's combine the best rank value with other optimal parameters:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'alpha': 1.0,\n", + " 'epsilon': 0.03,\n", + " 'weight_func': ,\n", + " 'regularization': 0.003,\n", + " 'rank': 60}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ials_best_config.update(ials_best_rank)\n", + "ials_best_config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### visualizing rank tuning results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now see how all three algorithms compare to each other." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_rank_scores(scores):\n", + " ax = None\n", + " for sc in scores:\n", + " ax = sc.sort_index().plot(label=sc.name, ax=ax)\n", + " ax.set_ylim(0, None)\n", + " ax.set_title('Recommendations quality')\n", + " ax.set_xlabel('# of latent factors')\n", + " ax.set_ylabel(target_metric.upper());\n", + " ax.legend()\n", + " return ax" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xl8lfWd9//X55yT5GQHsgEJS0D2RZYoioJaC9qWaq12IDjqtFrbDrTqtDOj992qxWld6m9sufEe67SOrdNKHTtOqf2hValKFZUg+yqbkABZyb6ecz73H9eVcBISEggnC/k8H4/zuLbvdZ3vdRHO+3yv6zrfS1QVY4wx5lx5ersCxhhj+jcLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbrEgMcYY0y0WJMb0ABF5W0TuOsd1R4pItYh4z3e9eoOIHBaRz7rj/0tEftHbdTLdY0Fizgv3w6HO/cA7ISLPi0hCb9erPwr/oAVQ1SOqmqCqwd6sVySo6o9V9S4AERktIioivt6ulzk7FiTmfPqiqiYAM4CZwAO9XB9jTA+wIDHnnaqeAF7HCRQARCRGRJ4UkSMiUigiz4hIbNjyG0Vki4hUisgBEbnenT9cRNaISJmI7BeRr4et87CI/JeI/KeIVInIdhEZLyIPiEiRiBwVkYVh5d8WkX8RkffdltMfRSRFRH7jvu9GERkdVn6iiLzhvvdeEfmbsGXPi8jTIvIn970/FJGxYcsXiMgeEakQkVWAhC0bKyLrRKRURErc9x/kLnsBGAn80a3jP7X9pt6FY/KSiPzarddOEckJW/7PIlLgLtsrIte292/oHpc17nH5SEQeEZG/ustOazmEn7o70/618z4Pi8h/upPvusNyd9+vcvdxWlj5dLflm9be9kzvsCAx552IZAGfA/aHzX4cGI8TLhcBmcCDbvlLgV8D/wgMAuYDh931XgTygeHALcCP23z4fRF4ARgMbMYJMI+7/RXAz9tUbwlwm7t8LLAB+A9gCLAbeMitUzzwBvBbIB3IBf6viEwJ21Yu8EP3vfcDP3LXTQV+D3wfSAUOAFeEHyLgUXefJgEjgIcBVPU24Ahu605Vn+B0nR2TG4DV7rFcA6xy6zUBWA5coqqJwHWcOs5tPQ3UA8OAr7mvrupw/zox3x0Ocvf9HXc//jasTC7wpqoWn0V9TKSpqr3s1e0XzgdSNVAFKPAWzgcCOB8sNcDYsPKXA4fc8Z8DT7WzzRFAEEgMm/co8Lw7/jDwRtiyL7p18LrTiW5dmuvxNvC/w8r/f8DaNutvcccXA+vb1OfnwEPu+PPAL8KWfR7Y447fDnwQtkxwPvjv6uDYfQnY3OZYfjZserS7H74uHpM3w5ZNBurc8YuAIuCzQNQZ/i29QBMwMWzej4G/tq1P2PK3z2X/3Pr+5xm2Owc4Cnjc6Tzgb3r7791erV/WIjHn05fU+aZ7NTAR59s4QBoQB2wSkXIRKQdec+eD8+F4oJ3tDQfKVLUqbN6nOK2JZoVh43VAiZ66KF3nDhPOUL7tdHPZUcCc5vq6db4VGBpW/kTYeG3YusNxPvwAUOcTsGXaPT2z2j3FVAn8J6eOVWe6ckza1ssvIj5V3Q/ci/PhXeTWYXg775GGE1pHw+Z92sX6dXf/WlHVD3G+hFwlIhNxwnDNuWzLRI4FiTnv1Dkl8TzwpDurBOdDeoqqDnJfyepcmAfnA2vs6VviGDBERBLD5o0ECiJT81aOAu+E1bf5dMu3urDucZxwBEBEJHwapwWhwHRVTcI5dSNhy8/UJXe3jomq/lZVr8QJSsU55dhWMRBoU+eRYeM17jAubF54wHa2fx1Wr4P5v3K3cRvwsqrWd2FbpgdZkJhI+SmwQERmqGoI+HfgKRFJBxCRTBG5zi37S+CrInKtiHjcZRNV9SjwPvCoiPhFZDpwJ/CbHqj/q8B4EblNRKLc1yUiMqkL6/4JmCIiX3YvSH+H1h+0iTin4MpFJBPn2lC4QmBMexvuzjERkQki8hkRicG5/lGHc5qs7XsEgf8GHhaROBGZDNwRtrwYJ7j+VkS8IvI1Wn8R6Gz/OlIMhDh9318AbsIJk193cVumB1mQmIhwP2x+DfzAnfXPOBekP3BPd7wJTHDLfgR8FXgKqADewfnGDM7F1dE438RfwblG8UYP1L8KWIhzcf4Yzumix4GYLqxbAnwFeAwoBcYB74UV+SEwC2df/4TzoR3uUeD77im177XzFud6TGLcOpW4+5MO/K8Oyi7HOVV3Aqd1+R9tln8dJyBKgSk44dass/1rl6rW4tyw8J6775e58/OBj3FaLOu7si3Ts8Q5fWuMMR0Tkb/DuZh+ZS+9/3PAMVX9fm+8vzkz+wWpMaZPE+e3PV/G+ZGr6YPs1JYxps8SkUeAHcBPVPVQb9fHtM9ObRljjOkWa5EYY4zplgFxjSQ1NVVHjx7d29Uwxph+ZdOmTSWq2mm/ZgMiSEaPHk1eXl5vV8MYY/oVEelSjwYDIkiMMaYnqSplNY2ICLFRXmJ8Hjyervy4v3+yIDHGmLMUCilFVQ0UlNeSf7Iu7FVLwck6CsrraAiEWq0T4/Pgj/ISG+UlNtoJl9hoL36fMx0b5SUmyuMsj/I6ZcPKtcxzy8VF+0iJjyYtMQZ/VO8+PNOCxBhj2giGlMLKevJP1jlhUeYGRbkTFMfK62kMtg6KlPhosgbHMmlYEp+dnMGwZD8eEeqagtQ1BqkPBKlvDFLfFHLmNQWpd1/FVYGWcg0BZ1jXFCTUxZtqk/w+0pP8pCXEkJ4UQ3piDGmJMaQn+klPdOalJfhJivXhdP12flmQGGMueKpKYzBEYyBEQ+DUsCUs3NZEc1gcL68n0OZTPC0xhsxBsUzNTOb6qcPIHBxL1uBYRgyOZfigWOKiz+/HqarSFFTqmoI0uMHjhE/IDZoAJVWNFFc3UFRZT1FVA8VVDWw+Uk5RVT31TaHTthnj85DWEjKngiYtsTmA/KQlxpASH43P2/WbeiMaJOI85e5nOM83+IWqPtZmeQxOf0yzcfrsWayqh90HHT3bXAx4WFVfcdc5jPPMiyAQUNUcjDH9XjCklFQ3cLyinhMVdRyvqKe6PkBjsPWHf0MgSGOgdSg4ZdqZHwjR4AbImYhAemIMWYPjmDVyMJnTY8kaHEfW4FgyB8eSOSi2x08fiQjRPiHa54HYqLNaV1WpaghQXNVAUWUDRVX1FLtB0xw4h0pq+PBQGeW1Taet7xEYEt9pt3ItIhYkIuLFecraApyH+mwUkTWquius2J3ASVW9SESW4HSKtxjnl6w5qhoQkWHAVhH5o6oG3PWucTvGM8b0A4FgiKKq5pCo53hFnTOsrOd4uTNeWNVAsJ1zOR6BaJ+HaK+HmCivM/R5iPadGvqjPCTHRhHtbT0/uqWcc63B2YanpVx6op+swbEMG+Qnxte71xnOJxEhyR9Fkj+KsWkJZyzbEAhSUt1IUWV9S9A4YVPPpi6+XyRbJJcC+1X1IICIrAZuBMKD5EZOPYLzZWCViIjbC2gzP2d+PoMxphc1uqeITlTWt2pNOIHhhEZxVcNp5/v9UR6GJ8cyNNnP5WNTGZbsZ2iyv2U4NMlPcmzUWZ1iMWcvxuclc5DT6mrrsXbKtyeSQZJJ6yes5eM8NrPdMm7rowJIAUpEZA7wHE534reFtUYU+LOIKPBzVX0WY0xE1TcFOVxaw8HiGg6V1HCguJqDxTUUlNdRUt1A256W4qO9DBsUy7BkP+PS09xwcL75D0v2MywpNmIXfk3Pi2SQtPcX0rZl0WEZ9xGbU9wHCf1KRNa6T0a7QlWPuQ9IekNE9qjqu6e9ucjdwN0AI0eObLvYGNNGKKScqKznYHENB0uq3WENB4urKSivaxUWQ5P8jEmL5zMT0lvCYWiyExzDkv0k+s/unL7p3yIZJPm0flRnFs6DeNork+8+SS4ZKAsvoKq7RaQGmArkqeoxd36RiLyCcwrttCBxWyrPAuTk5NipMWNcVfVNHCpxWhcHi6s54I4fLqmhrunUAxPjo71kp8Uza+RgbpmdxZi0BMakxpOdGk98jN3waU6J5F/DRmCciGTjPJZzCbC0TZk1OI/w3ADcAqxTVXXXOeqe7hqF8yS9wyISD3hUtcodXwisiOA+GNMvVdU3caKiniNlte6pKCc0DpbUUFzV0FLOIzBiSBzZqfFcPiaFMWnxjEmLZ2xaAumJMXbqyXRJxILEDYHlwOs4t/8+p6o7RWQFTstiDc6zul8Qkf04LZEl7upXAveLSBPOM5z/XlVLRGQM8Ir7x+0Dfquqr0VqH4zpa5pvkT1R4VzcLnQvcBe60ycqnfGaxtaPYh8cF8WYtASuHp9Gdlo8Y1ITGJsWz8iUuAvqbiXTOwbE80hycnLUOm00fV1dY9AJg4p6TlTWcaKiwbkbKiw0itq5RdbnETKS/GQkxTA02U9GknPH09Bk59bWMakJDI6P7qW9Mv2ZiGzqym/17ESnMT2otLqBfYXVfFJUxSeF1XxaVssJ9zcVlfWB08onxvjIcG+FHeveIts8PTTJT0ZyDKnxMRd0h4Cm77MgMSYCTtY0sq+win1F1XxSWMW+Qic4SmsaW8okxvgYnRrPqJR4LhuT0qolkeEOE+yitukH7K/UmG4or21kX2G1GxRVbmujmpLqUxe0E2J8XJSewLWT0hmfkci4jETGZyQwNMlvF7PNBcGCxJguqKht4pOiqlOh4Y6H3wEVH+3looxErpmQ5gZGAuMzEhmWbIFhLmwWJMa0oarsOVHFG7sK+ehQGfsKqygKC4y4aC/j0hOYPy6N8W5YjMtIIHNQrAWGGZAsSIzB6VQw79OT/HlnIW/sPsHRsjpEYNLQJK4cl8p493TUuPREMgfF2sVtY8JYkJgBq7YxwLv7SvjzrhOs21NEeW0T0T4PV4xN4e+vvohrJ6WTnujv7Woa0+dZkJgBpaS6gbd2F/LnnYX8dX8JDYEQybFRfGZiOgsnZzB/fJp1/2HMWbL/MeaCd7C4mjd2FfLGrkI2HTmJKmQOiiX30pEsnJLBJaOHEGVdlRtzzixIzAUnFFK25Je3hMf+omoApgxP4p5rx7FgcgaThyXZhXFjzhMLEnNBaAgEef9AKX/eWcibuwsprmrA6xEuGzOEv50zks9OziBrcFxvV9OYC5IFiemXAsEQ+Sfr2HK0nD/vOsE7e4upaQwSH+3l6gnpLJicwTUT0kmOs+diGBNpFiSmz1JViqsa3Icr1XCopNp5jkZJDUdKawm4nRemJcZw48xMFkzOYO7YFOvN1pgeZkFiel1lfROHS2panptxqMQNjeKaVt2hR/s8ZKfEMz49keunDCU7NZ4JQxOZOjzZftdhTC+yIDE9oiEQ5GhZbcszv1ue0FdS06pfKhHIGhxLdmoCOaOGMCbNeSJfdmo8w5Pth4DG9EUWJOa8C4WUXccreWdfMRsPl3GopIajZbWEP0YjNSGa7NR4PjMxjTFpCWSnxjMmNZ4RQ+LwR9mpKWP6EwsSc14UVzXwzr4TvL53JxuP76CWo3j9x/H7K4nPSGbayFSGJWQwetAwxqdmMmrQENJj00mNSyXWF9vb1TfGdIMFiTkn5XXV/HHPJv5ycAu7SvdQGfoUT8wJxNMEqRCLl9HJYxiVNJGKhgqKag+Td3Ij75U0wP7W20qMSiQtLo20uDTSY9NIixlMelQiab540r1+UokiTbzENDVAYxU0VENjNTTWgHgg6xIYeTkkpJ31fqgqAQ3QEGigPlhPY7Dx1DBQT3xUPGMHjcUj9oNFYzpiQWLOSFUprC1kT9kePszfwUfHdvBp1X7qKULEOVfliY5jhH8MF6dfzuUjpjEpZSJj4jOJLt4LRbugoQpiq9GGSirryyluOElRYyXFgWqKA7UUNdZQXF1Bkexjk0CRz0ugnR8LJgeDpAWDpAfcoXpICgZo3Pk89R6hMXYw9YkZNMSn0hA7iAZvFA3BhpZXeEA0B0ZDsIGQhs54DFL8KVw+/HLmDp/L5cMvJzU2NSLH2pj+yp7Zblo0BZs4WHGQvSf3srdsLztLdrO7bC+1gcqWMqHGFGJCWYxNHs8VI6ezaOJsxg7KRMoPQ36e8yrIgxPbIdjY+g18fohOgJgEiE50hwlhw0SITiAUHU+F10eRRynWJopDjRQF6ygO1FDUVElxQzlF9aWU1pUSVOeuLgH8CDGhINGhEH5VYsRLTFQCMf5k/HGpRMem4PfFEu2Nxu/zE+ONaXn5fX5nvjdsvi+GkroSNhzbwAfHP6CsvgyA8YPHt4TKrPRZ+H3WsaO5MHX1me0WJAOUqrK/fD8fHv+Q3WW72XdyH/vL9xMIOc8NF40iWJ9BoH4YvkAmU9Mm8dmxM1gwcRSj4xuhYBPkb3JCo2AT1JY6G46Kg+EzIXM2ZOXA0OkQO9gJCu/5bQAHQ0HqAnXEeGPweXxOlyehEJTsgyMbTr3Kj7h1i4cRl8DIuTDyMqd+0fFdeq+Qhthbtpf3j73PhmMb+LjoY5pCTcR4Y5idMbslWMYNGmddr5gLRp8IEhG5HvgZ4AV+oaqPtVkeA/wamA2UAotV9bCIXAo821wMeFhVX+nKNttjQeKobarloxMfsT5/PesL1nO85jgA8b7BxASzKC9Po6Y6nVD9cCamZnPV+KFcNXYQs/3HiDq+yQ2PPCj9xN2iQNoEyMxxPpSzciBt0nkPjG6rKICjH8CnG+DIB1C4A1AQLwy7GEa5wTLisi5fZ6ltqmVT4aaWYDlQcQCAtNi0ltNglw27jJTYlAjumDGR1etBIiJeYB+wAMgHNgK5qrorrMzfA9NV9ZsisgS4SVUXi0gc0KiqAREZBmwFhgPa2TbbM5CD5GjVUd7Nf5f1BevZeHwjjaFGYn2xTBmcw8mSsWz7ZCihpkGkJkQz76JUrhvRxNyYwySVbnVC4/gWCNQ7G4tPcy5sN7c2hs8Ef3Kv7t85qa+Aox+5LZYPnP0Mur9lSbnIuXA/8nInXIaMcX7c0okTNSfYcGyDEyzHN1DRUAHApCGTWoJlZvpMor3RkdwzY86rvhAkl+O0JK5zpx8AUNVHw8q87pbZICI+4ASQpmGVEpFs4AMgE7iks222ZyAFSVOwiU1Fm1ifv55389/lcOVhAEYljWJe5jzSvDN4a0s8731SwRA//NPkk1wd/ykZlTuQgk1QXehsyOd3vq1n5kDWbGc4aGSXPlT7nUADHNtyKliObID6cmdZXIrTykqbAGkTTw0T0js8FsFQkD1le3j/2Pu8f+x9thRtIaAB/F4/OUNzmDt8LnOHz2VM8hg7DWb6tK4GSSTPQWQCR8Om84E5HZVxWx8VQApQIiJzgOeAUcBt7vKubBMAEbkbuBtg5MiR3d+bPqy4tpj1BetZn7+eDcc3UNNUQ5QnikuGXsLiCYu5MvNKDhyL5em/7GfnkSK+EPcxr4/azvjy9cgu90J6ykUw5hqnpZE5GzKmgm+AfHv2xcDIOc4L3Osse51AKfjYueay/WVwWxmAc90nPFjSJjiBkzgUr8fLlNQpTEmdwtenf52aphryTuS1BMsTG58AID0unYvTLmZo/FAy4jLIiMsgPS6djPgM0mPTifL2jQ4nQxriZP1JimqLKKotorC2sGU82hvNiMQRjEwcyYjEEWQlZtnNBwNQJIOkva9abZs/HZZR1Q+BKSIyCfiViKzt4jZx138W9zpLTk7OBXVHQTAUZHvJ9pbw2F22G4CMuAw+l/055mfOZ86wOcR4Y/nT9uPc89wOhhW/x7f8eVwd/zFRwVqoGASTvggTFzmncOKG9PJe9SEeD6RPcl45X3PmqULVCSjeA8V7Tw13/QHqnj+1bkyyGypuwKRPJD5tIldlzeeqEVcBcKz6GBuObeC9Y++x7+Q+/lrwV+oCdadVY4h/SEvAZMS7IRMWNhlxGcRHde1mgY7UB+opri1uFQ7h40W1RRTVFbXchNFyiMRDij+F+mA9VY1VrZalx6W3CpcRSSOcYeIIkqKTulXfrgppiLL6MgprCjlRe6L1sOZEyz76PD6SopNIikkiKTqJ5OjkU+Mxyc6y8PEYp0xidCJej/XA0CySQZIPjAibzgKOdVAm3z21lQyUhRdQ1d0iUgNM7eI2L0gVDRW8V/Ae7xa8y3sF71HeUI5HPMxIm8E9s+5hXuY8xg8ej4jQEAiy5qN97H7nJXJq/8rvvFvwRzei/hRk4ldg8o2QPR/6yDfefkEEkoY5r7HXnJqvCjXFrQOmaA/sXQubXzhVLjqhJVyGp03g5rRJ3HzxPZA4DA0FqGqooLDmBEW1hRTWFFJYV0RhbTFF9cUcqzzMlsJNlDdVnVatBG8s6TGDyIgZRHp0MhnRyWREJZIRnUx6VALiiaJIQhSqcwt1UX0ZhXWnQqIivJXlivXFtgTY7IzZpMeltwqx9Lh0UmJT8Hmcj4+KhgqOVh3lSOURZ1h1hPyqfNYXrKekrqTVtgfFDGJk4kiyErOcsEka2RIyKf6ULp3qawmJWjcUOgiJplBTq/WiPFFkxGUwNH4oM9JnkB6XTigUoqKxgsqGSioaKzhafZTK0koqGyvbDfdwiVGJLaHTMgwLneSYZMYkj2HikInERV3Yz8KJ5DUSH86F8WuBApwL40tVdWdYmWXAtLCL7V9W1b9xr4scdU9njQI2ANOB8s622Z7+eI2ktqmWnaU72VK0hfUF69lavJWQhhgcM5grMq9gftZ85g6fS3LMqYvdtZWlfPTaf+LZ/UfmhLYQI03U+9OImXojMuVG57bXvnZH1YWspiSs9bLnVNg0X4c6S/UiFHu9nPB5KfJ6KfR5KfL63KEzv8TrJdTBh7GokhIKkR7ykC5RZPhiSY9KJD16MOlxaWTEDyU9aQQJCcOQuBTn+lDcECcEz/FaTm1TLUerjpJflc+RqtZBc7zmeKsfg8b6Yk87TRYIBc4qJDLiM1pOFbYdDvEPOatrUk3BppaQqWx0XhUNFc64GzzNy5rnNw/D6ycIYxKymDxkApNTpjA57WImpk7pF+HS6xfb3Up8Hvgpzq26z6nqj0RkBZCnqmtExA+8AMzEaYksUdWDInIbcD/QBISAFar6Px1ts7N69PUgCYQC7C/fz/aS7Wwv3s72ku0crDjY8p9s0pBJzM+az7yseUxNmdq6SV1TSu32NRR9+BKZJz8kiiClnlTqxy1i+NzFyIjLnFM1pu+oLXOuuxTthtoS5zZkjxc8vlPj4nGH3jZDj1PutHnOdAAoDdRQ1FhJYWM5GmwkXb1kBIOkNDUQVVfuvH9dmfPbn9pSd/okHZwlBm+0EyqxQ5xgiRvihkwKxKVCfKpz80F8ujP0D+rS31xTsImC6oJW4dIcNvlV+S0fxmcKiYz4DIbGDWWwf3DvdWMTbHJ6cCj4GAo2occ+pr54Lyc9yr6oaHbFuK/oKIp9zhc5UWVMIMTkoDBZfUz2xDLRm0BcVILzW6zoOIiKdX77FBXrToe9ouOc30ANGuW8IvQFsU8ESV/Rl4JEVTlec5xtJdvYUbyD7SXb2VW6i/qgc4ttckwy01KnMS11GlNTpzItdRqD/YNbb6S6CHb/kcbt/4PvyHt4CHIklMauwdcwal4uk2ZdbeFhzk4o6NwWHR4utaVtAudk63l1J6G97mU8PidgEtKccIlPOzWekO4ET/N4XGq7H4LBUJDiumKiPFG9GxJthUJQdhCOOaFBwcdwYtupW+RjBzs3qwyfBcNngDcGmmpbXkW1xeyqKWBX3Ql2NZSys6mcEnV6gPAA2epzwqUpyOT6BibW1xLXWANtrlG14omClLHODTOp4yF1nDNMuQhiB3Vrdy1IwvRmkFQ2VrKjZAfbi7c7w5LtlNY7vwKP9kQzMWUi01OnMzV1KtNTp5OVmNV+87vyGOz+I+z6A/rp+wjKIR3K/x+cQ83YL/DF665n0vB++JsO03+FQk6Y1BQ5X25qip1XdZE7r7j1vGBD+9uJHeIGTNqpYct487Tb6onq4Z6iK4+1tDSc8Nh86u69qDgYNgMyZzmv4bNg8OizPg1YVFvErtJd7Crdxc7Snewq3dVybckjHrKTspk8ZCKTk8cyJSmbCXHDiNMQ1FfCyUNQ8onzKv3ECbnw0IlPd4NlHKS4AZN6kdOK6cLNAhYkYXoqSJqCTew9uZdtxdtaQqP5dxwA2cnZLa2NaanTGD94/Om3eDbVOXcHVRc559LLDsKeP0H+RwAcj8nmpdpZvBGaw7RZl/ONq8YyOrV7d+4YE3Gq0FDpXDdqCZoiZ/q0ICp2enluT3TCqRZNeMA0j8eHBVHs4LNrmdeddEKjOTAKNkH1CWeZxwcZU5ywyJzltDpSJ0TslFJzuDQHS3vhMmHIBPw+P8FQEEUJapBQMECosZpQfQWhxiqCDdXOdFMNoWATQREUCIoHjfIT9PkJ+fyEfDGEfNEEPT5CIoQ0REhDvPrlVy1ImkUqSEIa4q0jb5F3Io8dJTvYXba75bxuij+FaWnTnNZGyhSmxg8nsaEmLCTCwqKq0BlWFzr/2dqoGzKZ17mM/3N8Esd8I7l1zkjumjeGocl2v765QDXVhQVNc8smbDp8WW1J+6fYxBsWLqlhrRx3PC4Fyg6dOk1VdvDUuikXnTpFlTkbhk7t+dZQG0W1Rews2cmuMidYPjn5CUEN4hEPXvEiCF6PF4948ODB4wmbL148GsQTaMQTaMATqMPbVI+nqRZPUx0eVTyARxWvLwaJTsAbk8iTS9dZkDSLRJDUNNXw/b9+nzePvEmsJ5rJ8cOZFjWEaRLLtECQoTUVSHXhqW9f7Z3jjE5w/rAThjrDxKE0+lMp1MEcbUxgf108fz0RxZ8/DZIcG8Udc0fz1bmjGRw/QH4oaExXhILuKbbisJaN29JpHg+f31TTev3E4adOT2XOdk5XdfPaQr8SaISTh51TYyX7oGS/O9yHPHDEgqTZ+Q6So5VH+c5fvs3B8oP8Q9lJbq2oPPWDHPGc+uaTkNEqJJpDQxPSORFM5kAFHCiuPvUqquFEZX3L+3gERqfGs+SSESydM4qEGLt115hua6xxQ6UUkoY7vw0yp1NFPJ5e7yLlgvR+wft8753v4Wmq5ZnjJ7h8zOdg0d+ealnEpbScN60L/yAVAAAgAElEQVRvCnKopKYlJA4ecgLjYPEn1DYGW7aZGONjTHoCcy9KYWxaAmPT4hmblsDIlDhifPbrWWPOq+h45zV4dG/XpG87i5sGLEi6SFX51c5f8dSmpxgbVH52/DgjrnkQvWwZJTVNTlgcqeZA0T4nLEqqyT9ZR3iDL3NQLGPTE7hk9BA3MJzQSEuMsc77jDH9lgVJF9QF6njo/YdYe2gtC2sbeKQqSNzS/+bVqrGseHQdRVWnbmuMjfIyJi2eGSMGc/OsrJbAyE6NJzbaWhfGmAuPBUknjlUf495197Dn5B7uKSvnzoRxVN7xHN9Zd5I1Wzdz8YhBfOvqsU5gpCcwLMmPx2OtC2PMwGFBcgYbT2zku3+5j6bGSladKGL+lKWsH/tdvvfLPZRWN/LdBeP51tVj8Xn7yK9ujTGmF1iQtENVeXHPizyx8XFGNgX5WVEpwxf8hB/kz+KFX21lXHoCv7zjEqZm2i/JjTHGgqSNhmAD//LBv/A/+/+Hq2vrebTez9HrXuL6dUEOl37KXVdm873rJuCPsusdxhgDFiStFNYU8g9/uZdtpTv45skKvjF4Fs+MeIB//a8ShiXH8tu7LuPysSm9XU1jjOlTLEhcW4q2cN+671Bbf5KfFhUzY+LXuenQtWzfXcIts7N46IuTSfTbg6CMMaYtCxLg9/t+z7988AjDmpp4tqyaQ2N/xOUfDScxpomf3zab66YM7e0qGmNMnzWgg6Qp2MTjHz3G7/a9xBV19fyocRCPxD3M/2yMY8HkNB798jRSE2J6u5rGGNOnDdggKakr4bt/uZePi7fy1fJKboqdw/Wlt1KHn5/cMplbZnfwXBBjjDGtDMgg2Vmyk3veWkZFXSmPl5RRFX0b1xy8kjnZKTz5lYsZMaTvP0vZGGP6igEXJH888Ed++N6DDGlq4BdljTxd9z3eLp/I978wga9dkW2/SjfGmLM0YIIkEArwrxuf5IU9v+GSunruqUzhG2V/T8rwbF5dPIPxGYm9XUVjjOmXBkSQBDXIN1+7kw+LP+bWiirGVlzK4tqlfP0zk/j2Z8YR7bMuTowx5lxFNEhE5HrgZ4AX+IWqPtZmeQzwa2A2UAosVtXDIrIAeAyIBhqBf1TVde46bwPDgDp3MwtVtehM9ThYfgBPURMPl1byfmkurw/5HL+942JmjRx83vbVGGMGqogFiYh4gaeBBUA+sFFE1qjqrrBidwInVfUiEVkCPA4sBkqAL6rqMRGZCrwOZIatd6uqdvmRhxps4snjjTxZ+c/Mvuwq/vS5icRFD4jGmDHGRFwkP00vBfar6kEAEVkN3AiEB8mNwMPu+MvAKhERVd0cVmYn4BeRGFVt4BxkNHp5NPQE3//qFcwfn3YumzDGGNOBSAZJJnA0bDofmNNRGVUNiEgFkILTIml2M7C5TYj8h4gEgd8D/6LtPHheRO4G7gZIHj6GvPu+QHKcdXFijDHnWySvMrd3H23bD/wzlhGRKTinu74RtvxWVZ0GzHNft7X35qr6rKrmqGrORcMGW4gYY0yERDJI8oERYdNZwLGOyoiID0gGytzpLOAV4HZVPdC8gqoWuMMq4Lc4p9CMMcb0kkgGyUZgnIhki0g0sARY06bMGuAOd/wWYJ2qqogMAv4EPKCq7zUXFhGfiKS641HAImBHBPfBGGNMJyIWJKoaAJbj3HG1G3hJVXeKyAoRucEt9ksgRUT2A/8A3O/OXw5cBPxARLa4r3QgBnhdRLYBW4AC4N8jtQ/GGGM6J+1cp77g5OTkaF5el+8WNsYYA4jIJlXN6ayc/aTbGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbrEgMcYY0y0WJMYYY7rFgsQYY0y3WJAYY4zpFgsSY4wx3WJBYowxplssSIwxxnSLBYkxxphusSAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdYkFijDGmWyxIjDHGdEtEg0RErheRvSKyX0Tub2d5jIj8zl3+oYiMducvEJFNIrLdHX4mbJ3Z7vz9IrJSRCSS+2CMMebMIhYkIuIFngY+B0wGckVkcptidwInVfUi4CngcXd+CfBFVZ0G3AG8ELbOvwF3A+Pc1/WR2gdjjDGdi2SL5FJgv6oeVNVGYDVwY5syNwK/csdfBq4VEVHVzap6zJ2/E/C7rZdhQJKqblBVBX4NfCmC+2CMMaYTkQySTOBo2HS+O6/dMqoaACqAlDZlbgY2q2qDWz6/k20CICJ3i0ieiOQVFxef804YY4w5s0gGSXvXLvRsyojIFJzTXd84i206M1WfVdUcVc1JS0vrQnWNMcaci06DRES8IpIaNh3tftvf3cmq+cCIsOks4FhHZUTEByQDZe50FvAKcLuqHggrn9XJNo0xxvSgMwaJiCzB+WDfJiLviMg1wEGcC+i3drLtjcA4EckWkWhgCbCmTZk1OBfTAW4B1qmqisgg4E/AA6r6XnNhVT0OVInIZe7dWrcDf+jKjhpjjIkMXyfLvw/MVtX9IjIL2AAsUdVXOtuwqgZEZDnwOuAFnlPVnSKyAshT1TXAL4EXRGQ/TmAtcVdfDlwE/EBEfuDOW6iqRcC3gOeBWGCt+zLGGNNLxLn5qYOFIh+r6qyw6T2qOrFHanYe5eTkaF5eXm9Xwxhj+hUR2aSqOZ2V66xFki4i/xA2nRA+rar/eq4VNMYYc2HoLEj+HUg8w7QxxpgB7oxBoqo/7GiZiMSf/+oYY4zpb7py+2+miOS4d14hIuki8mPgk4jXzhhjTJ/X2e2/9wJbgP8DfCAidwC7ce6Ymh356hljjOnrOrtGcjcwQVXLRGQksB+Yr6ofRL5qxhhj+oPOTm3Vq2oZgKoeAfZZiBhjjAnXWYskS0RWhk2nh0+r6nciUy1jjDH9RWdB8o9tpjdFqiLGGGP6p85u//3VmZYbY4wxZwwSEWnbyWIrqnrD+a2OMcaY/qazU1uX4zx46kXgQ9p/HogxxpgBrLMgGQosAHKBpThdu7+oqjsjXTFjjDH9wxlv/1XVoKq+pqp3AJfh/I7kbRH5do/UzhhjTJ/XWYsEEYkBvoDTKhkNrAT+O7LVMsYY0190drH9V8BUnIdH/VBVd/RIrYwxxvQbnbVIbgNqgPHAd5yn2wLORXdV1aQI1s0YY0w/0NnvSDrtHdgYY8zAZkFhjDGmWyxIjDHGdEtEg0RErheRvSKyX0Tub2d5jIj8zl3+oYiMdueniMhfRKRaRFa1Wedtd5tb3Fd6JPfBGGPMmXV6+++5EhEv8DTODxrzgY0iskZVd4UVuxM4qaoXicgS4HFgMVAP/ADnjrGp7Wz+VlXNi1TdjTHGdF0kWySXAvtV9aCqNgKrgRvblLkRaO4Y8mXgWhERVa1R1b/iBIoxxpg+LJJBkonTT1ezfHdeu2VUNQBUACld2PZ/uKe1fiBh9yQbY4zpeZEMkvY+4PUcyrR1q6pOA+a5r9vafXORu0UkT0TyiouLO62sMcaYcxPJIMkHRoRNZwHHOiojIj4gGSg700ZVtcAdVgG/xTmF1l65Z1U1R1Vz0tLSzmkHjDHGdC6SQbIRGCci2SISDSwB2j7fZA1whzt+C7BOVTtskYiIT0RS3fEoYBFg3bYYY0wvithdW6oaEJHlwOuAF3hOVXeKyAogT1XXAL8EXhCR/TgtkSXN64vIYSAJiBaRLwELgU+B190Q8QJvAv8eqX0wxhjTOTlDA+CCkZOTo3l5drewMcacDRHZpKo5nZWzX7YbY4zpFgsSY4wx3WJBYowxplssSIwxxnSLBYkxxphusSAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdYkFijDGmWyxIjDHGdIsFiTHGmG6xIDHGGNMtFiTGGGO6xYLEGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbolokIjI9SKyV0T2i8j97SyPEZHfucs/FJHR7vwUEfmLiFSLyKo268wWke3uOitFRCK5D8YYY84sYkEiIl7gaeBzwGQgV0Qmtyl2J3BSVS8CngIed+fXAz8AvtfOpv8NuBsY576uP/+1N8YY01WRbJFcCuxX1YOq2gisBm5sU+ZG4Ffu+MvAtSIiqlqjqn/FCZQWIjIMSFLVDaqqwK+BL0VwH4wxxnQikkGSCRwNm85357VbRlUDQAWQ0sk28zvZJgAicreI5IlIXnFx8VlW3RhjTFdFMkjau3ah51DmnMqr6rOqmqOqOWlpaWfYpDHGmO6IZJDkAyPCprOAYx2VEREfkAyUdbLNrE62aYwxpgdFMkg2AuNEJFtEooElwJo2ZdYAd7jjtwDr3Gsf7VLV40CViFzm3q11O/CH8191Y4wxXeWL1IZVNSAiy4HXAS/wnKruFJEVQJ6qrgF+CbwgIvtxWiJLmtcXkcNAEhAtIl8CFqrqLuBbwPNALLDWfRljjOklcoYGwAUjJydH8/LyersaxhjTr4jIJlXN6ayc/bLdGGNMt0Ts1FZf19TURH5+PvX19Z0XNmfF7/eTlZVFVFRUb1fFGNMDBmyQ5Ofnk5iYyOjRo7FeVs4fVaW0tJT8/Hyys7N7uzrGmB4wYE9t1dfXk5KSYiFynokIKSkp1tIzZgAZsEECWIhEiB1XYwaWAR0kxhhjus+CpBd5vV5mzJjB1KlT+cpXvkJtbe152W5hYSGLFi3i4osvZvLkyXz+858HIDs7m71797Yqe++99/LEE0/w9ttvk5yczMyZM5kwYQLz58/n1VdfPS/1McZc2CxIelFsbCxbtmxhx44dREdH88wzz3R53UAg0OGyBx98kAULFrB161Z27drFY489BsCSJUtYvXp1S7lQKMTLL7/M4sWLAZg3bx6bN29m7969rFy5kuXLl/PWW2+d494ZYwaKAXvXVrgf/nEnu45VntdtTh6exENfnNLl8vPmzWPbtm0cPnyYRYsWsWPHDgCefPJJqqurefjhh7n66quZO3cu7733HjfccAO333473/zmNzly5AgAP/3pT7niiis4fvw4CxcubNn29OnTAcjNzWXx4sU89NBDALz77ruMHj2aUaNGcejQoVb1mTFjBg8++CCrVq3i2muvbbUsGAxy5513kpeXh4jwta99jfvuu+/sD5Ix5oJgQdIHBAIB1q5dy/XXd/6MrvLyct555x0Ali5dyn333ceVV17JkSNHuO6669i9ezfLli1j8eLFrFq1is9+9rN89atfZfjw4UyfPh2Px8PWrVu5+OKLWb16Nbm5uR2+16xZs/jJT35y2vwtW7ZQUFDQEnbl5eXnuOfGmAuBBQmcVcvhfKqrq2PGjBmA0yK58847OXbszJ0ZN5+GAnjzzTfZtWtXy3RlZSVVVVVcd911HDx4kNdee421a9cyc+ZMduzYQVpaGrm5uaxevZopU6bwhz/8gRUrVnT4Xh11nzNmzBgOHjzIt7/9bb7whS+0av0YYwYeC5Je1HyNJJzP5yMUCrVMt/09Rnx8fMt4KBRiw4YNxMbGnrbtIUOGsHTpUpYuXcqiRYt49913ufnmm8nNzWXhwoVcddVVTJ8+nfT09A7rt3nzZiZNmkQwGGT27NkA3HDDDaxYsYKtW7fy+uuv8/TTT/PSSy/x3HPPndMxMMb0f3axvY/JyMigqKiI0tJSGhoaznjn1MKFC1m1alXLdHMorVu3ruUOsKqqKg4cOMDIkSMBGDt2LCkpKdx///1nPK21bds2HnnkEZYtW4bX62XLli1s2bKFFStWUFJSQigU4uabb+aRRx7h448/Ph+7bozpp6xF0sdERUXx4IMPMmfOHLKzs5k4cWKHZVeuXMmyZcuYPn06gUCA+fPn88wzz7Bp0yaWL1/e0rq56667uOSSS1rWy83N5YEHHuCmm25qtb3169czc+ZMamtrSU9PZ+XKladdaAcoKCjgq1/9akvL6dFHHz1Pe2+M6Y8GbDfyu3fvZtKkSb1UowufHV9j+j/rRt4YY0yPsCAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdEtEgEZHrRWSviOwXkfvbWR4jIr9zl38oIqPDlj3gzt8rIteFzT8sIttFZIuI5LXdZn8yd+7cVtNPPfUUfr+fioqKlnlvv/02ixYtOm3dV199lZkzZ7Z0Ff/zn/884vU1xpj2ROwHiSLiBZ4GFgD5wEYRWaOqu8KK3QmcVNWLRGQJ8DiwWEQmA0uAKcBw4E0RGa+qQXe9a1S1JFJ17ynvv/9+q+kXX3yRSy65hFdeeYW/+7u/63C9pqYm7r77bj766COysrJoaGjg8OHDka2sMcZ0IJK/bL8U2K+qBwFEZDVwIxAeJDcCD7vjLwOrxHlO643AalVtAA6JyH53exsiUtO198OJ7ed3m0OnweceO2ORhIQEqqurAThw4ADV1dX85Cc/4cc//vEZg6SqqopAIEBKSgoAMTExTJgw4bxV3RhjzkYkT21lAkfDpvPdee2WUdUAUAGkdLKuAn8WkU0icndHby4id4tInojkFRcXd2tHesKLL75Ibm4u8+bNY+/evRQVFXVYdsiQIdxwww2MGjWK3NxcfvOb37Tq6NEYY3pSJFsk0s68tv2xdFTmTOteoarHRCQdeENE9qjqu6cVVn0WeBacLlLOWNNOWg49YfXq1bzyyit4PB6+/OUv81//9V8sW7asw/K/+MUv2L59O2+++SZPPvkkb7zxBs8//3zPVdgYY1yRDJJ8YETYdBbQ9mEbzWXyRcQHJANlZ1pXVZuHRSLyCs4pr9OCpD/Ztm0bn3zyCQsWLACgsbGRMWPGnDFIAKZNm8a0adO47bbbyM7OtiAxxvSKSJ7a2giME5FsEYnGuXi+pk2ZNcAd7vgtwDp1epFcAyxx7+rKBsYBH4lIvIgkAohIPLAQ2BHBfegRL774Ig8//DCHDx/m8OHDHDt2jIKCAj799NN2y1dXV/P222+3TG/ZsoVRo0b1UG2NMaa1iLVIVDUgIsuB1wEv8Jyq7hSRFUCeqq4Bfgm84F5ML8MJG9xyL+FcmA8Ay1Q1KCIZwCvO9Xh8wG9V9bVI7UNPWb16NWvXrm0176abbmL16tXMmTOHt956i6ysrJZlL774Ik888QTf+MY3iI2NJT4+3lojxpheY93Im4iw42tM/2fdyBtjjOkRFiTGGGO6xYLEGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEh6kdfrZcaMGUydOpWvfOUr1NbWnpftFhYWsmjRopYu5j//+c8DkJ2dzd69e1uVvffee3niiSd4++23SU5OZubMmUyYMIH58+fz6quvnpf6GGMubBYkvSg2NpYtW7awY8cOoqOjeeaZZ7q8biAQ6HDZgw8+yIIFC9i6dSu7du3iscecvsSWLFnC6tWrW8qFQiFefvllFi9eDMC8efPYvHkze/fuZeXKlSxfvpy33nrrHPfOGDNQRLKvrX7j8Y8eZ0/ZnvO6zYlDJvLPl/5zl8vPmzePbdu2cfjwYRYtWsSOHU7PL08++STV1dU8/PDDXH311cydO5f33nuPG264gdtvv51vfvObHDlyBICf/vSnXHHFFRw/fpyFCxe2bHv69OkA5ObmsnjxYh566CEA3n33XUaPHs2oUaM4dOhQq/rMmDGDBx98kFWrVnHttdd261gYYy5s1iLpAwKBAGvXrmXatGmdli0vL+edd97hu9/9Lvfccw/33XcfGzdu5Pe//z133XUXAMuWLePOO+/kmmuu4Uc/+hHHjjl9ZU6fPh2Px8PWrVsBp2uW3NzcDt9r1qxZ7NlzfgPWGHPhsRYJnFXL4Xyqq6tjxowZgNMiufPOO1s+9DvSfBoK4M0332TXrlPPCausrKSqqorrrruOgwcP8tprr7F27VpmzpzJjh07SEtLIzc3l9WrVzNlyhT+8Ic/sGLFig7fayB0n2OM6T4Lkl7UfI0knM/na/WQqvr6+lbL4+PjW8ZDoRAbNmwgNjb2tG0PGTKEpUuXsnTpUhYtWsS7777LzTffTG5uLgsXLuSqq65i+vTppKend1i/zZs3W39ZxphO2amtPiYjI4OioiJKS0tpaGg4451TCxcuZNWqVS3TzaG0bt26ljvAqqqqOHDgACNHjgRg7NixpKSkcP/995/xtNa2bdt45JFHOn0mijHGWIukj4mKiuLBBx9kzpw5ZGdnM3HixA7Lrly5kmXLljF9+nQCgQDz58/nmWeeYdOmTSxfvryldXPXXXdxySWXtKyXm5vLAw88wE033dRqe+vXr2fmzJnU1taSnp7OypUr7UK7MaZT1o28iQg7vsb0f9aNvDHGmB5hQWKMMaZbBnSQDITTer3BjqsxA8uADRK/309paal96J1nqkppaSl+v7+3q2KM6SED9q6trKws8vPzKS4u7u2qXHD8fj9ZWVm9XQ1jTA8ZsEESFRVFdnZ2b1fDGGP6vYie2hKR60Vkr4jsF5H721keIyK/c5d/KCKjw5Y94M7fKyLXdXWbxhhjelbEgkREvMDTwOeAyUCuiExuU+xO4KSqXgQ8BTzurjsZWAJMAa4H/q+IeLu4TWOMMT0oki2SS4H9qnpQVRuB1cCNbcrcCPzKHX8ZuFZExJ2/WlUbVPUQsN/dXle2aYwxpgdF8hpJJnA0bDofmNNRGVUNiEgFkOLO/6DNupnueGfbBEBE7gbudierRWRve+XOk1SgJILbj6T+XHew+vc2q3/vinT9R3WlUCSDRNqZ1/Ze247KdDS/vRZUu/fvquqzwLNnquD5IiJ5XelGoC/qz3UHq39vs/r3rr5S/0ie2soHRoRNZwFtH7bRUkZEfEAyUHaGdbuyTWOMMT0okkGyERgnItkiEo1z8XxNmzJrgDvc8VuAder8QnANsMS9qysbGAd81MVtGmOM6UERO7XlXvNYDrwOeIHnVHWniKwA8lR1DfBL4AUR2Y/TElnirrtTRF4CdgEBYJmqBgHa22ak9uEs9MgptAjpz3UHq39vs/r3rj5R/wHRjbwxxpjIGbB9bRljjDk/LEiMMcZ0iwXJWRCRESLyFxHZLSI7ReQed/4QEXlDRD5xh4N7u65n4vYSsFlEXnWns90uaj5xu6yJ7u06dkREBonIyyKyx/13uLy/HH8Ruc/9u9khIi+KiL+vH3sReU5EikRkR9i8do+3OFa63RdtE5FZvVfzDuv+E/dvZ5uIvCIig8KWtdstU29pr/5hy74nIioiqe50rx57C5KzEwC+q6qTgMuAZW4XLfcDb6nqOOAtd7ovuwfYHTb9OPCUW/+TOF3X9FU/A15T1YnAxTj70eePv4hkAt8BclR1Ks7NIkvo+8f+eZxuisJ1dLw/h3OH5TicHwP/Ww/VsSPPc3rd3wCmqup0YB/wAHTcLVPPVbVdz3N6/RGREcAC4EjY7N499qpqr3N8AX9w/0H3AsPcecOAvb1dtzPUOQvnP/9ngFdxfvxZAvjc5ZcDr/d2PTuoexJwCPcmkbD5ff74c6oXhyE4d0u+ClzXH449MBrY0dnxBn4O5LZXrq/Uvc2ym4DfuOMPAA+ELXsduLyvHXt33ss4X6IOA6l94dhbi+QcuT0VzwQ+BDJU9TiAO0zvvZp16qfAPwEhdzoFKFfVgDsd3h1NXzMGKAb+wz019wsRiacfHH9VLQCexPkWeRyoADbRf459uI6Od3vdIvXl/fkasNYd7xd1F5EbgAJV3dpmUa/W34LkHIhIAvB74F5Vrezt+nSViCwCilR1U/jsdor21XvCfcAs4N9UdSZQQx88jdUe9zrCjUA2MByIxzkd0VZfPfZd0W/+lkTkf+Ocqv5N86x2ivWpuotIHPC/gQfbW9zOvB6rvwXJWRKRKJwQ+Y2q/rc7u1BEhrnLhwFFvVW/TlwB3CAih3F6Tv4MTgtlkNtFDfTtbmfygXxV/dCdfhknWPrD8f8scEhVi1W1CfhvYC7959iH6+h494sujETkDmARcKu654HoH3Ufi/NFZKv7fzgL+FhEhtLL9bcgOQsiIji/xt+tqv8atii8q5c7cK6d9Dmq+oCqZqnqaJwLi+tU9VbgLzhd1EDfrv8J4KiITHBnXYvT+0F/OP5HgMtEJM79O2que7849m10dLzXALe7dxBdBlQ0nwLrK0TkeuCfgRtUtTZsUUfdMvUZqrpdVdNVdbT7fzgfmOX+v+jdY9/bF5P60wu4Eqe5uA3Y4r4+j3Od4S3gE3c4pLfr2oV9uZr/197dhVhRxnEc//4Q2Y3YIMheCFxJkcUoVsjKaksJiopAKYoKyV4uIkgUhS4Eca+KuopeDI2UNhLxIsEgqGxly5fwZTc1MDDYyLskC4mSXv5dPP9lj+ueo2cn2O30+8Cwz87MM/PMLGf+Z57Z+T/wUZavo3xoTgDbgbbJbl+DdncDB/NvsAO4/L9y/oFe4DhwDOgD2qb6uQe2Up7p/EG5cD1T73xTulfeBL4DjlL+Q22qtf0E5VnCyOf37Zr112bbvwXum4rnfszyYUYftk/quXeKFDMzq8RdW2ZmVokDiZmZVeJAYmZmlTiQmJlZJQ4kZmZWiQOJtSRJL0laJGmJpKbefpc0IzPyDkrqGbNst6SbLlB/Zb6FPCHZ7tvqLGuT9JmkIUmPNrndWZIen2i7zOpxILFWdQslD9pdwBdN1r0bOB4R8yOi2boAK4EJBxLKOz7jBhJKfrfpEdEdEdua3O4soKlAUvPWvVldDiTWUnK8iSPAAmAf8CywQdJ5+YkkdUraleM37JI0U1I38Apwf37rv6TBvjZIOqgyxkhvzltByaXVL6k/590jaZ+kw5K2Z642JA1L6s35RyV1ZTLQ54BVuf+emv1dCbwPdOey2ZLWSTqgMsbJxnxrHklz8s7l69z+bOBloCfrrlIZD2Vz7ntQ0uKsuzzbuRP4RNI1kgay3rGxd2lmk/7mrCdP//YE3Ay8DkwH9jRYbyfwZJafBrxQoqgAAAJhSURBVHZkeTnwRp06u8m3hhl9o3tazr8xfx9m9I3jK4AB4NL8/UVgXc16L2T5eeCdLK8H1tTZ/yIyI0FtG7LcBzyY5a+ApVlup9whja27Gtic5S5KGpf2PP6TNce3Glhbc6wdk/039jS1Jt+RWCuaT0l/0UXJZ1XPQuCDLPdRUuA04xFJh4FByoBI88ZZ59acv0fSECU3VWfN8pHEn4coXU/NWpzPc45SknBeL6kDuDYiPgSIiN/j3LxSI+6gHDcRcRz4Hpibyz6NiJ+yfAB4StJ64IaIODOBdloLc/+ntYzsltpCyXx6ivItXHkBXxgRv11gExedLygT+60BFkTEaUlbKN/mz1uVclF+rM6mzubPv2jy8yipHXiLcof0Q17o2xk/pfi4m2iw7NeRQkQMSLoTeADok/RqRLzXTFuttfmOxFpGRAxFRDdlCNV5wOfAvVEeTI8XRPZSsiADPAF82cTuLqNcbH+RdBXnji1yBujI8n7gdklzoIwpIWkujdXWb2QkcJ3K5y4PA0QZI+ekpCW5z7b8L7Kx2x2gHDfZppmUhIXnkNRJGcdmEyX79aSOxW5TjwOJtRRJM4DTEfE30BURjbq2VlC6bI4Ayyhj2V+UKCPUDQLfAO8Ce2oWbwQ+ltQfET9Snjlszf3sp3S5NbITWDr2Yfs4bfgZ2ETJ9rqD0gU1YhmwIve5F7iakjH5z3wAv4pyNzMtu8W2Acsj4iznWwQMSRoEHgJeu0D77X/G2X/NzKwS35GYmVklDiRmZlaJA4mZmVXiQGJmZpU4kJiZWSUOJGZmVokDiZmZVfIP0+s0buDScsEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_rank_scores([ssvd_rank_scores,\n", + " ials_rank_scores,\n", + " psvd_rank_scores]);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It can be seen that scaling has a significant impact on the quality of recommendations. This is, however, a preliminary result, which is yet to be verified via cross-validation." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Models comparison" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The results above were computed only with a single split into train-test corresponding to a single fold. In order to verify the obtained results, perform a full CV with optimal parameters fixed. It can be achieved with the built-in `run_cv_experiment` function from Polara's evaluation engine as shown below." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## cross-validation experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.evaluation import evaluation_engine as ee" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Fixing optimal configurations:" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [], + "source": [ + "set_config(psvd, {'rank': psvd_best_rank})\n", + "set_config(ssvd, ssvd_best_config)\n", + "set_config(ials, ials_best_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Performing 5-fold CV:" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "5/5\n", + "[04:15<00:51, 50.93s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 5/5 [04:15<00:51, 50.93s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "models = [psvd, ssvd, ials]\n", + "metrics = ['ranking', 'relevance', 'experience']\n", + "\n", + "# run experiments silently\n", + "data_model.verbose = False\n", + "for model in models:\n", + " model.verbose = False\n", + "\n", + "# perform cross-validation on models, report scores according to metrics\n", + "cv_results = ee.run_cv_experiment(models,\n", + " metrics=metrics,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output contains results for all folds:" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
typerelevancerankingexperience
metrichrmrrcoverage
foldmodel
1PureSVD0.0768570.0291010.085902
PureSVD-s0.0847290.0322210.148946
iALS0.0737800.0272740.089461
2PureSVD0.0798680.0293850.089836
PureSVD-s0.0869530.0340590.150539
\n", + "
" + ], + "text/plain": [ + "type relevance ranking experience\n", + "metric hr mrr coverage\n", + "fold model \n", + "1 PureSVD 0.076857 0.029101 0.085902\n", + " PureSVD-s 0.084729 0.032221 0.148946\n", + " iALS 0.073780 0.027274 0.089461\n", + "2 PureSVD 0.079868 0.029385 0.089836\n", + " PureSVD-s 0.086953 0.034059 0.150539" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cv_results.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## plotting results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will plot average scores and confidence intervals for them. The following function will do this based on raw input from CV:" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_cv_results(scores, subplot_size=(6, 3.5)):\n", + " scores_mean = scores.mean(level='model')\n", + " scores_errs = ee.sample_ci(scores, level='model')\n", + " # remove top-level columns with classes of metrics (for convenience)\n", + " scores_mean.columns = scores_mean.columns.get_level_values(1)\n", + " scores_errs.columns = scores_errs.columns.get_level_values(1)\n", + " # plot results\n", + " n = len(scores_mean.columns)\n", + " return scores_mean.plot.bar(yerr=scores_errs, rot=0,\n", + " subplots=True, layout=(1, n),\n", + " figsize=(subplot_size[0]*n, subplot_size[1]),\n", + " legend=False);" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X203WV95/33x0RAUUFjsMpTYKC24UEsR2jHh6ZSMKyKsSOUAFPgvrlXtMo4q962hZlCI2M70DVTagfaMRUQcRAoLdO0RNGK+DQWEwQJgXI3xgAxWsJDEVDAwPf+Y/9O3GzOydnJ2efsffZ5v9Y6K799/a7ftb+/vc+52Hz39ZCqQpIkSZIkqVde1O8AJEmSJEnScDHZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIOGRpKNSX6133FIkiRJ0mxnskGSJEmSJPWUyQbNOknm9jsGSZrNxuqH7ZslaXolmdPvGDTcTDZo2ByR5M4kjyW5NsluSRYl2ZTk95L8ALii30FK0jBopq/9TtPvPpnksiSvSfLZJI8n+Yckr0yyIEklOSvJ/cDNY5X1+34kabol2TfJ3yTZkuThJJckeVGS309yX5IHk3wqyR5N/c8lObujjW8n+XfN8c8l+UKSR5Lcm+Q32up9MslfJFmV5EngV5L8WpLbk/wwyQNJlne0fXoTx8NJzmufttzEeU6S7zTnr0vyqql+zTRzmGzQsPkNYDFwAHA4cGZT/jPAq4D9gWV9iUyShtN7gGOBnwVOAD4L/Cfg1bQ+Z3ywre4vAz8PvGOCMkkaes3Igr8H7gMWAHsD19D6/Hom8CvAgcDLgEuay64GTmlrYyGtz7c3Jtkd+EJTZ6+m3p8nOaTtaU8F/hB4OfA14EngdGBP4NeA30ry7ra2/xw4DXgtsEcT46gPAu+m1Y+/DngUuHQSL4mGjMkGDZs/q6rNVfUI8HfAEU35c8AfVNXTVfXj/oUnSUPnf1TVv1TV94CvArdW1e1V9TRwA/DGtrrLq+rJjn54rDJJmg2OovU/6b/T9INPVdXXaP3P/Z9U1YaqegI4F1jaTDe7gdZI3v2bNk4D/qbpc98JbKyqK6pqa1V9C/hr4MS25/zbqvp6VT3XPN8tVbW2eXwn8BlayQOa6/6uqr5WVc8A5wPV1tZ7gf9cVZua518OnOi0OI0y2aBh84O24x/RygQDbKmqp/oQjyQNu39pO/7xGI9f1vb4gTGuH6tMkmaDfYH7qmprR/nraI12GHUfMBd4TVU9DtwILG3OLQX+V3O8P3B0kn8d/aGVjPiZtrae1+cmOTrJl5ppHI8B76M1Mm00jm31q+pHwMNtl+8P3ND2XPcAzwKv6foV0FAz2aDZoiauIkmaYmP1xfbPkmarB4D9xhgJsJnW/8iP2g/Yyk+TuZ8BTknyS8BLgC+1tfflqtqz7edlVfVbbW119rlXAyuBfatqD+B/AmnOfR/YZ7RikpcA8zriP77j+XZrRrpJJhskSZIkqQ++Set/6C9MsnuzsPmbaSUTfjvJAUleBvwRcG3bCIhVtJIRFzTlzzXlfw/8bJLfTPLi5udNSX5+OzG8HHikqp5KchStNR1GXQ+ckOTfJtkF+Ag/TURAKzHxh6NTOpLMT7JkEq+HhozJBkmSJEmaZlX1LK2FdQ8C7gc2AScDlwNXAV8Bvgs8BfyHtuueBv4G+FVaIxNGyx8HjqM1tWIzrenFFwG7bieM9wMXJHmc1poM17W1t6553mtoJUUeBx4Enm6qfIzWqIjPN9f/I3D0Dr8QGlqpcvSiJEmSJGl8zSiLfwUOrqrv9jseDT5HNkiSJEmSXiDJCUle2myr+d+AtcDG/kalmcJkgyRJkiRpLEtoTcnYDBwMLC2HxqtLTqOQJEmSJEk95cgGSZIkSZLUUyYbJEmSJElST83tdwCdXv3qV9eCBQv6HYYkvcBtt932UFXN73cc08G+WNIgsh+WpP7rti8euGTDggULWLNmTb/DkKQXSHJfv2OYLvbFkgaR/bAk9V+3fbHTKCRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGDa1FixaxaNGifochSbOafbEk9Zf9sPrFZIMkDZAki5Pcm2R9knPGOL9rkmub87cmWdCUH5Xkjubn20l+ve2ajUnWNufWTN/dSJIkabaa2+8AJEktSeYAlwLHApuA1UlWVtXdbdXOAh6tqoOSLAUuAk4G7gJGqmprktcC307yd1W1tbnuV6rqoem7G0mSJM1mJhskaXAcBayvqg0ASa4BlgDtyYYlwPLm+HrgkiSpqh+11dkNqKkPV5IkTdZhVx42pe1v+MGGaXmetWesndL2NfM4jUKSBsfewANtjzc1ZWPWaUYtPAbMA0hydJJ1wFrgfW2jGgr4fJLbkiwb78mTLEuyJsmaLVu29OSGJEmSNDs5skF9s+CcG6e0/R9seHhanmfjhb82pe1rVskYZZ0jFMatU1W3Aock+XngyiSfraqngDdX1eYkewFfSPJPVfWVFzRStQJYATAyMuLICEmSJO00RzZI0uDYBOzb9ngfYPN4dZLMBfYAHmmvUFX3AE8ChzaPNzf/PgjcQGu6hiRJkjRlTDZI0uBYDRyc5IAkuwBLgZUddVYCZzTHJwI3V1U118wFSLI/8HpgY5Ldk7y8Kd8dOI7WYpKSJGkWOPDcAznw3AP7HYZmIadRSNKAaHaSOBu4CZgDXF5V65JcAKypqpXAZcBVSdbTGtGwtLn8LcA5SX4CPAe8v6oeSnIgcEMSaPX5V1fV56b3ziRJkjTbmGzQ0PqZUy/sdwjSDquqVcCqjrLz246fAk4a47qrgKvGKN8AvKH3kWpoLN9jatvf+OTUP8/yx6aubalNksXAx2glhD9RVRd2nH8b8KfA4cDSqrq+4/wrgHuAG6rq7OmJWpL6w2kUkiRJ0gSSzAEuBY4HFgKnJFnYUe1+4Ezg6nGa+S/Al6cqRkkaJCYbJEmSpIkdBayvqg1V9QxwDbCkvUJVbayqO2lNZ3ueJEcCrwE+Px3BSlK/mWyQJEmSJrY38EDb401N2YSSvAj478DvTFBvWZI1SdZs2bJlpwOVpEFgskGSJEmaWMYoqy6vfT+wqqoe2F6lqlpRVSNVNTJ//vwdDlCSBokLREqSJEkT2wTs2/Z4H2Bzl9f+EvDWJO8HXgbskuSJqjqnxzFK0sDoamRDksVJ7k2yPskLOsUkuya5tjl/a5IFTfmLk1yZZG2Se5Kc29vwB9OiRYtYtGhRv8OQJElS76wGDk5yQJJdaG09vLKbC6vqtKrar6oWAB8GPmWiQdKwmzDZ0OXKu2cBj1bVQcDFwEVN+UnArlV1GHAk8N7RRIQkSZI0U1TVVuBs4CZa21deV1XrklyQ5F0ASd6UZBOtz8AfT7KufxFLUn91M41i28q7AElGV969u63OEmB5c3w9cEmS0JrHtnuSucBLgGeAH/YmdEmSJGn6VNUqYFVH2fltx6tpTa/YXhufBD45BeFJ0kDpJtkw1sq7R49Xp6q2JnkMmEcr8bAE+D7wUuC3q+qRyQYtaWYZnVZ0yy239DUOSdPvljN373cIkiSpD7pZs6GblXfHq3MU8CzwOuAA4P9NcuALnsBtfiRJkiRJGhrdJBu6WXl3W51mysQewCPAqcDnquonVfUg8HVgpPMJ3OZHkiRJkqTh0c00im0r7wLfo7Xy7qkddVYCZwDfAE4Ebq6qSnI/8PYkn6Y1jeIXgT/tVfA7a8E5N05p+z/Y8PCUP8/GC39tytqWpJnOqTuSJEn9NeHIhm5W3gUuA+YlWQ98CBjdyudSWnsJ30UraXFFVd3Z43uQJEmSJEkDpJuRDd2svPsUrS1+Oq97YqxySZIkSZI0vLpZs0GSJEmSJKlrXY1skDTcXMdEkoab65hIkqabIxskaYAkWZzk3iTrk5wzxvldk1zbnL81yYKm/KgkdzQ/307y6922KUmSJPWaIxumwM+cemG/Q5A0AyWZQ2th3WNpbSm8OsnKqrq7rdpZwKNVdVCSpcBFwMm0FuIdqaqtSV4LfDvJ3wHVRZuSJElSTzmyQZIGx1HA+qraUFXPANcASzrqLAGubI6vB45Jkqr6UbN7EMButJIM3bYpSZIk9ZQjGyRpcOwNPND2eBNw9Hh1mlEMjwHzgIeSHA1cDuwP/GZzvps2AUiyDFgGsN9++03+brZn+R5T2/7GJ6fneZY/NrXtS5IkzVCObJCkwZExyqrbOlV1a1UdArwJODfJbl22SXP9iqoaqaqR+fPn70DYkiRJ0vM5skHSlHMdk65tAvZte7wPsHmcOpuSzAX2AB5pr1BV9yR5Eji0yzYlSZKknnJkgyQNjtXAwUkOSLILsBRY2VFnJXBGc3wicHNVVXPNXIAk+wOvBzZ22aYkSZLUUyYbJGlANAs8ng3cBNwDXFdV65JckORdTbXLgHlJ1gMfAka3snwLrR0o7gBuAN5fVQ+N1+b03ZUkDY8utid+W5JvJdma5MS28iOSfCPJuiR3Jjl5eiOXpOnnNApJGiBVtQpY1VF2ftvxU8BJY1x3FXBVt21KGjAumjrwutye+H7gTODDHZf/CDi9qv45yeuA25LcVFX/Og2hS1JfmGyQJEmSJrZtK2GAJKNbCW9LNlTVxubcc+0XVtX/13a8OcmDwHzAZIOkoWWyQZI0dG45c/d+hyBp+HS9lfD2JDkK2AX4zhjnpm8LYkmaYq7ZIEmSJE2s662Ex20geS2tKW//V1U913neLYglDROTDZIkSdLEJrWVcJJXADcCv19V/9jj2CRp4JhskCRJkia201sJN/VvAD5VVX81hTFK0sAw2SBJkiRNoJvtiZO8KckmWrsGfTzJ6FbDvwG8DTgzyR3NzxF9uA1JmjYuEClJkiR1oYvtiVfTml7Red2ngU9PeYCSNEAc2SBJkiRJknrKkQ2SJElDzu1gJUnTzZENkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkDZAki5Pcm2R9knPGOL9rkmub87cmWdCUH5vktiRrm3/f3nbNLU2bdzQ/e03fHUmSJGk2mtvvACRJLUnmAJcCxwKbgNVJVlbV3W3VzgIeraqDkiwFLgJOBh4CTqiqzUkOBW4C9m677rSqWjMtNyJJkqRZz5ENkjQ4jgLWV9WGqnoGuAZY0lFnCXBlc3w9cEySVNXtVbW5KV8H7JZk12mJWpIkSepgskGSBsfewANtjzfx/NEJz6tTVVuBx4B5HXXeA9xeVU+3lV3RTKE4L0l6G7YkSZL0fCYbJGlwjJUEqB2pk+QQWlMr3tt2/rSqOgx4a/Pzm2M+ebIsyZoka7Zs2bJDgUuSJEntuko27OyCZc25w5N8I8m6ZuGy3XoXviQNlU3Avm2P9wE2j1cnyVxgD+CR5vE+wA3A6VX1ndELqup7zb+PA1fTmq7xAlW1oqpGqmpk/vz5PbkhSZIkzU4TJhvaFiw7HlgInJJkYUe1bQuWARfT+lZt9IPwp4H3VdUhwCLgJz2LXpKGy2rg4CQHJNkFWAqs7KizEjijOT4RuLmqKsmewI3AuVX19dHKSeYmeXVz/GLgncBdU3wfkjSUuvgC7m1JvpVka5ITO86dkeSfm58zOq+VpGHTzciGnV6wDDgOuLOqvg1QVQ9X1bO9CV2ShkuzBsPZtHaSuAe4rqrWJbkgybuaapcB85KsBz4EjH7YPRs4CDivY4vLXYGbktwJ3AF8D/jL6bsrSRoOXX4Bdz9wJq1RZO3Xvgr4A+BoWp+t/yDJK6c6Zknqp262vhxrwbKjx6tTVVuTjC5Y9rNAJbkJmA9cU1V/3PkESZYBywD222+/Hb0HSRoaVbUKWNVRdn7b8VPASWNc91Hgo+M0e2QvY5SkWWrbF3AASUa/gNu2PXFVbWzOPddx7TuAL1TV6LS3LwCLgc9MfdiS1B/djGyYzIJlc4G3AKc1//56kmNeUNF5wpIkSRps3ewYNKlrXahX0jDpJtkwmQXLNgFfrqqHqupHtL6t+4XJBi1JkiRNs26+gJvUtX4BJ2mYdJNs2OkFy2jNOz48yUubJMQv0zbUTJIkSZohuvkCbiqulaQZacJkw2QWLKuqR4E/oZWwuAP4VlXd2PvbkCRJkqZUN1/Ajecm4Lgkr2wWhjyuKZOkodXNApE7vWBZc+7TtLa/lCRJkmakZhH00S/g5gCXj34BB6ypqpVJ3gTcALwSOCHJR6rqkKp6JMl/oZWwALhgdLFISRpWXSUbJEmSpNmuiy/gVtOaIjHWtZcDl09pgJI0QLpZs0GSJEmSJO2kRYsWsWjRon6HMa1MNkiSJEmSpJ4y2SBJkiRJknrKNRskSZIkSbPeYVceNmVtb/jBhil/DoC1Z6yd0vZ3hCMbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZRrNkiSJEnqudFt/m655Za+xiENggPPPbDfIUw7RzZIkiRJkqSecmSDJEmSNAtN9ar407H6/iCtvC/p+RzZIEmSJEmSespkgyQNkCSLk9ybZH2Sc8Y4v2uSa5vztyZZ0JQfm+S2JGubf9/eds2RTfn6JH+WJNN3R5IkSZqNnEYhSQMiyRzgUuBYYBOwOsnKqrq7rdpZwKNVdVCSpcBFwMnAQ8AJVbU5yaHATcDezTV/ASwD/hFYBSwGPjsd9yRJmr1m44J4kn7KkQ2SNDiOAtZX1Yaqega4BljSUWcJcGVzfD1wTJJU1e1VtbkpXwfs1oyCeC3wiqr6RlUV8Cng3VN/K5IkSZrNTDZI0uDYG3ig7fEmfjo64QV1qmor8Bgwr6POe4Dbq+rppv6mCdoEIMmyJGuSrNmyZctO34QkSZJkskGSBsdYaynUjtRJcgitqRXv3YE2W4VVK6pqpKpG5s+f30W4kjS7TGJdnRcnubJZP+eeJOdOd+ySNN1MNkjS4NgE7Nv2eB9g83h1kswF9gAeaR7vA9wAnF5V32mrv88EbUqSJtC2rs7xwELglCQLO6ptW1cHuJhW8hfgJGDXqjoMOBJ472giQpKGlckGSRocq4GDkxyQZBdgKbCyo85K4Izm+ETg5qqqJHsCNwLnVtXXRytX1feBx5P8YrMLxenA3071jUjSENrpdXVojSjbvUkSvwR4Bvjh9IQtSf1hskGSBkSzBsPZtHaSuAe4rqrWJbkgybuaapcB85KsBz4EjA7jPRs4CDgvyR3Nz17Nud8CPgGsB76DO1FI0s6YzLo61wNPAt8H7gf+W1U90vkErp0jaZi49aUkDZCqWkVre8r2svPbjp+iNRy387qPAh8dp801wKG9jVSSZp3JrKtzFPAs8DrglcBXk/xDVW14XsWqFcAKgJGRkTHX15GkmcKRDZIkSdLEJrOuzqnA56rqJ1X1IPB1YGTKI5akPjLZIEmSJE1sp9fVoTV14u1p2R34ReCfpiluSeoLkw2SJEnSBCa5rs6lwMuAu2glLa6oqjun9QYkaZq5ZoMkSZLUhUmsq/PEWOWSNMwc2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqqa6SDUkWJ7k3yfok54xxftck1zbnb02yoOP8fkmeSPLh3oQtSZIkSZIG1YTJhiRzgEuB44GFwClJFnZUOwt4tKoOAi4GLuo4fzHw2cmHK0mSJEmSBl03IxuOAtZX1Yaqega4BljSUWcJcGVzfD1wTJIAJHk3sAFY15uQJUmSJEnSIOsm2bA38EDb401N2Zh1qmor8BgwL8nuwO8BH5l8qJIkSZIkaSboJtmQMcqqyzofAS6uqie2+wTJsiRrkqzZsmVLFyFJkiRJkqRBNbeLOpuAfdse7wNsHqfOpiRzgT2AR4CjgROT/DGwJ/Bckqeq6pL2i6tqBbACYGRkpDORIUmSJEmSZpBuRjasBg5OckCSXYClwMqOOiuBM5rjE4Gbq+WtVbWgqhYAfwr8UWeiQZL0Uzu7+0+SeUm+1Oz8c0nHNbc0bd7R/Ow1PXcjSZKk2WrCkQ1VtTXJ2cBNwBzg8qpal+QCYE1VrQQuA65Ksp7WiIalUxm0JA2jtt1/jqU1Ymx1kpVVdXdbtW27/yRZSmv3n5OBp4DzgEObn06nVdWaKb0BSZIkqdHNNAqqahWwqqPs/Lbjp4CTJmhj+U7EJ0mzybbdfwCSjO7+055sWAIsb46vBy5Jkqp6EvhakoOmMV5JkiRpTN1Mo5AkTY+d3v2ni7avaKZQnDe6NXEnF+uVpO3b2aluzbnDk3wjyboka5PsNp2xS9J0M9kgSYNjMrv/bM9pVXUY8Nbm5zfHqlRVK6pqpKpG5s+fP2GwkjSbtE11Ox5YCJySZGFHtW1T3YCLaU11o1lA/dPA+6rqEGAR8JNpCl2S+sJkgyQNjh3Z/Wf0w+vo7j/jqqrvNf8+DlxNa7qGJGnHbJvqVlXPAKNT3dotAa5sjq8HjmlGkx0H3FlV3waoqoer6tlpiluS+sJkgyQNjp3e/We8BpPMTfLq5vjFwDuBu3oeuSQNv8lMdftZoJLclORbSX53rCdwOpukYdLVApGSpKk32d1/kmwEXgHskuTdtL5Juw+4qUk0zAH+AfjLabwtSRoWk5nqNhd4C/Am4EfAF5PcVlVffF7FqhXACoCRkZGJpshJ0kAz2SBJA2Qyu/9U1YJxmj2yV/FJ0iy2I1PdNnVMddsEfLmqHgJIsgr4BeCLSNKQchqFJEmSNLHJTHW7CTg8yUubJMQv8/xtjSVp6DiyQZIkSZrAZKa6VdWjSf6EVsKigFVVdWNfbkSSponJBkmSJKkLk5zq9mla219K0qzgNApJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskaYAkWZzk3iTrk5wzxvldk1zbnL81yYKmfF6SLyV5IsklHdccmWRtc82fJcn03I0kSZJmK5MNkjQgkswBLgWOBxYCpyRZ2FHtLODRqjoIuBi4qCl/CjgP+PAYTf8FsAw4uPlZ3PvoJUmSpJ8y2SBJg+MoYH1VbaiqZ4BrgCUddZYAVzbH1wPHJElVPVlVX6OVdNgmyWuBV1TVN6qqgE8B757Su5CkIbWzo8/azu/XjEAbKzEsSUPFZIMkDY69gQfaHm9qysasU1VbgceAeRO0uWmCNgFIsizJmiRrtmzZsoOhS9Jwm+Tos1EXA5+d6lglaRCYbJCkwTHWWgq1E3V2qn5VraiqkaoamT9//naalKRZaadHnwEkeTewAVg3TfFKUl+ZbJCkwbEJ2Lft8T7A5vHqJJkL7AE8MkGb+0zQpiRpYjs9+izJ7sDvAR+ZhjglaSCYbJCkwbEaODjJAUl2AZYCKzvqrATOaI5PBG5u1mIYU1V9H3g8yS82366dDvxt70OXpKE3mdFnHwEurqontvsETmeTNETm9jsASVJLVW1NcjZwEzAHuLyq1iW5AFhTVSuBy4CrkqynNaJh6ej1STYCrwB2aYbrHldVdwO/BXwSeAmtucLOF5akHbcjo882dYw+Oxo4MckfA3sCzyV5qqqet1VxVa0AVgCMjIxsb4qcJA08kw2SNECqahWwqqPs/Lbjp4CTxrl2wTjla4BDexelJM1K20afAd+jlew9taPO6Oizb/D80WdvHa2QZDnwRGeiQZKGjckGSZIkaQKTHX0mSbNNV8mGJIuBj9HqWD9RVRd2nN+V1t7tRwIPAydX1cYkxwIXArsAzwC/U1U39zB+SZIkaVpMZvRZW53lUxKcJA2YCReInOSewg8BJ1TVYbSGlF3Vq8AlSZIkSdJg6mY3ip3eU7iqbq+q0YVz1gG7NaMgJEmSJEnSkOom2bDTewp31HkPcHtVPd35BG7zI0mSJEnS8Ogm2TCZPYVbJ5NDaE2teO9YT1BVK6pqpKpG5s+f30VIkiRJkiRpUHWTbNiRPYXp2FOYJPsANwCnV9V3JhuwJEmSJEkabN0kG7btKZxkF1pb+KzsqDO6pzC07SmcZE/gRuDcqvp6r4KWJEmSJEmDa8JkQ7MGw+iewvcA143uKZzkXU21y4B5zZ7CHwLOacrPBg4CzktyR/OzV8/vQpIkSZIkDYy53VTa2T2Fq+qjwEcnGaMkSZIkSZpBuplGIUmSJEmS1DWTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZI0gBJsjjJvUnWJzlnjPO7Jrm2OX9rkgVt585tyu9N8o628o1J1ia5I8ma6bkTSZIkzWYmGyRpQCSZA1wKHA8sBE5JsrCj2lnAo1V1EHAxcFFz7UJgKXAIsBj486a9Ub9SVUdU1cgU34YkDa2dTQgnOTbJbU3i97Ykb5/u2CVpuplskKTBcRSwvqo2VNUzwDXAko46S4Arm+PrgWOSpCm/pqqerqrvAuub9iRJPTCZhDDwEHBCVR0GnAFcNT1RS1L/mGyQpMGxN/BA2+NNTdmYdapqK/AYMG+Cawv4fPNt2rIpiFuSZoOdTghX1e1VtbkpXwfslmTXaYlakvrEZIMkDY6MUVZd1tnetW+uql+g9W3cB5K8bcwnT5YlWZNkzZYtW7qNWZJmi8kkhNu9B7i9qp6eojglaSCYbJCkwbEJ2Lft8T7A5vHqJJkL7AE8sr1rR79Nq6oHgRsYZ3pFVa2oqpGqGpk/f/6kb0aShsxkEsKtk8khtKZWvHfMJzDpK2mImGyQpMGxGjg4yQFJdqG14OPKjjorac33BTgRuLmqqilf2ixOdgBwMPDNJLsneTlAkt2B44C7puFeJGnYTCYhTJJ9aCV8T6+q74z1BCZ9JQ2Tuf0OQJLUUlVbk5wN3ATMAS6vqnVJLgDWVNVK4DLgqiTraX2AXdpcuy7JdcAg4wkmAAAKIElEQVTdwFbgA1X1bJLXADe01pBkLnB1VX1u2m9Okma+bQlh4Hu0+t9TO+qMJoS/QVtCOMmewI3AuVX19WmMWZL6xmSDJA2QqloFrOooO7/t+CngpHGu/UPgDzvKNgBv6H2kkjS7TCYhDJwNHAScl+S8puy4ZnqbJA0lkw2SJElSF3Y2IVxVHwU+OuUBStIAcc0GSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPdVVsiHJ4iT3Jlmf5Jwxzu+a5Nrm/K1JFrSdO7cpvzfJO3oXuiQNn6nobydqU5LUHT8TS1L3Jkw2JJkDXAocDywETkmysKPaWcCjVXUQcDFwUXPtQmApcAiwGPjzpj1JUoep6G+7bFOSNAE/E0vSjulmZMNRwPqq2lBVzwDXAEs66iwBrmyOrweOSZKm/Jqqerqqvgusb9qTJL3QVPS33bQpSZqYn4klaQd0k2zYG3ig7fGmpmzMOlW1FXgMmNfltZKklqnob+2HJak3/EwsSTtgbhd1MkZZdVmnm2tJsgxY1jx8Ism9XcQ16F4NPDRVjeeiqWp56Ezp+wC+FztgGP4m9p/i9qeivx0rqfyCfhiGsi+e8r9/PjLWy64xTO174fvQrWH4m5jqfnh7/Ey8c6b2v/9n+vffpan/TOx70Y1heR+66ou7STZsAvZte7wPsHmcOpuSzAX2AB7p8lqqagWwopuAZ4oka6pqpN9xzHa+D4PD96IrU9XfTtgPw/D1xf7ODQ7fi8Hg+zBpfibeCf7eDQbfh8Ew296HbqZRrAYOTnJAkl1oLW6zsqPOSuCM5vhE4OaqqqZ8abMy7wHAwcA3exO6JA2dqehvu2lTkjQxPxNL0g6YcGRDVW1NcjZwEzAHuLyq1iW5AFhTVSuBy4Crkqynlb1d2ly7Lsl1wN3AVuADVfXsFN2LJM1oU9XfjtXmdN+bJM10fiaWpB2TVrJVvZZkWTMUTn3k+zA4fC803fydGxy+F4PB90H94O/dYPB9GAyz7X0w2SBJkiRJknqqmzUbJEmSJEmSumayAUjybJI7ktyV5K+SvLRH7b4myd8n+XaSu5Osasq/m+T1HXX/NMnvJlmU5LEktye5N8lXkryzF/EMGl/34ZDk/3Q8/u0kTyXZo61sUZK/H+Padzav+eh79d7piFmDx/6gP3zdh4P9sHrFPqE/fN2Hg33xC5lsaPlxVR1RVYcCzwDv6/bCtLY1Gs8FwBeq6g1VtRA4pym/hmbBoKaNF9FasfjapuirVfXGqno98EHgkiTHdH87M4av+xCoqn/bUXQKrRW7f3171yV5Ma3tvU6oqjcAbwRumYoYNSPYH/SHr/sQsB9WD9kn9Iev+xCwL34hkw0v9FXgoCQLktw1Wpjkw0mWN8e3JPmjJF8G/mOS+Un+Osnq5ufNzWWvpbWvMgBVdWdz+Bna/sCBtwEbq+q+zmCq6g5aHcXZPbzHQTRjXvckc5J8ssk+r03y25O68xkuyRNtx/8GeBnw+7Q62O15Oa0dcR4GqKqnq+reqYpTM8qM6Q+GzIx53e2Hn89+WFNkxvQJQ2bGvO72xc9nX/xCJhvaNJnB44G1XVTfs6p+uar+O/Ax4OKqehPwHuATTZ1LgcuSfCnJf07yOtj2h/5ckjc09ZbS+qMfz7eAn9vxO5oZZuDrfgSwd1UdWlWHAVd0EfdscQqt1/SrwOuT7DVexap6hNa+4/cl+UyS05rMumaxGdgfDIUZ+LrbD4/PfliTNgP7hKEwA193++Lx2RdjsmHUS5LcAawB7qe1R/JErm07/lVaw4vuoPWL8ookL6+qm4ADgb+k9Qd6e5L5zTWfAZY2ncoS4K+281zZobuZOWbq674BODDJ/0iyGPhhF3HPFkuBa6rqOeBvgJO2V7mq/h/gGOCbwIeBy6c8Qg2qmdofzHQz9XW3Hx6f/bAmY6b2CTPdTH3d7YvHZ19Ma7iGmnlS7QVJtvL8ZMxuHdc82Xb8IuCXqurHnQ03maqrgavTWgzkbcBf0/oD/zzwZeDOqnpwO/G9Ebiny3uZSWbE655kDnBbU7ayqs5vMsHvAD4A/Abwf2/3TmeBJIcDBwNfSAKwC63/CF26veuqai2wNslVwHeBM6c2Ug2oGdEfdHkvM8mMeN3th7tjP6wemBF9Qpf3MpPMiNfdvrg79sU/5ciG8f0LsFeSeUl2Bba3CuvnaZvHlOSI5t+3p1lNNsnLgX9DK1tJVX2H1rycC9nOsKXml/U8JvjlHCID97pX1bPNoj1HNJ3qq4EXVdVfN3V+Yedvd6icAiyvqgXNz+uAvZPsP1blJC9Lsqit6AjgBXMFNasNXH8wqbuZOQbudbcf7pr9sKbCwPUJk7qbmWPgXnf74q7ZFzcc2TCOqvpJkguAW2lllv5pO9U/CFya5E5ar+lXaK0ieyStIU2jmclPVNXqtus+A/xX4IaO9t6a5HbgpcCDwAer6os9uK2BN0Ne972BK9rmUp27I/c4xJbSmmfY7oam/FbgmCSb2s6dAvxuko8DP6aVoT9zGuLUDDFD+oOhM0Ned/vhsdkPq+dmSJ8wdGbI625fPDb74kaqqt8xSJIkSZKkIeI0CkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhs0qyTZ2OwJPKk6kqSdZ18sSf1lP6zpYLJBkiRJkiT1lMkGDbwkC5L8U5JPJLkryf9K8qtJvp7kn5McleRVSf53kjuT/GOSw5tr5yX5fJLbk3wcSFu7/z7JN5PckeTjSeb07SYlacDZF0tSf9kPa6Yx2aCZ4iDgY8DhwM8BpwJvAT4M/CfgI8DtVXV48/hTzXV/AHytqt4IrAT2A0jy88DJwJur6gjgWeC0absbSZqZ7Islqb/shzVjzO13AFKXvltVawGSrAO+WFWVZC2wANgfeA9AVd3cZG/3AN4G/Lum/MYkjzbtHQMcCaxOAvAS4MFpvB9JmonsiyWpv+yHNWOYbNBM8XTb8XNtj5+j9Xu8dYxrquPfdgGurKpzexahJA0/+2JJ6i/7Yc0YTqPQsPgKzZCvJIuAh6rqhx3lxwOvbOp/ETgxyV7NuVcl2X+6g5akIWNfLEn9ZT+sgeHIBg2L5cAVSe4EfgSc0ZR/BPhMkm8BXwbuB6iqu5P8PvD5JC8CfgJ8ALhvugOXpCGyHPtiSeqn5dgPa0CkaqzRNJIkSZIkSTvHaRSSJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6qn/H/5Eue+p2MGUAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_cv_results(cv_results);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The difference between PureSVD and iALS is not significant. In contrast, the advantage of the scaled version of PureSVD denoted as `PureSVD-s` over the other models is much more pronounced making it a clear favorite. Interestingly, the difference is especially pronounced in terms of the `coverage` metric, which is defined as the ratio of unique recommendations generated for all test users to the total number of items in the training data. This indicates that generated recommendations are not only more relevant but also are significantly more diverse. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## comparing training time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another important practical aspect is how long does it take to compute a model? Sometimes the best model in terms of quality of recommendations can be the slowest to compute. You can check each model's training time by accessing the `training_time` list attribute. It holds the history of trainings, hence, if you have just performed 5-fold CV experiment, the last 5 entries in this list will correspond to the training time on each fold. This information can be used to get average time with some error bounds as shown below." + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "timings = {}\n", + "for model in models:\n", + " timings[f'{model.method} rank {model.rank}'] = model.training_time[-5:]" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHNhJREFUeJzt3XuYHFWd//H3BxJQIYCS4ZKQkBWzcdGFgHkiLsqOgiw3xXVRE1wgKxpvrOJlFV1XWRCN/lS8RMWAWeAnBBY1muUisCjLRUAHNoSAIDEGEwIkXAQiICR8949zBiqd7plOd89MJufzep5+prrqVJ1TVd2frjpd1aOIwMzMyrHFUDfAzMwGl4PfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn7rGEmvk3TXINX1TklXDFJd+0u6W9IaSW8ZjDo3Rm7XSwdguTMkXdfp5fZT586SrpH0uKSvSvq0pLMGsw0lcPAPAklHS+rJb9D7JF0m6bVD3a4qSd2SVmzkPCHpZb3PI+LaiJg0AG2bkOsaUanrvIg4uNN1NXAKMDsito2InwxSnXVJulrSu6vjcruWDlWbOmwm8CCwXUR8LCK+EBHv7m8m2zgO/gEm6aPA14EvADsD44HvAEcOZbtso+wO3N7KjNUPK2vK7sAd4TtLB1ZE+DFAD2B7YA3wtj7KbE36YFiZH18Hts7TuoEVwCeAVcB9wFuAw4DfAg8Dn64s62Tgh8CFwOPALcDelekBvKzy/Gzg88A2wJPAs7m9a4AxwFTgBuCPue7ZwFZ53mvy8v6Uy7+jt72V5f8VcHWe/3bgzTV1fxu4JLf1JmCPBtvoD7mu3ra9BpgBXFezbh8A7s7LOxXYI7f/MeA/e9ueyx8BLMxt+yWwV4O6f5e3y5O57q3ztlmQt/8S4D119sEPcr3vbvC6OBdYDdwDfAbYIk+bAVwPfAt4FLgTODBPOw1YBzyV2zK7dr/m7fod4LJc5npgF9Lr6pG8vH0qbTkpr+PjwB3A31emrbeN66zHa/O2+yOwHJjR5PpdB3wlt+f3wKGVtj8DPJ3bflDenj+o1HlsXuZDwL8By4CDmnw/fhK4N6/rXb3btcTHkDdgc34AhwBrgRF9lDkFuBHYCejKb6RT87TuPP9ngZHAe/Kb6XxgFPCKHAIvzeVPzm+co3L5j+c31sg8vW7wV+paUdO2VwH7ASOACcBvgBMr02uX99wycv1LgE8DWwFvyG+4SZW6HyZ9uIwAzgMuaLCNJuS6RlTGzWDD4F8AbJe3y5+Bq4CXkoLoDuC4XHZf0gfpq4EtgeNygGzdoP71wgX4H1K4vgCYnPfJgTX74C2kM+oX1lneucBP8z6cQPoQP76yXmuBj+Rt+A7SB8BL8vSrqfkwYcPgfzDvuxcAP8+vgWPzun4e+EVl3reRPsi2yHX9Cdi13jauqXN83p/Tczt3BCY3uX7PkF7LWwLvJx3wqPY1WdmeP8jDe5I+EF5Lek19JS+r3+AHJpE+nMZUXlN1DzRKeLirZ2DtCDwYEWv7KPNO4JSIWBURq4F/B46pTH8GOC0ingEuAEYD34iIxyPidtKR9F6V8jdHxA9z+a+R3vz7tdL4iLg5Im6MiLURsQz4HvC3Tc6+H7AtMCsino6InwMXk4Ki148j4ld5+5xHCtF2fCkiHsvbZTFwRUQsjYhHSUfA++Ry7wG+FxE3RcS6iDiH9EHR73aSNI4UPJ+MiKciYiFwFuvvsxsi4icR8WxEPFkz/5akgP1U3ofLgK/WzL8K+HpEPBMRF5KOTg/fiO0wP++7p4D5wFMRcW5ErCOdDfZuByLioohYmdt6IemMaWoTdbwT+O+ImJfb+VBELGxy/e6JiDNze84BdiV1g/bnKOC/IuK6iHiadEDUbJfQOtLZ2p6SRkbEsoj4XZPzbnYc/APrIWB0P/28Y0inrr3uyeOeW0Z+g0DqbgB4oDL9SVLA9lreOxARz5K6iqrLa5qkv5R0saT7JT1G+p5idJOzjwGW5zb0ugcYW3l+f2X4CdZfj1bUbpdG22l34GOS/tj7AMbR3HYaAzwcEY9XxtWu13IaG006Wq3d59X57418WFqZvjH7sNntgKRjJS2sbIdX0tw+HkfqIqrVzPo9t98j4ok82My+H8P6r+8nSO+xfkXEEuBE0hnEKkkXSGrpfbE5cPAPrBtIXTF9XQK4khREvcbnca0a1zsgaQtgt8ryngBeVCm7S2W43pHTd0l9whMjYjtSt42abMdKYFxuQ6/xpD7WjdXpL/qWk86idqg8XhQR85qYdyXwEkmjKuNq16uv9j5IOour3efV+cdKUs303n3YsW0haXfgTOAEYMeI2IF0ptTMPl5O+g6lVjPr16r7SK9nACS9kHRW3ZSIOD8iXpvbFsCXOtCmYcnBP4ByF8NngW9LeoukF0kaKelQSV/OxeYBn5HUJWl0Lv+DNqp9laS35rOME0ldGDfmaQuBoyVtKekQ1u+2eQDYUdL2lXGjSF9QrpH0clJ/LDXzNLp+/CZSf/En8jp3A28idVdtrNWkL1g7da36mcD7JL1ayTaSDq8J87oiYjnpe5gvSnqBpL2A40ldVf3KZ2//CZwmaVQO34+y/j7fCfhQ3m5vI31Jfmme1tc231jbkAJwNYCkfyId8TfjPOAgSW+XNELSjpImN7l+rfoh8CZJfyNpK1K36HMfUvmS5LofjJImSXqDpK1JB2NPkrp/iuTgH2AR8TXSC/8zpDfYctIRVu/14J8HeoBFwG2kK3E+30aVPyX1sT5C6ld9a+7vB/gwKXz/SOqjfe6a9Ii4k/QhtDSf9o8hfTl8NOlLvDNJ/cNVJwPn5PJvr1nvp4E3A4eSjgK/Axyb69ko+ZT+NOD6XFdL31lUltdD6uefTdpOS0hfOjZrOunLwZWkPvTPRcSVGzH/P5M+FJeSrnA5H5hbmX4TMJG03U4DjoqI3i6NbwBHSXpE0jc3os4NRMQdpP73G0gfKH9NugqomXn/QLq67GOkL+kXAnvnyf2tX6vtvT0v+wLS0f/jpO9D/pyLjMvrUs/WwCzSNr2f9OH66XbbNFxp/a5EG84knUy6uuMfh7ot1hpJM0hX7WxSN/htiiRtSzqImRgRv893+F4UEZcPcdM2eb65xMyGDUlvIl2mK9LlnLeRLrclfIdv09zVY2bDyZE8f7PjRGBauNtio7mrx8ysMD7iNzMrzCbZxz969OiYMGHCUDfDzGzYuPnmmx+MiK5mym6SwT9hwgR6enqGuhlmZsOGpHv6L5W4q8fMrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjMrQnd3N93d3UPdjE2Cg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDCb5P/cNbMyTDjpkkGr6/6lDw1qnctmHT4o9bTCR/xmZoVx8JuZFabfrh5Jc4EjgFUR8co87kJgUi6yA/DHiJhcZ95lwOPAOmBtREzpULvNzKxFzfTxnw3MBs7tHRER7+gdlvRV4NE+5n99RDzYagPNzKyz+g3+iLhG0oR60yQJeDvwhs42y8zMBkq7ffyvAx6IiLsbTA/gCkk3S5rZ14IkzZTUI6ln9erVbTbLzMwaaTf4pwPz+pi+f0TsCxwKfFDSAY0KRsSciJgSEVO6urrabJaZmTXScvBLGgG8FbiwUZmIWJn/rgLmA1Nbrc/MzDqjnSP+g4A7I2JFvYmStpE0qncYOBhY3EZ9ZmbWAf0Gv6R5wA3AJEkrJB2fJ02jpptH0hhJl+anOwPXSboV+BVwSUT8rHNNNzOzVjRzVc/0BuNn1Bm3EjgsDy8F9m6zfWZmHbHL0bOGugmbDN+5a2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhmvmfu3MlrZK0uDLuZEn3SlqYH4c1mPcQSXdJWiLppE423MzMWtPMEf/ZwCF1xp8eEZPz49LaiZK2BL4NHArsCUyXtGc7jTUzs/b1G/wRcQ3wcAvLngosiYilEfE0cAFwZAvLMTOzDmqnj/8ESYtyV9CL60wfCyyvPF+Rx9UlaaakHkk9q1evbqNZZmbWl1aD/7vAHsBk4D7gq3XKqM64aLTAiJgTEVMiYkpXV1eLzTIzs/60FPwR8UBErIuIZ4EzSd06tVYA4yrPdwNWtlKfmZl1TkvBL2nXytO/BxbXKfZrYKKkv5C0FTANWNBKfWZm1jkj+isgaR7QDYyWtAL4HNAtaTKp62YZ8N5cdgxwVkQcFhFrJZ0AXA5sCcyNiNsHZC3MzKxp/QZ/REyvM/r7DcquBA6rPL8U2OBSTzMzGzq+c9fMrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwvQb/JLmSlolaXFl3P+TdKekRZLmS9qhwbzLJN0maaGknk423MzMWtPMEf/ZwCE1464EXhkRewG/BT7Vx/yvj4jJETGltSaamVkn9Rv8EXEN8HDNuCsiYm1+eiOw2wC0zczMBkAn+vjfBVzWYFoAV0i6WdLMDtRlZmZtGtHOzJL+FVgLnNegyP4RsVLSTsCVku7MZxD1ljUTmAkwfvz4dpplZmZ9aPmIX9JxwBHAOyMi6pWJiJX57ypgPjC10fIiYk5ETImIKV1dXa02y8zM+tFS8Es6BPgk8OaIeKJBmW0kjeodBg4GFtcra2Zmg6eZyznnATcAkyStkHQ8MBsYReq+WSjpjFx2jKRL86w7A9dJuhX4FXBJRPxsQNbCzMya1m8ff0RMrzP6+w3KrgQOy8NLgb3bap2ZmXWc79w1MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjQV/JLmSlolaXFl3EskXSnp7vz3xQ3mPS6XuVvScZ1quJmZtabZI/6zgUNqxp0EXBURE4Gr8vP1SHoJ8Dng1cBU4HONPiDMzGxwNBX8EXEN8HDN6COBc/LwOcBb6sz6d8CVEfFwRDwCXMmGHyBmZjaI2unj3zki7gPIf3eqU2YssLzyfEUetwFJMyX1SOpZvXp1G80yM7O+DPSXu6ozLuoVjIg5ETElIqZ0dXUNcLPMzMrVTvA/IGlXgPx3VZ0yK4Bxlee7ASvbqNPMzNrUTvAvAHqv0jkO+GmdMpcDB0t6cf5S9+A8zszMhkizl3POA24AJklaIel4YBbwRkl3A2/Mz5E0RdJZABHxMHAq8Ov8OCWPMzOzITKimUIRMb3BpAPrlO0B3l15PheY21LrzMys43znrplZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv1mTuru76e7uHupmmLXNwW9mVhgHv5lZYZr6rR6zTdWEky4ZtLruX/rQoNe5bNbhg1aXlcNH/GZmhXHwm5kVxsFvZlYY9/GbNWmXo2cNdRPMOsJH/GZmhXHwm5kVpuXglzRJ0sLK4zFJJ9aU6Zb0aKXMZ9tvspmZtaPlPv6IuAuYDCBpS+BeYH6dotdGxBGt1mNmZp3Vqa6eA4HfRcQ9HVqemZkNkE4F/zRgXoNpr5F0q6TLJL2i0QIkzZTUI6ln9erVHWqWmZnVajv4JW0FvBm4qM7kW4DdI2Jv4FvATxotJyLmRMSUiJjS1dXVbrPMzKyBThzxHwrcEhEP1E6IiMciYk0evhQYKWl0B+o0M7MWdSL4p9Ogm0fSLpKUh6fm+h7qQJ1mZtaitu7clfQi4I3Aeyvj3gcQEWcARwHvl7QWeBKYFhHRTp1mZtaetoI/Ip4AdqwZd0ZleDYwu506zMyss3znrplZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFaTv4JS2TdJukhZJ66kyXpG9KWiJpkaR9263TzMxa19b/3K14fUQ82GDaocDE/Hg18N3818zMhsBgdPUcCZwbyY3ADpJ2HYR6zcysjk4EfwBXSLpZ0sw608cCyyvPV+Rx65E0U1KPpJ7Vq1d3oFlmZlZPJ4J//4jYl9Sl80FJB9RMV515YoMREXMiYkpETOnq6upAs8zMrJ62gz8iVua/q4D5wNSaIiuAcZXnuwEr263XzMxa01bwS9pG0qjeYeBgYHFNsQXAsfnqnv2ARyPivnbqNTOz1rV7Vc/OwHxJvcs6PyJ+Jul9ABFxBnApcBiwBHgC+Kc26zQzsza0FfwRsRTYu874MyrDAXywnXrMzKxzfOeumVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBP4i6u7vp7u4e6maYWeHa/Q9cw96Eky4ZtLruX/rQoNe5bNbhg1aXmQ0PPuI3MytMy0f8ksYB5wK7AM8CcyLiGzVluoGfAr/Po34cEae0Wudwt8vRs4a6CWZmbXX1rAU+FhG3SBoF3Czpyoi4o6bctRFxRBv1mJlZB7Xc1RMR90XELXn4ceA3wNhONczMzAZGR/r4JU0A9gFuqjP5NZJulXSZpFd0oj4zM2td21f1SNoW+BFwYkQ8VjP5FmD3iFgj6TDgJ8DEBsuZCcwEGD9+fLvNMjOzBto64pc0khT650XEj2unR8RjEbEmD18KjJQ0ut6yImJOREyJiCldXV3tNMvMzPrQcvBLEvB94DcR8bUGZXbJ5ZA0Ndf3UKt1mplZ+9rp6tkfOAa4TdLCPO7TwHiAiDgDOAp4v6S1wJPAtIiINuo0M7M2tRz8EXEdoH7KzAZmt1qHmZl1nu/cNTMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8K0FfySDpF0l6Qlkk6qM31rSRfm6TdJmtBOfWZm1r6Wg1/SlsC3gUOBPYHpkvasKXY88EhEvAw4HfhSq/WZmVlntHPEPxVYEhFLI+Jp4ALgyJoyRwLn5OEfAgdKUht1mplZm0a0Me9YYHnl+Qrg1Y3KRMRaSY8COwIP1i5M0kxgZn66RtJdbbRtUzaaOus/UORzrE7z/hveBm3/DcG+273Zgu0Ef70j92ihTBoZMQeY00Z7hgVJPRExZajbYa3x/hvevP+Sdrp6VgDjKs93A1Y2KiNpBLA98HAbdZqZWZvaCf5fAxMl/YWkrYBpwIKaMguA4/LwUcDPI6LuEb+ZmQ2Olrt6cp/9CcDlwJbA3Ii4XdIpQE9ELAC+D/x/SUtIR/rTOtHoYW6z787azHn/DW/ef4B8AG5mVhbfuWtmVhgHv5lZYTar4Je0TtJCSYslXSTpRR1a7s6SLpZ0q6Q7JF2ax/9e0qSasl+X9AlJ3ZIelfS/+WctrpF0RCfa06CNyySN7qfM1bktC/Njpzx+UH5ao+T90wxJa5ooM1fSKkmLa8afKmlR3r5XSBqTx0vSN/O+XSRp34Fqf7Mk/bLm+UckPSVp+8q4bkkX15n3iLzPevf1eweojc28n7aSNEfSbyXdKekf8vhN/6dqImKzeQBrKsPnAR/diHlH9DHte8CHK8/3yn+/CHyuMn4L0iWsuwPdwMWVaZOBZcCBrbajn/mWAaP7KXM1MKXO+A8AZ+ThacCF3j9trWer+3BNE2UOAPYFFteM364y/KHK/jwMuIx0T81+wE0DsW/b3F6/Aq4FZlTGrbd/8riRpEvGd8vPtwYmDdC+aOb99O/A5yuvrdF5eFDeT+08Nqsj/hrXAi+TNKF6dCTp45JOzsNXS/qCpP8BPiypS9KPJP06P/bPs+1KCgwAImJRHpzH+lcqHQAsi4h7ahsTEQuBU4ATaqdJOjkfOVwBnJvbfK2kW/Ljb3K57tzmH+YjjPOk9X8CQ9ILJf1M0ns2YlsNxU9rDKf9s6Wks/OZym2SPlKnzNmSvibpF8CXJE2V9Mt8dPrL3jMPSTMk/Tjvo7slfbnOskZLukHS4XXaeQ117oWJiMcqT7fh+RsljwTOjeRGYAdJu9bOP5iqZzaS9gC2BT4DTO9n1lGkKxEfAoiIP0fEBnf4D+L76V2kgwsi4tmI6L0jeJP/qZp27tzdZCndLHYo8LMmiu8QEX+b5zsfOD0irpM0nnSp6l+RfozuQqXLV/8b+I+IWBkRiyQ9K2nviLiVFDLz+qjrFuBfGkx7FfDaiHhSqQvkjRHxlKSJeZm9dxvuA7yCdORzPbA/cF2eti3pN5POjYhzG9TzH5LWAT8iHa0EG/HTGp0wDPfPZGBsRLwyt2OHBvP/JXBQRKyTtB1wQN6eBwFfAP6hsrx9gD8Dd0n6VkQsz8vemXT/y2ci4somts9zJJ0GHAs8Crw+j6730ypjgfs2ZtkDaDppn1wLTJK0U0SsqlcwIh6WtAC4R9JVwMXAvIh4tk7xAX0/VV4Dp0rqBn4HnBARDzDI76dWbG5H/C+UtBDoAf5Auo+gPxdWhg8CZudlLAC2kzQqIi4HXgqcCbwc+F9JXXmeecC0HGZHAhf1UVdfn/oLIuLJPDwSOFPSbXl51V89/VVErMgv9oXAhMq0n5JCr1HovzMi/hp4XX4c00e7BuI63+G6f5YCL5X0LUmHAI81KHdRRKzLw9sDF+WzmdNJ4dLrqoh4NCKeAu7g+d9YGQlcBXxiY0MfICL+NSLGkbrRes9cBmvftmoacEF+Pf8YeFtfhSPi3cCBpO6hjwNzGxQd6PfTCNKvFVwfEfsCNwBfydM29W2+2QX/kxExOT/+OdKvhq5l/fV8Qc08f6oMbwG8prKMsRHxOKSjjYg4PyKOId21fECeZx7wdlIoLWp0tJLtA/ymwbRqOz4CPADsTToy2aoy7c+V4XWsf9Z2PXBoo9PKiLg3/30cOJ/0C6sweD+tMSz2T+7a6f0C/JSIeIS0L64GPgic1WD+altPBX6RzxLeVLNejfbhWuBm4O/6aGMzzuf5s4tmflplSEjaC5gIXClpGelDoL/uHiLitog4HXgjz69nrYF+Pz0EPAHMz88vIn33AsPgp2o2t+Cv5wFgJ0k7Stoa6OvKjSuo9PFKmpz/viGfLiJpFLAH6YiViPgd6UUwiz66EfKL/N9I3RL92R64Lx+FHEO6M7oZn81t+U6d+kcoX6UgaSRpO/T2rQ/lT2tscvsnItZVPlw+m7fbFhHxo1ymmStjtgfuzcMzmigP6ajwXcDLVecfG/Uld2H0ejNwZx5eAByrZD/g0YjYlLp5To6ICfkxBhgrqe6vTEraNner9JoMbPB9TR0dfz/l98d/kb6EhnQWckce3uR/qmazD/6IeIb0pd1NpD7BO/so/iFgitJlb3cA78vjXwX0SFpEOqU7KyJ+XZlvHqmLYT7re13+cu8uUuB/KCKuaqLZ3wGOk3Qjqd/4T/2UrzoReEGdLw23Bi7P67CQFEpn5mnfB3ZU+mmNjwIbFTrtGCb7Zyxwde5iOhv4VBOr9mXgi5Kup/mgIXcVTQNeL+kDtdMlzSOt4yRJKyQdnyfNUvryeRFwMPDhPP5SUlfVEtL+3mCZQ2gaG+6T+Tz/hfyBeR1XSFpBOiP7hPIlyaSramY0Uc9AvJ8APgmcnLf5McDH8vghez81yz/ZYGZWmM3+iN/MzNbn4DczK4yD38ysMA5+M7PCOPjNzArj4DczK4yD38ysMP8HCFh3sSdTyLEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "time_df = pd.DataFrame(timings)\n", + "time_df.mean().plot.bar(yerr=time_df.std(), rot=0, title='Computation time for optimal config, s');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`PureSVD-s` compares favoribly to the iALS, even though it requires higher rank value, which results in a longer training time comparing to `PureSVD`. Another interesting measure is what time does it take to achieve approximately the same quality by all models. \n", + "Note that all models give approximately the same quality at the optimal rank of iALS. Let's compare training time for this value of rank." + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF7RJREFUeJzt3X24XVVh5/HvjxcRJIKYC0ggpgpNBxgTbIwv1DbKi7yKjoxNxrGo+ESd8ogdHYpFAamttFNfWmBKI0TUkcAgDYMlCCkVgVGRAIGAQOUlSAiSQCSAvBn4zR97XTkczsm9Oeckl2T9Ps9zn7vP2mvvtfbZyW/vvfY+58o2ERFRj83GugMREbFhJfgjIiqT4I+IqEyCPyKiMgn+iIjKJPgjIiqT4I91JulkSf97rPuxqZJ0q6QZG6itxyW9bkO0FS8dCf5NlKSlkp4s/7F/KekcSduOUV/eKOmq0pcHJR3bMm+SpB9IekLS7ZL2H4s+Dkp530e9DWW/fLG1zPZetq9cD327UtJH29ra1vbdg24rXtoS/Ju2w21vC0wF9gE+u6E7IGk88H3gn4BXA7sDl7dUmQfcWOadAHxX0tCG7mdETRL8FbD9S+AymgMAAJIOlXSjpEcl3Sfp5JZ5kyRZ0lGSfiHpIUkndFq3pC0lzZN0oaSXdajy34HLbH/H9tO2H7N9W1n2d4E3AifZftL2hcAS4H1d2tpa0pcl3StptaRrJG1d5r27DJE8Us5s/0PLcksl/Q9JN0v6taSzJe0k6VJJj0n6V0mvatv22ZKWS3pA0qdb1vWCM3RJMyQtK9PfBiYC3ytXN8eV8gvKVdfqcuWzVymfDXwAOK7U/15Lf/cv01tJ+lrpy/IyvVVr25I+LWlF6euHu7x3fwW8HTi9tHV6Kbek3Vu27X+V9+VxSf9P0s6lzV+VK7J9Wta5S9nvKyXdI+mTndru0p8/l3R/ef/vkLTfaJeN/iX4KyBpV+Bg4M6W4l8DfwJsDxwKfELSe9oW/QNgMrAfcGJrmJb1bg1cBDwNvN/2Mx2afwuwStKPSjh9T9LEMm8v4G7bj7XUv6mUd/J3wO8DbwN2AI4DnisHkHnAp4AhYAFN+LYeiN4HHAD8LnA4cCnwF8B4mv8H7aH1DmAP4EDg+NEM39j+IPALypWW7b8tsy4t69oRuAH4Tqk/p0z/bal/eIfVnkDzHk4FpgDTgc+1zN8Z2A6YABwNnDF8EGvr2wnA1cAxpa1jumzG+8v6x9Ps1x+XPo8Hvgt8BUDSZsD3aPbXBJp/I5+S9K61vUdl2cnAMcCbbI8D3gUsHWm5GJwE/6btIkmPAfcBK4CThmfYvtL2EtvP2b6ZJjj/qG35L5Qz8Zto/oNPaZn3SpohnLuAD9t+tksfdgWOAo6lORu+p7QFsC2wuq3+amBc+0pK0HwEONb2/baftf0j208DfwxcYnuh7d/QHCC2pjlADDvN9oO276cJwGtt31iWn08zFNa+7b+2vQT4BjCry/aNyPbccqXzNHAyMEXSdqNc/APAKbZX2F4JfAH4YMv835T5v7G9AHic5mDdq/m2r7f9FM378pTtb5X9ez7Pv09vAoZsn2L7mXKf4OvAzFG08SywFbCnpC1tL7V9Vx99jnWU4N+0vaecUc0Afo/mrA0ASW9Wc1N1paTVwMdb5xe/bJl+giaoh70FeANwqtf+TX9P0oTJdSVMvgC8rQTf4zQHkFavBB7jxcYDL6c50LTbBbh3+IXt52gOdhNa6jzY1qf21+03vu9rmb63tLHOJG0u6VRJd0l6lOfPbNvf625esG0d+vKw7TUtr9v307oa7fv0WmCXMrT2iKRHaK6gdhqpAdt30lydnQyskHSepJ7e3+hNgr8Ctn8InENzJjzsXOBiYDfb2wFnAlqH1V4OfAm4QtLa/rPfDLQeGIanBdwKvE5S6xn+lFLe7iHgKeD1HeYtpwmiZsWSgN2A+0fYhrXZrWV6YmkDmiGybVrm7dy2XPtB8L8ARwD70wzJTBruZpf67V6wbW19WVeD/Cre+4B7bG/f8jPO9iGj6oh9ru0/oNk2A38zwL7FCBL89fgacICk4Ru844BVtp+SNJ0moNZJGcM+lyb8u53BfgN4r6SpkrYEPg9cY/sR2/8OLAZOkvRySe+luYq4sENbzwFzga+Um4qbS3prudH5f4BDJe1X2vg0zfj0j9Z1m1p8XtI25Ubsh2mGOSj9PUTSDpJ2pjlzbfUg0Ppc/LjSl4dpDhh/PUL9dvOAz0kaKu/xiUCvn6EYqa118VPg0XKTduuyP/aW9Cb47Y3njgcaSZMlvbPsu6doriS6DRXGepDgr0QZH/4WTfAC/DfglHIP4ESa8OxlvX9Jc4P3XyXt0GH+v9EMAVxCc59hd154kJkJTAN+BZwKHFn62slnaJ76uQ5YRXOWuJntO4D/CpxGc2VwOM0N1k43m0frhzQ3w68A/s728COo36a537GU5qrn/LblvkQT1I9I+gzNe34vzdXHz4CftNU/m2as+xFJF3XoxxeBRTRXTktobrR+sUO90fh74MjyhM4/9LgOAMqY/+E0N53voXnfz6K5qoHmiunHXRbfimZfP0QznLgjzb+R2ECUP8QS8TxJk2iCbMu2sfNYB5LOAi6wfdlY9yVeLMEf0SLBHzXIUE9ERGVyxh8RUZmc8UdEVGaLse5AJ+PHj/ekSZPGuhsRERuN66+//iHbo/qCw5dk8E+aNIlFixaNdTciIjYaku4duVYjQz0REZVJ8EdEVCbBHxFRmQR/RERlEvwREZUZ8akeSXOBw4AVtvcuZefz/B972B54xPbUDssupflu9WeBNbanDajfERHRo9E8znkOcDrNtwwCYPuPh6clfZkX/xWlVu+w/VCvHYyIiMEaMfhtX1W+uOpFyh+8eD/wzsF2KyIi1pd+x/jfDjxo++dd5hu4XNL1kmavbUWSZktaJGnRypXdvo49IiL61e8nd2fx/B/O7mRf28sl7QgslHS77as6VbQ9B5gDMG3atHxzXEQFJh1/yVh3Yb1ZeuqhY92Frno+45e0BfCfePFfIPot28vL7xXAfGB6r+1FRMRg9DPUsz9wu+1lnWZKesXwH9GW9ArgQOCWPtqLiIgBGDH4Jc2j+duZkyUtk3R0mTWTtmGe8kewF5SXOwHXSLqJ5g8zX2L7+4PrekRE9GI0T/XM6lL+oQ5ly4FDyvTdwJQ++xcREQOWT+5GRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFRmxOCXNFfSCkm3tJSdLOl+SYvLzyFdlj1I0h2S7pR0/CA7HhERvRnNGf85wEEdyr9qe2r5WdA+U9LmwBnAwcCewCxJe/bT2YiI6N+IwW/7KmBVD+ueDtxp+27bzwDnAUf0sJ6IiBigfsb4j5F0cxkKelWH+ROA+1peLytlHUmaLWmRpEUrV67so1sREbE2vQb/PwKvB6YCDwBf7lBHHcrcbYW259ieZnva0NBQj92KiIiR9BT8th+0/azt54Cv0wzrtFsG7NbyeldgeS/tRUTE4PQU/JJe0/LyvcAtHapdB+wh6XckvQyYCVzcS3sRETE4W4xUQdI8YAYwXtIy4CRghqSpNEM3S4GPlbq7AGfZPsT2GknHAJcBmwNzbd+6XrYiIiJGbcTgtz2rQ/HZXeouBw5peb0AeNGjnhERMXbyyd2IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyIwa/pLmSVki6paXsf0q6XdLNkuZL2r7LskslLZG0WNKiQXY8IiJ6M5oz/nOAg9rKFgJ7234D8O/AZ9ey/DtsT7U9rbcuRkTEII0Y/LavAla1lV1ue015+RNg1/XQt4iIWA8GMcb/EeDSLvMMXC7pekmzB9BWRET0aYt+FpZ0ArAG+E6XKvvaXi5pR2ChpNvLFUSndc0GZgNMnDixn25FRMRa9HzGL+ko4DDgA7bdqY7t5eX3CmA+ML3b+mzPsT3N9rShoaFeuxURESPoKfglHQT8OfBu2090qfMKSeOGp4EDgVs61Y2IiA1nNI9zzgN+DEyWtEzS0cDpwDia4ZvFks4sdXeRtKAsuhNwjaSbgJ8Cl9j+/nrZioiIGLURx/htz+pQfHaXusuBQ8r03cCUvnoXEREDl0/uRkRUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUZlTBL2mupBWSbmkp20HSQkk/L79f1WXZo0qdn0s6alAdj4iI3oz2jP8c4KC2suOBK2zvAVxRXr+ApB2Ak4A3A9OBk7odICIiYsMYVfDbvgpY1VZ8BPDNMv1N4D0dFn0XsND2Ktu/Ahby4gNIRERsQP2M8e9k+wGA8nvHDnUmAPe1vF5Wyl5E0mxJiyQtWrlyZR/dioiItVnfN3fVocydKtqeY3ua7WlDQ0PruVsREfXqJ/gflPQagPJ7RYc6y4DdWl7vCizvo82IiOhTP8F/MTD8lM5RwP/tUOcy4EBJryo3dQ8sZRERMUZG+zjnPODHwGRJyyQdDZwKHCDp58AB5TWSpkk6C8D2KuAvgevKzymlLCIixsgWo6lke1aXWft1qLsI+GjL67nA3J56FxERA5dP7kZEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVKbn4Jc0WdLilp9HJX2qrc4MSatb6pzYf5cjIqIfW/S6oO07gKkAkjYH7gfmd6h6te3Dem0nIiIGa1BDPfsBd9m+d0Dri4iI9WRQwT8TmNdl3lsl3STpUkl7dVuBpNmSFklatHLlygF1KyIi2vUd/JJeBrwbuKDD7BuA19qeApwGXNRtPbbn2J5me9rQ0FC/3YqIiC4GccZ/MHCD7QfbZ9h+1PbjZXoBsKWk8QNoMyIiejSI4J9Fl2EeSTtLUpmeXtp7eABtRkREj3p+qgdA0jbAAcDHWso+DmD7TOBI4BOS1gBPAjNtu582IyKiP30Fv+0ngFe3lZ3ZMn06cHo/bURExGDlk7sREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERl+g5+SUslLZG0WNKiDvMl6R8k3SnpZklv7LfNiIjo3RYDWs87bD/UZd7BwB7l583AP5bfERExBjbEUM8RwLfc+AmwvaTXbIB2IyKig0EEv4HLJV0vaXaH+ROA+1peLytlERExBgYx1LOv7eWSdgQWSrrd9lUt89VhGbcXlIPGbICJEycOoFsREdFJ32f8tpeX3yuA+cD0tirLgN1aXu8KLO+wnjm2p9meNjQ01G+3IiKii76CX9IrJI0bngYOBG5pq3Yx8Cfl6Z63AKttP9BPuxER0bt+h3p2AuZLGl7Xuba/L+njALbPBBYAhwB3Ak8AH+6zzYiI6ENfwW/7bmBKh/IzW6YN/Gk/7URExODkk7sREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlBvU3dzdak46/ZKy7sF4tPfXQse5CRLzEVB/8sXHLgTti3WWoJyKiMgn+iIjKJPgjIiqT4I+IqEyCPyKiMgn+iIjKJPgjIirTc/BL2k3SDyTdJulWScd2qDND0mpJi8vPif11NyIi+tXPB7jWAJ+2fYOkccD1khba/llbvattH9ZHOxERMUA9n/HbfsD2DWX6MeA2YMKgOhYREevHQMb4JU0C9gGu7TD7rZJuknSppL3Wso7ZkhZJWrRy5cpBdCsiIjroO/glbQtcCHzK9qNts28AXmt7CnAacFG39dieY3ua7WlDQ0P9disiIrroK/glbUkT+t+x/c/t820/avvxMr0A2FLS+H7ajIiI/vTzVI+As4HbbH+lS52dSz0kTS/tPdxrmxER0b9+nurZF/ggsETS4lL2F8BEANtnAkcCn5C0BngSmGnbfbQZERF96jn4bV8DaIQ6pwOn99pGREQMXj65GxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmb6CX9JBku6QdKek4zvM30rS+WX+tZIm9dNeRET0r+fgl7Q5cAZwMLAnMEvSnm3VjgZ+ZXt34KvA3/TaXkREDEY/Z/zTgTtt3237GeA84Ii2OkcA3yzT3wX2k6Q+2oyIiD5t0ceyE4D7Wl4vA97crY7tNZJWA68GHmpfmaTZwOzy8nFJd/TRt5ey8XTY/vVFucYatOy/jdsG239jsO9eO9qK/QR/pzN391CnKbTnAHP66M9GQdIi29PGuh/Rm+y/jVv2X6OfoZ5lwG4tr3cFlnerI2kLYDtgVR9tRkREn/oJ/uuAPST9jqSXATOBi9vqXAwcVaaPBP7Ndscz/oiI2DB6HuopY/bHAJcBmwNzbd8q6RRgke2LgbOBb0u6k+ZMf+YgOr2R2+SHszZx2X8bt+w/QDkBj4ioSz65GxFRmQR/RERlEvwjkPSspMWSbpF0gaRtBrTenST9i6SbJP1M0oJSfo+kyW11vybpOEkzJK2WdGP5qoyrJB02iP5sCrKv6iXpR22v/0zSU5K2aymbIelfOix7WNlPw/v3Yxuiz2MpwT+yJ21Ptb038Azw8dEuWB5h7eYUYKHtKbb3BIa/6+g8Wm6CS9qM5omo80vR1bb3sT0Z+CRwuqT9Rr85m7Tsq0rZfltb0SyaJw/fu7blJG1Jc8P3cNtTgH2AK9dHH19KEvzr5mpgd0mTJN0yXCjpM5JOLtNXSvprST8EjpU0JOlCSdeVn33LYq+h+ZwDALZvLpPzeOHTT38ILLV9b3tnbC+mCaVjBriNm4qNZl9J2lzSOeVKZYmkP+tryysk6fGW6dcD2wKfozkArM04mqcbHwaw/bTtTfVbA34rwT9K5YzwYGDJKKpvb/uPbH8Z+Hvgq7bfBLwPOKvUOQM4W9IPJJ0gaRf4bag8J2lKqTeTJmC6uQH4vXXfok3XRrivpgITbO9t+z8C3xhFv6O7WTT74WpgsqQdu1W0vYrm80b3Spon6QPlym2Ttslv4ABsLWkxsAj4Bc1nE0Zyfsv0/jSX+Itp/oG9UtI425cBrwO+ThMGN0oaKsvMA2aWADsCuGAtbeVL7563se6ru4HXSTpN0kHAo6Pod3Q3EzjP9nPAPwP/eW2VbX8U2A/4KfAZYO567+EY6+e7emrxpO2prQWS1vDCg+bL25b5dcv0ZsBbbT/ZvuJytnEucG656fSHwIU0YXI58EPgZtsr1tK/fYDbRrktm7qNYl+p+Urz60vZxbZPLFcN7wL+FHg/8JG1bml0JOkNwB7AQjVfBPwymgPrGWtbzvYSYImkbwP3AB9avz0dWznj782DwI6SXi1pK2BtT2tcTsu4rqSp5fc7h586kTQOeD3NWSq276IZczyVtQwdlH/kn2eEf9SVe8ntK9vPlpvQU0vojwc2s31hqfPG3je3erOAk21PKj+7ABMkdfzmSknbSprRUjQVeNE9mk1Nzvh7YPs3ar6a4lqas4Pb11L9k8AZkm6meb+vonna5PdphhWGz0jPsn1dy3LzgC8B89vW93ZJNwLbACuAT9q+YgCbtUnaSPbVBOAbLWPLn12XbYwXmElzf6fV/FJ+Lc3fBFnWMm8WcJykfwKepLkC/NAG6OeYylc2RERUJkM9ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZn/D4l8dOyzlPZ0AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fixed_rank_timings = {}\n", + "for model in models:\n", + " model.rank = ials_best_config['rank']\n", + " model.build()\n", + " fixed_rank_timings[model.method] = model.training_time[-1]\n", + "\n", + "pd.Series(fixed_rank_timings).plot.bar(rot=0, title=f'Rank {ials.rank} computation time, s')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By all means computing SVD on this dataset is much faster than ALS. This may, however, vary on other datasets due to a different sparsity structure. Nevertheless, you can still expect, that SVD-based models will be perfroming well due the usage of to highly optimized BLAS and LAPACK routines." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Bonus: scaling for iALS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You may reasonably question whether that scaling trick also works for non SVD-based models. Let's verify its applicability for iALS. We will reuse Polara's built-in scaling functions in order to create a new class of the *scaled iALS-based model*." + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.models import ScaledMatrixMixin" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [], + "source": [ + "class ScaledIALS(ScaledMatrixMixin, ImplicitALS): pass # similarly to how PureSVD is extended to its scaled version" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "sals = ScaledIALS(data_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to save time, we will utilize the optimal configuration for scaling, found by tuning scaled version of PureSVD. Alternatively, you could include scaling parameters into the grid search step by extending `als_param_grid` and `als_param_names` variables. However, taking configuration of PureSVD-s should be a good enough approximation at least for verifying the effect of scaling. The tuning itself has to be repeated from the beginning." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### hyper-parameter tuning" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "60/60\n", + "[22:42<00:24, 22.70s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 60/60 [22:42<00:24, 22.70s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sals_best_config, sals_param_scores = find_optimal_config(sals,\n", + " als_param_grid,\n", + " als_param_names,\n", + " target_metric,\n", + " init_config=[init_config,\n", + " ssvd_best_config], # the rank value will be overriden\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "15/15\n", + "[07:03<00:39, 28.18s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 15/15 [07:03<00:39, 28.18s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sals_best_rank, sals_rank_scores = find_optimal_config(sals,\n", + " rank_grid,\n", + " 'rank',\n", + " target_metric,\n", + " init_config=[init_config,\n", + " ssvd_best_config,\n", + " sals_best_config],\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### visualizing rank tuning results" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XmcVNWd///Xp/bqfe8GGuim2ZcWsN3ABWPAJUTjkgF01BgdkxlNJs6q80uMMd9Ekzjf5OsXv2OcxCwmkSRmnBATYlAHSVwB2TeBBrsb6H3vrr3O7497u6luummgKZrl83w86lF3OffWuQVd7zrn3jpXjDEopZRSJ8sx0hVQSil1dtMgUUopNSwaJEoppYZFg0QppdSwaJAopZQaFg0SpZRSw6JBotRpICJrROS+k9x2nIh0iojzVNdrJIjIARH5uD39byLyg5GukxoeDRJ1StgfDgH7A69WRH4sImkjXa+zUeIHLYAxpsoYk2aMiY1kvZLBGPNNY8x9ACJSIiJGRFwjXS91YjRI1Kn0SWNMGjAbmAM8MsL1UUqdBhok6pQzxtQCr2IFCgAi4hWRp0SkSkTqRORZEfEnrL9JRDaJSLuI7BOR6+zlo0VkpYg0i8heEfmbhG0eE5Ffi8jPRKRDRLaKyGQReURE6kWkWkQWJZRfIyL/S0TetltOvxORXBH5uf2660SkJKH8VBFZbb/2bhH5q4R1PxaRZ0Tk9/ZrvyciZQnrF4rILhFpE5HlgCSsKxORN0SkSUQa7dfPste9AIwDfmfX8V/6f1M/jvfkVyLyU7te20WkImH9v4rIQXvdbhG5ZqB/Q/t9WWm/L++LyNdF5C/2uqNaDoldd8c6vgFe5zER+Zk9u9Z+brWP/Sr7GGcllC+wW775A+1PjQwNEnXKiUgxcD2wN2Hxt4DJWOEyERgDPGqXvxj4KfDPQBZwJXDA3u5FoAYYDdwGfLPfh98ngReAbGAjVoA57P0/Dny/X/WWAnfa68uAd4AfATnATuCrdp1SgdXAL4ACYBnw/0RkRsK+lgFfs197L/ANe9s84DfAl4E8YB8wP/EtAp6wj2kaMBZ4DMAYcydQhd26M8Z8m6MN9Z7cCKyw38uVwHK7XlOAB4GLjDHpwLUceZ/7ewYIAqOAz9qP4zXo8Q3hSvs5yz72N+3j+OuEMsuA14wxDSdQH5Vsxhh96GPYD6wPpE6gAzDA61gfCGB9sHQBZQnlLwP229PfB747wD7HAjEgPWHZE8CP7enHgNUJ6z5p18Fpz6fbdempxxrg/0so/+/Aqn7bb7KnlwB/7lef7wNftad/DPwgYd0NwC57+i7g3YR1gvXBf98g792ngI393suPJ8yX2MfhOs735LWEddOBgD09EagHPg64j/Fv6QQiwNSEZd8E/tK/Pgnr15zM8dn1/dkx9nsJUA047Pn1wF+N9P93ffR9aItEnUqfMtY33QXAVKxv4wD5QAqwQURaRaQV+KO9HKwPx30D7G800GyM6UhY9hFWa6JHXcJ0AGg0R05KB+zntGOU7z/fU3Y8cElPfe063wEUJZSvTZjuTth2NNaHHwDG+gTsnbe7Z1bYXUztwM848l4N5Xjek/718omIyxizF/gS1od3vV2H0QO8Rj5WaFUnLPvoOOs33OPrwxjzHtaXkKtEZCpWGK48mX2p5NEgUaecsbokfgw8ZS9qxPqQnmGMybIfmcY6MQ/WB1bZ0XviEJAjIukJy8YBB5NT8z6qgTcT6tvT3fK3x7HtYaxwBEBEJHEeqwVhgHJjTAZW140krD/WkNzDek+MMb8wxlyOFZQGq8uxvwYg2q/O4xKmu+znlIRliQE71PENWr1Blv/E3sedwEvGmOBx7EudRhokKlm+BywUkdnGmDjwn8B3RaQAQETGiMi1dtkfAveIyDUi4rDXTTXGVANvA0+IiE9EyoF7gZ+fhvq/AkwWkTtFxG0/LhKRacex7e+BGSJyi31C+ov0/aBNx+qCaxWRMVjnhhLVARMG2vFw3hMRmSIiHxMRL9b5jwBWN1n/14gB/wU8JiIpIjIduDthfQNWcP21iDhF5LP0/SIw1PENpgGIc/SxvwDcjBUmPz3OfanTSINEJYX9YfNT4Cv2on/FOiH9rt3d8RowxS77PnAP8F2gDXgT6xszWCdXS7C+ib+MdY5i9WmofwewCOvk/CGs7qJvAd7j2LYR+DTwJNAETALeSijyNWAu1rH+HutDO9ETwJftLrV/GuAlTvY98dp1arSPpwD4t0HKPojVVVeL1br8Ub/1f4MVEE3ADKxw6zHU8Q3IGNONdcHCW/axX2ovrwE+wGqx/Pl49qVOL7G6b5VSanAi8hmsk+mXj9DrPw8cMsZ8eSReXx2b/oJUKXVGE+u3Pbdg/chVnYG0a0spdcYSka8D24DvGGP2j3R91MC0a0sppdSwaItEKaXUsJwX50jy8vJMSUnJSFdDKaXOKhs2bGg0xgw5rtl5ESQlJSWsX79+pKuhlFJnFRE5rhENzosgUUqp08kYQ3NXGBHB73bidTlwOI7nx/1nJw0SpZQ6QfG4ob4jxMHWbmpaAgmPbg62BDjYGiAUjffZxuty4HM78bud+D1WuPg9Tnwua97vduJ1O6z1bqdVNqFc7zK7XIrHRW6qh/x0Lz73yN48U4NEKaX6icUNde1BaloCVlg020HRagXFodYg4VjfoMhN9VCc7WfaqAw+Pr2QUZk+HCIEIjEC4RjBaIxgOEYwEreWRWIE7UdDR7S3XChqPQciMeLHeVFths9FQYaP/DQvBRleCtK95Kd7KUj3UZBuLctP85Hhd2EN/XZqaZAopc55xhjCsTjhaJxQ9Mhzb1jYrYmesDjcGiTa71M8P93LmCw/M8dkct3MUYzJ9lOc7Wdstp/RWX5SPKf249QYQyRmCERihOzgscInbgdNlMaOMA2dIerbg9R3hGjoCLGxqpX6jiDBSPyofXpdDvJ7Q+ZI0OSn9wSQj/x0L7mpHlzO47+oN6lBItZd7v4P1v0NfmCMebLfei/WeEwXYo3Zs8QYc8C+0dFzPcWAx4wxL9vbHMC650UMiBpjKlBKnfVicUNjZ4jDbUFq2wIcbgvSGYwSjvX98A9FY4SjfUPBKjPA8mickB0gxyICBeleirNTmDsumzHlfoqzUyjO9jMm28+YLP9p7z4SETwuweNygN99QtsaY+gIRWnoCFHfHqK+I0iDHTQ9gbO/sYv39jfT2h05anuHQE7qkMPK9UpakIiIE+suawuxbuqzTkRWGmN2JBS7F2gxxkwUkaVYg+Itwfola4UxJioio4DNIvI7Y0zU3u5qe2A8pdRZIBqLU9/RExJBDrcFrOf2IIdbrem6jhCxAfpyHAIelwOP04HX7bSeXQ48riPPPreDTL8bj7Pvck9vOetcg7UPR2+5gnQfxdl+RmX58LpG9jzDqSQiZPjcZPjclOWnHbNsKBqjsTNMfXuwN2issAmy4ThfL5ktkouBvcaYSgARWQHcBCQGyU0cuQXnS8ByERF7FNAePo59fwal1AgK211Ete3BPq0JKzCs0GjoCB3V3+9zOxid6aco08dlZXmMyvRRlOnrfS7K8JHpd59QF4s6cV6XkzFZVqurvycHKD+QZAbJGPreYa0G67aZA5axWx9tQC7QKCKXAM9jDSd+Z0JrxAB/EhEDfN8Y8xxKqaQKRmIcaOqisqGL/Y1d7GvopLKhi4OtARo7Q/QfaSnV42RUlp9RmT4mFeTb4WB98x+V6WNUhj9pJ37V6ZfMIBnof0j/lsWgZexbbM6wbyT0ExFZZd8Zbb4x5pB9g6TVIrLLGLP2qBcXuR+4H2DcuHH9Vyul+onHDbXtQSobuqhs7LSfu6hs6ORga6BPWBRl+JiQn8rHphT0hkNRphUcozJ9pPtOrE9fnd2SGSQ19L1VZzHWjXgGKlNj30kuE2hOLGCM2SkiXcBMYL0x5pC9vF5EXsbqQjsqSOyWynMAFRUV2jWmlK0jGGF/o9W6qGzoZJ89faCxi0DkyA0TUz1OSvNTmTsum9suLGZCfhoT8lIpzUsl1asXfKojkvm/YR0wSURKsW7LuRS4vV+ZlVi38HwHuA14wxhj7G2q7e6u8Vh30jsgIqmAwxjTYU8vAh5P4jEodVbqCEaobQtS1dxtd0VZoVHZ2EVDR6i3nENgbE4KpXmpXDYhlwn5qUzIT6UsP42CdK92PanjkrQgsUPgQeBVrMt/nzfGbBeRx7FaFiux7tX9gojsxWqJLLU3vxx4WEQiWPdw/jtjTKOITABetv9zu4BfGGP+mKxjUOpM03OJbG2bdXK7zj7BXWfP17Zb013hvrdiz05xMyE/jQWT8ynNT2VCXhpl+amMy005p65WUiPjvLgfSUVFhdFBG9WZLhCOWWHQFqS2PUBtW8i6GiohNOoHuETW5RAKM3wUZngpyvRRmGFd8VSUaV3aOiEvjexUzwgdlTqbiciG4/mtnnZ0KnUaNXWG+LCukz31Heyp6+Sj5m5q7d9UtAejR5VP97ootC+FLbMvke2ZL8rwUZjpJS/Ve04PCKjOfBokSiVBS1eYD+s6+LC+kz11HXxYZwVHU1e4t0y610VJXirjc1O5dEJun5ZEof2cpie11VlA/5cqNQyt3WE+rOu0g6LDbm100th55IR2mtfFxII0rplWwOTCdCYVpjO5MI2iDJ+ezFbnBA0SpY5DW3eEPfUdR0LDnk68AirV42RiYTpXT8m3AyONyYXpjMrUwFDnNg0SpfoxxrCrtoPVO+p4f38zH9Z1UJ8QGCkeJ5MK0rhyUj6T7bCYVJjGmCy/BoY6L2mQKIU1qOD6j1r40/Y6Vu+spbo5gAhMK8rg8kl5TLa7oyYVpDMmy68nt5VKoEGizlvd4ShrP2zkTztqeWNXPa3dETwuB/PLcvm7BRO5ZloBBem+ka6mUmc8DRJ1XmnsDPH6zjr+tL2Ov+xtJBSNk+l387GpBSyaXsiVk/N1+A+lTpD+xahzXmVDJ6t31LF6Rx0bqlowBsZk+Vl28TgWzSjkopIc3DpUuVInTYNEnXPiccOmmtbe8Nhb3wnAjNEZ/P01k1g4vZDpozL0xLhSp4gGiTonhKIx3t7XxJ+21/HazjoaOkI4HcKlE3L460vG8fHphRRnp4x0NZU6J2mQqLNSNNBOw4fvU11Tza+aSli1N0RXOEaqx8mCKQUsnF7I1VMKyEzR+2IolWwaJOqMZYyhoSNEZUMHTfu3YmrWkd60ieKuHZTEPmKUGEYBc3Hwd+kXEJt8PeMuuxVvwcSRrrpS5xUNEjXi2oMRDjR29d43o7G2Gl/9RoratzIjvodyRyXpEgCgg1Sq/NN4L+djxEZfyKjCQspa32XCh6tg4zetR/5UmHI9TLkBxlwIDh0mXalk0mHk1WkRisaobu7uved3zx36DjY0U9j9IXMce5nj2Mtsx16KpRGAGE5aMyYTKpyLr/RisibOw5E3ERyDXGHVcgB2/xF2/wE+egviUUjNh8nXWqEy4Wrw6HkSpY7X8Q4jr0GiTrl43LDjcDtvftjAugPN7G/sorq5m7gxlEots2Uvl3krudBZSUm0EifWTZgi6cU4iitwjr0Iiitg1AXg9p9cJQItsPd1K1T2rIZQO7h8VphMuR4mXwfphafwqJU692iQJNAgSb6GjhB/3l3L5ne30bBxG/kNVUxqq6Es1IA3BdJSQqR5WvF6A7j9MVzpPlyl03BNvgjHhEthTEXyPtijYah6G3avgl1/gLYqa/mYiiNdYAXTQC8HVqoPDZIEGiSnXrC9g81rN7D/3U107dhJzuEDlLQfxheLWAUc4M2I4k6NEAs7iYZ9RLvARONH7cuRno6roABXQT6u/ALIyyaWm0kkO41gTgqBTD+dGW66JExXpIvuSDddUevZKU7K88uZUzCHXH/u0BU3Bup3WC2V3aswNRvAQDxtHKbkGszYK4nnzcREY8SDQRypqXgnTkQG605T6hymd0hUp4QxhmhdHcEdO6nduJX6jVth3x6yWurJwHABEPL6cBamkT3eRZqnHl92BBk3it2Tr2Jv7ng60/LoMlG6w12E21qhsQlHYxuulnY8zV34WwOktB0kveoAGTti5HSAy84bB5BqPzp84EqHWJrQnQbxDDdtvjhrI3HejRrynJmMcuVS4Mwi15GOP+bEhEKYUIh4KIQJBo9M9y4rhngciAKv2o++nHl5pM67jLT580mdNw9Xfv7pevuVOitoi0T1MuEwof37Ce7cSWjXbrp27KB75y6cHe29ZQ6l5lKXV0zu+Awmjuqg2LUdd/QjDrpdbB41ha05Y9gqUXZ1fEQkHumzf6/TS6o7lRRXCqnuVGvanTDds9yZQkZISG+PkNoawt8axNvSjaulHWdjG9LUQryhiWhjI8Ss8ytGIOp2EHQawi5D2AUxjwuPLxV/agZp6bmkpWbj9PkRrxfxeXF4fYjXi8PnRbw+xAWOtv1I03akfjOOWDvRkJuuznF0VYWJtXdbxzFlCqnz55M6fx4pF16Iw6cDO6pzk3ZtJdAgOZoxhtCePXS/+y7BHTsJ7t5NaM8eiFr3DY843ezPKGJfxmhqcorJmV7KlRO7mWs+IFb9BtsIssWXwtbsQrY54rTErMtz/S4/03OnU55Xzqz8WUzNmUqmN5MUVwoux6ltAJtYjHgggMPjAbcbESFu4uxv288H9R/wQd0HbKzfyMHOg711uyD/AuYWzmVuwVxm5c0ixT3IVVzxGNSsg52/g+3/jWmrIdSeQmd4Ol11XgK7azCRCOL1klJRYQfLfLyTJ+nQK+qccUYEiYhcB/wfwAn8wBjzZL/1XuCnwIVAE7DEGHNARC4GnuspBjxmjHn5ePY5EA0SS7y7m65336Nz7Zt0rl1L9NBhAKKZ2RzKG8tmbwHbUwqpzBxN1qQJXF8qfNy1gXDja2xr3sZWt5Mt/hQOuKzzBYIwIXMCs/JnUZ5fTnleOWVZZac8MIartquWTfWb2FC3gY31G/mw5UMMBqc4mZYzrTdYZhfMHvg8SzwOB9fD9pdh+39DxyHixku362K6Wgvo3FlLuHI/AK78fFLnzSP18stJnXcZrtzjOG+j1BlqxINERJzAh8BCoAZYBywzxuxIKPN3QLkx5vMishS42RizRERSgLAxJioio4DNwGjADLXPgZzPQRKurqZzjRUc3e+9hwmHkZQUYnMqeCNzEj+NjaLBn0VemofLy3K5PH8P7q43+LBxA1ujbezweAjZJ5pz3OmUF86lPP8CZuXPYkbuDNI96SN8hCeuI9zBpvpNbKzfyAf1H7C1YSvheBiAkowS5hbOZU7BHOYWzGVs+ti+LYx4HGreh23/BTt+C5214PITKVxAV6CMrsoOut55n1hbGwDe6dOscyvz5+OfO9dqPSl1ljgTguQyrJbEtfb8IwDGmCcSyrxql3lHRFxALZBvEiolIqXAu8AY4KKh9jmQ8ylITDhM94YNdL65ls433yS83/qm7Bk/ntSrrqJm8hyWN6WxZn8b6T64dk4HBe711LRtZGuwjkb7zn9eA9P8BcwqupjysVcyq6Cc0amjz8lum3AszI6mHXxQ/wEb66xwaQ9b54WyvdmUZZVRllXGhMwJvdO5vlzExKHqXaulsuO30FUP7hTMxEUE/ZfQVROn65336d64EaJRxOcj5eKLeoPFU1Z2Tr6f6txxJly1NQaoTpivAS4ZrIzd+mgDcoFGEbkEeB4YD9xprz+efQIgIvcD9wOMGzdu+EdzBovU19O1di2db66l6+23iXd1IW43KRdfTPaypaRecQV/CabwzP/s5YMNDZTmruNjMzawM76fV1uty6NKIlEu8+Yxq+hCyqfeyuTRl+J2nh8DHnqcHmYXzGZ2wWyYCXETp7K1kg/qP2B703YqWyv5Q+Uf6Ih09G6T6c2kLLOMCVkTKCudzYQLPsXEzmby97yB7Pwd/u7/xu9OJe+m64k99FW6mzPoencdXW+9Rd0TVm+sq7AQ/+zZuIuKcBUW4i4qxFVYiKuwCHdBPnKGtF5MPE6spYVoXR2R2jqiBw8QrTlA5PAhHG7BXZSHZ3Q+nqI83EV5OLwJ9T4qKPvND7XekwYp2eDPAW/G4KManEmMgUgAgm3gyzwvRlNIZpAM9FWrf/Nn0DLGmPeAGSIyDfiJiKw6zn1ib/8c9nmWioqKc+qKAhOLEdiyhc61VqsjtGMnAK6iIjI+8QnSFlxF6iWXYPwp/H7rYZ79700Q+D1pWVspmNxCowPCkRjXRIRrcmcxd+ptZE6+4bz4D388HOJgYvZEJmYfGfzRGENDoIF9rfuobKtkX+s+9rXuY/VHq3kp9FJvuXR3OhNmXEqZM5UJ7Q1MrH6Tsp0vU+hMIX36DfDpLxDxT6XzvfV0/eUtQrt20bl2LSYQOKoeztxc3IV2uBQV2tNFuAsLcBUV4SooxJmWOqxjjQeDRGsPE63aS6S6kujBj4jW1hJpaCDa2EK0pYtIewji/f6ExODyxonHhHik74e7yx/DkxbFnRbFkxbDkx7FnWYtc3qG8acoTvBnWaGSkpPwnG09EpYZbxaxkINIW4hoY4sVgLWHidTWEak9TLS2jmhdHbhcODMzcWZk4MzIwJGViTPDnk9PxeF34/Q7cHoFpzuG0x3F4QrjlG4k3A6BVmsUhWBr3+lY+Ei9fZmQPgrSiyB9tP08CjJGHVmeVghn8Re3ZAZJDTA2Yb4YODRImRq7aysTaE4sYIzZKSJdwMzj3Oc5KdbaSudf3qLzzTfp+vOfibW2gsOBf84c8v/hH0i76kq8kycjIoSiMV76859Zu+mHtHh2UJ8VIpQjZMdi3CCZLBo1n4tm3Ym7qFx/zX2cRISClAIKUgq4bPRlvcuNMTQFm6hsrWRf277egFnTtpv/CjZDjh9yxpCCk7KmvzBh9WrK4k7KiuZSdu/NjCv7MtF4nFhHG+HaWiL19UTq6onVNxCtbyTa0Ei4ah/xDeugvfOoepkUHyY3E5OXSTwnnVhuBrGcNKL2wx2Pk9bQQmpjK56GZmhsIdrUTrS1i2hHhFjw6A92ccVx++O4UuKk5HtwTUrBlZOBKz/PCrMxY3GNGo9kFIInlVh7J+HD9YQP1hE5VE/4UB3hww10Hqojtr+tz76dGWm4RxfgGVWAZ1S+NT06H8/oApyZ6f26+gyEOiHQbH1AdzdDoBnT2USsuYFI9QGizZuJtHQT7YgRCTiJdDuJdjuJBpyYeN//2+IEV7oLd5Yff34a7qllmFiMWHsnsa464nVVRPZHCAZjxIIGEzv234bDbXD6BIfPiTPVgzPVhzN1PM6MmTgyM3BmZuHN8+DLjuIIN0D7YWhca51Xi0f7v+vWuHDpRZCREDbpCWGTMdoKyjOwVZbMcyQurBPj1wAHsU6M326M2Z5Q5gFgVsLJ9luMMX9lnxeptruzxgPvAOVA61D7HMjZeI4k3tVFYNt2Ahs30rl2LYFNmyAex5mdTeoVl5N21VWkzZ+PMyvL2iASpGH3n/jV+z/hg+BuNvkh7BByY4aPpxSzqPQG5pbficufPbIHdh5pDjZT2VpJZVsle1v3Utm6l31Nu2hM6CI7Ee6IIacTcjogp8OQm/Cc3fPcCY5B/qTjQHsqtKdDV5qDUIaLaKYXslNx5mXjKSwkZfR4MgtKyMoaT1bGWLJ82aS4Uk76XE68q4twTQ3hqioiVVWEq6oJV31EpKqayOHD9o9BLZKSgmfsWDzjxuEeNxbP2HGYaHTAloSJ9P2NkrjduArycOdl4cpJx53px5Xpxp0muPxR3L4wTulAgi0JodQCLq/VmvFl2a2dI9PGnU4s5iMW9RCLuoiFHMRDhlgwTqw7TKy9g3h7G7G2dmJtbcTared4W1vf+ongKBmHe/pU/DNmkDZzFqklRThibdBRC+2HrOeOwwmPWuhqOPoNdbgTQsZuyaQVQlq+9ZxaAGn2w+U9qX+zPu/rSJ9stytxA/A9rEt1nzfGfENEHgfWG2NWiogPeAGYg9USWWqMqRSRO4GHgQjW///HjTH/Pdg+h6rHmR4kJholtGcPgS1bCWzZTHDLVkL79vX+kfmmTydtwVWkXXklvlmzEKc9LHpzJS07f8dre3/H64Eq3vN5iIqQE3MwL2Uyt82+nTmTb8Shw6ifUdpCbVQ27WbvvlW0tO7HIU6cIjjFgRMHDnHgFPvZnnfY6x3iwCXOI+sc/baJC86OEM7WAI6WABGHg47cTFry0mnM8tFKmJZQC62hVlqDrdZzqJW2UBtm4F5i3A432d5sMn2Z1rPXes7yZZHjyyHHl0OuL5dcfy65vlwyvBk4ZOhvzSYcJnzwIJHqasIfVRGuriLyURXh6moi1dW9H8biduMqKrJaQ6NGWeeSiorsc0tFuEcV4czOPvFhbIw5JS3ySDzC3pa9bGvaxrbGbWxr2Ep14z5Su2KMrzdMqDVMqIXSWuuLAFgfaofzHdSM9lI7NoWGcRm0jM3ClZqGz+XD7/Ljd3jwG4MvFsUfDeGLBPGHu/AFO0kJtuHvbqG4s5nRXS0Ddy35MvuFyyCBk5o/aLfaGREkZ4qkBUlnPbz2GESDR77R9D5nHr3Mk44RIXroEIEtW+zg2EJw+3ZMMAiAMzMT3wXl+GeV4y+fha+8HFe23YoId8OBv9D44e95o2YNq+linc9HTITMiJtxTOOWCz/LLbOvPq4/ZKV6xOIxOsIdA4ZMS6ilz3zP+rZwG3Fz9NhpLnGR7cvuDZYcX07vdK6/73y2L3vA3x2ZWIxofT3i8ZxcSCRJ3MSpaq86EhqN29jVvItQLARYF2HMzJvJzNyZTM+djsfpIRgNEogGCEQDROvrce+pxrf3IGmVdWTub8TfZv3txwWaCnzUjPHy0SgXlUWwJz9GmyNI1PTvCjvC5XAxPnU0Jb58St2ZlIiH0rhQEg6T0d0MnQ3QWWd9XoUHaQ2n5PYNl7RCSCtALv+SBkmPpARJaxX89FPQftDquwy0WldpmFifYrGwEGj2EGxyE2j2EGj2EgtY34LEKfhGpeArycVfNgr/lFLcY4sRf/aRAHI44cBb1O1ZxWtNW1jt9/CBz4sRIS3sJ9Y+kxlFn+Sfr/4Y00dnntpjVOoY4iaTd3BdAAAgAElEQVROW6iNpkATTcEmmgJNNAebe6ebgk00B47M9/xWp78sb1afFk2Ov28LJ8eX07vM5zq9w9HUddX1CY3tjdt7r97zu/xMy5nGzLyZzMqbxYy8GRSnFZ9wN2Ckrp7g9u3WY9s2Aju2E2uw7smDw4FnQine6dNwTJ2EmVpGpKyYkNv6PVR1RzUH2g+wv20/B9oPUN1e3Sd0cn25lGSWUJpZSklGCaWpoyl1pTE6Lji7Gq2A6UoIms76I9PRAPK1dg2SHqc8SBr3WCES7oDbfw3jrCuQTShEcOtGAhvXEdy6jcD23YQP1vVu5inMwD82A98oD/588GUGkXCbFUD2lR5BERqcDpqcThqdTqrcLt5ISWGzz+rvzDE5NDZdQLRzNjfPrODzV5VRkje8K3eUSjZjDJ2RTito+gfPAEHUFekacD8prpTeFk1vt1pPKyehxZPjyyHTm3lCLfO2UBvbG7f3CY6GgHWewiUuJmVPslob9mNC5oSkjeLQGy7bthHcvn3AcPFNnYb4vBCLQzyOiceJxyIEwt10BTvoDnfSHe4iGO4iEO4mFoviMNY5NKcR/A4vPocXv3jwOjx4ceMRJw4jmHgMYjEmvvqqBkmPUxokhzfDC7eACOaO39CxtY7udesIbN1CaMfO3n5dZ14e/vJy/OXleGfNJDR5LM2uII3djTQGG2nobqAx0EhToImGgDXdGGigc4A/oJKU8cQDF7JzbwleM4o7LhnHfVdMoChTBwtU56ZgNNjbomkONvdp6fRMNwebaQ400xJqGbCLzSlOq4utX6umJ2iyfdlUd1T3hkZVR1XvtiUZJX1CY0r2lNPeGurPCpdtBLfZrZc9H1oh4nBYXX89z04nOARxOMHptFpITicx4oRMhKAJE4iFCJgg3bEggXiQmFgDn8YFXG4PfncqKd40rnvhTxokPU5ZkHz0Dvzir8CbQey2Fzn8nR/QsXo1xuclPGksbWX51JVkcGCMmyp/Nw3BRhoDjTQHmgfs40xxpZDnz+t95Kfkk+HOJh7JIBBMoaXDy56DTtZVhsn0u7l7Xgn3zCshO/XM+KGaUmeCWDxGW7ittxutN3gSQydhXSDa9zc7BSkFzMqb1Rsa03Onk+HJGKGjOf0isQjVndUcaDvSRba/bT/72/bz9u1va5D0OCVBsuc1+OVfQ+YYwlf/P6of/l+E9u3jhY85+EMFxO2hRRziIMeXQ54/j1x/Lvn+/L5h4c8n15dLNJLG4RbDvobOI4/6Lmrbg70v6RAoyUtl6UVjuf2S8aR5z6zBEJU6G3VHumkONtMSbKEwtZCClIKRrtIZyRiDw+EY8SFSzh3bX4bf/A0UTKVz4iPU3PMFAvEg31kijFpwLcsnfqq3RZHlzertNw1GYuxv7LJC4mAX7zdagVHZUE93+MhJ+XSviwkFacybmEtZfhpl+amU5acxLjcFr0sv3VXqVEpxp5DiTqE4vXikq3JGO5GLBjRIhvLBC/C7L2LGXERz7Cbqv/gvHC5w8eTNwh0f/xfunHYnTV0R9jV08sHeTvbV11th0dhJTUuAxAbfmCw/ZQVpXFSSYweGFRr56V4dvE8pddbSIDmWd56BV/+N+LgFHN5aSvuq5bw/zcXPbsnkmwv/nbr6MVz6xBvUd4R6N/G7nUzIT2X22GxunVvcGxilean4Pdq6UEqdezRIBmIMrHkC3vwWkVHXUr0yTHD3q7x4lYPKT87imUu/xfLVjazcvJELxmbxtwvKrMAoSGNUhg+HQ1sXSqnzhwZJf/E4vPoIvPcsXRmfoPpHlQSDnfzv2xxMumEpn83+LHc+t5OmzjD/uHAyf7ugDJfzzPjVrVJKjQQNkkSxKKz8AmbTL2iJXkftDzdTl+3g3z/j5q7rvsLWXZO597ebmFSQxg/vvoiZY/SX5EoppUHSIxqClz5LfPsr1B68ira3t/DBJAe/WlLEZ+d8nf/7SpADTR9x3+Wl/NO1U/C59XyHUkqBBokl1Am/vIPItrVUb5tDqHIPv54vVH96Hhebz/FPP6tjVKafX9x3KZeV5Y50bZVS6oyiQRJogZ9/mu5NW6haN4FgsIGnb3Ew7vp7OLjxIlYfquO2C4v56ienk+47e+9gppRSyXJ+B0lHHfzsFlre/YjD6/NpyAjy9D2pTJryEL94NYt0b5jv33kh184oGumaKqXUGev8DZLWKsyPbqT2f9pp/TCdzRMM/3XHBGJd97HiTTcLp+fzxC2zyEsb/l3GlFLqXHZ+BknDh0S/fxNVq2OE6rz89lJh0ycXsHPrdWC8fOe26dx24YnfV0Appc5H51+QHN5M4Lu3cuB1F+Gwi/+4yUnN9KVsW3cBl5Tm8tSnL2BsTspI11Ippc4a51eQfPQ2bV+/g5p3fDSnCsvvzuTD+F107Snhy5+Ywmfnl+qv0pVS6gSdN0Fidr1K7cOfp3WXjx3jhB/eXMbug8uYXjCW7949m8mF6SNdRaWUOiudF0FiuprZ9/m/I1Lr4w8VwopLLqep+hM8uGAqX/jYJDwuHeJEKaVOVlI/QUXkOhHZLSJ7ReThAdZ7ReSX9vr3RKTEXr5QRDaIyFb7+WMJ26yx97nJfgx5V5pQ1SG6Gzz8xyfc/MeMpaSFbuelz1/JPy6aoiGilFLDlLQWiYg4gWeAhUANsE5EVhpjdiQUuxdoMcZMFJGlwLeAJUAj8EljzCERmQm8CoxJ2O4OY8xx3/IwivC127PZGP4Mfz19Hg9fP5UUz3nRGFNKqaRL5qfpxcBeY0wlgIisAG4CEoPkJuAxe/olYLmIiDFmY0KZ7YBPRLzGmBAnoSbbS6bzYX58x2VcOTn/ZHahlFJqEMns1xkDVCfM19C3VdGnjDEmCrQB/QezuhXY2C9EfmR3a31FBvmxh4jcLyLrRWS90+Typy9eryGilFJJkMwgGegD3pxIGRGZgdXd9bmE9XcYY2YBV9iPOwd6cWPMc8aYCmNMxaRR2WSm6DhZSimVDMkMkhpgbMJ8MXBosDIi4gIygWZ7vhh4GbjLGLOvZwNjzEH7uQP4BVYXmlJKqRGSzCBZB0wSkVIR8QBLgZX9yqwE7ranbwPeMMYYEckCfg88Yox5q6ewiLhEJM+edgOLgW1JPAallFJDSFqQ2Oc8HsS64mon8CtjzHYReVxEbrSL/RDIFZG9wD8APZcIPwhMBL7S7zJfL/CqiGwBNgEHgf9M1jEopZQamhjT/7TFuaeiosKsX3/cVwsrpZQCRGSDMaZiqHL6azyllFLDokGilFJqWDRIlFJKDYsGiVJKqWHRIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOiQaKUUmpYNEiUUkoNiwaJUkqpYdEgUUopNSwaJEoppYZFg0QppdSwaJAopZQaFg0SpZRSw6JBopRSalg0SJRSSg2LBolSSqlh0SBRSik1LEkNEhG5TkR2i8heEXl4gPVeEfmlvf49ESmxly8UkQ0istV+/ljCNhfay/eKyNMiIsk8BqWUUseWtCARESfwDHA9MB1YJiLT+xW7F2gxxkwEvgt8y17eCHzSGDMLuBt4IWGb/wDuBybZj+uSdQxKKaWGlswWycXAXmNMpTEmDKwAbupX5ibgJ/b0S8A1IiLGmI3GmEP28u2Az269jAIyjDHvGGMM8FPgU0k8BqWUUkNIZpCMAaoT5mvsZQOWMcZEgTYgt1+ZW4GNxpiQXb5miH0CICL3i8h6EVnf0NBw0gehlFLq2JIZJAOduzAnUkZEZmB1d33uBPZpLTTmOWNMhTGmIj8//ziqq5RS6mQMGSQi4hSRvIR5j/1tf+cQm9YAYxPmi4FDg5UREReQCTTb88XAy8Bdxph9CeWLh9inUkqp0+iYQSIiS7E+2LeIyJsicjVQiXUC/Y4h9r0OmCQipSLiAZYCK/uVWYl1Mh3gNuANY4wRkSzg98Ajxpi3egobYw4DHSJyqX211l3Ab4/nQJVSSiWHa4j1XwYuNMbsFZG5wDvAUmPMy0Pt2BgTFZEHgVcBJ/C8MWa7iDwOrDfGrAR+CLwgInuxAmupvfmDwETgKyLyFXvZImNMPfC3wI8BP7DKfiillBohYl38NMhKkQ+MMXMT5ncZY6aelpqdQhUVFWb9+vUjXQ2llDqriMgGY0zFUOWGapEUiMg/JMynJc4bY/73yVZQKaXUuWGoIPlPIP0Y80oppc5zxwwSY8zXBlsnIqmnvjpKKaXONsdz+e8YEamwr7xCRApE5JvAnqTXTiml1BlvqMt/vwRsAv4v8K6I3A3sxLpi6sLkV08ppdSZbqhzJPcDU4wxzSIyDtgLXGmMeTf5VVNKKXU2GKprK2iMaQYwxlQBH2qIKKWUSjRUi6RYRJ5OmC9InDfGfDE51VJKKXW2GCpI/rnf/IZkVUQppdTZaajLf39yrPVKKaXUMYNERPoPstiHMebGU1sdpZRSZ5uhurYuw7rx1IvAewx8PxCllFLnsaGCpAhYCCwDbsca2v1FY8z2ZFdMKaXU2eGYl/8aY2LGmD8aY+4GLsX6HckaEfnCaamdUkqpM95QLRJExAt8AqtVUgI8DfxXcqullFLqbDHUyfafADOxbh71NWPMttNSK6WUUmeNoVokdwJdwGTgi9bdbQHrpLsxxmQksW5KKaXOAkP9jmTI0YGVUkqd3zQolFJKDYsGiVJKqWFJapCIyHUisltE9orIwwOs94rIL+3174lIib08V0T+R0Q6RWR5v23W2PvcZD8KknkMSimljm3Iy39Plog4gWewftBYA6wTkZXGmB0Jxe4FWowxE0VkKfAtYAkQBL6CdcXYzAF2f4cxZn2y6q6UUur4JbNFcjGw1xhTaYwJAyuAm/qVuQnoGRjyJeAaERFjTJcx5i9YgaKUUuoMlswgGYM1TlePGnvZgGWMMVGgDcg9jn3/yO7W+ookXJOslFLq9EtmkAz0AW9Ookx/dxhjZgFX2I87B3xxkftFZL2IrG9oaBiyskoppU5OMoOkBhibMF8MHBqsjIi4gEyg+Vg7NcYctJ87gF9gdaENVO45Y0yFMaYiPz//pA5AKaXU0JIZJOuASSJSKiIeYCnQ//4mK4G77enbgDeMMYO2SETEJSJ59rQbWAzosC1KKTWCknbVljEmKiIPAq8CTuB5Y8x2EXkcWG+MWQn8EHhBRPZitUSW9mwvIgeADMAjIp8CFgEfAa/aIeIEXgP+M1nHoJRSamhyjAbAOaOiosKsX69XCyul1IkQkQ3GmIqhyukv25VSSg2LBolSSqlh0SBRSik1LBokSimlhkWDRCml1LBokCillBoWDRKllFLDokGilFJqWDRIlFJKDYsGiVJKqWHRIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOiQaKUUmpYknaHxDNdJBKhpqaGYDA40lU55/h8PoqLi3G73SNdFaXUaXDeBklNTQ3p6emUlJQgIiNdnXOGMYampiZqamooLS0d6eoopU6D87ZrKxgMkpubqyFyiokIubm52tJT6jxy3gYJoCGSJPq+KnV+Oa+DRCml1PAlNUhE5DoR2S0ie0Xk4QHWe0Xkl/b690SkxF6eKyL/IyKdIrK83zYXishWe5un5Sz++ut0Opk9ezYzZ87k05/+NN3d3adkv3V1dSxevJgLLriA6dOnc8MNNwBQWlrK7t27+5T90pe+xLe//W3WrFlDZmYmc+bMYcqUKVx55ZW88sorp6Q+SqlzW9KCREScwDPA9cB0YJmITO9X7F6gxRgzEfgu8C17eRD4CvBPA+z6P4D7gUn247pTX/vTw+/3s2nTJrZt24bH4+HZZ5897m2j0eig6x599FEWLlzI5s2b2bFjB08++SQAS5cuZcWKFb3l4vE4L730EkuWLAHgiiuuYOPGjezevZunn36aBx98kNdff/0kj04pdb5I5lVbFwN7jTGVACKyArgJ2JFQ5ibgMXv6JWC5iIgxpgv4i4hMTNyhiIwCMowx79jzPwU+BawaTkW/9rvt7DjUPpxdHGX66Ay++skZx13+iiuuYMuWLRw4cIDFixezbds2AJ566ik6Ozt57LHHWLBgAfPmzeOtt97ixhtv5K677uLzn/88VVVVAHzve99j/vz5HD58mEWLFvXuu7y8HIBly5axZMkSvvrVrwKwdu1aSkpKGD9+PPv37+9Tn9mzZ/Poo4+yfPlyrrnmmj7rYrEY9957L+vXr0dE+OxnP8tDDz104m+SUuqckMwgGQNUJ8zXAJcMVsYYExWRNiAXaDzGPmv67XPMQAVF5H6slgvjxo070bqfVtFolFWrVnHddUM3rlpbW3nzzTcBuP3223nooYe4/PLLqaqq4tprr2Xnzp088MADLFmyhOXLl/Pxj3+ce+65h9GjR1NeXo7D4WDz5s1ccMEFrFixgmXLlg36WnPnzuU73/nOUcs3bdrEwYMHe8OutbX1JI9cKXUuSGaQDHTuwpxEmZMqb4x5DngOoKKi4lj7PKGWw6kUCASYPXs2YLVI7r33Xg4dOnTMbXq6oQBee+01duw40sBrb2+no6ODa6+9lsrKSv74xz+yatUq5syZw7Zt28jPz2fZsmWsWLGCGTNm8Nvf/pbHH3980NcyZuC3bcKECVRWVvKFL3yBT3ziE31aP0qp808yg6QGGJswXwz0/5TsKVMjIi4gE2geYp/FQ+zzrNFzjiSRy+UiHo/3zvf/PUZqamrvdDwe55133sHv9x+175ycHG6//XZuv/12Fi9ezNq1a7n11ltZtmwZixYt4qqrrqK8vJyCgoJB67dx40amTZtGLBbjwgsvBODGG2/k8ccfZ/Pmzbz66qs888wz/OpXv+L5558/qfdAKXX2S+ZVW+uASSJSKiIeYCmwsl+ZlcDd9vRtwBtmsK/BgDHmMNAhIpfaV2vdBfz21Fd95BQWFlJfX09TUxOhUOiYV04tWrSI5cuPXNTWE0pvvPFG7xVgHR0d7Nu3r7d7r6ysjNzcXB5++OFjdmtt2bKFr3/96zzwwAM4nU42bdrEpk2bePzxx2lsbCQej3Prrbfy9a9/nQ8++OBUHLpS6iyVtBaJfc7jQeBVwAk8b4zZLiKPA+uNMSuBHwIviMherJbI0p7tReQAkAF4RORTwCJjzA7gb4EfA36sk+zDOtF+pnG73Tz66KNccskllJaWMnXq1EHLPv300zzwwAOUl5cTjUa58sorefbZZ9mwYQMPPvhgb+vmvvvu46KLLurdbtmyZTzyyCPcfPPNffb35z//mTlz5tDd3U1BQQFPP/30USfaAQ4ePMg999zT23J64oknTtHRK6XORnKMBsA5o6Kiwqxfv77Psp07dzJt2rQRqtG5T99fpc5+IrLBGFMxVDn9ZbtSSqlh0SBRSik1LBokSimlhkWDRCml1LBokCillBoWDRKllFLDokEygubNm9dn/rvf/S4+n4+2trbeZWvWrGHx4sVHbfvKK68wZ86c3qHiv//97ye9vkopNZDz9p7tZ4K33367z/yLL77IRRddxMsvv8xnPvOZQbeLRCLcf//9vP/++xQXFxMKhThw4EByK6uUUoPQIAFY9TDUbj21+yyaBdc/ecwiaWlpdHZ2ArBv3z46Ozv5zne+wze/+c1jBklHRwfRaJTc3FwAvF4vU6ZMOaqcDveulDodNEjOEC+++CLLli3jiiuuYPfu3dTX1w86oGJOTg433ngj48eP55prrmHx4sUsW7YMh6NvT6UO966UOh00SGDIlsPpsGLFCl5++WUcDge33HILv/71r3nggQcGLf+DH/yArVu38tprr/HUU0+xevVqfvzjH/cpo8O9K6VOBz3ZfgbYsmULe/bsYeHChZSUlLBixQpefPHFIbebNWsWDz30EKtXr+Y3v/kNsViM2bNn997dMDs7m82bN7NgwQKeeeYZ7rvvvtNwNEqp8422SM4AL774Io899hiPPPJI77LS0lI++uijAct3dnayfv16FixYAFhdWOPHj+8d7r1HY2MjHo+HW2+9lbKysmOed1FKqZOlQXIGWLFiBatW9R0N/+abb2bFihVccsklvP766xQXH7mf14svvsi3v/1tPve5z+H3+0lNTT2qWwt0uHel1Omhw8irpND3V6mznw4jr5RS6rTQIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOS1CARketEZLeI7BWRhwdY7xWRX9rr3xORkoR1j9jLd4vItQnLD4jIVhHZJCLr++/zbKLDyCulzgVJ+0GiiDiBZ4CFQA2wTkRWGmN2JBS7F2gxxkwUkaXAt4AlIjIdWArMAEYDr4nIZGNMzN7uamNMY7LqfrroMPJKqXNBMn/ZfjGw1xhTCSAiK4CbgMQguQl4zJ5+CVguImIvX2GMCQH7RWSvvb93klHRb73/LXY17zql+5yaM5V/vfhfj1km2cPIK6XU6ZDMrq0xQHXCfI29bMAyxpgo0AbkDrGtAf4kIhtE5P7BXlxE7heR9SKyvqGhYVgHcjoMNIz8YBKHkV+2bBk///nPe4dBUUqp0y2ZLRIZYFn/8VgGK3OsbecbYw6JSAGwWkR2GWPWHlXYmOeA58AaIuVYFR2q5XA6JGMYeaWUOh2SGSQ1wNiE+WLg0CBlakTEBWQCzcfa1hjT81wvIi9jdXkdFSRnk8Rh5AHC4TATJkw4ZpCANYz8rFmzuPPOOyktLdUgUUqNiGR2ba0DJolIqYh4sE6er+xXZiVwtz19G/CGsUaRXAksta/qKgUmAe+LSKqIpAOISCqwCNiWxGM4LXqGkT9w4AAHDhzg0KFDHDx48JjDyK9Zs6Z3vmcYeaWUGglJa5EYY6Ii8iDwKuAEnjfGbBeRx4H1xpiVwA+BF+yT6c1YYYNd7ldYJ+ajwAPGmJiIFAIvW+fjcQG/MMb8MVnHcLokaxh5pZQ6HXQYeZUU+v4qdfbTYeSVUkqdFhokSimlhkWDRCml1LBokCillBoWDRKllFLDokGilFJqWDRIRpDT6WT27NnMnDmTT3/603R3d5+S/dbV1bF48eLeIeZvuOEGAEpLS9m9e3efsl/60pf49re/zZo1a8jMzGTOnDlMmTKFK6+8kldeeeWU1EcpdW7TIBlBfr+fTZs2sW3bNjweD88+++xxbxuNRgdd9+ijj7Jw4UI2b97Mjh07ePLJJwFYunQpK1as6C0Xj8d56aWXWLJkCQBXXHEFGzduZPfu3Tz99NM8+OCDvP766yd5dEqp80Uyx9o6a9R+85uEdp7aYeS906ZS9G//dtzlr7jiCrZs2cKBAwdYvHgx27ZZI7889dRTdHZ28thjj7FgwQLmzZvHW2+9xY033shdd93F5z//eaqqqgD43ve+x/z58zl8+DCLFi3q3Xd5eTkAy5YtY8mSJXz1q18FYO3atZSUlDB+/Hj279/fpz6zZ8/m0UcfZfny5VxzzTXDei+UUuc2bZGcAaLRKKtWrWLWrFlDlm1tbeXNN9/kH//xH/n7v/97HnroIdatW8dvfvMb7rvvPgAeeOAB7r33Xq6++mq+8Y1vcOiQNVZmeXk5DoeDzZs3A9bQLMuWLRv0tebOncuuXac2YJVS5x5tkcAJtRxOpUAgwOzZswGrRXLvvff2fugPpqcbCuC1115jx44j9wlrb2+no6ODa6+9lsrKSv74xz+yatUq5syZw7Zt28jPz+f/b+/+g6wq6ziOvz8DK4ukKeAisQIrYhvYBo4I+ANQC39EKoNTLI5hyjhNEMroJDvMMIp/lNlkEgqhKUUmJqkgM2YGOJAiCS4ukJCQpOtYwIpEEuTKtz+eZ+Gy3Ls/uOyee3e/r5k7e87znHPu9z57733uec4531NeXs6iRYsYOHAgS5YsYdasWRmfqz2kz3HOZc87kgTVHSNJ1bFjx6NuUnXgwIGj6rt06XJ4+tChQ6xZs4bOnTsfs+2uXbsyYcIEJkyYwJgxY1i1ahXjxo2jvLyc0aNHM3LkSMrKyigqKsoYX2VlpefLcs41yoe2ckyPHj3YuXMnNTU1HDx4sMEzp0aPHs2cOXMOz9d1SitWrDh8Bti+ffvYvn07vXv3BqBfv35069aN6dOnNzisVVVVxX333dfoPVGcc873SHJMQUEBM2fOZOjQoZSUlFBaWppx2dmzZzN58mTKysqora1lxIgRzJs3j/Xr1zNlypTDezeTJk1iyJAhh9crLy+noqKCsWPHHrW91atXM3jwYPbv309RURGzZ8/2A+3OuUZ5GnnXIrx9nct/nkbeOedcq/COxDnnXFbadUfSHob1kuDt6lz70m47ksLCQmpqavxL7wQzM2pqaigsLEw6FOdcK2m3Z20VFxdTXV3Nrl27kg6lzSksLKS4uDjpMJxzraTddiQFBQWUlJQkHYZzzuW9Fh3aknSVpK2Stkmanqa+k6SnY/1aSX1T6ipi+VZJVzZ1m84551pXi3UkkjoADwNXAwOAckkD6i12K7DHzM4BHgTuj+sOAMYDA4GrgEckdWjiNp1zzrWiltwjuRDYZmZ/N7P/AYuA6+otcx3wqzi9GLhCkmL5IjM7aGbvAtvi9pqyTeecc62oJY+R9ALeT5mvBoZmWsbMaiXtBbrF8tfrrdsrTje2TQAk3QbcFmf/I2lruuVOkO7A7hbcfkvK59jB40+ax5+slo6/T1MWasmORGnK6p9rm2mZTOXp9qDSnr9rZvOB+Q0FeKJIWteUNAK5KJ9jB48/aR5/snIl/pYc2qoGzkqZLwbq32zj8DKSOgKfBz5qYN2mbNM551wrasmO5A2gv6QSSScRDp4vrbfMUmBinL4BWGHhCsGlwPh4VlcJ0B/4SxO36ZxzrhW12NBWPOYxBXgJ6AA8bmabJc0C1pnZUuCXwEJJ2wh7IuPjupsl/Q74K1ALTDazzwDSbbOlXkMztMoQWgvJ59jB40+ax5+snIi/XaSRd84513Laba4t55xzJ4Z3JIP7m+4AAAeLSURBVM4557LiHUkzSDpL0kpJb0vaLOn2WN5V0suS3ol/T0861obELAGVkpbF+ZKYouadmLLmpKRjzETSaZIWS9oS/w/D86X9JU2L75tNkp6SVJjrbS/pcUk7JW1KKUvb3gpmx/RFVZLOTy7yjLE/EN87VZKek3RaSl3atExJSRd/St1dkkxS9zifaNt7R9I8tcCdZvYlYBgwOaZomQ4sN7P+wPI4n8tuB95Omb8feDDGv4eQuiZXPQT8wcxKga8QXkfOt7+kXsBU4AIzO49wssh4cr/tFxDSFKXK1N5XE86w7E+4GHhuK8WYyQKOjf1l4DwzKwP+BlRA5rRMrRdqWgs4Nn4knQV8DXgvpTjZtjczfxznA1gS/6FbgZ6xrCewNenYGoi5mPDhvxxYRrj4czfQMdYPB15KOs4MsZ8KvEs8SSSlPOfbnyNZHLoSzpZcBlyZD20P9AU2NdbewC+A8nTL5Urs9erGAk/G6QqgIqXuJWB4rrV9LFtM+BG1A+ieC23veyTHKWYqHgysBXqY2YcA8W9RcpE16mfAD4BDcb4b8LGZ1cb51HQ0ueZsYBfwRByae0xSF/Kg/c3sA+AnhF+RHwJ7gfXkT9unytTe6dIi5fLruQV4MU7nReySrgU+MLO36lUlGr93JMdB0ueA3wN3mNm/k46nqSSNAXaa2frU4jSL5uo54R2B84G5ZjYY+IQcHMZKJx5HuA4oAb4AdCEMR9SXq23fFHnzXpI0gzBU/WRdUZrFcip2SScDM4CZ6arTlLVa/N6RNJOkAkIn8qSZPRuL/yWpZ6zvCexMKr5GXAxcK2kHIXPy5YQ9lNNiihrI7bQz1UC1ma2N84sJHUs+tP9XgXfNbJeZfQo8C1xE/rR9qkztnRcpjCRNBMYAN1ocByI/Yu9H+CHyVvwMFwNvSjqThOP3jqQZJIlwNf7bZvbTlKrUVC8TCcdOco6ZVZhZsZn1JRxYXGFmNwIrCSlqILfj/yfwvqQvxqIrCNkP8qH93wOGSTo5vo/qYs+Ltq8nU3svBb4dzyAaBuytGwLLFZKuAu4GrjWz/SlVmdIy5Qwz22hmRWbWN36Gq4Hz4+ci2bZP+mBSPj2ASwi7i1XAhvi4hnCcYTnwTvzbNelYm/BaRgHL4vTZhA/NNuAZoFPS8TUQ9yBgXfwfPA+cni/tD9wLbAE2AQuBTrne9sBThGM6nxK+uG7N1N6E4ZWHge3ARsIZarkW+zbCsYS6z++8lOVnxNi3AlfnYtvXq9/BkYPtiba9p0hxzjmXFR/acs45lxXvSJxzzmXFOxLnnHNZ8Y7EOedcVrwjcc45lxXvSFybJOmHkkZJul5Ss65+l3RGzMhbKenSenWvSLqgkfXviFchH5cY90UZ6jpJ+pOkDZK+1czt9pU04Xjjci4T70hcWzWUkAdtJLC6meteAWwxs8Fm1tx1Ae4AjrsjIVzjk7YjIeR3KzCzQWb2dDO32xdoVkeSctW9cxl5R+LalHi/iSpgCLAGmATMlXRMfiJJfSQtj/dvWC6pt6RBwI+Ba+Kv/s4NPNdcSesU7jFybyybSsiltVLSylg2WtIaSW9KeibmakPSDkn3xvKNkkpjMtDvAtPi81+a8nxFwG+AQbGun6SZkt5QuMfJ/HjVPJLOiXsub8Xt9wN+BFwa152mcD+UJ+JzV0q6LK57c4zzBeCPknpKWhXX21R/L825xK+c9Yc/TvQDuBD4OVAAvNrAci8AE+P0LcDzcfpmYE6GdV4hXjXMkSu6O8Tysji/gyNXHHcHVgFd4vzdwMyU5b4fp78HPBan7wHuyvD8o4gZCVJjiNMLgW/E6bXA2DhdSNhDqr/uncATcbqUkMalML7+6pTXdycwI+W1npL0/9gfufXwPRLXFg0mpL8oJeSzymQ48Ns4vZCQAqc5vinpTaCScEOkAWmWGRbLX5W0gZCbqk9KfV3iz/WEoafmuiwez9lISMI5UNIpQC8zew7AzA7Y0Xml6lxCeN2Y2RbgH8C5se5lM/soTr8BfEfSPcCXzWzfccTp2jAf/3RtRhyWWkDIfLqb8Ctc8Qt8uJn9t5FNNDlfUEzsdxcwxMz2SFpA+DV/zKKEL+XyDJs6GP9+RjM/j5IKgUcIe0jvxy/6QtKnFE+7iQbqPqmbMLNVkkYAXwcWSnrAzH7dnFhd2+Z7JK7NMLMNZjaIcAvVAcAK4EoLB6bTdSKvEbIgA9wI/LkZT3cq4ct2r6QeHH1vkX3AKXH6deBiSedAuKeEpHNpWOr6DanruHbH4y43AFi4R061pOvjc3aKZ5HV3+4qwusmxtSbkLDwKJL6EO5j8ygh+3Wi92J3ucc7EtemSDoD2GNmh4BSM2toaGsqYcimCriJcC/7JrFwh7pKYDPwOPBqSvV84EVJK81sF+GYw1PxeV4nDLk15AVgbP2D7Wli+Bh4lJDt9XnCEFSdm4Cp8TlfA84kZEyujQfgpxH2ZjrEYbGngZvN7CDHGgVskFQJjAMeaiR+18549l/nnHNZ8T0S55xzWfGOxDnnXFa8I3HOOZcV70icc85lxTsS55xzWfGOxDnnXFa8I3HOOZeV/wNfw82jPPH+zQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_rank_scores([ssvd_rank_scores,\n", + " sals_rank_scores,\n", + " ials_rank_scores,\n", + " psvd_rank_scores]);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There seem to be no difference between the original and scaled versions of iALS. Let's verify this with CV experiment." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### cross-validation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You only need to perform CV computations for the new model. Configuration of data will be the same as previously, as the `data_model` instance ensures reproducible data state." + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'alpha': 0.3,\n", + " 'epsilon': 0.1,\n", + " 'weight_func': ,\n", + " 'regularization': 1.0,\n", + " 'rank': 60}" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sals_best_config.update(sals_best_rank)\n", + "sals_best_config" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "set_config(sals, sals_best_config)" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "5/5\n", + "[03:00<00:36, 36.06s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 5/5 [03:00<00:36, 36.06s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sals.verbose = False\n", + "sals_cv_results = ee.run_cv_experiment([sals],\n", + " metrics=metrics,\n", + " iterator=track)" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+0nVV97/v3x0RAUVExWuVX4EBtg1BattBeq02lKIyqsUesQW6Fe7kjWuX2jHptC+cUGji2RzrOKbUH2msqIGIVLC23qUSjFfHXtZggSAyU2xijxGgJQlFQwMD3/rGejYuVneyV7Gevtfba79cYa+xnzWfOueYz2Xuy893zR6oKSZIkSZKktjxl2A2QJEmSJEnjxWCDJEmSJElqlcEGSZIkSZLUKoMNkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhls0NhIsiXJrw27HZIkSZI03xlskCRJkiRJrTLYoHknycJht0GS5rOpxmHHZkkarCQLht0GjTeDDRo3xyW5PckDSa5Nsl+SpUm2JvmDJN8Frhx2IyVpHDTL136vGXcfSnJ5khck+XiSHyT5pyTPSbI4SSU5O8m3gBunShv280jSoCU5JMnfJ9me5HtJLk3ylCR/mOSbSe5J8sEkBzT5P5HknJ46vprkPzbXP5PkU0nuS3JXkt/syveBJH+VZE2Sh4BfTfLrSW5N8v0kdydZ2VP3W5p2fC/J+d3Llpt2npvk6839jyZ57mz3meYOgw0aN78JnAIcDhwLnNWk/xTwXOAwYMVQWiZJ4+kNwMnATwOvBT4O/GfgeXR+z/idrry/Avws8Opp0iRp7DUzCz4GfBNYDBwEXEPn99ezgF8FjgCeAVzaFPswcHpXHUvo/H57Q5L9gU81eZ7f5PvLJEd3feybgT8Gngl8AXgIeAvwbODXgd9O8vquuv8SOAN4IXBA08ZJvwO8ns44/iLgfuCyGXSJxozBBo2bv6iqbVV1H/CPwHFN+uPAH1XVI1X1o+E1T5LGzv+sqn+rqm8Dnwdurqpbq+oR4Hrg57vyrqyqh3rG4anSJGk+OIHOP9J/rxkHH66qL9D5x/2fVdXmqnoQOA9Y3iw3u57OTN7DmjrOAP6+GXNfA2ypqiurakdVfQX4O+C0rs/8h6r6YlU93nzeTVW1oXl/O/AROsEDmnL/WFVfqKpHgQuA6qrrrcB/qaqtzeevBE5zWZwmGWzQuPlu1/UP6USCAbZX1cNDaI8kjbt/67r+0RTvn9H1/u4pyk+VJknzwSHAN6tqR0/6i+jMdpj0TWAh8IKq+gFwA7C8ubcc+Jvm+jDgxCT/PvmiE4z4qa66njTmJjkxyWeaZRwPAG+jMzNtsh1P5K+qHwLf6yp+GHB912fdCTwGvKDvHtBYM9ig+aKmzyJJmmVTjcWOz5Lmq7uBQ6eYCbCNzj/kJx0K7OAnwdyPAKcn+SXgacBnuur7bFU9u+v1jKr67a66esfcDwOrgUOq6gDg/wbS3PsOcPBkxiRPAw7saf+pPZ+3XzPTTTLYIEmSJElD8GU6/6B/T5L9m43NX0YnmPC7SQ5P8gzgT4Bru2ZArKETjLioSX+8Sf8Y8NNJfivJU5vXS5P87G7a8Ezgvqp6OMkJdPZ0mHQd8Nok/0uSfYAL+UkgAjqBiT+eXNKRZFGSZTPoD40Zgw2SJEmSNGBV9RidjXWPBL4FbAXeBFwBXA18DvgG8DDwf3aVewT4e+DX6MxMmEz/AfAqOksrttFZXnwxsO9umvF24KIkP6CzJ8NHu+rb2HzuNXSCIj8A7gEeabK8l86siE825f8ZOHGPO0JjK1XOXpQkSZIk7Vozy+LfgaOq6hvDbo9GnzMbJEmSJEk7SfLaJE9vjtX878AGYMtwW6W5wmCDJEmSJGkqy+gsydgGHAUsL6fGq08uo5AkSZIkSa1yZoMkSZIkSWqVwQZJkiRJktSqhcNuQK/nPe95tXjx4mE3Q5J2csstt9xbVYuG3Y5BcCyWNIochyVp+Podi0cu2LB48WLWr18/7GZI0k6SfHPYbRgUx2JJo8hxWJKGr9+x2GUUkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBrVi6dKlLF26dNjNkKR5zbFYkobLcVj6CYMNkjRCkpyS5K4km5KcO8X9fZNc29y/OcniJv2EJLc1r68m+Y2uMluSbGjurR/c00iSJGm+WjjsBkiSOpIsAC4DTga2AuuSrK6qO7qynQ3cX1VHJlkOXAy8CfgaMFFVO5K8EPhqkn+sqh1NuV+tqnsH9zSSJEmazww2SNLoOAHYVFWbAZJcAywDuoMNy4CVzfV1wKVJUlU/7MqzH1Cz31xJksbHMVcdM+M6Nn93c2t1AWw4c0Mr9UjD4DIKSRodBwF3d73f2qRNmaeZtfAAcCBAkhOTbAQ2AG/rmtVQwCeT3JJkxa4+PMmKJOuTrN++fXsrDyRJkqT5yZkNYvG5N8y4ju9u/l5rdQFsec+vt1KPNMdkirTeGQq7zFNVNwNHJ/lZ4KokH6+qh4GXVdW2JM8HPpXkX6rqcztVUrUKWAUwMTHhzAhJkiTtNWc2SNLo2Aoc0vX+YGDbrvIkWQgcANzXnaGq7gQeAl7SvN/WfL0HuJ7Ocg1JkiRp1hhskKTRsQ44KsnhSfYBlgOre/KsBs5srk8DbqyqasosBEhyGPBiYEuS/ZM8s0nfH3gVnc0kJUlSy4447wiOOO+IYTdDGgkuo5CkEdGcJHEOsBZYAFxRVRuTXASsr6rVwOXA1Uk20ZnRsLwp/svAuUl+DDwOvL2q7k1yBHB9EuiM+R+uqk8M9skkSZI03xhsUCt+6s3vGXYTpLFQVWuANT1pF3RdPwy8cYpyVwNXT5G+Gfi59luqVq08oJ16tjzUXn0rH5h5HdKYSXIK8F46AeH3V9V7eu6/Avhz4FhgeVVd13P/WcCdwPVVdc5gWi1Jw+EyCkmSJGkaSRYAlwGnAkuA05Ms6cn2LeAs4MO7qOa/Ap+drTZK0igx2CBJkiRN7wRgU1VtrqpHgWuAZd0ZqmpLVd1OZznbkyQ5HngB8MlBNFaShs1ggyRJkjS9g4C7u95vbdKmleQpwP8Afm+afCuSrE+yfvv27XvdUEkaBQYbJEmSpOllirTqs+zbgTVVdffuMlXVqqqaqKqJRYsW7XEDJWmUuEGkJEmSNL2twCFd7w8GtvVZ9peAlyd5O/AMYJ8kD1bVuS23UZJGRl8zG5KckuSuJJuS7DQoJtk3ybXN/ZuTLG7Sn5rkqiQbktyZ5Lx2mz/7li5dytKlS4fdDEmSJA3XOuCoJIcn2YfO0cOr+ylYVWdU1aFVtRh4F/BBAw2Sxt20wYY+d949G7i/qo4ELgEubtLfCOxbVccAxwNvnQxESJIkSXNFVe0AzgHW0jm+8qNVtTHJRUleB5DkpUm20vkd+H1JNg6vxZI0XP0so3hi512AJJM7797RlWcZsLK5vg64NEnorGPbP8lC4GnAo8D322m6pHE1OZvopptuGmo7JEnqVlVrgDU9aRd0Xa+js7xid3V8APjALDRPkkZKP8GGqXbePXFXeapqR5IHgAPpBB6WAd8Bng78blXdN9NGS3OV/4iWNJtuOmv/YTdBkiQJ6G/Phn523t1VnhOAx4AXAYcD/1eSI3b6AI/5kSRJkiRpbPQTbOhn590n8jRLJg4A7gPeDHyiqn5cVfcAXwQmej/AY34kSZIkSRof/SyjeGLnXeDbdHbefXNPntXAmcCXgNOAG6uqknwLeGWSD9FZRvGLwJ+31fjpLD73hhnX8d3N32utLoAt7/n1VuqRpPnEJUiSJElzy7TBhmYPhsmddxcAV0zuvAusr6rVwOXA1Uk20ZnRsLwpfhlwJfA1Okstrqyq22fhOSSNgLaCcm0G+QzwSZIkSYPXz8yGfnbefZjOET+95R6cKl2SJEmjy9lEkqSZ6mfPBkmSJEmSpL71NbNBmu9cHiBJ48G/2EuSNBjObJCkEZLklCR3JdmU5Nwp7u+b5Nrm/s1JFjfpJyS5rXl9Nclv9FunJEmS1DZnNkzjp978nmE3QdI8kWQBnY11T6ZzpPC6JKur6o6ubGcD91fVkUmWAxcDb6KzEe9Es6nvC4GvJvlHoPqoU5IkSWqVwQZJI2ceB/lOADZV1WaAJNcAy4DuwMAyYGVzfR1waZJU1Q+78uxHJ8jQb52SJElSqww2SNLoOAi4u+v9VuDEXeVpZjE8ABwI3JvkROAK4DDgt5r7/dQJQJIVwAqAQw89dOZPA7DygHbq2fJQe/WtfGDmdUiSJGm33LNBkkZHpkirfvNU1c1VdTTwUuC8JPv1WSdN+VVVNVFVE4sWLdqDZkuSJElP5swGaYDm8fIA9WcrcEjX+4OBbbvIszXJQuAA4L7uDFV1Z5KHgJf0WaekcdLGDKA2ZxOBM4okaR5yZoMkjY51wFFJDk+yD7AcWN2TZzVwZnN9GnBjVVVTZiFAksOAFwNb+qxTkiRJapXBBkkaEVW1AzgHWAvcCXy0qjYmuSjJ65pslwMHJtkEvBOYPMryl+mcQHEbcD3w9qq6d1d1Du6pJGl89HE88SuSfCXJjiSndaUfl+RLSTYmuT3JmwbbckkaPJdRSNIIqao1wJqetAu6rh8G3jhFuauBq/utU5qTXB6gIerzeOJvAWcB7+op/kPgLVX1r0leBNySZG1V/fsAmi5JQ2GwQZIkSZretEcJV9WW5t7j3QWr6v/rut6W5B5gEWCwQdLYMtggSRp5N521/7CbIEl9HyW8O0lOAPYBvj7FvfaPIJakIXHPBkmSJGl6fR8lvMsKkhfSWfL2v1XV4733PYJY0jgx2CBJkiRNb0ZHCSd5FnAD8IdV9c8tt02SRo7BBkmSJGl6e32UcJP/euCDVfW3s9hGSRoZ7tkgSZKkJ3GflJ1V1Y4kk0cJLwCumDyeGFhfVauTvJROUOE5wGuTXFhVRwO/CbyCztHFZzVVnlVVtw3+SSRpMAw2SJIkSX3o43jidXSWV/SW+xDwoVlvoCSNEJdRSJIkSZKkVjmzQZIkzRsuD5AkaTCc2SBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWyQpBGS5JQkdyXZlOTcKe7vm+Ta5v7NSRY36ScnuSXJhubrK7vK3NTUeVvzev7gnkiSJEnz0cJhN0CS1JFkAXAZcDKwFViXZHVV3dGV7Wzg/qo6Msly4GLgTcC9wGuraluSlwBrgYO6yp1RVesH8iCSJEma95zZIEmj4wRgU1VtrqpHgWuAZT15lgFXNdfXASclSVXdWlXbmvSNwH5J9h1IqyVJkqQeBhskaXQcBNzd9X4rT56d8KQ8VbUDeAA4sCfPG4Bbq+qRrrQrmyUU5ydJu82WJEmSnsxggySNjqmCALUneZIcTWdpxVu77p9RVccAL29evzXlhycrkqxPsn779u171HBJkiS1a+nSpSxdunTYzdhrfQUb9nbDsubesUm+lGRjs3HZfu01X5LGylbgkK73BwPbdpUnyULgAOC+5v3BwPXAW6rq65MFqurbzdcfAB+ms1xjJ1W1qqomqmpi0aJFrTyQJEmS5qdpgw1dG5adCiwBTk+ypCfbExuWAZfQ+ava5C/CHwLeVlVHA0uBH7fWekkaL+uAo5IcnmQfYDmwuifPauDM5vo04MaqqiTPBm4AzquqL05mTrIwyfOa66cCrwG+NsvPIUljqY8/wL0iyVeS7EhyWs+9M5P8a/M6s7esJI2bfk6jeGLDMoAkkxuWde+OvgxY2VxfB1zarAl+FXB7VX0VoKq+11K7JWnsVNWOJOfQOUliAXBFVW1MchGwvqpWA5cDVyfZRGdGw/Km+DnAkcD5Sc5v0l4FPASsbQINC4B/Av56YA8lSWOizxODvgWcBbyrp+xzgT8CJugsfbulKXv/INouabCOueqYVurZ/N3NrdW34cwNM65jT/UTbJhqw7ITd5Wn+WV5csOynwYqyVpgEXBNVf1p7wckWQGsADj00EP39BkkaWxU1RpgTU/aBV3XDwNvnKLcu4F376La49tsoyTNU9P+Aa6qtjT3Hu8p+2rgU1U1ueztU8ApwEdmv9mSNBz97Nkwkw3LFgK/DJzRfP2NJCftlNF1wpIkSRpt/ZwYNKOybtQraZz0M7NhTzYs29qzYdlW4LNVdS9AkjXALwCfnmG7JUmSpEHq5w9wMypbVauAVQATExP91i1pTB1x3hHDbsKM9DOzYa83LKOz7vjYJE9vghC/wpP3epAkSZLmgn7+ADcbZSVpTpo22FBVO+hsPLYWuBP46OSGZUle12S7HDiw2bDsncC5Tdn7gT+jE7C4DfhKVd3Q/mNIkiRJs6qfP8DtylrgVUmek+Q5dDbwXTtL7ZSkkdDPMoq93rCsufchOsdfSpIkSXNSPycGJXkpcD3wHOC1SS6sqqOr6r4k/5VOwALgosnNIiVpXPUVbJAkSZLmuz7+ALeOzhKJqcpeAVwxqw2UpBHSz54NkiRJkiS1YunSpSxdunTYzdAsM9ggSZIkSZJaZbBBkiRJkiS1yj0bJEmSJEnTOuaqY1qpZ/N3N7dW34YzN8y4Ds0OZzZIkiRJkqRWGWyQJEmSJEmtMtggSZIkSZJa5Z4NkiRJknZr8pjCm266aajt0Hg44rwjht0EDYAzGyRJkiRJUquc2SBJkiSNsTZ2/G/z9ADwBAFpPnBmgyRJkiRJapXBBkkaIUlOSXJXkk1Jzp3i/r5Jrm3u35xkcZN+cpJbkmxovr6yq8zxTfqmJH+RJIN7IkmSJM1HLqOQpBGRZAFwGXAysBVYl2R1Vd3Rle1s4P6qOjLJcuBi4E3AvcBrq2pbkpcAa4GDmjJ/BawA/hlYA5wCfHwQzyRJGg9u6CdpTzmzQZJGxwnApqraXFWPAtcAy3ryLAOuaq6vA05Kkqq6taq2Nekbgf2aWRAvBJ5VVV+qqgI+CLx+9h9FkiRJ85nBBkkaHQcBd3e938pPZifslKeqdgAPAAf25HkDcGtVPdLk3zpNnQAkWZFkfZL127dv3+uHkCRJkgw2SNLomGovhdqTPEmOprO04q17UGcnsWpVVU1U1cSiRYv6aK4kzS8z2FfnqUmuavbPuTPJeYNuuyQNmsEGSRodW4FDut4fDGzbVZ4kC4EDgPua9wcD1wNvqaqvd+U/eJo6JUnT6NpX51RgCXB6kiU92Z7YVwe4hE7wF+CNwL5VdQxwPPDWyUCEJI0rgw2SNDrWAUclOTzJPsByYHVPntXAmc31acCNVVVJng3cAJxXVV+czFxV3wF+kOQXm1Mo3gL8w2w/iCSNob3eV4fOjLL9myDx04BHge8PptmSNBwGGyRpRDR7MJxD5ySJO4GPVtXGJBcleV2T7XLgwCSbgHcCk9N4zwGOBM5Pclvzen5z77eB9wObgK/jSRSStDdmsq/OdcBDwHeAbwH/varu6/0A986RNE48+lKSRkhVraFzPGV32gVd1w/TmY7bW+7dwLt3Ued64CXttlSS5p2Z7KtzAvAY8CLgOcDnk/xTVW1+UsaqVcAqgImJiSn315GkucKZDZIkSdL0ZrKvzpuBT1TVj6vqHuCLwMSst1iShshggyRJkjS9vd5Xh87SiVemY3/gF4F/GVC7JWkoDDZIkiRJ05jhvjqXAc8AvkYnaHFlVd0+0AeQpAFzzwZJkiSpDzPYV+fBqdIlaZw5s0GSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRW9RVsSHJKkruSbEpy7hT3901ybXP/5iSLe+4fmuTBJO9qp9mSJEmSJGlUTRtsSLIAuAw4FVgCnJ5kSU+2s4H7q+pI4BLg4p77lwAfn3lzJUmSJEnSqOtnZsMJwKaq2lxVjwLXAMt68iwDrmqurwNOShKAJK8HNgMb22myJEmSJEkaZf0EGw4C7u56v7VJmzJPVe0AHgAOTLI/8AfAhTNvqiRJkiRJmgv6CTZkirTqM8+FwCVV9eBuPyBZkWR9kvXbt2/vo0mSJEmSJGlULewjz1bgkK73BwPbdpFna5KFwAHAfcCJwGlJ/hR4NvB4koer6tLuwlW1ClgFMDEx0RvIkCRJkiRJc0g/MxvWAUclOTzJPsByYHVPntXAmc31acCN1fHyqlpcVYuBPwf+pDfQIEn6ib09/SfJgUk+05z8c2lPmZuaOm9rXs8fzNNIkiRpvpp2ZkNV7UhyDrAWWABcUVUbk1wErK+q1cDlwNVJNtGZ0bB8NhstSeOo6/Sfk+nMGFuXZHVV3dGV7YnTf5Isp3P6z5uAh4HzgZc0r15nVNX6WX0ASZIkqdHPMgqqag2wpiftgq7rh4E3TlPHyr1onyTNJ0+c/gOQZPL0n+5gwzJgZXN9HXBpklTVQ8AXkhw5wPZKkiRJU+pnGYUkaTD2+vSfPuq+sllCcf7k0cS93KxXknZvb5e6NfeOTfKlJBuTbEiy3yDbLkmDZrBBkkbHTE7/2Z0zquoY4OXN67emylRVq6pqoqomFi1aNG1jJWk+6VrqdiqwBDg9yZKebE8sdQMuobPUjWYD9Q8Bb6uqo4GlwI8H1HRJGgqDDZI0Ovbk9J/JX14nT//Zpar6dvP1B8CH6SzXkCTtmSeWulXVo8DkUrduy4CrmuvrgJOa2WSvAm6vqq8CVNX3quqxAbVbkobCYIMkjY69Pv1nVxUmWZjkec31U4HXAF9rveWSNP5mstTtp4FKsjbJV5L8/lQf4HI2SeOkrw0iJUmzb6an/yTZAjwL2CfJ6+n8Je2bwNom0LAA+Cfgrwf4WJI0Lmay1G0h8MvAS4EfAp9OcktVffpJGatWAasAJiYmplsiJ0kjzWCDJI2QmZz+U1WLd1Ht8W21T5LmsT1Z6ra1Z6nbVuCzVXUvQJI1wC8An0aSxpTLKCRJkqTpzWSp21rg2CRPb4IQv8KTjzWWpLHjzAZJkiRpGjNZ6lZV9yf5MzoBiwLWVNUNQ3kQSRoQgw2SJElSH2a41O1DdI6/lKR5wWUUkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSRkiSU5LclWRTknOnuL9vkmub+zcnWdykH5jkM0keTHJpT5njk2xoyvxFkgzmaSRJkjRfGWyQpBGRZAFwGXAqsAQ4PcmSnmxnA/dX1ZHAJcDFTfrDwPnAu6ao+q+AFcBRzeuU9lsvSZIk/YTBBkkaHScAm6pqc1U9ClwDLOvJswy4qrm+DjgpSarqoar6Ap2gwxOSvBB4VlV9qaoK+CDw+ll9CkkaU3s7+6zr/qHNDLSpAsOSNFYMNkjS6DgIuLvr/dYmbco8VbUDeAA4cJo6t05TJwBJViRZn2T99u3b97DpkjTeZjj7bNIlwMdnu62SNAoMNkjS6JhqL4Xaizx7lb+qVlXVRFVNLFq0aDdVStK8tNezzwCSvB7YDGwcUHslaagMNkjS6NgKHNL1/mBg267yJFkIHADcN02dB09TpyRpens9+yzJ/sAfABcOoJ2SNBIMNkjS6FgHHJXk8CT7AMuB1T15VgNnNtenATc2ezFMqaq+A/wgyS82f117C/AP7TddksbeTGafXQhcUlUP7vYDXM4maYwsHHYDJEkdVbUjyTnAWmABcEVVbUxyEbC+qlYDlwNXJ9lEZ0bD8snySbYAzwL2aabrvqqq7gB+G/gA8DQ6a4VdLyxJe25PZp9t7Zl9diJwWpI/BZ4NPJ7k4ap60lHFVbUKWAUwMTGxuyVykjTyDDZI0gipqjXAmp60C7quHwbeuIuyi3eRvh54SXutlKR56YnZZ8C36QR739yTZ3L22Zd48uyzl09mSLISeLA30CBJ48ZggyRJkjSNmc4+k6T5pq9gQ5JTgPfSGVjfX1Xv6bm/L52z248Hvge8qaq2JDkZeA+wD/Ao8HtVdWOL7ZckSZIGYiazz7ryrJyVxknSiJl2g8gZnil8L/DaqjqGzpSyq9tquCRJkiRJGk39nEax12cKV9WtVTW5cc5GYL9mFoQkSZIkSRpT/QQb9vpM4Z48bwBurapHej/AY34kSZIkSRof/QQbZnKmcOdmcjSdpRVvneoDqmpVVU1U1cSiRYv6aJIkSZIkSRpV/QQb9uRMYXrOFCbJwcD1wFuq6uszbbAkSZIkSRpt/QQbnjhTOMk+dI7wWd2TZ/JMYeg6UzjJs4EbgPOq6ottNVqSJEmSJI2uaYMNzR4Mk2cK3wl8dPJM4SSva7JdDhzYnCn8TuDcJv0c4Ejg/CS3Na/nt/4UkiRJkiRpZCzsJ9PenilcVe8G3j3DNkqSJEmSpDmkn2UUkiRJkiRJfTPYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRphCQ5JcldSTYlOXeK+/smuba5f3OSxV33zmvS70ry6q70LUk2JLktyfrBPIkkSZLmM4MNkjQikiwALgNOBZYApydZ0pPtbOD+qjoSuAS4uCm7BFgOHA2cAvxlU9+kX62q46pqYpYfQ5LG1t4GhJOcnOSWJvB7S5JXDrrtkjRoBhskaXScAGyqqs1V9ShwDbCsJ88y4Krm+jrgpCRp0q+pqkeq6hvApqY+SVILZhIQBu4FXltVxwBnAlcPptWSNDwGGyRpdBwE3N31fmuTNmWeqtoBPAAcOE3ZAj7Z/DVtxSy0W5Lmg70OCFfVrVW1rUnfCOyXZN+BtFqShsRggySNjkyRVn3m2V3Zl1XVL9D5a9w7krxiyg9PViRZn2T99u3b+22zJM0XMwkId3sDcGtVPTJL7ZSkkWCwQZJGx1bgkK73BwPbdpUnyULgAOC+3ZWd/GtaVd0DXM8ulldU1aqqmqiqiUWLFs34YSRpzMwkINy5mRxNZ2nFW6dMJgyOAAALAklEQVT8AIO+ksaIwQZJGh3rgKOSHJ5kHzobPq7uybOaznpfgNOAG6uqmvTlzeZkhwNHAV9Osn+SZwIk2R94FfC1ATyLJI2bmQSESXIwnYDvW6rq61N9gEFfSeNk4bAbIEnqqKodSc4B1gILgCuqamOSi4D1VbUauBy4OskmOr/ALm/KbkzyUeAOYAfwjqp6LMkLgOs7e0iyEPhwVX1i4A8nSXPfEwFh4Nt0xt839+SZDAh/ia6AcJJnAzcA51XVFwfYZkkaGoMNkjRCqmoNsKYn7YKu64eBN+6i7B8Df9yTthn4ufZbKknzy0wCwsA5wJHA+UnOb9Je1Sxvk6SxZLBBkiRJ6sPeBoSr6t3Au2e9gZI0QtyzQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWyQJEmSJEmt6ivYkOSUJHcl2ZTk3Cnu75vk2ub+zUkWd907r0m/K8mr22u6JI2f2Rhvp6tTktQffyeWpP5NG2xIsgC4DDgVWAKcnmRJT7azgfur6kjgEuDipuwSYDlwNHAK8JdNfZKkHrMx3vZZpyRpGv5OLEl7pp+ZDScAm6pqc1U9ClwDLOvJswy4qrm+DjgpSZr0a6rqkar6BrCpqU+StLPZGG/7qVOSND1/J5akPdBPsOEg4O6u91ubtCnzVNUO4AHgwD7LSpI6ZmO8dRyWpHb4O7Ek7YGFfeTJFGnVZ55+ypJkBbCieftgkrv6aNcgPQ+4t42KcnEbtYws+6k/rfTTmPcRjGY/HdZqbTubjfF2qqDyTuMwzKOx+MKpumqs2E/Ta+3/V/ZTn9rrp9keh3fH34nb/F3vLH92+mE/TW/M+whGs5/6Gov7CTZsBQ7pen8wsG0XebYmWQgcANzXZ1mqahWwqp8GD0OS9VU1Mex2jDr7qT/2U3/maT/N1ng77TgMjsXjwn6ann3UH/tpJ/5O7PdEX+yn/thP/ZnL/dTPMop1wFFJDk+yD53NbVb35FkNnNlcnwbcWFXVpC9vduY9HDgK+HI7TZeksTMb420/dUqSpufvxJK0B6ad2VBVO5KcA6wFFgBXVNXGJBcB66tqNXA5cHWSTXSit8ubshuTfBS4A9gBvKOqHpulZ5GkOW22xtup6hz0s0nSXOfvxJK0Z9IJtmp3kqxoprVpN+yn/thP/bGf1Mvvif7YT9Ozj/pjP6mX3xP9sZ/6Yz/1Zy73k8EGSZIkSZLUqn72bJAkSZIkSerbWAYbkjyW5LYkX0vyt0me3lK9L0jysSRfTXJHkjVN+jeSvLgn758n+f0kS5M8kOTWJHcl+VyS17TRnj1su30yy5L8vz3vfzfJw0kO6EpbmuRjU5R9TdMfk/341kG0eRDsl/nJMWfKttsnA+CYszP7ZH5yzNll++2XWeaYM7V52S9VNXYv4MGu678B3rkHZRfu5t77gP/U9f7Y5ut/A/6oK/0pdI44OgxYCnys695xwBbgJPtkuH0ygD7/MvB54KyutCc9e5P2VDrHXx3cvN8XePGw22+/+Jrhf2fHHPtkJF6OOfbJfH055tgvo/JyzJm//TKWMxt6fB44MsniJF+bTEzyriQrm+ubkvxJks8C/ynJoiR/l2Rd83pZU+yFdAYHAKrq9ubyIzS7DTdeAWypqm/2NqaqbgMuAs5p8Rn31JzpkyQLknygiT5vSPK7M3ryWZTkwa7r/wA8A/hD4PRpij6Tzskw3wOoqkeq6q7ZauegzXa/zKXvkXlszow5AzRn+mSu/Yw5Fu/McVjMoTFnwOZMv8ylnzPH4anNx7F4rIMNSRYCpwIb+sj+7Kr6lar6H8B7gUuq6qXAG4D3N3kuAy5P8pkk/yXJi+CJweTxJD/X5FtOZ2DZla8AP7PnTzRzc7BPjgMOqqqXVNUxwJV9tHsUnE7neT8PvDjJ83eVsaruo3P+9jeTfCTJGUnG9WdzNvplrn6PzAtzcMyZdXOwT+byz5hj8c4ch+eZOTjmDMQc7Je5+nPmODy1eTEWj+t/vKcluQ1YD3yLzpnH07m26/rXgEubOlYDz0ryzKpaCxwB/DWdQeDWJIuaMh8BljcD1zLgb3fzWdmjp2nHXO2TzcARSf5nklOA7/fR7lGwHLimqh4H/h544+4yV9X/AZxEZzrVu4ArZr2FwzEb/TJXv0fG3Vwdc2bTXO2Tufwz5li8M8fh+WOujjmzba72y1z9OXMcntq8GIsXDrsBs+RHVXVcd0KSHTw5uLJfT5mHuq6fAvxSVf2ot+ImsvRh4MPpbN7xCuDv6AwinwQ+C9xeVffspn0/D9zZ57O0ZU70SZIFwC1N2uqquqCJBL8aeAfwm8D/vtsnHbIkxwJHAZ9KArAPnR/+y3ZXrqo2ABuSXA18Azhrdls6WG31S5KzmePfI/PEnBhz+nyWtsyJPhmHcRgci6fiODzvzIkxp89nadOc6JdxGIsdh6c2n8bicZ3ZMJV/A56f5MAk+wK72+n1k3StlUpyXPP1lWl2rE3yTOA/0ImIUlVfp7OO5j3sZmpU8811PtN8Mw3IyPVJVT1WVcc1rwuSPA94SlX9XZPnF/b+cQfmdGBlVS1uXi8CDkpy2FSZkzwjydKupOOAndbxjYFW+mVMvkfmq5Ebc2b0NO0YuT4Zo58xx+KdOQ5r5MacGT1Ne0auX8bk58xxeGrzZiwe15kNO6mqHye5CLiZToTsX3aT/XeAy5LcTqePPge8DTiezrSpyejn+6tqXVe5j9DZcfb6nvpenuRW4OnAPcDvVNWnW3isGZkjfXIQcGV+si7pvD15xiFZTmcNYLfrm/SbgZOSbO26dzrw+0neB/yITvT8rAG0c9Bmq1/m4vfIvDRHxpyBmiN9Mld/xhyLd+Y4PM/NkTFn4OZIv8zFnzPH4anNm7E41Tk+Q5IkSZIkqRXzaRmFJEmSJEkaAIMNkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYoHklyZbmDNoZ5ZEk7T3HYkkaLsdhDYLBBkmSJEmS1CqDDRp5SRYn+Zck70/ytSR/k+TXknwxyb8mOSHJc5P8P0luT/LPSY5tyh6Y5JNJbk3yPiBd9f6vSb6c5LYk70uyYGgPKUkjzrFYkobLcVhzjcEGzRVHAu8FjgV+Bngz8MvAu4D/DFwI3FpVxzbvP9iU+yPgC1X188Bq4FCAJD8LvAl4WVUdBzwGnDGwp5GkucmxWJKGy3FYc8bCYTdA6tM3qmoDQJKNwKerqpJsABYDhwFvAKiqG5vo7QHAK4D/2KTfkOT+pr6TgOOBdUkAngbcM8DnkaS5yLFYkobLcVhzhsEGzRWPdF0/3vX+cTrfxzumKFM9X7sFuKqqzmuthZI0/hyLJWm4HIc1Z7iMQuPiczRTvpIsBe6tqu/3pJ8KPKfJ/2ngtCTPb+49N8lhg260JI0Zx2JJGi7HYY0MZzZoXKwErkxyO/BD4Mwm/ULgI0m+AnwW+BZAVd2R5A+BTyZ5CvBj4B3ANwfdcEkaIytxLJakYVqJ47BGRKqmmk0jSZIkSZK0d1xGIUmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa36/wG5EhUVaMdJbgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_cv_results(cv_results.append(sals_cv_results));" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Surprisingly, the iALS model remains largely insensitive to the scaling trick. At least in the current settings and for the current dataset. \n", + "**Remark**: You may want to repeat all experiments in a different setting with `random_holdout` set to `True`. My own results indicate that in this case iALS becomes more responsive to scaling and gives the same result as the scaled version of SVD. However, SVD is still much easier to compute and tune." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conclusion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With a proper tuning the quality of recommendations of one of the simplest SVD-based models can be substantially improved. Despite common beliefs, it turns out that PureSVD with simple scaling trick compares favorably to a much more popular iALS algorithm. It not only generates more relevant recommendations, but also makes them more diverse and potentially more interesting. In addition to that it has a number of unique advantages and merely requires to simply do `from scipy.sparse.linalg import svds` to start using it. Of course, the obtained results may not necessarily hold on all other datasets and require further verification. However, in the view of all its features, the scaled SVD-based model can be certainly considered as at least one of the default baseline candidates. The result obtained here resembles situation in the Natural Language Processing field, where simple SVD-based model with proper tuning [turns out to be competitive](http://www.aclweb.org/anthology/Q15-1016) on a variety of downstream tasks even in comparison with Neural Network-based models. In the end it's all about fair comparison.\n", + "\n", + "As a final remark, Polara is designed to support opennes and reproducibility of research. It allows to perform thorough experiments with minimal efforts. It can be used to quickly test known ideas or implement something new by providing controlled environment and rich functionality with high level abstractions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + }, + "toc": { + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 608478336b0981d4af5e126a79719e5761b87a31 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 2 May 2019 07:07:51 +0300 Subject: [PATCH 05/82] add new tutorial on hyper-parameter tuning and comprehensive model evaluation --- ...ing_and_cross_validation_experiments.ipynb | 1729 +++++++++++++++++ 1 file changed, 1729 insertions(+) create mode 100644 examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb diff --git a/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb b/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb new file mode 100644 index 0000000..f4792c8 --- /dev/null +++ b/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb @@ -0,0 +1,1729 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial we will go through a full cycle of model tuning and evaluation with Polara. This will include 2 phases: grid-search for finding (almost) optimal values of hyper-parameters and verification of results via 5-fold cross-validation.\n", + "\n", + "
We will focus on performing a fair comparison of popular ALS-based matrix factorization (MF) model called Weighted Regularized Matrix Factorization (WRMF a.k.a. iALS) [Hu2008] with less popular model called PureSVD [Cremonesi2010] based on standard SVD.
\n", + "\n", + "We will use standard *Scipy*'s implementation for the latter and a great library called [*implicit*](https://github.com/benfred/implicit) for iALS. Both are wrapped by Polara and can be accessed via the corresponding classes. Due to its practicality the *implicit* library is often recommended to beginners and sometimes even serves as a default tool in production. On the other hand, there are some important yet often overlooked features, which make SVD-based models stand out. Ignoring them in my opinion leads to certain misconceptions and myths, not to say that it also overcomplicates things quite a bit.\n", + "\n", + "Note that by saying SVD I really mean *Singular Value Decomposition*, not just an arbitrary matrix factorization. In that sense, **methods like FunkSVD, SVD++, SVDFeature, etc., are not SVD-based at all**, even though historically they use SVD acronym in their names and are often referenced as if they are real substitutes for SVD. These methods utilize another optimization algorithm, typically based on stochastic gradient descent, and do not preserve the algebraic properties of SVD. This is really an important distinction, especially in the view of the following remarks:\n", + "\n", + "1. **SVD-based approach has a number of unique and beneficial properties**. To name a few, it produces stable and determenistic output with global guarantees. It admits the same prediction formula for both known and previously unseen users (as long as at least one user rating is known). It can take a hybrid form to include side information via the generalized formulation (see Chapter 6 of [my thesis](https://www.skoltech.ru/en/2018/09/phd-thesis-defense-evgeny-frolov)). Even without hybridization it can be quite successfully applied in the cold start regime (paper on this is coming). It requires minimal tuning and allows to compute and store a single latent feature matrix - either for users or for items - instead of computing and storing both of them. This luxury is not available in the majority of other matrix factorization approaches, definitely not in popular ones. \n", + "2. Computational complexity of truncated SVD **scales linearly with the number of known observations** and quadratically with the rank of decomposition (thanks to Lanczos procedure). There are [open source implementations](https://github.com/criteo/rsvd), allowing to handle in a distributed manner nearly *billion-scale problems* with its [efficient randomized version](https://medium.com/criteo-labs/sparkrsvd-open-sourced-by-criteo-for-large-scale-recommendation-engines-6695b649f519). \n", + "3. At least **in some cases PureSVD outperforms other more sophisticated matrix factorization methods** [Cremonesi2010].\n", + "4. Moreover, **PureSVD can be quite easily tuned to perform even better** [Nikolakopoulos2019].\n", + "\n", + "Despite that impresisve list, PureSVD technique (and especially its modifications) rarely gets into the list of baseline models to compare with. Hence, this tutorial also aims at performing a thorough assessment of the default choice of many practitioners to see whether it really provides the celebrated advantages over the simpler approach." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* [Hu 2008], Hu Y., Koren, Y. and Volinsky, C., 2008, December. *Collaborative Filtering for Implicit Feedback Datasets*. In ICDM (Vol. 8, pp. 263-272). [Link](http://yifanhu.net/PUB/cf.pdf). \n", + "* [Cremonesi2010] Cremonesi, P., Koren, Y. and Turrin, R., 2010, September. *Performance of recommender algorithms on top-n recommendation tasks*. In Proceedings of the fourth ACM conference on Recommender systems (pp. 39-46). ACM. [Link](https://www.researchgate.net/publication/221141030_Performance_of_recommender_algorithms_on_top-N_recommendation_tasks). \n", + "* [Nikolakopoulos2019] Nikolakopoulos, A.N., Kalantzis, V., Gallopoulos, E. and Garofalakis, J.D., 2019. *EigenRec: generalizing PureSVD for effective and efficient top-N recommendations*. Knowledge and Information Systems, 58(1), pp.59-81. [Link](https://arxiv.org/abs/1511.06033)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Downloading data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use the **[Movielens-10M](https://grouplens.org/datasets/movielens/)** dataset for our experiments. It's large enough to perform reliable evaluation; however, not that large that you would spend too many hours waiting for results. If you don't plan to play with the code, you may want to run all cells with a single command and leave the notebook running in the background, while reading the text. It takes around 1.5h to complete this notebook on a modern laptop." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that you'll need an internet connection in order to run the code below. It will automatically download data, store it in a temporary location, and convert into a `pandas` dataframe. \n", + "Alternatively, if you have already downloaded the dataset, you can use path to it as an input for the `get_movielens_data` function instead of `tmp_file`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import urllib\n", + "from polara import (get_movielens_data, # returns data in the pandas dataframe format\n", + " RecommenderData) # provides common interface to access data " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
useridmovieidratingtimestamp
011225.0838985046
111855.0838983525
212315.0838983392
312925.0838983421
413165.0838983392
\n", + "
" + ], + "text/plain": [ + " userid movieid rating timestamp\n", + "0 1 122 5.0 838985046\n", + "1 1 185 5.0 838983525\n", + "2 1 231 5.0 838983392\n", + "3 1 292 5.0 838983421\n", + "4 1 316 5.0 838983392" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url = 'http://files.grouplens.org/datasets/movielens/ml-10m.zip'\n", + "tmp_file, _ = urllib.request.urlretrieve(url) # this may take some time depending on your internet connection\n", + "\n", + "data = get_movielens_data(tmp_file, include_time=True)\n", + "data.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The resulting dataframe has a bit more than 10M ratings, as expected." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(10000054, 4)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Basic data stats:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "userid 69878\n", + "movieid 10677\n", + "rating 10\n", + "timestamp 7096905\n", + "dtype: int64" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.apply('nunique')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Preparing data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As always, you need to firstly define a data model that will provide a common interface for all recommendation algorithms used in experiments:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Fields(userid='userid', itemid='movieid', feedback='rating')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_model = RecommenderData(data, 'userid', 'movieid', 'rating', custom_order='timestamp', seed=0)\n", + "data_model.fields" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting `seed=0` ensures controllable randomization when sampling test data, which enhances reproducibility; `custom_order` allows to select observations for evaluation based on their timestamp, rather than on rating value (more on that later). " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's look at the default configuration of data model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'shuffle_data': False,\n", + " 'test_fold': 5,\n", + " 'test_ratio': 0.2,\n", + " 'permute_tops': False,\n", + " 'warm_start': True,\n", + " 'random_holdout': False,\n", + " 'negative_prediction': False,\n", + " 'test_sample': None,\n", + " 'holdout_size': 3}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_model.get_configuration()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default, `Polara` samples 20% of users and marks them for test (`test_ratio` attribute). These users would be excluded from the training dataset, if `warm_start` attribute remained set to `True` (strong generalization test). However, in the iALS case such setting would require running additional half-step optimization (folding-in) for each test user in order to obtain their latent representation. To avoid that we turn the \"warm start\" setting off and perform standard evaluation (weak generalization test). In that case test users are part of the training (except for the ratings that were held out for evaluation) and one can directly invoke scalar products of latent factors. Note that *SVD recommendations do not depend on this setting due to uniform projection formula applicable to both known and \"warm\" users*: $r = VV^\\top p$, where $r$ is a vector of predicted relevance scores, $p$ is a vector of *any* known preferences and $V$ is an item latent features matrix." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing data...\n", + "Done.\n", + "There are 9986078 events in the training and 13976 events in the holdout.\n" + ] + } + ], + "source": [ + "data_model.holdout_size = 1 # hold out 1 item from every test user\n", + "data_model.random_holdout = False # take items with the latest timstamp\n", + "data_model.warm_start = False # standard case\n", + "data_model.prepare()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `holdout_size` attribute controls how many user preferences will be used for evaluation. Current configuration instructs data model to holdout one item from every test user. The `random_holdout=False` setting along with `custom_order` input argumment of data model make sure that only the latest rated item is taken for evaluation, allowing to avoid \"recommendations from future\". All these items are available via `data_model.test.holdout`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# General configuration" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## technical settings" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sometimes due to the size of dataset evaluation make take considerably longer than actual training time. If that's the case, you may want to give Polara more resources to perform evaluation, which is mostly controlled by changing `memory_hard_limit` and `max_test_workers` settings from their defaults. The former defines how much memory is allowed to use when generating predictions for test users. Essentially, Polara avoids running an inefficient by-user-loop for that task and makes calculation in bulk, which allos to invoke linear algebra kernels and speed up calculations. This, however, generates dense $M \\times N$ matrix, where $M$ is the number of test users and $N$ is the number of all items seen during training. If this matrix doesn't fit into the memory constraint defined by `memory_hard_limit` (1Gb by default), calculations will be perfomed on a sequence of groups of $m\n", + "\n", + "100%\n", + "15/15\n", + "[00:57<00:04, 3.79s/it]" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 15/15 [00:57<00:04, 3.79s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# the model will be computed only once for the max value of rank\n", + "psvd_best_rank, psvd_rank_scores = find_optimal_svd_rank(psvd,\n", + " rank_grid,\n", + " target_metric,\n", + " config=init_config,\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that in this case the most of the time is spent on evaluation, rather than on model computation. The model was computed only once. You can verify it by calling `psvd.training_time` list attribute and seeing that it contains only one entry:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[9.0811659]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "psvd.training_time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's see how quality of recommendations changes with rank (number of latent features)." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8XOV97/HPV4sly5ssW17wbmwCNlATBIa0pEkoYEISwy2vYkoJaWndBUqTNmnh5iZNuG1vc9tXszQ0LQn7bQIpLYlLCiYJZGkCxjI2XnEQxmDZxguSV3mRpd/94xyZsZA0kg/jkezv+/Wa15zlOc88c2zNd855znlGEYGZmdnxKil2A8zMbGBzkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDIpK3YDToTRo0fH1KlTi90MM7MBZdmyZTsjojZfuVMiSKZOnUp9fX2xm3FKaW8PXn1zP6+/2ULtsAomVA+muqocScVumpn1kqTXelPulAgSK7ym/YdZsamZFa/vYvmmXby4aRd7Dh45pszg8lJOq67ktOrBTKgezGlHH5VMqB7MuBGVVJSVFukdmNnxcpBYnx060saaLXtY8fouVmxKHq83tQBQInjXuOFcde5pnDepmum1Q9i57xCbdx1ky64DRx/rtu5l575Db6u7dlhFGjSVnDbirbCZkAZOzZBBmY5qIoJDR9o5cLiNA63p43AbB3OmD7Sm84fbGFJRxvvPHMPooRXH/ZpmJzsHifUoInjtzRaWp0cbKzbtYu3WPbS2JaNGjxteyXmTq7lh7mTmTKrmnIkjqBrUu/9WB1vbeGN3EjCbdx1gS0fY7D7AS2/s5emXtnOwtf2YbSrKSnKOZioZOWQQh1rbuw6CoyHR/tb0kTb6OuC1BBdMqeHy2WO5YvY4JtVU9a0Cs5OcToVh5Ovq6sJ9JL2zq+Xw0aOMFekpquaWVgCqBpVyzoQRzJlczXmTqpkzaSTjRlQWrC0RQXNLa07QdDwOHp3f1dJKRXkJVYNKGVxeSmV5KYPT6cHlpVTmTA8elK4vL2Vwecmx891sv2X3ARav2cZTa97gpTf2AjBr/HCumD2OeWeP44yxQ93vYyctScsioi5vOQfJqau9PVi9ZTfLc05RvbpzP5B8Cz9jzDDmTKpmzuRq5kyqZuaYoZSVnrpXjG/cuZ+n1r7Bk6vf4IXXdwEwdVQVV8wex+Wzx3HepGpKShwqdvLoF0EiaR7wZaAU+EZE/G2n9RXAg8D5wJvAdRGxUdKFwN0dxYDPRcRjvamzKw6St7S3B8s37eLxlVv4r1Vb2bYn6acYM6zimNA4d2I1Qyt85rM72/cc5Km121i85g2efeVNjrQHY4ZVcNmsscw7exwXTR9F+SkcunZyKHqQSCoFfgFcBjQCS4HrI2JtTpk/As6NiD+QtAC4JiKuk1QFHI6II5LGAy8CpwGRr86unOpBEhG82Libx19MwmPL7oMMKivhfWfUcuU545g7bRTjR1T6FM1x2t3SytPrt7F49TZ+/IsdHGhtY3hlGZeeNZYrZo/lvWfU9rrfyKw/6W2QFPJ/94VAQ0RsSBv0MDAfyP3Qnw98Lp1+FPiqJEVES06ZSpIA6W2dRhIeqzfv4fGVW3h85VY27zpAean41TNq+dS8d/FrZ41lWGV5sZt5UhhRVc41503kmvMmcrC1jZ/8YgeL12zjhy9t47Hlm6ksL+GSmbXMmz2OS88aQ3XVoGI32ewdVcggmQBsyplvBOZ2VyY9+tgNjAJ2SpoL3AtMAW5M1/emzlNWRLB26x6+t3Ir31u1ldfebKGsRFwyczSfuOwMLps1lhGDHR6FVFleyuVpn0lrWztLX23iyTVv8NSabXx/7TZKS8RF02uSfpVZ4wp6sYLZiVLIIOnqPEnn82jdlomIJcBsSWcBD0h6opd1JhVLC4GFAJMnT+5tmweciGD9tr1JeKzcyoad+yktEe85fRS3vG8Gl88e62/ARVJeWsJ7ZozmPTNG87kPz2bl5t0sXvMGi9e8wWe/u4bPfncNM8YMZXJNFRNHDk4fVUyoTqaz3jPzTmk5fITNzQdo3HUgeW5OrqLb3NxCZXkpM8YMTR61yXPtsIp+0e58Dra2UVaiU/oCkndKIYOkEZiUMz8R2NJNmUZJZcAIoCm3QESsk7QfOLuXdXZsdzdph31dXd1Jd2nay9v28nh65NGwfR8lgotPH8XvvXc6V8weR80Qh0d/UlKi5GKGSdX8xbwzadi+lydXv8GLjbvZ3HyA+o1NXY4EkBswR4MmXTbqHQqa3Qda04BoSQMiJyx2HaBp/+FjypeX6uhNoi2H23jshc3sPfRW24dVlh0TLB2PiSOrKD2BV7W1trWzZdcBNjUdYFNzC5uaWtjUfIBNTS00Nrewc1/yvkYMLqdmyCBGViXPNUMGMXLIIGqqOj2n08Mqy3x1XieF7GwvI+kYvxTYTNIx/psRsSanzC3AOTmd7f8jIn5D0jRgU3o6awrwLHAusCtfnV05WTrbN+zYl4THyq2s37YXCeZOq+Gqc0/jyrPH+e7rAa7zB3pjOt2YfrDvPtB6TPnK8pKcgDn2aGbiyCpGD02+TDTtP5xzFHFs/ZubDxwTAh31JvUkofVWnYOZUF3FmGEVx3yQRgTb9x6iYfs+Grbv4+Xte9Pp/ceMXlBRVsL02qFvC5mpo6uOa2ic9vbkdY+GRE5gNDYfYOvuA7TnfLyVlSQBOKlmMJPSfdUWQfP+wzS1tCbP+w/T3HKYN/cf5vCR9i5ft7REjKwqZ2SnoBnVEUBDknWjhlQwY8xQBg8auMP+FP2qrbQRHwS+RHKp7r0R8deS7gTqI2KRpErgIeA8kiORBRGxQdKNwO1AK9AO3BkR3+muznztGMhBsnHnfr63aiuPr9zKuq17jt5lfdW547nynHGMGeZz7KeKPQdb3zpayAmYxl3J9K6WtweNEAda245ZPqyi7OhRTVeB8U6eUtvd0krDjr1HQ6Zh+z4aduyjsfnA0REGSkvE5JoqTu90BDNjzFBaj7Sn4XDsUUVjUwuNuw687cN+7PAKJo2sYlJNFZNGDmZiTVU6P5hxwyt7fRorIjjQ2sab+5Jg6QiYpv1p4LQcPiZ4mva30txymLb2Yz9Py0rE7AkjqJsykgumjuT8KTXUDhs4X/j6RZD0FwMlSCKC15taWLKhiedefZMlG5rYvOsAAOdPGclV54zng+eMdwetdWnvwdZjTk1tamoh4G2B0R8uuDhwuI0NO5NgeWX7Pl5OQ2bjm/uPDr/Tleqq8qPBMGlkVRoUg5lUkxxhVJYX79t/RLDn4JGjQbN9zyFWNu6i/rVmXty0i0Np6E0ZVUXdlBrqpo6kbspITq8d2m9PlTlIcvTXIIkINuzcz5INTSxJg+ONPQcBGDVkEBdOq+Gi6aP4tVljmVA9uMitNSu81rZ2Xm9qSQJmxz4qykqPBsXEkYMH7CXrh4+0s3rLbpZtbGbpxiaWvdbMm2nfU3VVOedPHsn5U0dywdQazpkwoqiBmMtBkqO/BElE0LB9H8+92sRzG97k+Veb2LE3OYc8emgFF02vYe70UVw0rYYZYzyGk9nJKiJ4ded+6l9rTsLltSY27EiGJxpUWsLZE4ZzwdQazp8ykvOnjGRUkfo/HSQ5ihUk7e3JpblLNrzJklebeP7VpqPfQsYNr2Tu9BrmThvF3Ok1TB89xMFhdgpr2n+YZa81U7+xifrXmlnVuJvDbcnpsOm1Q6ibMvLoKbFpJ+jzoj/c2X7KaWsP1m3dw3NpcCzd2HS0A3RC9WB+9V21XJQGx+SaKgeHmR1VM2QQl80ay2WzxgLJfS6rNu+mfmMSLk+t3ca36xuB5NT3nEnVVA4qpb09aGsP2iN5bguOLmuLSKZzntva6WJZp/Xpst5ykGS0busefvryDpZsaOL5jU3sTe8FmFxTxWVnjWXu9FHMnVbj37Awsz6pLC/lgqk1XDC1Bjid9vbglR37qH+tmfqNzazavIsjbUFJiSiVkucS3ppOn8tLSxhcKkokSks6nsmZ1tu26ahnZS/b6lNbGdz9k1f4m/96CYDpo4ccc6pq/Ah3jpvZwOZTWwUUEfz9U+u565lXuOqc8fzlh2cxZrgvyTWzU5ODpI/a24O/XLSGh557jQUXTOKvrznnhA77YGbW3zhI+qC1rZ1P/duLfGfFFn7/vdO5/coz3WFuZqc8B0kvHWxt49ZvvsAP1m3nU1e8iz963+kOETMzHCS9svdgK7/7QD3Pb2zif199NjdeNKXYTTIz6zccJHk07T/Mx+57njVb9vCl6+Ywf86EYjfJzKxfcZD04I3dB/mte5awqamFu288n0vPGlvsJpmZ9TsOkm5s3Lmf37pnCbtaWnngdy7koumjit0kM7N+yUHShXVb93DjPc/T1t7ON39vLudOrC52k8zM+i0HSSfLXmvmt+97nqpBZTy88GJmjBlW7CaZmfVrDpIcP315BwsfXMbY4RU8dPNcj49lZtYLDpLUE6u2ctvDyzm9digP3nyhf8LWzKyXHCTAt+s3cfu/r2TOpGru+9iFjKgamL/CZmZWDKd8kHzjpxv4q++t45KZo/mXG8+natApv0vMzPrklP3UjAi++P1f8JWnG7jy7HF8acEcKsr6x+8km5kNJKdkkLS3B3c+vpb7f76R36ibyN9ccw5lpSXFbpaZ2YB0ygXJkbZ2/vzRlfzH8s387q9M49NXneXBF83MMjilguRgaxt//K3lfH/tNj55+Rnc8v4ZDhEzs4xOmSDZd+gICx+s5+evvMmd82fz0YunFrtJZmYnhVMiSNragxu+sYTVm3fzxet+iWvOm1jsJpmZnTQK2sMsaZ6k9ZIaJN3exfoKSY+k65dImpouv0zSMkmr0ucP5Gzzo7TOFeljTL52vLJjH+u27uGff+t8h4iZ2TusYEckkkqBu4DLgEZgqaRFEbE2p9jNQHNEzJC0APgCcB2wE/hwRGyRdDawGMj9IZAbIqK+t21pbQvu/+0LeM/pozO+KzMz66yQRyQXAg0RsSEiDgMPA/M7lZkPPJBOPwpcKkkRsTwitqTL1wCVkiqOtyHTRw9xiJiZFUghg2QCsClnvpFjjyqOKRMRR4DdQOcf/vh1YHlEHMpZdl96Wusz6uayK0kLJdVLqt+3uynL+zAzsx4UMki6+oCPvpSRNJvkdNfv56y/ISLOAS5JHzd29eIRcXdE1EVEXW1tbZ8abmZmvVfIIGkEJuXMTwS2dFdGUhkwAmhK5ycCjwEfjYhXOjaIiM3p817gmySn0MzMrEgKGSRLgZmSpkkaBCwAFnUqswi4KZ2+Fng6IkJSNfA94I6I+FlHYUllkkan0+XAh4DVBXwPZmaWR8GCJO3zuJXkiqt1wLcjYo2kOyV9JC12DzBKUgPwp0DHJcK3AjOAz3S6zLcCWCxpJbAC2Ax8vVDvwczM8lNE526Lk09dXV3U1/f6amEzMwMkLYuIunzlPOStmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZllUtAgkTRP0npJDZJu72J9haRH0vVLJE1Nl18maZmkVenzB3K2OT9d3iDpK5JUyPdgZmY9K1iQSCoF7gKuBGYB10ua1anYzUBzRMwAvgh8IV2+E/hwRJwD3AQ8lLPN14CFwMz0Ma9Q78HMzPIr5BHJhUBDRGyIiMPAw8D8TmXmAw+k048Cl0pSRCyPiC3p8jVAZXr0Mh4YHhHPRkQADwJXF/A9mJlZHoUMkgnAppz5xnRZl2Ui4giwGxjVqcyvA8sj4lBavjFPnWZmdgKVFbDurvouoi9lJM0mOd11eR/q7Nh2IckpMCZPnpyvrWZmdpwKeUTSCEzKmZ8IbOmujKQyYATQlM5PBB4DPhoRr+SUn5inTgAi4u6IqIuIutra2oxvxczMulPIIFkKzJQ0TdIgYAGwqFOZRSSd6QDXAk9HREiqBr4H3BERP+soHBFbgb2SLkqv1voo8N0CvgczM8ujYEGS9nncCiwG1gHfjog1ku6U9JG02D3AKEkNwJ8CHZcI3wrMAD4jaUX6GJOu+0PgG0AD8ArwRKHeg5mZ5afk4qeTW11dXdTX1xe7GWZmA4qkZRFRl6+c72w3M7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkJiZWSYOEjMzy8RBYmZmmThIzMwsEweJmZllkjdIJJVKGp0zP0jSQknrCts0MzMbCHoMEkkLgCZgpaQfS3o/sAG4ErjhBLTPzMz6ubI86/8XcH5ENEh6N/AssCAiHit808zMbCDId2rrcEQ0AETEC8CrDhEzM8uV74hkjKQ/zZkfmjsfEf9QmGaZmdlAkS9Ivg4M62HezMxOcT0GSUR8vrt1koa8880xM7OBpjeX/06QVCdpUDo/RtLfAC8XvHVmZtbv5bv89+PACuAfgeck3QSsAwYD5xe+eWZm1t/l6yNZCLwrIpokTQYagPdGxHOFb5qZmQ0E+U5tHYyIJoCIeB34RV9CRNI8SeslNUi6vYv1FZIeSdcvkTQ1XT5K0jOS9kn6aqdtfpTWuSJ9jOlte8zM7J2X74hkoqSv5MyPyZ2PiNu621BSKXAXcBnQCCyVtCgi1uYUuxlojogZ6V30XwCuAw4CnwHOTh+d3RAR9XnabmZmJ0C+IPlUp/llfaj7QqAhIjYASHoYmA/kBsl84HPp9KPAVyUpIvYD/y1pRh9ez8zMiiDf5b8PZKh7ArApZ74RmNtdmYg4Imk3MArYmafu+yS1Af8O/FVEROcCkhaS9PEwefLk43oDZmaWX49BImlRT+sj4iM9bd7VJsdRprMbImKzpGEkQXIj8GAXbbsbuBugrq4uX51mZnac8p3aupjkiOFbwBK6/uDvTiMwKWd+IrClmzKNksqAESSjDXcrIjanz3slfZPkFNrbgsTMzE6MfFdtjQP+J0mH95dJOs53RsSPI+LHebZdCsyUNC29mXEB0PkIZxFwUzp9LfB0V6epOkgq6/htFEnlwIeA1XnaYWZmBZSvj6QNeBJ4UlIFcD3wI0l3RsQ/5tn2iKRbgcVAKXBvRKyRdCdQHxGLgHuAhyQ1kByJLOjYXtJGYDgwSNLVwOXAa8DiNERKgR+QjP9lZmZFoh4OAJICSYBcRRIiU0mOIu7tOMU0ENTV1UV9va8WNjPrC0nLIqIuX7l8ne0PkJzWegL4fET4NJKZmR0jX2f7jcB+4AzgNuloX7uAiIjhBWybmZkNAPn6SPKODmxmZqc2B4WZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLpKBBImmepPWSGiTd3sX6CkmPpOuXSJqaLh8l6RlJ+yR9tdM250talW7zFUkq5HswM7OeFSxIJJUCdwFXArOA6yXN6lTsZqA5ImYAXwS+kC4/CHwG+GQXVX8NWAjMTB/z3vnWm5lZbxXyiORCoCEiNkTEYeBhYH6nMvOBB9LpR4FLJSki9kfEf5MEylGSxgPDI+LZiAjgQeDqAr4HMzPLo5BBMgHYlDPfmC7rskxEHAF2A6Py1NmYp04zMzuBChkkXfVdxHGUOa7ykhZKqpdUv2PHjh6qNDOzLAoZJI3ApJz5icCW7spIKgNGAE156pyYp04AIuLuiKiLiLra2to+Nt3MzHqrkEGyFJgpaZqkQcACYFGnMouAm9Lpa4Gn076PLkXEVmCvpIvSq7U+Cnz3nW+6mZn1VlmhKo6II5JuBRYDpcC9EbFG0p1AfUQsAu4BHpLUQHIksqBje0kbgeHAIElXA5dHxFrgD4H7gcHAE+nDzMyKRD0cAJw06urqor6+vtjNMDMbUCQti4i6fOV8Z7uZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wKGiSS5klaL6lB0u1drK+Q9Ei6fomkqTnr7kiXr5d0Rc7yjZJWSVohqb6Q7Tczs/zKClWxpFLgLuAyoBFYKmlRRKzNKXYz0BwRMyQtAL4AXCdpFrAAmA2cBvxA0hkR0ZZu9/6I2FmotpuZWe8V8ojkQqAhIjZExGHgYWB+pzLzgQfS6UeBSyUpXf5wRByKiFeBhrQ+MzPrZwoZJBOATTnzjemyLstExBFgNzAqz7YBPCVpmaSF3b24pIWS6iXV79ixI9MbMTOz7hUySNTFsuhlmZ62/eWIeDdwJXCLpPd29eIRcXdE1EVEXW1tbW/bbGZmfVTIIGkEJuXMTwS2dFdGUhkwAmjqaduI6HjeDjyGT3mZmRVVIYNkKTBT0jRJg0g6zxd1KrMIuCmdvhZ4OiIiXb4gvaprGjATeF7SEEnDACQNAS4HVhfwPZiZWR4Fu2orIo5IuhVYDJQC90bEGkl3AvURsQi4B3hIUgPJkciCdNs1kr4NrAWOALdERJukscBjSX88ZcA3I+LJQr0HMzPLT8kBwMmtrq4u6ut9y4mZWV9IWhYRdfnK+c52MzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXiIDEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLpKBBImmepPWSGiTd3sX6CkmPpOuXSJqas+6OdPl6SVf0tk4zMzuxChYkkkqBu4ArgVnA9ZJmdSp2M9AcETOALwJfSLedBSwAZgPzgH+SVNrLOs3M7AQq5BHJhUBDRGyIiMPAw8D8TmXmAw+k048Cl0pSuvzhiDgUEa8CDWl9vanTzMxOoEIGyQRgU858Y7qsyzIRcQTYDYzqYdve1GlmZidQWQHrVhfLopdlulveVfB1rjOpWFoILExn90la30073wmjgZ0FrL+QBnLbwe0vNre/uArd/im9KVTIIGkEJuXMTwS2dFOmUVIZMAJoyrNtvjoBiIi7gbuPt/F9Iak+IupOxGu90wZy28HtLza3v7j6S/sLeWprKTBT0jRJg0g6zxd1KrMIuCmdvhZ4OiIiXb4gvaprGjATeL6XdZqZ2QlUsCOSiDgi6VZgMVAK3BsRayTdCdRHxCLgHuAhSQ0kRyIL0m3XSPo2sBY4AtwSEW0AXdVZqPdgZmb5KTkAsCwkLUxPpQ04A7nt4PYXm9tfXP2l/Q4SMzPLxEOkmJlZJg6SPpA0SdIzktZJWiPpT9LlNZK+L+nl9Hlksdvak3SUgOWSHk/np6VD1LycDlkzqNht7I6kakmPSnop/Xe4eKDsf0mfSP/frJb0LUmV/X3fS7pX0nZJq3OWdbm/lfhKOnzRSknvLl7Lu23736X/d1ZKekxSdc66LodlKpau2p+z7pOSQtLodL6o+95B0jdHgD+LiLOAi4Bb0iFabgd+GBEzgR+m8/3ZnwDrcua/AHwxbX8zydA1/dWXgScj4kzgl0jeR7/f/5ImALcBdRFxNsnFIgvo//v+fpJhinJ1t7+vJLnCcibJPVxfO0Ft7M79vL3t3wfOjohzgV8Ad0D3wzKduKZ26X7e3n4kTQIuA17PWVzcfR8RfhznA/hu+g+6HhifLhsPrC9223po80SSP/4PAI+T3Py5EyhL118MLC52O7tp+3DgVdK+vZzl/X7/89aoDDUkV0s+DlwxEPY9MBVYnW9/A/8CXN9Vuf7S9k7rrgH+NZ2+A7gjZ91i4OL+tu/TZY+SfInaCIzuD/veRyTHSclIxecBS4CxEbEVIH0eU7yW5fUl4M+B9nR+FLArkiFqoH8POzMd2AHcl56a+4akIQyA/R8Rm4G/J/kWuZVkOKBlDJx9n6u7/T3QhjD6HeCJdHpAtF3SR4DNEfFip1VFbb+D5DhIGgr8O/DxiNhT7Pb0lqQPAdsjYlnu4i6K9tdL+cqAdwNfi4jzgP30w9NYXUn7EeYD04DTgCEkpyM666/7vjcGzP8lSZ8mOVX9rx2LuijWr9ouqQr4NPDZrlZ3seyEtd9B0keSyklC5F8j4j/SxdskjU/Xjwe2F6t9efwy8BFJG0lGTv4AyRFKtZIhaqCHYWf6gUagMSKWpPOPkgTLQNj/vwa8GhE7IqIV+A/gPQycfZ+ru/3dm2GRik7STcCHgBsiPQ/EwGj76SRfRF5M/4YnAi9IGkeR2+8g6QNJIrkbf11E/EPOqtyhXm4i6TvpdyLijoiYGBFTSToWn46IG4BnSIaogf7d/jeATZLelS66lGT0g4Gw/18HLpJUlf4/6mj7gNj3nXS3vxcBH02vILoI2N1xCqy/kDQP+AvgIxHRkrOqu2GZ+o2IWBURYyJiavo33Ai8O/27KO6+L3Zn0kB6AL9Ccri4EliRPj5I0s/wQ+Dl9Lmm2G3txXt5H/B4Oj2d5I+mAfg3oKLY7euh3XOA+vTf4DvAyIGy/4HPAy8Bq4GHgIr+vu+Bb5H06bSSfHDd3N3+Jjm9chfwCrCK5Aq1/tb2BpK+hI6/33/OKf/ptO3rgSv7477vtH4jb3W2F3Xf+852MzPLxKe2zMwsEweJmZll4iAxM7NMHCRmZpaJg8TMzDJxkNhJSdL/kfQ+SVdL6tPd75Jq0xF5l0u6pNO6H0nq8TeyJX08vQv5uKTtfk836yok/UDSCknX9bHeqZJ+83jpISyNAAADkUlEQVTbZdYdB4mdrOaSjIP2q8BP+7jtpcBLEXFeRPR1W4CPA8cdJCT3+HQZJCTju5VHxJyIeKSP9U4F+hQkOXfdm3XLQWInlfT3JlYCFwDPAr8LfE3S28YnkjRF0g/T32/4oaTJkuYA/xf4YPqtf3APr/U1SfVKfmPk8+my20jG0npG0jPpssslPSvpBUn/lo7VhqSNkj6fLl8l6cx0MNA/AD6Rvv4lOa83Bvh/wJx03emSPitpqZLfOLk7vWseSTPSI5cX0/pPB/4WuCTd9hNKfg/lvvS1l0t6f7rtx9J2/ifwlKTxkn6Sbre681GaWdHvnPXDj3f6AVwI/CNQDvysh3L/CdyUTv8O8J10+mPAV7vZ5kekdw3z1h3dpenyc9P5jbx1x/Fo4CfAkHT+L4DP5pT743T6j4BvpNOfAz7Zzeu/j3REgtw2pNMPAR9Op5cA16TTlSRHSJ23/TPgvnT6TJJhXCrT99+Y8/7+DPh0znsdVux/Yz/618NHJHYyOo9k+IszScaz6s7FwDfT6YdIhsDpi9+Q9AKwnOQHkWZ1UeaidPnPJK0gGZtqSs76joE/l5Gceuqr96f9OatIBuGcLWkYMCEiHgOIiINx7LhSHX6F5H0TES8BrwFnpOu+HxFN6fRS4LclfQ44JyL2Hkc77STm85920khPS91PMvLpTpJv4Uo/wC+OiAN5quj1eEHpwH6fBC6IiGZJ95N8m39bUZIP5eu7qepQ+txGH/8eJVUC/0RyhLQp/aCvpOshxbusood1+zsmIuInkt4LXAU8JOnvIuLBvrTVTm4+IrGTRkSsiIg5JD+hOgt4Grgiko7prkLk5ySjIAPcAPx3H15uOMmH7W5JYzn2t0X2AsPS6eeAX5Y0A5LflJB0Bj3L3b4nHcG1M+13uRYgkt/IaZR0dfqaFelVZJ3r/QnJ+yZt02SSAQuPIWkKye/YfJ1k9Oui/ha79T8OEjupSKoFmiOiHTgzIno6tXUbySmblcCNJL9l3yuR/ELdcmANcC/ws5zVdwNPSHomInaQ9Dl8K32d50hOufXkP4FrOne2d9GGXcDXSUZ7/Q7JKagONwK3pa/5c2AcyYjJR9IO+E+QHM2UpqfFHgE+FhGHeLv3ASskLQd+HfhynvbbKcaj/5qZWSY+IjEzs0wcJGZmlomDxMzMMnGQmJlZJg4SMzPLxEFiZmaZOEjMzCwTB4mZmWXy/wEPZfeUlgnphAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ax = psvd_rank_scores.plot(ylim=(0, None))\n", + "ax.set_xlabel('# of latent factors')\n", + "ax.set_ylabel(target_metric.upper());" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scaled PureSVD model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will employ a simple scaling trick over the rating matrix $R$ that was proposed by the authors of [EIGENREC model](https://arxiv.org/abs/1511.06033) [Nikolakopoulos2019]: $R \\rightarrow RD^{f-1},$\n", + "where $D$ is a diagonal scaling matrix with elements corresponding to the norm of the matrix columns (or square root of the number of nonzero elements in each column for the binary case). Parameter $f$ controls the effect of scaling and typically lies in the range [0, 1]. Finding the optimal value is an experimental task and will be performed via grid-search. We will use built-in support for such model in Polara." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Grid search" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.models import ScaledSVD\n", + "from polara.evaluation.pipelines import find_optimal_config # generic routine for grid-search" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we have to compute SVD model for every value of $f$. However, we can still avoid computing the model for each rank value by the virtue of rank truncation." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def fine_tune_scaledsvd(model, ranks, scale_params, target_metric, config=None):\n", + " rev_ranks = sorted(ranks, key=lambda x: -x) # descending order helps avoiding model recomputation\n", + " param_grid = [(s1, r) for s1 in scale_params for r in rev_ranks]\n", + " param_names = ('col_scaling', 'rank')\n", + " return find_optimal_config(model,\n", + " param_grid,\n", + " param_names,\n", + " target_metric,\n", + " init_config=config,\n", + " return_scores=True,\n", + " force_build=False, # avoid recomputing the model\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We already know an approximate range of values for the scaling factor. You may also want to play with other values, especially when working with a different dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "60/60\n", + "[04:17<00:03, 4.29s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 60/60 [04:17<00:03, 4.29s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ssvd = ScaledSVD(data_model) # create model\n", + "scaling = [0.2, 0.4, 0.6, 0.8]\n", + "\n", + "ssvd_best_config, ssvd_scores = fine_tune_scaledsvd(ssvd,\n", + " rank_grid,\n", + " scaling,\n", + " target_metric,\n", + " config=init_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that during this grid search the model was computed only `len(scaling)=4` number of times, other points were found via rank truncation. Let's see how quality changes with different values of scaling parameter $f$." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXd8VfXdx9/fO7MHGZAQICyVIajgQqtSFVctVmmlrtZRnlrtbrV9qtVa22rr83TpU2sfd4ciLtQ6akWfVlEBBwoOQMCEMLL3Te74Pn+ck+QmZEFyITHf9+t1Xue37/ece+/5nN8WVcUwDMMw9hbP/jbAMAzDGN6YkBiGYRgDwoTEMAzDGBAmJIZhGMaAMCExDMMwBoQJiWEYhjEgTEiMfYKIvCgil+1l3vEi0iAi3sG2a38gIltE5CTX/Z8i8r+DWPaNIlIhIjsGq0wj8YiIisiUfqQ7QURK94VNe4IJySDhPhya3QfeDhG5R0TS9rddw5H4By2Aqn6sqmmqGt2fdiUCVf25ql4GICLF7gPFtzdlicg44LvAdFUdMxj2ufY0ur/rbSLy34kSdBE5VkReEZFaEakSkZdF5HAROdq1Ib2bPG+KyJVx967BPXaKyJMicnIibDU6Y0IyuJypqmnAIcChwA/3sz3GyGICUKmqu/Y0Yx/iNdv9XZ8InAd8ZZDLR0QygCeB3wOjgLHAT4AWVV0JlALndMkzE5gO/C0uOMu1dTbwD+BREfnyntpr7BkmJAlAVXcAz+IICgAiEhSRW0TkY/dt6XYRSY6LXygib4lInYhsEpFT3fBCEVnuvqFtFJGvxOW5XkQeEpE/i0i9iLwjIgeIyA9FZJeIlIjIgrj0L7pNH6+4b21PiEiOiPzF/dxVIlIcl/4gEfmH+9kfiMgX4uLuEZHbROQp97NfE5HJcfEni8j77tvlrYDExU0WkRdEpNJthvmLiGS5cfcD44EnXBuv6vqm3o97slRE7nPtWicic+Pir3bfrOvdazqxu+/QvS/L3fvyuoj8VET+7cbtVnOQuKa73q6vm8+5XkT+7Hr/zz3XuNd+vHuNB8elzxen5pvXpZyTcB6chW7ee9zwz7r3oMa1cVpcni3u/VgLNPb1sFfV94F/ATPd/J2aY9zfxI2u+wQRKXXL3wHc7YZ/xv2d17i/w1lu9gPcz/ibqkZVtVlVn1PVtW78vcBFXUy6CHhKVSu7sXWHqv4WuB64WUS6fdaJyBEistr9nneKyH/3kK7teq5y/1vbReQsETldRD50v6f/jEsfFJHfiEiZe/xGRIJx8d93yygTkUu6fFavz4ohiaraMQgHsAU4yXUXAe8Av42L/w2wHOdtKx14AviFG3cEUAucjCPuY4GD3LiXgP8BknCEqRw40Y27HggBpwA+4D5gM/AjwI/z5rg5zoYXgY3AZCATWA98CJwUl/9uN20qUAJc7MYdBlQAM9z4e4Aq13Yf8BfgATcuF6gDFrl2fBuIAJe58VPcaw0CeTgP0N90dy9dfzGggG8P7snpgBf4BfCqG3ege02FceVO7uH7fABY6t6HmcA24N/d2RN3b/f4+lx7/9xLuf8D3Bzn/ybwRA82nwCUxvkPABpdW/zAVe73H4iz4y1gHJDcQ5kKTHHd04EdwKVd4+J+EzfG2RIBbnbvQzLOb2gXcKT73XzJtSEIZACVOIJxGpDdxY5xQBgY7/o9OLWUs3q6d274JDd8Wg/XtxK40HWnAUf1cm8jwI/p+G+VA3/F+T/PwPndTXLT3wC8CuS7v4FXgJ+6cacCO3F+V6luGfH3ubdnRafveKgc+92AT8rh/iEagHr3R/FPnGo2OG/jjcQ9tICjcR/ywB+BX3dT5jggCqTHhf0CuMd1Xw/8Iy7uTNcGr+tPd21ps+NF4Edx6f8LeLpL/rdc97nAv7rY80fgOtd9D/C/cXGnA++77otwH95x11+K+6Dt5jrPAt7sci+7FZJ+3pPn4+KmA82uewrOg+wkwN/Ld+nFeWgdFBf2c/opJHtyffQtJEfiiJ/H9a8GvtDD55xAZyG5Flga5/fgCOIJcXZc0sfvWnFeCqqBTcCNcbb0JSStQFJc/B9wH6ZxYR8Ax7vuaW4ZpTgP7eXA6Li0zwP/6bpPxnmx8fd079zwJDf8mB6u7/9wmtBy+7gPJwDN7P7fOjIuzRo6hG0TcHpc3CnAFtd9F3BTXNwBbfeSvp8Vnb7joXJY09bgcpaqpuN82QfhvJmD80aSAqxxq/Q1wDNuODgPx03dlFcIVKlqfVzYVpwaSxs749zNQIV2dEo3u+e0XtJ39belnQAc2Wava/P5QHwnbvzIoKa4vIU4Dz8A1PkHtPvd5pkH3CamOuDPdNyrvujPPelqV5KI+FR1I/AtnIf3LteGwm4+Iw9HtEriwrb2076BXl8nVPU1nAfL8SJyEM7DZnk/sxcSZ7eqxnCuKf5elXTN1A2HqWq2qk5W1WvccvpDuaqG4vwTgO92+U2Nc+1EVd9T1S+rahHO23ohztt5G/HNWxcCf1XVcB82tF1rlYicLx2d8U+74ZfiPMjfF6dp9zO9lFXZzX+rp/9Pp3vvugvj4nr6bfX1rBiSmJAkAFV9CefN6hY3qALnRzZDVbPcI1OdTkFwflSTdy+JMmCUdB6tMh7nrTLRlAAvxdmbpc7Iqcv7kXc7zgMCABGReD9ODUKBWaqaAVxAXB+KG9cTA7onqvpXVT0W56GmOE0vXSnHeSOOt3l8nLvRPafEhcULbF/X16N5PYTf65ZxIbCsy8O5N8pwrhPo9D3E36ve7nVfNNHzPeiu7BLgZ11+Uymq+je6ZnT6Y+7B7Y9xeQQYKyLzgbNxmmL74nM4tdAPVPUv7m84TVVPcz9ng6p+EacJ6mZgmYik9qPcvuh073F+P2Wuu9P/g86/rb6eFUMSE5LE8RvgZBE5xH2D+xPwaxHJBxCRsSJyipv2TuBiETlRRDxu3EGqWoLTtvoLEUlyOyYvxemPSDRPAgeIyIUi4nePw+M7a3vhKWCGiJztduB+g84PmXScJrgaERkLfL9L/p04bdu7MZB7IiIHisin3U7PEM4fdrchxe5b5yPA9SKSIiLTcdrz2+LLcR7GF4iI1+0sjX8R6Ov6eqIciLH7td+P80C8gP49PNtYCpzh/q78OEODW3Du32DwFnCeew9OBY7vI/2fgK+KyJHikCoiZ4hIujgDO74rIkXQPpT5izj9DACoaiOwDKfjfquqru7pg0RktIhcCVwH/LCnWpSIXCAieW58jRs8GMPM/wZcIyJ5IpKL07fSNqhiKfBlEZkuIimujUB7rbG3Z8WQxIQkQbgPm/tw2qkBrsbp6HzVbe54HqfzF1V9HadT+9c4ne4v0fE280Wc9t8y4FGcPop/7AP764EFwGL3s3fQ0XHaV94K4PPATTgdqFOBl+OS/ASn47UWR3Qe6VLEL3D+hDUi8r1uPmJv70nQtanCvZ584D97SHslTjPFDpw347u7xH8FRyAqcTpa4x/OfV1ft6hqE/Az4GX32o9yw0uBN3De8P/Vn7LcfB/giM/vca75TJwh6q39LaMPvumW2dbs+Vgf9qzGuW+34vS5bAS+7EbX4/QHvSYijTgC8i6O+MVzL85/oydBrXHzv4PTb/d5Vb2rF7NOBdaJSAPwW2DxHtT4euNGnP6sta4tb7hhqOrTOC+aL+Dcgxe65O3xWTFUEbcDxzCMXhBnLsJlbrPY/vj8u4AyVb1mf3y+YfTGXs2gNQxj3yHO3J6zcSa5GsaQw5q2DGMIIyI/xWni+ZWqbt7f9hhGd1jTlmEYhjEgrEZiGIZhDIgR0UeSm5urxcXF+9sMwzCMYcWaNWsqVLXPyZAjQkiKi4tZvbrHIeeGYRhGN4hIv1Z0sKYtwzAMY0CYkBiGYRgDYkQ0bRmGYSSKmMaobK6krLGMsoaOY2fTTgQh6AuS5E0iyZfUcXbd3ca54cneZCfeDfN7/DjLpQ09TEgMwzB6IRqLUt5c7ghEF7Eoayxje8N2WmOdV53JDGYyJmUMHvHQHGmmJdpCKBIiFA0RioTQvVgr0yMegt4gyb7k9nN2UjZ5yXnkJueSm5xLXkoeuUm55KbkkpecR2YwE0/3e3oNKiYkhmGMaCKxCDubdu4mEG3uHY07iGikU55RSaMYmzaWA7MP5NPjPk1BWgFj08ZSkFpAYVohqf6eFxBWVVpjrY6wREK0RFtojjQTioZoibS0i037Od4dl6Y50kxlcyXrK9dT3lxOc6R5t8/yiY+c5BxHZJLzyE3JbXfnJOd0EqGAN7DX99CExDCMIYeqUtdaR0O4gdZoa8cRc87hWJiWaEt7eCd/rJVwtMMfjoXbw1uiLYSj4XZ3eVM5O5t2EuuyOHB+cj6FaYUcnHcwp048lYJUVyjSCihILSDZt/c734oIQW+QoDdIZjBzoLeqnaZwE+XN5VQ0V1DeXE5lcyXlTY6/ormC7Y3bWVuxlupQdbc1osxgZnttpk1s+ktChcRdWvq3ODvO/a+q3tQlPoiziuccnFVUz1XVLSJyBHBHWzLgelV91M2zBWel0CgQUdW5GIYxbIhpjKpQFTubdrKzcWfnc9NOdjXtYmfjTkLRvV+E1yc+/F4/AW+AoCfY7g54AgS9jj/Vl8qE0RM61SbGpo1lTOqYAb2d7y9S/ClM8E9gQsaEXtNFYhGqQlXtAlPRXNFJcMqby3lr11tUNFf0+7MTtkSKiHhx9gM/GWfrzFXAF1V1fVyar+Fs/vNVEVkMfE5Vz3XX6G9V1YiIFABv4+yzHXGFZK67VHm/mDt3rto8EsNIPNFYlIrminZR6E4odjbtJBLr3FTk8/gYnTKa0SmjyU/Jbz9nBDMIeAKOCHgD+D1+gt5guzvgDezmD3gCeD3e/XQHPjmoKh6PZ01/XtYTWSM5Atioqh8BiMgDwEJgfVyahTjbnoKzYc2tIiLuvgxttO25bBjGfiYai1LWUMaWui1sqdvSPjqpTSwqmiuIaud9oYLeoCMSqaM5NP/Qdnf8eVTSqH3SKWz0nz0ZIZZIIRlL532JS3E2ruk2jVvbqAVygAoRORK4C2cTmwtV23u7FHhORBT4o6reQTeIyBJgCcD48eO7S2IYRjeoKtUt1Wyp3cLWuq1srtvM1tqtbKnbQkl9CeFYxzbpyb5kxqSOYXTKaI4qOKpdGNrCRqeMJjOYOWSHrRqDQyKFpLtfTteaRY9pVPU1nO1apwH3isjT7s5lx6hqmbsN5T9E5H1V/b/dCnEE5g5wmrYGciGG8UkkFAmxtW4rW+sckdhat5UttVvYXLeZ+tb69nQ+j4/x6eMpzijm+HHHMzFjIsWZxUzImEB2MNtEwkiokJTSeYP7IpytUbtLU+ru7Z0JVMUnUNX33K0zZwKrVbXMDd8lIo/iNKHtJiSGYTgd29sbt7O11q1ZuGKxpW4LOxp3dBq9k5+Sz8SMiZw+8XQmZEygOKOY4oxiCtIK8HlsgKfRM4n8dawCporIRGAbzt7f53VJsxz4ErASWAS8oKrq5ilxm7sm4OxXvEVEUgGPqta77gXADQm8BsMYsrR1bO9q2uWMdHJHPMX7tzdupyXa0p4n1Z9KcUYxh40+jAkZE5iYMZEJGc5InxR/yn68GmM4kzAhcUXgSuBZnOG/d6nqOhG5AadmsRy4E7hfRDbi1EQWu9mPBX4gImEgBnxNVStEZBLwqFuV9gF/VdVnEnUNhrG/aAw37i4MjZ39FaGK3eY/+MRHXkoe+Sn5TM2eyvFFx7c3Q03MnEhOUo41RRmDzojYIdGG/xpDCVVlZ9NONtVsoqS+pGPuRJxwNIYbd8uX7k8nPyW/09E2TDY/Nd9GPxmDjojs9+G/hpFQog0NtHzwAd6MDLyjRuHNzER8Q+cn3SYYG2s2sqlmU8dRu6mTUHjFS25yLqNTRjM5czJHFxy9u1Ck5FvTkzFkGTr/OsPoJ83vvEvN0gepffIptDlufSERR1Sysx1hyc7Gm52FL9t1j8rGFxfny85GUlIG3NSjquxo3MGm2k29CsaopFFMyZrCmZPOZErWFCZlTaI4o5hRSaNsAp0xrDEhMYYFscZGap98ipoHHyS0fj2SnEzGGaeT/ukT0VAzkapqotXVRKuriFRXE62qJlxSQmjtWiLV1RCJdFuuBAKdhMXbdrii48nIwJOUhASCSCBAlTZQ0rKDktadfNRcykdNJWxo3kqNNhL1ACLtgvHZyZ9lcuZkJmc5R3ZS9r69aYaxjzAhMYY0ofXrqX5wKXVPPEGsqYngAQcw+sfXknnmmXjT0/tVhqoSa2ggWlVFtLq6XWiiNdVEqqqIVte0x7VuKyVaVU2svr7H8rLdY1bXz/EInkAQT7AFCW5EgiVI8BU8gSC1SUnUBQN4AkEk2HYEEL8f8XjB6+l89ngQrwc83h7PeATxtqX19pzW50O8PsTvc9LFu30+xOe4xedz08a5fXHprJN++KEKkRCE6qClzj3Xdvhb6nuP6ycmJMaQI9bURN3TT1P94FJCa9ciwSAZp51G1rlfIPmQQ/b4gSYieNPTHeGZ0POCdi3RFl7b/horSlbw780vEqouJzUEud5MipPGMj5pDEWB0RT4cxjjzSZZfcRaWtCWVrSlhVhLqN2trS1OXKjFiWt10kXq6p04N1zDYTQWg2i089l1Dym83t0FJykJT0oKntRU59yXOzWlx7ih1L81ZGltguotUPWRc26udh747UJQB6HaOH89xK1E0COBNAhmQFKGc07JgeyJwBv9Msu+OWPIEPrgQ2oefJDa5cuJNTQQmDKZ0f/5n2Qu/CzezMFbbjue2pZa/q/0/1hRsoKXt71MU6SJZF8yx449lvlHzmde4TxyknMS8tn9oVuRiRMbjUbbRWf3NArRCBqNouFIJ7dGIxCJoJFoZ3ekLS7e3UO6SBhtaSXW1ESssZFYUxPhmpp2d6ypqXMfVh9IILCbwHjzcgmMLcJfVIS/aCz+sWMJFBXhSfkEDzxoaYDqzY5YtB+boXIT1Hed0y2dBSApA9ILIO/AuPB01525e9qgG99jH909/TLZhMTYr8RCIeqeeYaaBx6k+a23kECA9FNOIXvxuSQfdlhCmlNK6kt4seRFVpSs4I2dbxDVKHnJeZwx6Qzmj5vPEQVHEPQGB/1z9wbxeJymq/1tyF6i0Six5mZijU3Emho7znHio01NRBsbUVd82oWosZHWzVto/PfLaKjzkvLeUaPwFxURcMXF74pNoGgsvsJCPIEhvgx8qG53oaj6CKo2QcPOzmlT82DUJJh0vHNuO7KLITkbhkCTowmJsV9o2bSJ6gcfpPbx5cRqawkUF5N/9dVknrUQX/bgdkrHNMb6yvW88PELrChZwcaajQBMyZrCJTMvYf64+czInWHzLxKAeL1409LwpqXtdRmqSrSyknBpKa2l2whv20a4tJTwtlKa311H3T+eh3Bc840Ivvz8noVm9OjENqO19Uu0NEDtx3Ei8ZFTq6j6CJq67IKRNsYRhyknw6iJjjtnstO8lJSROFsHCZuQaOwzYi0t1D/3HNUPPkjz6jXg95Nx8slknXsuKUccPqi1j9ZoK6/veJ0VH6/gxZIX2dW8C494OCz/MOaPm8/8cfMZlzGu74KMIY9Go0R27eoQmtLSdrFp3baNyI4dzsO9DZ8P/+jR+MfkEZxQQNLkIpKKRxMsyEa0BcLNEG5yz/HuPQjrjoyxbm3CFYpRkztqFsG9F9pEYhMSjSFDy+bN1Cx9iNpHHyVaU4N//Hjyv/ddMj/3OXw5g9f/0Ft/xwnjTuC4sceRlZQ1aJ9nDA3E68VfUIC/oICUQw6Ghh1Q33FozTbCJR8TLiujdUcl4co6wjUbaN28mZq3fWjEqYmKRwlmhknK7jiCWRE8QT/4k8Gf0uWcDEkFu4f5UzqOzLEdYuHf++15hzomJEZCiLW0UP+P56l56CGaXnsNfD7STzyR7HO/QMpRRzlt/4PAtoZtrPh4BStKVrBm5xqiGiU3OZfTJ53O/HHzObLgyCHT32EMgHCzIwwNO6F+O9S7567+UM1uWcXjJ5A+hkDhaFIPnAbpY5wjbQzqT6O1vJHQ1p2ENpcR2lRK/YbN1HzkDn31eAhMmkjStOkkTZ9O0rRpJE2fhjdj6Dc37UtMSIxBpXndOmoffoTaJ58kVleHf+xY8r79bbLO/hy+vLy9KjMUCVFSX9Jp34y2oyrk7DowJWsKF8+8mPnj5jMzd6b1dww3YjGoLYHKjU4/QuVG56jb5ghINwKBx98hCjmTofhYSB/tjFpyhYL0AqdDuocXFwGC7tE2LlBViZSVEXrvPULr1xNa/x5Nr79O3RNPtOfzFxU5wjJ9WrvA7O3vu79oJOIMXGhqItbYhIaandFtGRl409MRvz+hn98b1kdiDJhoTY0z6/zhh2l57z1n5NWCBWSdczYpRx7Zr9pHNBalrLGsY6Ol2g7B2N64vdO+GbnJue37ZUzJmsJxRccxPsN2wRzyqEJTZYdItB+bnCNuuXsC6ZAzCbLGu4IQfxQ4Yb0IRCKIVFYSWu+Ky3vvEXpvPeGtH7fHe/Ny42ot00k66CDE620fCt3paOwaFjdkutv4JrSlpRfrQFJSnCWCMjLwZKTjzcjEm56OJzMDb3oG3swMR3Ta0rhh3vT0HpcK6m8fiQmJsVdoLEbTq69Ss+xh6p9/Hm1tJWn6dDIXnUPmGWd0O+9DValoruhUq2hzl9SXEIl1LGOS5k+jOKOYCZnOXhkT0ic47vQJpAWGZsek4dLaGFer2NRZNOJrFh6/0/GcM8WpUeRMcY+pkJY/JIa19kW0vp6W999vr7mE3nuPlk2b9mgyafukzq6HO3lTOoW3TepMxZMUJNbURLS2jmh9HbHaOqL19UTraju76+qJNTT0boTP5whMejqeTEeAvJkZFP3619bZbgw+4bIyah55lNpHHiFcVoYnM5OsL3yBrHPOJmnatPZ0LdEW3tj5Bm/seqN9v++tdVtpinSMaAl4AozPGM+kzEnMHzffEQ53k6VRSaNsSY6hTCzqzKzurnZRt61z2oyxjkDMPCdOLCZD1gTwDu9HkDc9nZTDDyfl8MPbw2KhEC0bNtDywQcg0kkcpKsgJCc7S9AkGI1EiNbXE6uvJ1pbR6y+jmidc8Tq6jrEqK6OaF090bo6wqWl/S7faiRGn8RaW2l4/nlqlj1M48qVAKQefTSZ55xN+kkn4QkGUVU+qv2Il7e9zCvbX2HNjjWEoiE84qEwtZAJmRM6CUVxRjGjU0bbqrdDndYmqNwAFRug/AOo+NA5KjdCtLUjXVKmU5PIndq5djFqEgRS95/9xoCw4b/GgAm9/z41yx6m7okniNbW4issIPdrXyPzc58jUDSWmlANz21fwSvbXuGVslfY2eTMyJ2YOZFzDjiHeYXzmDt6ru2jMRxorHAEovwDRzQqPoDyD50JdW2IxxnGmnsATDnJOedOdQQkZdSwaIoyEoMJidGJaF0dtU8+Se3DjxBatw7x+0k/+SQyzzmHwBFzeLdqPQ+UPcLKt1fybsW7KEp6IJ2jCo5iXuE85hXOozCtcH9fhtEdsZgjDOVuraLig46aRnNVRzpfMuROgXFHwKEXQN4BjmiMmgz+pP1nvzFkMSExnI7z11+n5uFHqH/uObSlheBBBzH6Rz+i4YRDeblpHS9vW8rrD32PhnADHvFwcO7BXD77cuaNnceMnBn4PPZT2m+ousuB1zqd2aFaaK5x3LWlHbWMyg3O0h1tpORA7oEw7Uxnkb9cVzAyx+3T0VDG8Mf+/SOY8I4d1D76KDWPPEq4pARPejqpZ32WLcdN5qWUj1m5/W98/MLNABSmFnLqxFOZVziPIwuOJCNgE7IGlWjYFYI4EQjVuO5uBKLd7ebRnkYJiTOENvcAZ9G/3KmOeOQeAKn7b1Vj45NFQoVERE4Ffgt4gf9V1Zu6xAeB+4A5QCVwrqpuEZEjgDvakgHXq+qj/SnT6B2NRGh46SVqlj5Ew7/+5SxFPmcmH35uJk8WVbCm5gkipRGSfckcPuZwzpt2HvMK51GcUWyjqAZCqLZjwb7KTc4qr5Wb3NnYtdDax/BMjx+SsyApyzmn5DhNTUmZHeHduVPzIWB9VEZiSZiQiIgXuA04GSgFVonIclVdH5fsUqBaVaeIyGLgZuBc4F1grqpGRKQAeFtEngC0H2Ua3dBaUkLNsoepfeQRIuXlePJy+OjMQ7ireCsfprwPvM80mcZFMy7imMJjOCT/EALeIb4U91CjpT5OJNwlwduEo+tqr20L+E08vnchSMp0/P5k68w2hiyJrJEcAWxU1Y8AROQBYCEQ/9BfCFzvupcBt4qIqGr88plJ0D6tuT9lGi5tw3arH3qIppWvgseDb94RvHreDG5PXUWzvsP8cfP58oSTOLrwaHKTc/e3yUOfloa4PSTiBKNyEzTu6pw2vcCpNRx0unPOmeycs4utlmB8okikkIwFSuL8pcCRPaVxax+1QA5QISJHAncBE4AL3fj+lAmAiCwBlgCMHz+yls9o2bTJWW338ced1XYLC/EuuYCHplSwrOZFAD4z+TNcPPNiJmVO2r/GDjXCzU5zU91251xb0rlJqmFH5/Rpox1xOGBBZ7EYNdHmTxgjhkQKSXf18K6zH3tMo6qvATNEZBpwr4g83c8ycfPfgdvPMnfu3E/8rMtYczN1zzxLzUMP0fzGG+2r7daccjh/DL7OC6UPEqwLsvigxVw0/SIK0gr2t8n7llgUGnY5W5XW74A691y/vbNwdLc4YGqeIw5TTuzYcKhNLILp+/5aDGOIkUghKQXidw4qArpuONyWplREfDgLcFbFJ1DV90SkEZjZzzJHFKH166l+6CHqnnjS2ee8uJi8732XTfPG81+lS3lt+02kB9L5yqyvcP608xmVNGp/mzy4qDoP/zYh6CQMOzqEo2EnaKxzXvE6NYqMgrjVY8dARqG7OGCh4x4GO9QZxv4kkUKyCpgqIhOBbcBi4LwuaZYDXwJWAouAF1RV3TwlbnPWBOBAYAtQ048yP/FEGxqoe/JJapY+RGj9eiQYJP2UBWQuWsRro+u48927eGfVO+Qm5/KdOd/h8wd8fvgvdBhpdfpG3RmMAAAgAElEQVQiyj9wj/edc9VHEGnePX1ytruceAHkz3DEok0c2sQiNQ9siRbDGDAJExJXBK4EnsUZqnuXqq4TkRuA1aq6HLgTuF9ENuLURBa72Y8FfiAiYSAGfE1VKwC6KzNR1zCUUFWa33qLmoeWUff002hzM8EDD2T0NdeQcsapPFf1Mne983M2vbeJorQirj3qWhZOWTj8NnUKN3fMti5/312q4wOnf6J9roQ7NyLvIJh0giMKGQUdwpE+5hO9G51hDDVs0cYhTqS6mrrly6lZtoyWDRuRlBQyzzidrM9/Hp02hcc2PsY96+5he+N2Dsg+gEtnXsqC4gVDf6Z5S72zVEe8WJS/D9Vbae/2Eq/TJ5F3oCMaeQc6R85UG/VkGPsAW7RxGBFtaCRc8jGtH5cQLi1xzm3+7dshGiVp1izG/PQGMk47ncZAlD+//yB/fuTrVIWqODT/UK456ho+NfZTQ2/SYFOVuxjg+3FNUh9CXdwS1d6AIw6Fh8HsL3YIx6hJ4BtmNSrD2AeoKqFwjJhbEYj/24s7Jqnro6DNL3FjljrC2vzSyd9fTEj2AapKZFe5Iw4lpe0i0VryMeGSUqJVncYX4M3Kwj9+PMmzZ5P52c+SfsoCkg48kIrmCn63/g6WfrCUhnADx449lssOvow5o+fspyvrhkgLfLwSNj4PG56H8vc64vwpztIcxcd0iEXugc68imG+L4Vh7A2RaIza5jC1zWFqmsPUNrnuplbHHx/mhtc2R6htbiUcHTqtSfbvHSRira2ES7ftXqMoLaG1pBQNxS2W5/HgLyjAP34c6SedhH9cEYFx4wmMH4d/3Di86Z2HlJbWl3LPqzfy6IZHiWiEBRMWcOnBl3LQqIP28VX2QPUW2PAP2PhP2Px/EG50lvSYMA9mfQFGz3SEwxYDNIYYsZgSiSmRWIxITIlGlXAsRjSmRKJOXDQWIxxVJyymRKJu2pgSjsbiwp1y2vKGYzHqQxFXGMLUNre2u2uawtQ1h6lvifRqX3rQR2aKn8xkP1kpfg4ck05mcoCsFD/pST58HiG+d6LN2RbWtkV1dz0Ybd0aHWl3z/vtm/t3H01IBkBr6TZ23ngjoQ8/ILJ9R6dvS5KTCRQV4R8/gdRjjsU/fpwjFuOK8BcWIoHulx9RVXY07mBz7WY2127mrV1v8dzW5xARFk5eyMUzL2ZCxoR9dYndE26GLS/Dxn84NY/KjU541gQ45IvOXhXFn4LgMB8pZgwbojGltjlMdVMr1Y2tVDc57pom193Y6sQ1hdvDapr2zVu93ytkJgfITPaRlRJgTEYSB45OJzPFT1ZceLtgJPvJSgmQkeTD592/L17f7mc6E5K9pPnttyn52hVoOEzaCccTGDfeqVmMH09g3Di8ubm99lc0R5qdPctrtziiUbeZLbVb2FK3hea44azpgXTOn3Y+F02/iNGpo/fFpe2OqiMWG593ah5bX3aWI/clOXMvDv+KIx45k209KGPAqCp1zRF21oeoaGihpl0UwlS5gtA1rC4U7vatG5wHeVZKgFEpzpv8pNw0slOdh3WSz4vPK/g8gtcj+L0evB7H7/N62sN383sFn8fj5mlL4+lUls/jIT3JR0rAO/T6LgcZE5K9oO6ZZym7+mp8+fmM++PtBCd1v8yIqrKraVe7SGyu3cyWOue8vXF7ezpBKEwrpDh9PHPGHMPEcJiJdeUU79pIbsV6ZGsprH4MMoucxf4yizqOjLHO8Fevf3AvsqXBaaba+Lxz1Gx1wnOmwpyLYepJMOEYG2Zr7BGhcJSddSF21rWwoy7ErroQO2pD7KxvYWdtiJ31IXbWhQiFY93mTwl4yU4JkJ3qJzslQFF2Ctkpjjs7xU92aqCTaGSnBkgdAQ/y/Y0JyR6gqlTdeSe7bvkvkg89lKLbbsU3ahShSMipXdR1FosttVtoinSsP5niS6E4s5jDRh9Gcdo4JuKjuLGGCRVbSNr+Dry7rGOuRHoBFB4K085ylhmvLXWOktegubqLZeLMnehOaDKLIKMIUnN7ry2owq73Opqrtq6EWBj8qc4+Fsd8w6l1ZBcP+n01hj+RaIyKhlZXJEKdxCLeX9sc3i1vkt/DmIwkRmckMbsoizGZSeSnBxmdkUROWoBRqQGyXWEI+mwC6VDE5pH0Ew2H2XHDDdQ8tIyM00+j4Be/YGXFan656pdsrt3c3qkFUJBawMTMiRRnFDMxcyIT0ydQHI2RX7EZ2f4WlL0BO96FaIuTITnbGfo69jBHPAoPcybY9URrI9RucxYUrNvmiozrry11wuJ3wgPwBiFzbIewZBY5fn+KW/P4p7OcCDgzwaec6AjH+KNsCO4wRVUJR5XWaIyWcJSWSIyWSIzWSIyWiOsPx2iNRmkJx9z4qBvvHuEoLdFYe3ynvJEYNU2t7Kh1mqBiXR4lXo+Qnx4kPyOJMRmOMHQcQcZkJJGfkURGks9qDEOU/s4jMSHpB9H6erZ985s0vrKSnK/+B6mXX8Z/v/Frln64lEmZkzi1+FRHODKLGZ82jpT6HVD2pnNsewO2v+2MZAIIpEPhIe7hikfWhMHtW1CFpsqOWkxdm8hs6/DXb+9YeyqYCZNPcIRj8omOwBhDltZIjIqGFsrrnWOXey5vCLWHlbvxPTUR7QlBn8c5/F4CXg9Bv4egz0vA5yEr2d9JFNpqFqMzg+SkBvF6TCCGMzYhcZBoLd1GyVf/g9YtWyn4+c/56JgJ/OjJz7OtYRtfmv4lvj71XILb34JNr0HZHxzxCNU6mX1JMOZgOPSCjtpGztTED4EVcZqyUnMdweqOaLhjd768gwa/j8XYI2IxpaY5HCcEcaLQSSyczufuyE7xk5ceJC89yJzx2eSlB8lI8pPkdx76jhi4IhAnCG3hTpi3XTgCPifMagtGX5iQ9EL8yKyCO/7AXYHXueeZ6yhMK+SuU+5ibjgGv5/jNFF5fJA/HWZ8rqN5Kn/a0H1Ae/3OelXDjEg0RmNLlIbWCE0tEULhGCLgEcHjcQYueMSZoetpCxdx0nicOb0e6SGNx3G3pWl7fjrNOh3NOZ2bhTo3G3VqGorLt3tzkuOvD0Uor2+hoqGl26GoSX4P+elJ5KUHmZyXxlGTctrFIi8t2O7OTQsS8NkcHWP/YELSA3XPPkfZVVfhy8sj/PvruLjkv9hYs5FFByzie3O/R2pM4fZjnWXIF93p1DyG4QimaExJZPNmVJWmliiNrRFHAFoiNLVGaGyJ0NASpbEl4sY58W3+9rj2eMffEhl4U82+xO+V9mag9iYin7e9BpCdEuDA0ekd4hAnEPkZSTbiyBgWmJB0IX5kVtIhs1lxxZH8/t2ryE7K5rYTb+O4ouOchE98y5nR/eWnYNwR+9Xm/lDV2MrGXQ0dR3kDm3Y1sK2mmyXY9wNej5Aa8JIa9HUcAS+jUlNIC/pIDXpJDXSOSw36SPJ7UVVi6nx3CsTi/DFVYjEnTNU9d00Tc9xtaZQOP0BSvBB0afppaxpKius3aBOLgM9jfQTGiMCEJA5nZNZPqXnoITwnHcePT6rlrY/u4tTiU/nRkT8iKynLSfjhs7Dmbjjmm866UUMEVaWsNtRJMDa5olHV2NqeLsnvYXJeGnOLszln1Fj8CZw9KwIpAZ8rBq4gBH2kumEpQS9pQR9Bn7XFG8ZwxYTExRmZ9S0aX3mFHeccw1UHrSYYSuZXx/2KUyee2pGwsQIev9JZP2r+j/aLreFojK2VTY5QlMeJRnkDTa3R9nRZKX6m5KWxYPpopuSnMTk/jSl5aYzNSsZjb8qGYQwSJiTEjczavIVnvjiZu4pf41MFn+In835CXkpeR0JVeOKbztauFz22T+ZXfLiznvVldZ2apLZWNnbqmC3ITGJKfhpfmDuOKflp7UdOasDe8g3DSDgjXkia166l5PKv0Rpq5L/OS2J9cTnXHX4d50w9Z/eH8Ft/hfefhAU3wugZCbUrHI1x45PruXelszSJ1yNMGJXC5Pw0Tp4+mil5ae21jLTgiP8aDcPYj4zoJ1Dds8+x7arvU5vm5brzwxRMn83Dx9xIUXrR7omrt8DTV8OEY+GoKxJqV2VDC1/7yxu8trmKS46ZyOIjxjEhJ8WWhzAMY0gyIoVEVam66y52/eoWNhX5uGWRl4s/9X0unH4hHumm4zkWhUcvd9yf+0NCJxS+u62W/7h/DRUNLfz63Nl87tBuRM0wDGMIMeKERMNhPr7+WpoefpyXpwkvXHggd376ZiZnTe4508pb4eNX4KzbEzqJ77E3t3H1w2vJSQ2w7KvzOLgoM2GfZRiGMViMKCGJ1tez7qtfxr9mPY/O85J15eXce8gS/J5eZp/veAf++VOY9lmYvTghdkWiMW5+5n3+9K/NHDFxFP9z/mHkptlCiYZhDA8SKiQicirwW8AL/K+q3tQlPgjcB8wBKoFzVXWLiJwM3AQEgFbg+6r6gpvnRaAAaJtJt0BVd/VlS92Wjay/9AJSt9fy0KJ8zv7WbczMndl7pnAIHvkPSBkFn/lNQjZtqm5s5et/e5N/b6zgS0dP4JrPTE/ovA7DMIzBJmFCIiJe4DbgZKAUWCUiy1V1fVyyS4FqVZ0iIouBm4FzgQrgTFUtE5GZwLNA/JK056tqv5fzba6v5r1FZ+ELR3n9qlO46oKbSPIl9Z1xxY2wax2cvwxSc/r7cf3mve11LLl/NTtrW/jlObP4wuHjBv0zDMMwEk0iX32PADaq6keq2go8ACzskmYhcK/rXgacKCKiqm+qqrs5BuuAJLf2slfox2W0+oE7fsGlX/5N/0Rk87/glVth7qUw9eS9/ege+fs72zn7f16hJRzjgf84ykTEMIxhSyKbtsYCJXH+UuDIntKoakREaoEcnBpJG+cAb6pqS1zY3SISBR4GbtRuVh0UkSXAEoCp6SnMeuTvZBb0s6M8VAuPXQ6jJsGCn/YvTz+JxpT//scH3LZiE4eNz+L2C+aQn9EPYTMMwxiiJFJIuutQ6PrA7zWNiMzAae5aEBd/vqpuE5F0HCG5EKefpXMhqncAdwDMnTNH+y0i4MwXqSuDS5+DQGr/8/VBbXOYbz/4Fi+8v4vFh4/jJwtn2NwQwzCGPYkUklIgvr2mCCjrIU2piPiATKAKQESKgEeBi1R1U1sGVd3mnutF5K84TWi7CUkn9qSTfP3j8Pbf4PiroajPjcH6zcZd9Sy5bw0fVzVx41kzOf/I8bZ8iWEYnwgS2UeyCpgqIhNFJAAsBpZ3SbMc+JLrXgS8oKoqIlnAU8APVfXltsQi4hORXNftBz4DvDtoFtfvcJaHLzwUjvv+oBX7j/U7Oeu2V6gLhfnrV47igqMmmIgYhvGJIWE1ErfP40qcEVde4C5VXSciNwCrVXU5cCdwv4hsxKmJtE3UuBKYAlwrIte6YQuARuBZV0S8wPPAnwbJYGdV33AznP2nQdnZMBZTfv/CRn79/IccPDaTP144h8Ks4bf5lWEYRm9IInfHGyrMnTtXV6/uY7Twqjvhqe/A6bfAEV8Z8Gc2tET4zoNv8dz6nZx96Fh+fvbBJPmtP8QwjOGDiKxR1T7b+EfUzPYeqdgIz10Dkz8Nh1824OK2VDTylftW81FFIz/+zHQuPqbYmrIMw/jEYkISjcCjS8AbgIW3DXj2+osf7OIbf3sTr0e475IjOGZK7iAZahiGMTQxIfnXf8G2NbDobsgo3OtiVJXbX/qIXz77PgeNyeCOC+cwblTKIBpqGIYxNBnZQrJtDbx0Mxz8BZh59l4X09Qa4fvL1vLU2u18ZlYBv1w0i5TAyL61hmGMHEbu0661yVmQMX0MnP6rvS6mpKqJJfev4f0ddVx96kF89fhJ1h9iGMaIYuQKyfPXQeUGuGg5JGftVRHPvLudHzzyDrGYcveXD+eEA/MH2UjDMIyhz8gUko3Pw+t3wFFfg0nH73H2ulCY65ev45E3tjFzbAa//+JhTMwdvKVUDMMwhhMjT0iaquCxKyDvIDjxx3uc/ZVNFXz/obXsqAvxjU9P4esnTrX9QwzDGNGMLCFRdSYdNlXC+UvB3/9Z5qFwlF89+wF3/nszE3NTWfbVozl0fHYCjTUMwxgejCwheWcZrHvUqYkUzO53tne31fLtB99iw64GLjxqAj88/SAblWUYhuEycp6GtaXw1Hdh3JFwzLf6lSUSjXH7S5v4zfMbyEkLcO8lR3D8AXkJNtQwDGN4MXKE5LHLQaPwuT+Cp+81rzZXNPKdpW/x5sc1nDm7kJ8unEFWSmAfGGoYhjG8GBlC0lgOmzfAmb+DURN7Taqq/Pm1j/n5U+/h9wq/XXwICw8Z22sewzCMkczIEJK6MjjgHDjsol6T7awLcdWytbz0YTmfmprLrxbNZkymbYNrGIbRGyNDSMQDn/1drwsyPrm2jGsee5dQOMoNC2dwoW0+ZRiG0S9GhpBkjYe07med1zaF+fHyd3n8rTJmj8viv78wm8l5afvYQMMwjOHLyBCSpMxug/+9oYLvPfQ25Q0tfPukA7hi/mR8NrnQMAxjjxgZQtKF5tYoNz/zPve8soXJeanccdE8ZhXt3XpbhmEYI50RJyRvl9Tw7aVv8VF5I1+eV8wPTjvItsA1DMMYACNGSMLRGLe+sJFbV2wkPz3Iny89kmOn2u6FhmEYAyWhHQIicqqIfCAiG0XkB93EB0XkQTf+NREpdsNPFpE1IvKOe/50XJ45bvhGEfmd9GNoVUskxqI/vMJv/7mBz84u5JlvHWciYhiGMUgkrEYiIl7gNuBkoBRYJSLLVXV9XLJLgWpVnSIii4GbgXOBCuBMVS0TkZnAs0DbrMA/AEuAV4G/A6cCT/dmy4Zd9Xirmvif8w/j9IMLBu8iDcMwjITWSI4ANqrqR6raCjwALOySZiFwr+teBpwoIqKqb6pqmRu+Dkhyay8FQIaqrlRVBe4DzurLkLSAj+e+dZyJiGEYRgJIpJCMBUri/KV01Cp2S6OqEaAWyOmS5hzgTVVtcdOX9lHmbhTnppKfYTPUDcMwEkEiO9u767vQPUkjIjNwmrsW7EGZbXmX4DSBMX78+L5sNQzDMPaSPmskIuIVkdw4f0BElojIe31kLQXGxfmLgLKe0oiID8gEqlx/EfAocJGqbopLX9RHmQCo6h2qOldV5+bl2dLvhmEYiaJXIXE7wKuAtSLykojMBz4CTgPO76PsVcBUEZkoIgFgMbC8S5rlwJdc9yLgBVVVEckCngJ+qKovtyVW1e1AvYgc5Y7Wugh4vD8XahiGYSSGvpq2rgHmqOpGETkMWAksVtVH+ypYVSMiciXOiCsvcJeqrhORG4DVqrocuBO4X0Q24gjWYjf7lcAU4FoRudYNW6Cqu4DLgXuAZJzRWr2O2DIMwzASiziDn3qIFHlDVQ+L87+vqgftE8sGkblz5+rq1av3txmGYRjDChFZo6pz+0rXV40kX0S+E+dPi/er6n/vrYGGYRjGJ4O+hORPQHovfsMwDGOE06uQqOpPeooTkdTBN8cwDMMYbvRn+O9YEZnrjrxCRPJF5OfAhoRbZxiGYQx5+hr++y3gLeD3wKsi8iXgPZwRU3MSb55hGIYx1Omrj2QJcKCqVonIeGAjcJyqvpp40wzDMIzhQF9NWyFVrQJQ1Y+BD01EDMMwjHj6qpEUicjv4vz58X5V/UZizDIMwzCGC30Jyfe7+NckyhDDMAxjeNLX8N97e4s3DMMwjF6FRES6LrLYCVX97OCaYxiGYQw3+mraOhpn46m/Aa/R/X4ghmEYxgimLyEZg7Pn+heB83CWdv+bqq5LtGGGYRjG8KDX4b+qGlXVZ1T1S8BROPNIXhSRr+8T6wzDMIwhT59b7YpIEDgDp1ZSDPwOeCSxZhmGYRjDhb462+8FZuJsHvUTVX13n1hlGIZhDBv6qpFcCDQCBwDfcHa3BZxOd1XVjATaZhiGYQwD+ppH0ufqwIZhGMbIxoTCMAzDGBAmJIZhGMaAMCExDMMwBkRChUREThWRD0Rko4j8oJv4oIg86Ma/JiLFbniOiKwQkQYRubVLnhfdMt9yj/xEXoNhGIbRO33OI9lbRMQL3IYzM74UWCUiy1V1fVyyS4FqVZ0iIouBm4FzgRBwLc7Q45ndFH++qq5OlO2GYRhG/0lkjeQIYKOqfqSqrcADwMIuaRYCbSsMLwNOFBFR1UZV/TeOoBiGYRhDmEQKyVicBR/bKHXDuk2jqhGgFsjpR9l3u81a10rc5JZ4RGSJiKwWkdXl5eV7br1hGIbRLxIpJN094HUv0nTlfFU9GPiUe1zYXSJVvUNV56rq3Ly8vD6NNQzDMPaORApJKTAuzl8ElPWURkR8QCZQ1VuhqrrNPdcDf8VpQjMMwzD2E4kUklXAVBGZKCIBYDHQdaOs5cCXXPci4AVV7bFGIiI+Ecl13X7gM4Ct/2UYhrEfSdioLVWNiMiVwLOAF7hLVdeJyA3AalVdDtwJ3C8iG3FqIovb8ovIFiADCIjIWcACYCvwrCsiXuB54E+JugbDMAyjb6SXCsAnhrlz5+rq1TZa2DAMY08QkTWqOrevdDaz3TAMwxgQJiSGYRjGgDAhMQzDMAaECYlhGIYxIExIDMMwjAFhQmIYhmEMCBMSwzAMY0CYkBiGYRgDwoTEMAzDGBAmJIZhGMaAMCExDMMwBkTCFm0c6oTDYUpLSwmFbBPG4UhSUhJFRUX4/f79bYphjHhGrJCUlpaSnp5OcXExPWyyaAxRVJXKykpKS0uZOHHi/jbHMEY8I7ZpKxQKkZOTYyIyDBERcnJyrDZpGEOEESskgInIMMa+O8MYOoxoITEMwzAGjgmJYRiGMSBMSIYJ119/PbfccktCP+OEE06gbSfJ008/nZqamkH/jDVr1nDwwQczZcoUvvGNb9DdDp1/+ctfmDVrFrNmzWLevHm8/fbbg26HYRiDhwmJ0S1///vfycrKGvRyL7/8cu644w42bNjAhg0beOaZZ3ZLM3HiRF566SXWrl3Ltddey5IlSwbdDsMwBo8RO/w3np88sY71ZXWDWub0wgyuO3NGr2nuu+8+brnlFkSEWbNmcf/997N161YuueQSysvLycvL4+6772b8+PF9ft7vfvc7br/9dnw+H9OnT+eBBx6goaGBr3/966xevRoR4brrruOcc87h8ssvZ9WqVTQ3N7No0SJ+8pOf7FZecXExq1evpqGhgdNOO41jjz2WV155hbFjx/L444+TnJzMqlWruPTSS0lNTeXYY4/l6aef5t133+3Rxu3bt1NXV8fRRx8NwEUXXcRjjz3Gaaed1indvHnz2t1HHXUUpaWlfV6/YRj7D6uR7CfWrVvHz372M1544QXefvttfvvb3wJw5ZVXctFFF7F27VrOP/98vvGNb/SrvJtuuok333yTtWvXcvvttwPw05/+lMzMTN555x3Wrl3Lpz/9aQB+9rOfsXr1atauXdv+5t8bGzZs4IorrmDdunVkZWXx8MMPA3DxxRdz++23s3LlSrxeb3v6srIyTj/99N3K2bZtG0VFRe3+oqIitm3b1utn33nnnbsJjWEYQ4uE1khE5FTgt4AX+F9VvalLfBC4D5gDVALnquoWEckBlgGHA/eo6pVxeeYA9wDJwN+Bb2p3De17QF81h0TwwgsvsGjRInJzcwEYNWoUACtXruSRRx4B4MILL+Sqq67qV3mzZs3i/PPP56yzzuKss84C4Pnnn+eBBx5oT5OdnQ3A0qVLueOOO4hEImzfvp3169cza9asHsueOHEihxxyCABz5sxhy5Yt1NTUUF9f3157OO+883jyyScBKCws5O9///tu5XT3NfU2jHfFihXceeed/Pvf/+7r8g3D2I8krEYiIl7gNuA0YDrwRRGZ3iXZpUC1qk4Bfg3c7IaHgGuB73VT9B+AJcBU9zh18K1PPKrar7kQ/Z0v8dRTT3HFFVewZs0a5syZQyQS6fYzNm/ezC233MI///lP1q5dyxlnnNHnxL5gMNju9nq97WXvKUVFRZ2aqUpLSyksLOw27dq1a7nssst4/PHHycnJ2ePPMgxj35HIpq0jgI2q+pGqtgIPAAu7pFkI3Ou6lwEnioioaqOq/htHUNoRkQIgQ1VXurWQ+4CzEngNCePEE09k6dKlVFZWAlBVVQU4/QNttYi//OUvHHvssX2WFYvFKCkpYf78+fzyl7+kpqaGhoYGFixYwK233tqerrq6mrq6OlJTU8nMzGTnzp08/fTTe2V/dnY26enpvPrqqwCdaj49UVBQ0J5HVbnvvvtYuLDrTwI+/vhjzj77bO6//34OOOCAvbLPMIx9RyKFZCxQEucvdcO6TaOqEaAW6O31c6xbTm9lAiAiS0RktYisLi8v30PTE8+MGTP40Y9+xPHHH8/s2bP5zne+Azid5nfffXd753tb30lvRKNRLrjgAg4++GAOPfRQvv3tb5OVlcU111xDdXU1M2fOZPbs2axYsYLZs2dz6KGHMmPGDC655BKOOeaYvb6GO++8kyVLlnD00UejqmRmZgI995EA/OEPf+Cyyy5jypQpTJ48ub3/4/bbb2/v27nhhhuorKzka1/7Gocccghz587daxsNw0g8MsDuhZ4LFvk8cIqqXub6LwSOUNWvx6VZ56Ypdf2b3DSVrv/LwNy2PhIRORz4haqe5Po/BVylqmf2ZsvcuXO1bX5EG++99x7Tpk0blGsdqTQ0NJCWlgY4nf3bt2/vl/ANFvYdGkZiEZE1qtrnm1wiO9tLgXFx/iKgrIc0pSLiAzKBqj7KLIrzd1emsY946qmn+MUvfkEkEmHChAncc889+9skwzD2A4kUklXAVBGZCGwDFgPndUmzHPgSsBJYBLzQ2wgsVd0uIvUichTwGnAR8PtEGD9UueKKK3j55Zc7hX3zm9/k4osv3ue2nHvuuZx77rn7/HMNwxhaJExIVDUiIlcCz+IM/71LVdeJyA3AalVdDtwJ3C8iG3Xs2isAAA/PSURBVHFqIovb8ovIFiADCIjIWcACVV0PXE7H8N+n3WPEcNttt+1vEwzDMDqR0Hkkqvp3nLke8WE/jnOHgM/3kLe4h/DVwMzBs9IwDMMYCDaz3TAMwxgQJiSGYRjGgDAhMQzDMAaECckwYSTtR9LGqlWr8Hq9LFu2bNDtMAxj8DAhMbplf+5HAs5s/auvvppTTjll0G0wDGNwsf1IAJ7+Aex4Z3DLHHMwnHZTr0lsP5Lu9yMB/r+9uw+uqs7vOP7+Dgjp0u4KoitrREJFIJHL41BwO0I3rRtETYto8RFZ0ao4YmdXkXVc1mXcMeNO3dXCtrgCFZ/a4tpGVOiK66DYxYAIAvJUExXYlfAgDytaQ7/945xkr+He5Iabk3MufF4zmdxzzu+efO8vufnmPOT75dFHH+Xyyy+npqam1dcuIvHSEUlM1I8kez+SnTt38vzzz3PLLbfk9NpFJF46IoFWjxyioH4kgUxl8u+8806qqqq+lJxEJLmUSGISRT+SFStWUF1dzezZs9m4cWOL/Uhqamro3r07N9xwQ5v7kRw5ciTSfiSrV69m0qSgyMGePXt46aWX6Ny5c1OCFJFk0amtmKgfSfZ+JLW1tdTV1VFXV8fEiROZO3eukohIgimRxET9SLL3IxGRwhJZP5IkUT+SaKgficiJLQn9SOQEp34kIgJKJAVH/UhEJGmUSAqM+pGISNLoYruIiORFiURERPKiRCIiInlRIhERkbwokRSIk60fyWuvvcaQIUMoKytjzJgx7R6HiLSfSO/aMrMK4GdAJ+AX7v5gs+1dgSeA4cBe4G/dvS7cNhO4ETgK3OHuy8L1dcChcH1DLv8s05qqt6rYvG9zvrv5kgE9BjBj5Ix23WdHylR0sT009iMZNWoUF198MUuXLj2mjPwnn3zCbbfdxtKlS+nduze7d++OJBYRaR+RHZGYWSdgDjAOKAWuMrPSZsNuBPa7+7nAw0BV+NxSYBJQBlQAc8P9NfoLdx/SHkkkTk888QSpVIrBgwdz3XXXAfDBBx9QXl5OKpWivLycDz/8MKd9PfLII5SWlpJKpZoKHh4+fJgpU6YwaNAgUqlUU/n3W2+9lREjRlBWVsasWbMy7q9Pnz7s2bOHuro6Bg4cyE033URZWRkXXXQRR44cAYIOhqlUitGjR3PXXXdx/vnntxhjej8SM2vqR9Lc008/zYQJE5r6sJxxxhk5zYGIxCPKI5KRwHZ3fx/AzJ4FKoFNaWMqgR+GjxcD/2hBudpK4Fl3/xyoNbPt4f7+O4pA4zhyaOxHsnLlSnr27NlUtLGxH8nkyZOZP38+d9xxR8Zfts09+OCD1NbW0rVr16ZTUun9SCAo2ghBP5IePXpw9OhRysvLWb9+fYtl5Ldt28YzzzzDY489xpVXXslzzz3Htddey5QpU5g3bx4XXHAB99xzT9P4Xbt2MXXq1GOOanLtR7J161a++OILxo4dy6FDh5g+fTrXX399q3MgIvGI8hrJWcBHacs7wnUZx7h7A3AAOK2V5zrwX2a2xsxuzvbFzexmM1ttZqvr6+vzeiFRaKkfydVXXw0E/UjeeOONnPbX2I/kySefpHPn4O+DV155hWnTpjWNSe9HMmzYMIYOHcrGjRvZtGlTxn02yrUfSaN8+5E0NDSwZs0aXnzxRZYtW8bs2bPZunVra1MgIjGJMpFkaqTR/DdJtjEtPfeb7j6M4JTZNDO7MNMXd/d57j7C3UecfvrpucbcYaLoRzJt2jTWrFnD8OHDaWhoaLEfyfLly1m/fj3jx49vcz+Sxn23Va79SIqLi6moqKBbt2707NmTCy+8kHXr1rX564lIx4gykewAzk5bLgZ2ZRtjZp2BrwH7Wnquuzd+3g08T3DKq+CoH0n2fiSVlZW8/vrrNDQ08Omnn7Jq1SpV+RVJsCgTSQ3Qz8xKzKwLwcXz6mZjqoHJ4eOJwKse/KlbDUwys65mVgL0A94ys25m9icAZtYNuAjYEOFriIz6kWTvRzJw4EAqKipIpVKMHDmSqVOntnohX0TiE2k/EjO7GPgpwe2/8939ATP7EbDa3avNrAhYBAwlOBKZlHZx/l7gO0ADcKe7v2xmfQmOQiC4UeBpd3+gtTjUjyQa6kcicmJLRD8Sd38JeKnZuh+kPf4MuCLLcx8AHmi27n1gcPtHKsdD/UhEBFRGvuCoH4mIJI0SSYFRPxIRSRrV2hIRkbwokYiISF6USEREJC9KJCIikhclkgJxMvUjOXDgAJdeeimDBw+mrKyMBQsWtHscItJ+dNcW8Lsf/5jP32vffiRdBw7gzO9/v1332ZHi7EcyZ84cSktLeeGFF6ivr6d///5cc801dOnSJZKYRCQ/OiKJkfqRZO5HYmYcOnQId+fw4cP06NGjqaKxiCSQu5/wH8OHD/fmNm3adMy6jrRhwwY/77zzvL6+3t3d9+7d6+7ul1xyiS9cuNDd3R9//HGvrKx0d/dZs2b5Qw89lHV/vXr18s8++8zd3ffv3+/u7nfffbdPnz69acy+ffu+9LUaGhp8zJgxvm7dOnd3HzNmjNfU1Li7+znnnOP19fVeW1vrnTp18rVr17q7+xVXXOGLFi1yd/eysjJfuXKlu7vPmDHDy8rK3N19586dPm7cuGNirKmp8fLy8qblFStW+Pjx448Zd/DgQR87dqyfeeaZ3q1bN1+yZEnG1xz391DkREdQzqrV37E6IomJ+pEEMpXJX7ZsGUOGDGHXrl2888473H777Rw8eLC1KRCRmCiRxMTVjyRrP5IFCxYwYcIEzIxzzz2XkpISNm9u32tYItJ+lEhion4k2fuR9O7dm+XLlwPw8ccfs2XLFvr27XtccYpI9JRIYqJ+JNn7kdx33328+eabDBo0iPLycqqqqppOAYpI8kTajyQp1I8kGupHInJiS0Q/EjmxqR+JiIASScFRPxIRSZqTOpHkeudUkqgfSeBkOCUrUihO2ovtRUVF7N27V7+QCpC7s3fvXoqKiuIORUQ4iY9IGv+nob6+Pu5Q5DgUFRVRXFwcdxgiwkmcSE455RRKSkriDkNEpOBFemrLzCrMbIuZbTezezJs72pm/xpuX2VmfdK2zQzXbzGzb+e6TxER6ViRJRIz6wTMAcYBpcBVZlbabNiNwH53Pxd4GKgKn1sKTALKgApgrpl1ynGfIiLSgaI8IhkJbHf39939f4Fngeb1MCqBfwkfLwbKLbiNqhJ41t0/d/daYHu4v1z2KSIiHSjKayRnAR+lLe8A/izbGHdvMLMDwGnh+t80e+5Z4ePW9gmAmd0M3BwuHjazLcfxGnLVE9gT4f6jVMixg+KPm+KPV9Txn5PLoCgTSaZ/0Gh+r222MdnWZzqCynj/rrvPA+a1FGB7MbPVuZQRSKJCjh0Uf9wUf7ySEn+Up7Z2AGenLRcDu7KNMbPOwNeAfS08N5d9iohIB4oykdQA/cysxMy6EFw8r242phqYHD6eCLwaduWqBiaFd3WVAP2At3Lcp4iIdKDITm2F1zxuB5YBnYD57r7RzH5E0L6xGngcWGRm2wmORCaFz91oZv8GbAIagGnufhQg0z6jeg1t0CGn0CJSyLGD4o+b4o9XIuI/KcrIi4hIdE7aWlsiItI+lEhERCQvSiRtYGZnm9mvzew9M9toZtPD9T3M7Fdmti383D3uWFsSVglYa2ZLwuWSsETNtrBkTZe4Y8zGzE41s8Vmtjn8PowulPk3s78Pf242mNkzZlaU9Lk3s/lmttvMNqStyzjfFngkLF+03syGxRd51tgfCn921pvZ82Z2atq2jGWZ4pIp/rRt3zMzN7Oe4XKsc69E0jYNwHfdfSAwCpgWlmi5B1ju7v2A5eFykk0H3ktbrgIeDuPfT1C6Jql+Bix19wHAYILXkfj5N7OzgDuAEe5+PsHNIpNI/twvJChTlC7bfI8juMOyH8E/A/+8g2LMZiHHxv4r4Hx3TwFbgZmQvSxTx4Wa0UKOjR8zOxv4K+DDtNXxzr276+M4P4D/DL+hW4Be4bpewJa4Y2sh5mKCN/+3gCUE//y5B+gcbh8NLIs7ziyxfxWoJbxJJG194uefP1Rx6EFwt+QS4NuFMPdAH2BDa/MN/DNwVaZxSYm92ba/AZ4KH88EZqZtWwaMTtrch+sWE/wRVQf0TMLc64jkOIWViocCq4Cvu/tvAcLPZ8QXWat+CtwN/F+4fBrwibs3hMvp5WiSpi9QDywIT839wsy6UQDz7+47gZ8Q/BX5W+AAsIbCmft02eY7U1mkJL+e7wAvh48LInYzuwzY6e7rmm2KNX4lkuNgZn8MPAfc6e4H444nV2Z2CbDb3dekr84wNKn3hHcGhgE/d/ehwO9J4GmsTMLrCJVACfANoBvB6Yjmkjr3uSiYnyUzu5fgVPVTjasyDEtU7Gb2FeBe4AeZNmdY12HxK5G0kZmdQpBEnnL3X4arPzazXuH2XsDuuOJrxTeBy8ysjqBy8rcIjlBODUvUQLLLzuwAdrj7qnB5MUFiKYT5/0ug1t3r3f0L4JfABRTO3KfLNt8FUcLIzCYDlwDXeHgeiMKI/U8J/hBZF76Hi4G3zexMYo5fiaQNzMwI/hv/PXf/h7RN6aVeJhNcO0kcd5/p7sXu3ofgwuKr7n4N8GuCEjWQ7Ph/B3xkZv3DVeUE1Q8KYf4/BEaZ2VfCn6PG2Ati7pvJNt/VwPXhHUSjgAONp8CSwswqgBnAZe7+adqmbGWZEsPd33X3M9y9T/ge3gEMC98X8c593BeTCukD+HOCw8X1wDvhx8UE1xmWA9vCzz3ijjWH1zIWWBI+7kvwptkO/DvQNe74Woh7CLA6/B78B9C9UOYfuB/YDGwAFgFdkz73wDME13S+IPjFdWO2+SY4vTIH+B/gXYI71JIW+3aCawmN799/Sht/bxj7FmBcEue+2fY6/nCxPda5V4kUERHJi05tiYhIXpRIREQkL0okIiKSFyUSERHJixKJiIjkRYlEJIHM7Idm9r244xDJhRKJSMTCfxLTe01OWPrhFomAmfUJ+6XMBd4GHjez1WE/kvvTxtWZ2f1m9raZvWtmAzLs6yYze9nM/qgjX4NIrpRIRKLTH3jCgwKT33X3EUAKGGNmqbRxe9x9GEEPiS+dzjKz24FLgb929yMdFLdImyiRiETnA3f/Tfj4SjN7G1hL0DypNG1cY/HPNQT9JxpdR1Ah+HJ3/zziWEWOmxKJSHR+D0ErY4IjjXIPOvO9CBSljWtMEkcJSuU32kCQWIojj1QkD0okItH7KkFSOWBmXydzH5JM1gJ/B1Sb2TeiCk4kX0okIhHzoJvdWmAjMB9Y2YbnvkFwNPOimfWMJkKR/Kj6r4iI5EVHJCIikhclEhERyYsSiYiI5EWJRERE8qJEIiIieVEiERGRvCiRiIhIXv4fpMZzyi2PXncAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "for cs in scaling:\n", + " cs_scores = ssvd_scores.xs(cs, level='col_scaling')\n", + " ax = cs_scores.plot(label=f'col_scaling: {cs}')\n", + "ax.set_title(f'Recommendations quality for {ssvd.method} model')\n", + "ax.set_ylim(0, None)\n", + "ax.set_ylabel(target_metric.upper())\n", + "ax.legend();" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "ssvd_rank_scores = ssvd_scores.xs(ssvd_best_config['col_scaling'], level='col_scaling')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The optimal set of hyper-parameters:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'col_scaling': 0.6, 'rank': 130}" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ssvd_best_config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## iALS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using *implicit* library in Polara is almost as simple as using SVD-based models. Make sure you have it installed in your python environment (follow instructions at https://github.com/benfred/implicit )." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "import os; os.environ[\"MKL_NUM_THREADS\"] = \"1\" # as required by implicit\n", + "import numpy as np\n", + "\n", + "from polara.recommender.external.implicit.ialswrapper import ImplicitALS\n", + "from polara.evaluation.pipelines import random_grid, set_config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### defining hyper-parameter grid" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hyper-parameter space in that case is much broader. We will start by adjusting all hyper-parameters expect the rank value and then, once an optimal config is found, we will perform full grid-search over the range of rank values defined by `rank_grid`." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "als_params = dict(alpha = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100],\n", + " epsilon = [0.01, 0.03, 0.1, 0.3, 1],\n", + " weight_func = [None, np.sign, np.sqrt, np.log2, np.log10],\n", + " regularization = [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],\n", + " rank = [40] # enforce rank value for quick exploration of other parameters\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to avoid too long computation time, grid-search is performed over 60 random points, which is [enough to get within 5% of the optimum with 95% confidence](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf). The grid is generated with the built-in `random_grid` function." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "als_param_grid, als_param_names = random_grid(als_params, n=60)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### random grid search " + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "ials = ImplicitALS(data_model) # create model" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "60/60\n", + "[24:14<00:24, 24.23s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 60/60 [24:14<00:24, 24.23s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ials_best_config, ials_grid_scores = find_optimal_config(ials,\n", + " als_param_grid, # hyper-parameters grid\n", + " als_param_names, # hyper-parameters' names\n", + " target_metric,\n", + " init_config=init_config,\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### rank tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In contrast to the case of SVD-based algorithms, iALS requires recomputing the model for every new rank value, therefore in addition to the previous 60 times, the model will be computed `len(rank_grid)` more times for all rank values." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "15/15\n", + "[06:23<00:32, 25.56s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 15/15 [06:23<00:32, 25.56s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ials_best_rank, ials_rank_scores = find_optimal_config(ials,\n", + " rank_grid,\n", + " 'rank',\n", + " target_metric,\n", + " # configs are applied in the order they're provided\n", + " init_config=[init_config,\n", + " ials_best_config],\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's combine the best rank value with other optimal parameters:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'alpha': 1.0,\n", + " 'epsilon': 0.03,\n", + " 'weight_func': ,\n", + " 'regularization': 0.003,\n", + " 'rank': 60}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ials_best_config.update(ials_best_rank)\n", + "ials_best_config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### visualizing rank tuning results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now see how all three algorithms compare to each other." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_rank_scores(scores):\n", + " ax = None\n", + " for sc in scores:\n", + " ax = sc.sort_index().plot(label=sc.name, ax=ax)\n", + " ax.set_ylim(0, None)\n", + " ax.set_title('Recommendations quality')\n", + " ax.set_xlabel('# of latent factors')\n", + " ax.set_ylabel(target_metric.upper());\n", + " ax.legend()\n", + " return ax" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xl8lfWd9//X55yT5GQHsgEJS0D2RZYoioJaC9qWaq12IDjqtFrbDrTqtDOj992qxWld6m9sufEe67SOrdNKHTtOqf2hValKFZUg+yqbkABZyb6ecz73H9eVcBISEggnC/k8H4/zuLbvdZ3vdRHO+3yv6zrfS1QVY4wx5lx5ersCxhhj+jcLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbrEgMcYY0y0WJMb0ABF5W0TuOsd1R4pItYh4z3e9eoOIHBaRz7rj/0tEftHbdTLdY0Fizgv3w6HO/cA7ISLPi0hCb9erPwr/oAVQ1SOqmqCqwd6sVySo6o9V9S4AERktIioivt6ulzk7FiTmfPqiqiYAM4CZwAO9XB9jTA+wIDHnnaqeAF7HCRQARCRGRJ4UkSMiUigiz4hIbNjyG0Vki4hUisgBEbnenT9cRNaISJmI7BeRr4et87CI/JeI/KeIVInIdhEZLyIPiEiRiBwVkYVh5d8WkX8RkffdltMfRSRFRH7jvu9GERkdVn6iiLzhvvdeEfmbsGXPi8jTIvIn970/FJGxYcsXiMgeEakQkVWAhC0bKyLrRKRURErc9x/kLnsBGAn80a3jP7X9pt6FY/KSiPzarddOEckJW/7PIlLgLtsrIte292/oHpc17nH5SEQeEZG/ustOazmEn7o70/618z4Pi8h/upPvusNyd9+vcvdxWlj5dLflm9be9kzvsCAx552IZAGfA/aHzX4cGI8TLhcBmcCDbvlLgV8D/wgMAuYDh931XgTygeHALcCP23z4fRF4ARgMbMYJMI+7/RXAz9tUbwlwm7t8LLAB+A9gCLAbeMitUzzwBvBbIB3IBf6viEwJ21Yu8EP3vfcDP3LXTQV+D3wfSAUOAFeEHyLgUXefJgEjgIcBVPU24Ahu605Vn+B0nR2TG4DV7rFcA6xy6zUBWA5coqqJwHWcOs5tPQ3UA8OAr7mvrupw/zox3x0Ocvf9HXc//jasTC7wpqoWn0V9TKSpqr3s1e0XzgdSNVAFKPAWzgcCOB8sNcDYsPKXA4fc8Z8DT7WzzRFAEEgMm/co8Lw7/jDwRtiyL7p18LrTiW5dmuvxNvC/w8r/f8DaNutvcccXA+vb1OfnwEPu+PPAL8KWfR7Y447fDnwQtkxwPvjv6uDYfQnY3OZYfjZserS7H74uHpM3w5ZNBurc8YuAIuCzQNQZ/i29QBMwMWzej4G/tq1P2PK3z2X/3Pr+5xm2Owc4Cnjc6Tzgb3r7791erV/WIjHn05fU+aZ7NTAR59s4QBoQB2wSkXIRKQdec+eD8+F4oJ3tDQfKVLUqbN6nOK2JZoVh43VAiZ66KF3nDhPOUL7tdHPZUcCc5vq6db4VGBpW/kTYeG3YusNxPvwAUOcTsGXaPT2z2j3FVAn8J6eOVWe6ckza1ssvIj5V3Q/ci/PhXeTWYXg775GGE1pHw+Z92sX6dXf/WlHVD3G+hFwlIhNxwnDNuWzLRI4FiTnv1Dkl8TzwpDurBOdDeoqqDnJfyepcmAfnA2vs6VviGDBERBLD5o0ECiJT81aOAu+E1bf5dMu3urDucZxwBEBEJHwapwWhwHRVTcI5dSNhy8/UJXe3jomq/lZVr8QJSsU55dhWMRBoU+eRYeM17jAubF54wHa2fx1Wr4P5v3K3cRvwsqrWd2FbpgdZkJhI+SmwQERmqGoI+HfgKRFJBxCRTBG5zi37S+CrInKtiHjcZRNV9SjwPvCoiPhFZDpwJ/CbHqj/q8B4EblNRKLc1yUiMqkL6/4JmCIiX3YvSH+H1h+0iTin4MpFJBPn2lC4QmBMexvuzjERkQki8hkRicG5/lGHc5qs7XsEgf8GHhaROBGZDNwRtrwYJ7j+VkS8IvI1Wn8R6Gz/OlIMhDh9318AbsIJk193cVumB1mQmIhwP2x+DfzAnfXPOBekP3BPd7wJTHDLfgR8FXgKqADewfnGDM7F1dE438RfwblG8UYP1L8KWIhzcf4Yzumix4GYLqxbAnwFeAwoBcYB74UV+SEwC2df/4TzoR3uUeD77im177XzFud6TGLcOpW4+5MO/K8Oyi7HOVV3Aqd1+R9tln8dJyBKgSk44dass/1rl6rW4tyw8J6775e58/OBj3FaLOu7si3Ts8Q5fWuMMR0Tkb/DuZh+ZS+9/3PAMVX9fm+8vzkz+wWpMaZPE+e3PV/G+ZGr6YPs1JYxps8SkUeAHcBPVPVQb9fHtM9ObRljjOkWa5EYY4zplgFxjSQ1NVVHjx7d29Uwxph+ZdOmTSWq2mm/ZgMiSEaPHk1eXl5vV8MYY/oVEelSjwYDIkiMMaYnqSplNY2ICLFRXmJ8Hjyervy4v3+yIDHGmLMUCilFVQ0UlNeSf7Iu7FVLwck6CsrraAiEWq0T4/Pgj/ISG+UlNtoJl9hoL36fMx0b5SUmyuMsj/I6ZcPKtcxzy8VF+0iJjyYtMQZ/VO8+PNOCxBhj2giGlMLKevJP1jlhUeYGRbkTFMfK62kMtg6KlPhosgbHMmlYEp+dnMGwZD8eEeqagtQ1BqkPBKlvDFLfFHLmNQWpd1/FVYGWcg0BZ1jXFCTUxZtqk/w+0pP8pCXEkJ4UQ3piDGmJMaQn+klPdOalJfhJivXhdP12flmQGGMueKpKYzBEYyBEQ+DUsCUs3NZEc1gcL68n0OZTPC0xhsxBsUzNTOb6qcPIHBxL1uBYRgyOZfigWOKiz+/HqarSFFTqmoI0uMHjhE/IDZoAJVWNFFc3UFRZT1FVA8VVDWw+Uk5RVT31TaHTthnj85DWEjKngiYtsTmA/KQlxpASH43P2/WbeiMaJOI85e5nOM83+IWqPtZmeQxOf0yzcfrsWayqh90HHT3bXAx4WFVfcdc5jPPMiyAQUNUcjDH9XjCklFQ3cLyinhMVdRyvqKe6PkBjsPWHf0MgSGOgdSg4ZdqZHwjR4AbImYhAemIMWYPjmDVyMJnTY8kaHEfW4FgyB8eSOSi2x08fiQjRPiHa54HYqLNaV1WpaghQXNVAUWUDRVX1FLtB0xw4h0pq+PBQGeW1Taet7xEYEt9pt3ItIhYkIuLFecraApyH+mwUkTWquius2J3ASVW9SESW4HSKtxjnl6w5qhoQkWHAVhH5o6oG3PWucTvGM8b0A4FgiKKq5pCo53hFnTOsrOd4uTNeWNVAsJ1zOR6BaJ+HaK+HmCivM/R5iPadGvqjPCTHRhHtbT0/uqWcc63B2YanpVx6op+swbEMG+Qnxte71xnOJxEhyR9Fkj+KsWkJZyzbEAhSUt1IUWV9S9A4YVPPpi6+XyRbJJcC+1X1IICIrAZuBMKD5EZOPYLzZWCViIjbC2gzP2d+PoMxphc1uqeITlTWt2pNOIHhhEZxVcNp5/v9UR6GJ8cyNNnP5WNTGZbsZ2iyv2U4NMlPcmzUWZ1iMWcvxuclc5DT6mrrsXbKtyeSQZJJ6yes5eM8NrPdMm7rowJIAUpEZA7wHE534reFtUYU+LOIKPBzVX0WY0xE1TcFOVxaw8HiGg6V1HCguJqDxTUUlNdRUt1A256W4qO9DBsUy7BkP+PS09xwcL75D0v2MywpNmIXfk3Pi2SQtPcX0rZl0WEZ9xGbU9wHCf1KRNa6T0a7QlWPuQ9IekNE9qjqu6e9ucjdwN0AI0eObLvYGNNGKKScqKznYHENB0uq3WENB4urKSivaxUWQ5P8jEmL5zMT0lvCYWiyExzDkv0k+s/unL7p3yIZJPm0flRnFs6DeNork+8+SS4ZKAsvoKq7RaQGmArkqeoxd36RiLyCcwrttCBxWyrPAuTk5NipMWNcVfVNHCpxWhcHi6s54I4fLqmhrunUAxPjo71kp8Uza+RgbpmdxZi0BMakxpOdGk98jN3waU6J5F/DRmCciGTjPJZzCbC0TZk1OI/w3ADcAqxTVXXXOeqe7hqF8yS9wyISD3hUtcodXwisiOA+GNMvVdU3caKiniNlte6pKCc0DpbUUFzV0FLOIzBiSBzZqfFcPiaFMWnxjEmLZ2xaAumJMXbqyXRJxILEDYHlwOs4t/8+p6o7RWQFTstiDc6zul8Qkf04LZEl7upXAveLSBPOM5z/XlVLRGQM8Ir7x+0Dfquqr0VqH4zpa5pvkT1R4VzcLnQvcBe60ycqnfGaxtaPYh8cF8WYtASuHp9Gdlo8Y1ITGJsWz8iUuAvqbiXTOwbE80hycnLUOm00fV1dY9AJg4p6TlTWcaKiwbkbKiw0itq5RdbnETKS/GQkxTA02U9GknPH09Bk59bWMakJDI6P7qW9Mv2ZiGzqym/17ESnMT2otLqBfYXVfFJUxSeF1XxaVssJ9zcVlfWB08onxvjIcG+FHeveIts8PTTJT0ZyDKnxMRd0h4Cm77MgMSYCTtY0sq+win1F1XxSWMW+Qic4SmsaW8okxvgYnRrPqJR4LhuT0qolkeEOE+yitukH7K/UmG4or21kX2G1GxRVbmujmpLqUxe0E2J8XJSewLWT0hmfkci4jETGZyQwNMlvF7PNBcGCxJguqKht4pOiqlOh4Y6H3wEVH+3looxErpmQ5gZGAuMzEhmWbIFhLmwWJMa0oarsOVHFG7sK+ehQGfsKqygKC4y4aC/j0hOYPy6N8W5YjMtIIHNQrAWGGZAsSIzB6VQw79OT/HlnIW/sPsHRsjpEYNLQJK4cl8p493TUuPREMgfF2sVtY8JYkJgBq7YxwLv7SvjzrhOs21NEeW0T0T4PV4xN4e+vvohrJ6WTnujv7Woa0+dZkJgBpaS6gbd2F/LnnYX8dX8JDYEQybFRfGZiOgsnZzB/fJp1/2HMWbL/MeaCd7C4mjd2FfLGrkI2HTmJKmQOiiX30pEsnJLBJaOHEGVdlRtzzixIzAUnFFK25Je3hMf+omoApgxP4p5rx7FgcgaThyXZhXFjzhMLEnNBaAgEef9AKX/eWcibuwsprmrA6xEuGzOEv50zks9OziBrcFxvV9OYC5IFiemXAsEQ+Sfr2HK0nD/vOsE7e4upaQwSH+3l6gnpLJicwTUT0kmOs+diGBNpFiSmz1JViqsa3Icr1XCopNp5jkZJDUdKawm4nRemJcZw48xMFkzOYO7YFOvN1pgeZkFiel1lfROHS2panptxqMQNjeKaVt2hR/s8ZKfEMz49keunDCU7NZ4JQxOZOjzZftdhTC+yIDE9oiEQ5GhZbcszv1ue0FdS06pfKhHIGhxLdmoCOaOGMCbNeSJfdmo8w5Pth4DG9EUWJOa8C4WUXccreWdfMRsPl3GopIajZbWEP0YjNSGa7NR4PjMxjTFpCWSnxjMmNZ4RQ+LwR9mpKWP6EwsSc14UVzXwzr4TvL53JxuP76CWo3j9x/H7K4nPSGbayFSGJWQwetAwxqdmMmrQENJj00mNSyXWF9vb1TfGdIMFiTkn5XXV/HHPJv5ycAu7SvdQGfoUT8wJxNMEqRCLl9HJYxiVNJGKhgqKag+Td3Ij75U0wP7W20qMSiQtLo20uDTSY9NIixlMelQiab540r1+UokiTbzENDVAYxU0VENjNTTWgHgg6xIYeTkkpJ31fqgqAQ3QEGigPlhPY7Dx1DBQT3xUPGMHjcUj9oNFYzpiQWLOSFUprC1kT9kePszfwUfHdvBp1X7qKULEOVfliY5jhH8MF6dfzuUjpjEpZSJj4jOJLt4LRbugoQpiq9GGSirryyluOElRYyXFgWqKA7UUNdZQXF1Bkexjk0CRz0ugnR8LJgeDpAWDpAfcoXpICgZo3Pk89R6hMXYw9YkZNMSn0hA7iAZvFA3BhpZXeEA0B0ZDsIGQhs54DFL8KVw+/HLmDp/L5cMvJzU2NSLH2pj+yp7Zblo0BZs4WHGQvSf3srdsLztLdrO7bC+1gcqWMqHGFGJCWYxNHs8VI6ezaOJsxg7KRMoPQ36e8yrIgxPbIdjY+g18fohOgJgEiE50hwlhw0SITiAUHU+F10eRRynWJopDjRQF6ygO1FDUVElxQzlF9aWU1pUSVOeuLgH8CDGhINGhEH5VYsRLTFQCMf5k/HGpRMem4PfFEu2Nxu/zE+ONaXn5fX5nvjdsvi+GkroSNhzbwAfHP6CsvgyA8YPHt4TKrPRZ+H3WsaO5MHX1me0WJAOUqrK/fD8fHv+Q3WW72XdyH/vL9xMIOc8NF40iWJ9BoH4YvkAmU9Mm8dmxM1gwcRSj4xuhYBPkb3JCo2AT1JY6G46Kg+EzIXM2ZOXA0OkQO9gJCu/5bQAHQ0HqAnXEeGPweXxOlyehEJTsgyMbTr3Kj7h1i4cRl8DIuTDyMqd+0fFdeq+Qhthbtpf3j73PhmMb+LjoY5pCTcR4Y5idMbslWMYNGmddr5gLRp8IEhG5HvgZ4AV+oaqPtVkeA/wamA2UAotV9bCIXAo821wMeFhVX+nKNttjQeKobarloxMfsT5/PesL1nO85jgA8b7BxASzKC9Po6Y6nVD9cCamZnPV+KFcNXYQs/3HiDq+yQ2PPCj9xN2iQNoEyMxxPpSzciBt0nkPjG6rKICjH8CnG+DIB1C4A1AQLwy7GEa5wTLisi5fZ6ltqmVT4aaWYDlQcQCAtNi0ltNglw27jJTYlAjumDGR1etBIiJeYB+wAMgHNgK5qrorrMzfA9NV9ZsisgS4SVUXi0gc0KiqAREZBmwFhgPa2TbbM5CD5GjVUd7Nf5f1BevZeHwjjaFGYn2xTBmcw8mSsWz7ZCihpkGkJkQz76JUrhvRxNyYwySVbnVC4/gWCNQ7G4tPcy5sN7c2hs8Ef3Kv7t85qa+Aox+5LZYPnP0Mur9lSbnIuXA/8nInXIaMcX7c0okTNSfYcGyDEyzHN1DRUAHApCGTWoJlZvpMor3RkdwzY86rvhAkl+O0JK5zpx8AUNVHw8q87pbZICI+4ASQpmGVEpFs4AMgE7iks222ZyAFSVOwiU1Fm1ifv55389/lcOVhAEYljWJe5jzSvDN4a0s8731SwRA//NPkk1wd/ykZlTuQgk1QXehsyOd3vq1n5kDWbGc4aGSXPlT7nUADHNtyKliObID6cmdZXIrTykqbAGkTTw0T0js8FsFQkD1le3j/2Pu8f+x9thRtIaAB/F4/OUNzmDt8LnOHz2VM8hg7DWb6tK4GSSTPQWQCR8Om84E5HZVxWx8VQApQIiJzgOeAUcBt7vKubBMAEbkbuBtg5MiR3d+bPqy4tpj1BetZn7+eDcc3UNNUQ5QnikuGXsLiCYu5MvNKDhyL5em/7GfnkSK+EPcxr4/azvjy9cgu90J6ykUw5hqnpZE5GzKmgm+AfHv2xcDIOc4L3Osse51AKfjYueay/WVwWxmAc90nPFjSJjiBkzgUr8fLlNQpTEmdwtenf52aphryTuS1BMsTG58AID0unYvTLmZo/FAy4jLIiMsgPS6djPgM0mPTifL2jQ4nQxriZP1JimqLKKotorC2sGU82hvNiMQRjEwcyYjEEWQlZtnNBwNQJIOkva9abZs/HZZR1Q+BKSIyCfiViKzt4jZx138W9zpLTk7OBXVHQTAUZHvJ9pbw2F22G4CMuAw+l/055mfOZ86wOcR4Y/nT9uPc89wOhhW/x7f8eVwd/zFRwVqoGASTvggTFzmncOKG9PJe9SEeD6RPcl45X3PmqULVCSjeA8V7Tw13/QHqnj+1bkyyGypuwKRPJD5tIldlzeeqEVcBcKz6GBuObeC9Y++x7+Q+/lrwV+oCdadVY4h/SEvAZMS7IRMWNhlxGcRHde1mgY7UB+opri1uFQ7h40W1RRTVFbXchNFyiMRDij+F+mA9VY1VrZalx6W3CpcRSSOcYeIIkqKTulXfrgppiLL6MgprCjlRe6L1sOZEyz76PD6SopNIikkiKTqJ5OjkU+Mxyc6y8PEYp0xidCJej/XA0CySQZIPjAibzgKOdVAm3z21lQyUhRdQ1d0iUgNM7eI2L0gVDRW8V/Ae7xa8y3sF71HeUI5HPMxIm8E9s+5hXuY8xg8ej4jQEAiy5qN97H7nJXJq/8rvvFvwRzei/hRk4ldg8o2QPR/6yDfefkEEkoY5r7HXnJqvCjXFrQOmaA/sXQubXzhVLjqhJVyGp03g5rRJ3HzxPZA4DA0FqGqooLDmBEW1hRTWFFJYV0RhbTFF9cUcqzzMlsJNlDdVnVatBG8s6TGDyIgZRHp0MhnRyWREJZIRnUx6VALiiaJIQhSqcwt1UX0ZhXWnQqIivJXlivXFtgTY7IzZpMeltwqx9Lh0UmJT8Hmcj4+KhgqOVh3lSOURZ1h1hPyqfNYXrKekrqTVtgfFDGJk4kiyErOcsEka2RIyKf6ULp3qawmJWjcUOgiJplBTq/WiPFFkxGUwNH4oM9JnkB6XTigUoqKxgsqGSioaKzhafZTK0koqGyvbDfdwiVGJLaHTMgwLneSYZMYkj2HikInERV3Yz8KJ5DUSH86F8WuBApwL40tVdWdYmWXAtLCL7V9W1b9xr4scdU9njQI2ANOB8s622Z7+eI2ktqmWnaU72VK0hfUF69lavJWQhhgcM5grMq9gftZ85g6fS3LMqYvdtZWlfPTaf+LZ/UfmhLYQI03U+9OImXojMuVG57bXvnZH1YWspiSs9bLnVNg0X4c6S/UiFHu9nPB5KfJ6KfR5KfL63KEzv8TrJdTBh7GokhIKkR7ykC5RZPhiSY9KJD16MOlxaWTEDyU9aQQJCcOQuBTn+lDcECcEz/FaTm1TLUerjpJflc+RqtZBc7zmeKsfg8b6Yk87TRYIBc4qJDLiM1pOFbYdDvEPOatrUk3BppaQqWx0XhUNFc64GzzNy5rnNw/D6ycIYxKymDxkApNTpjA57WImpk7pF+HS6xfb3Up8Hvgpzq26z6nqj0RkBZCnqmtExA+8AMzEaYksUdWDInIbcD/QBISAFar6Px1ts7N69PUgCYQC7C/fz/aS7Wwv3s72ku0crDjY8p9s0pBJzM+az7yseUxNmdq6SV1TSu32NRR9+BKZJz8kiiClnlTqxy1i+NzFyIjLnFM1pu+oLXOuuxTthtoS5zZkjxc8vlPj4nGH3jZDj1PutHnOdAAoDdRQ1FhJYWM5GmwkXb1kBIOkNDUQVVfuvH9dmfPbn9pSd/okHZwlBm+0EyqxQ5xgiRvihkwKxKVCfKpz80F8ujP0D+rS31xTsImC6oJW4dIcNvlV+S0fxmcKiYz4DIbGDWWwf3DvdWMTbHJ6cCj4GAo2occ+pr54Lyc9yr6oaHbFuK/oKIp9zhc5UWVMIMTkoDBZfUz2xDLRm0BcVILzW6zoOIiKdX77FBXrToe9ouOc30ANGuW8IvQFsU8ESV/Rl4JEVTlec5xtJdvYUbyD7SXb2VW6i/qgc4ttckwy01KnMS11GlNTpzItdRqD/YNbb6S6CHb/kcbt/4PvyHt4CHIklMauwdcwal4uk2ZdbeFhzk4o6NwWHR4utaVtAudk63l1J6G97mU8PidgEtKccIlPOzWekO4ET/N4XGq7H4LBUJDiumKiPFG9GxJthUJQdhCOOaFBwcdwYtupW+RjBzs3qwyfBcNngDcGmmpbXkW1xeyqKWBX3Ql2NZSys6mcEnV6gPAA2epzwqUpyOT6BibW1xLXWANtrlG14omClLHODTOp4yF1nDNMuQhiB3Vrdy1IwvRmkFQ2VrKjZAfbi7c7w5LtlNY7vwKP9kQzMWUi01OnMzV1KtNTp5OVmNV+87vyGOz+I+z6A/rp+wjKIR3K/x+cQ83YL/DF665n0vB++JsO03+FQk6Y1BQ5X25qip1XdZE7r7j1vGBD+9uJHeIGTNqpYct487Tb6onq4Z6iK4+1tDSc8Nh86u69qDgYNgMyZzmv4bNg8OizPg1YVFvErtJd7Crdxc7Snewq3dVybckjHrKTspk8ZCKTk8cyJSmbCXHDiNMQ1FfCyUNQ8onzKv3ECbnw0IlPd4NlHKS4AZN6kdOK6cLNAhYkYXoqSJqCTew9uZdtxdtaQqP5dxwA2cnZLa2NaanTGD94/Om3eDbVOXcHVRc559LLDsKeP0H+RwAcj8nmpdpZvBGaw7RZl/ONq8YyOrV7d+4YE3Gq0FDpXDdqCZoiZ/q0ICp2enluT3TCqRZNeMA0j8eHBVHs4LNrmdeddEKjOTAKNkH1CWeZxwcZU5ywyJzltDpSJ0TslFJzuDQHS3vhMmHIBPw+P8FQEEUJapBQMECosZpQfQWhxiqCDdXOdFMNoWATQREUCIoHjfIT9PkJ+fyEfDGEfNEEPT5CIoQ0REhDvPrlVy1ImkUqSEIa4q0jb5F3Io8dJTvYXba75bxuij+FaWnTnNZGyhSmxg8nsaEmLCTCwqKq0BlWFzr/2dqoGzKZ17mM/3N8Esd8I7l1zkjumjeGocl2v765QDXVhQVNc8smbDp8WW1J+6fYxBsWLqlhrRx3PC4Fyg6dOk1VdvDUuikXnTpFlTkbhk7t+dZQG0W1Rews2cmuMidYPjn5CUEN4hEPXvEiCF6PF4948ODB4wmbL148GsQTaMQTaMATqMPbVI+nqRZPUx0eVTyARxWvLwaJTsAbk8iTS9dZkDSLRJDUNNXw/b9+nzePvEmsJ5rJ8cOZFjWEaRLLtECQoTUVSHXhqW9f7Z3jjE5w/rAThjrDxKE0+lMp1MEcbUxgf108fz0RxZ8/DZIcG8Udc0fz1bmjGRw/QH4oaExXhILuKbbisJaN29JpHg+f31TTev3E4adOT2XOdk5XdfPaQr8SaISTh51TYyX7oGS/O9yHPHDEgqTZ+Q6So5VH+c5fvs3B8oP8Q9lJbq2oPPWDHPGc+uaTkNEqJJpDQxPSORFM5kAFHCiuPvUqquFEZX3L+3gERqfGs+SSESydM4qEGLt115hua6xxQ6UUkoY7vw0yp1NFPJ5e7yLlgvR+wft8753v4Wmq5ZnjJ7h8zOdg0d+ealnEpbScN60L/yAVAAAgAElEQVRvCnKopKYlJA4ecgLjYPEn1DYGW7aZGONjTHoCcy9KYWxaAmPT4hmblsDIlDhifPbrWWPOq+h45zV4dG/XpG87i5sGLEi6SFX51c5f8dSmpxgbVH52/DgjrnkQvWwZJTVNTlgcqeZA0T4nLEqqyT9ZR3iDL3NQLGPTE7hk9BA3MJzQSEuMsc77jDH9lgVJF9QF6njo/YdYe2gtC2sbeKQqSNzS/+bVqrGseHQdRVWnbmuMjfIyJi2eGSMGc/OsrJbAyE6NJzbaWhfGmAuPBUknjlUf495197Dn5B7uKSvnzoRxVN7xHN9Zd5I1Wzdz8YhBfOvqsU5gpCcwLMmPx2OtC2PMwGFBcgYbT2zku3+5j6bGSladKGL+lKWsH/tdvvfLPZRWN/LdBeP51tVj8Xn7yK9ujTGmF1iQtENVeXHPizyx8XFGNgX5WVEpwxf8hB/kz+KFX21lXHoCv7zjEqZm2i/JjTHGgqSNhmAD//LBv/A/+/+Hq2vrebTez9HrXuL6dUEOl37KXVdm873rJuCPsusdxhgDFiStFNYU8g9/uZdtpTv45skKvjF4Fs+MeIB//a8ShiXH8tu7LuPysSm9XU1jjOlTLEhcW4q2cN+671Bbf5KfFhUzY+LXuenQtWzfXcIts7N46IuTSfTbg6CMMaYtCxLg9/t+z7988AjDmpp4tqyaQ2N/xOUfDScxpomf3zab66YM7e0qGmNMnzWgg6Qp2MTjHz3G7/a9xBV19fyocRCPxD3M/2yMY8HkNB798jRSE2J6u5rGGNOnDdggKakr4bt/uZePi7fy1fJKboqdw/Wlt1KHn5/cMplbZnfwXBBjjDGtDMgg2Vmyk3veWkZFXSmPl5RRFX0b1xy8kjnZKTz5lYsZMaTvP0vZGGP6igEXJH888Ed++N6DDGlq4BdljTxd9z3eLp/I978wga9dkW2/SjfGmLM0YIIkEArwrxuf5IU9v+GSunruqUzhG2V/T8rwbF5dPIPxGYm9XUVjjOmXBkSQBDXIN1+7kw+LP+bWiirGVlzK4tqlfP0zk/j2Z8YR7bMuTowx5lxFNEhE5HrgZ4AX+IWqPtZmeQzwa2A2UAosVtXDIrIAeAyIBhqBf1TVde46bwPDgDp3MwtVtehM9ThYfgBPURMPl1byfmkurw/5HL+942JmjRx83vbVGGMGqogFiYh4gaeBBUA+sFFE1qjqrrBidwInVfUiEVkCPA4sBkqAL6rqMRGZCrwOZIatd6uqdvmRhxps4snjjTxZ+c/Mvuwq/vS5icRFD4jGmDHGRFwkP00vBfar6kEAEVkN3AiEB8mNwMPu+MvAKhERVd0cVmYn4BeRGFVt4BxkNHp5NPQE3//qFcwfn3YumzDGGNOBSAZJJnA0bDofmNNRGVUNiEgFkILTIml2M7C5TYj8h4gEgd8D/6LtPHheRO4G7gZIHj6GvPu+QHKcdXFijDHnWySvMrd3H23bD/wzlhGRKTinu74RtvxWVZ0GzHNft7X35qr6rKrmqGrORcMGW4gYY0yERDJI8oERYdNZwLGOyoiID0gGytzpLOAV4HZVPdC8gqoWuMMq4Lc4p9CMMcb0kkgGyUZgnIhki0g0sARY06bMGuAOd/wWYJ2qqogMAv4EPKCq7zUXFhGfiKS641HAImBHBPfBGGNMJyIWJKoaAJbj3HG1G3hJVXeKyAoRucEt9ksgRUT2A/8A3O/OXw5cBPxARLa4r3QgBnhdRLYBW4AC4N8jtQ/GGGM6J+1cp77g5OTkaF5el+8WNsYYA4jIJlXN6ayc/aTbGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbrEgMcYY0y0WJMYYY7rFgsQYY0y3WJAYY4zpFgsSY4wx3WJBYowxplssSIwxxnSLBYkxxphusSAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdYkFijDGmWyxIjDHGdEtEg0RErheRvSKyX0Tub2d5jIj8zl3+oYiMducvEJFNIrLdHX4mbJ3Z7vz9IrJSRCSS+2CMMebMIhYkIuIFngY+B0wGckVkcptidwInVfUi4CngcXd+CfBFVZ0G3AG8ELbOvwF3A+Pc1/WR2gdjjDGdi2SL5FJgv6oeVNVGYDVwY5syNwK/csdfBq4VEVHVzap6zJ2/E/C7rZdhQJKqblBVBX4NfCmC+2CMMaYTkQySTOBo2HS+O6/dMqoaACqAlDZlbgY2q2qDWz6/k20CICJ3i0ieiOQVFxef804YY4w5s0gGSXvXLvRsyojIFJzTXd84i206M1WfVdUcVc1JS0vrQnWNMcaci06DRES8IpIaNh3tftvf3cmq+cCIsOks4FhHZUTEByQDZe50FvAKcLuqHggrn9XJNo0xxvSgMwaJiCzB+WDfJiLviMg1wEGcC+i3drLtjcA4EckWkWhgCbCmTZk1OBfTAW4B1qmqisgg4E/AA6r6XnNhVT0OVInIZe7dWrcDf+jKjhpjjIkMXyfLvw/MVtX9IjIL2AAsUdVXOtuwqgZEZDnwOuAFnlPVnSKyAshT1TXAL4EXRGQ/TmAtcVdfDlwE/EBEfuDOW6iqRcC3gOeBWGCt+zLGGNNLxLn5qYOFIh+r6qyw6T2qOrFHanYe5eTkaF5eXm9Xwxhj+hUR2aSqOZ2V66xFki4i/xA2nRA+rar/eq4VNMYYc2HoLEj+HUg8w7QxxpgB7oxBoqo/7GiZiMSf/+oYY4zpb7py+2+miOS4d14hIuki8mPgk4jXzhhjTJ/X2e2/9wJbgP8DfCAidwC7ce6Ymh356hljjOnrOrtGcjcwQVXLRGQksB+Yr6ofRL5qxhhj+oPOTm3Vq2oZgKoeAfZZiBhjjAnXWYskS0RWhk2nh0+r6nciUy1jjDH9RWdB8o9tpjdFqiLGGGP6p85u//3VmZYbY4wxZwwSEWnbyWIrqnrD+a2OMcaY/qazU1uX4zx46kXgQ9p/HogxxpgBrLMgGQosAHKBpThdu7+oqjsjXTFjjDH9wxlv/1XVoKq+pqp3AJfh/I7kbRH5do/UzhhjTJ/XWYsEEYkBvoDTKhkNrAT+O7LVMsYY0190drH9V8BUnIdH/VBVd/RIrYwxxvQbnbVIbgNqgPHAd5yn2wLORXdV1aQI1s0YY0w/0NnvSDrtHdgYY8zAZkFhjDGmWyxIjDHGdEtEg0RErheRvSKyX0Tub2d5jIj8zl3+oYiMdueniMhfRKRaRFa1Wedtd5tb3Fd6JPfBGGPMmXV6+++5EhEv8DTODxrzgY0iskZVd4UVuxM4qaoXicgS4HFgMVAP/ADnjrGp7Wz+VlXNi1TdjTHGdF0kWySXAvtV9aCqNgKrgRvblLkRaO4Y8mXgWhERVa1R1b/iBIoxxpg+LJJBkonTT1ezfHdeu2VUNQBUACld2PZ/uKe1fiBh9yQbY4zpeZEMkvY+4PUcyrR1q6pOA+a5r9vafXORu0UkT0TyiouLO62sMcaYcxPJIMkHRoRNZwHHOiojIj4gGSg700ZVtcAdVgG/xTmF1l65Z1U1R1Vz0tLSzmkHjDHGdC6SQbIRGCci2SISDSwB2j7fZA1whzt+C7BOVTtskYiIT0RS3fEoYBFg3bYYY0wvithdW6oaEJHlwOuAF3hOVXeKyAogT1XXAL8EXhCR/TgtkSXN64vIYSAJiBaRLwELgU+B190Q8QJvAv8eqX0wxhjTOTlDA+CCkZOTo3l5drewMcacDRHZpKo5nZWzX7YbY4zpFgsSY4wx3WJBYowxplssSIwxxnSLBYkxxphusSAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdYkFijDGmWyxIjDHGdIsFiTHGmG6xIDHGGNMtFiTGGGO6xYLEGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbolokIjI9SKyV0T2i8j97SyPEZHfucs/FJHR7vwUEfmLiFSLyKo268wWke3uOitFRCK5D8YYY84sYkEiIl7gaeBzwGQgV0Qmtyl2J3BSVS8CngIed+fXAz8AvtfOpv8NuBsY576uP/+1N8YY01WRbJFcCuxX1YOq2gisBm5sU+ZG4Ffu+MvAtSIiqlqjqn/FCZQWIjIMSFLVDaqqwK+BL0VwH4wxxnQikkGSCRwNm85357VbRlUDQAWQ0sk28zvZJgAicreI5IlIXnFx8VlW3RhjTFdFMkjau3ah51DmnMqr6rOqmqOqOWlpaWfYpDHGmO6IZJDkAyPCprOAYx2VEREfkAyUdbLNrE62aYwxpgdFMkg2AuNEJFtEooElwJo2ZdYAd7jjtwDr3Gsf7VLV40CViFzm3q11O/CH8191Y4wxXeWL1IZVNSAiy4HXAS/wnKruFJEVQJ6qrgF+CbwgIvtxWiJLmtcXkcNAEhAtIl8CFqrqLuBbwPNALLDWfRljjOklcoYGwAUjJydH8/LyersaxhjTr4jIJlXN6ayc/bLdGGNMt0Ts1FZf19TURH5+PvX19Z0XNmfF7/eTlZVFVFRUb1fFGNMDBmyQ5Ofnk5iYyOjRo7FeVs4fVaW0tJT8/Hyys7N7uzrGmB4wYE9t1dfXk5KSYiFynokIKSkp1tIzZgAZsEECWIhEiB1XYwaWAR0kxhhjus+CpBd5vV5mzJjB1KlT+cpXvkJtbe152W5hYSGLFi3i4osvZvLkyXz+858HIDs7m71797Yqe++99/LEE0/w9ttvk5yczMyZM5kwYQLz58/n1VdfPS/1McZc2CxIelFsbCxbtmxhx44dREdH88wzz3R53UAg0OGyBx98kAULFrB161Z27drFY489BsCSJUtYvXp1S7lQKMTLL7/M4sWLAZg3bx6bN29m7969rFy5kuXLl/PWW2+d494ZYwaKAXvXVrgf/nEnu45VntdtTh6exENfnNLl8vPmzWPbtm0cPnyYRYsWsWPHDgCefPJJqqurefjhh7n66quZO3cu7733HjfccAO333473/zmNzly5AgAP/3pT7niiis4fvw4CxcubNn29OnTAcjNzWXx4sU89NBDALz77ruMHj2aUaNGcejQoVb1mTFjBg8++CCrVq3i2muvbbUsGAxy5513kpeXh4jwta99jfvuu+/sD5Ix5oJgQdIHBAIB1q5dy/XXd/6MrvLyct555x0Ali5dyn333ceVV17JkSNHuO6669i9ezfLli1j8eLFrFq1is9+9rN89atfZfjw4UyfPh2Px8PWrVu5+OKLWb16Nbm5uR2+16xZs/jJT35y2vwtW7ZQUFDQEnbl5eXnuOfGmAuBBQmcVcvhfKqrq2PGjBmA0yK58847OXbszJ0ZN5+GAnjzzTfZtWtXy3RlZSVVVVVcd911HDx4kNdee421a9cyc+ZMduzYQVpaGrm5uaxevZopU6bwhz/8gRUrVnT4Xh11nzNmzBgOHjzIt7/9bb7whS+0av0YYwYeC5Je1HyNJJzP5yMUCrVMt/09Rnx8fMt4KBRiw4YNxMbGnrbtIUOGsHTpUpYuXcqiRYt49913ufnmm8nNzWXhwoVcddVVTJ8+nfT09A7rt3nzZiZNmkQwGGT27NkA3HDDDaxYsYKtW7fy+uuv8/TTT/PSSy/x3HPPndMxMMb0f3axvY/JyMigqKiI0tJSGhoaznjn1MKFC1m1alXLdHMorVu3ruUOsKqqKg4cOMDIkSMBGDt2LCkpKdx///1nPK21bds2HnnkEZYtW4bX62XLli1s2bKFFStWUFJSQigU4uabb+aRRx7h448/Ph+7bozpp6xF0sdERUXx4IMPMmfOHLKzs5k4cWKHZVeuXMmyZcuYPn06gUCA+fPn88wzz7Bp0yaWL1/e0rq56667uOSSS1rWy83N5YEHHuCmm25qtb3169czc+ZMamtrSU9PZ+XKladdaAcoKCjgq1/9akvL6dFHHz1Pe2+M6Y8GbDfyu3fvZtKkSb1UowufHV9j+j/rRt4YY0yPsCAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdEtEgEZHrRWSviOwXkfvbWR4jIr9zl38oIqPDlj3gzt8rIteFzT8sIttFZIuI5LXdZn8yd+7cVtNPPfUUfr+fioqKlnlvv/02ixYtOm3dV199lZkzZ7Z0Ff/zn/884vU1xpj2ROwHiSLiBZ4GFgD5wEYRWaOqu8KK3QmcVNWLRGQJ8DiwWEQmA0uAKcBw4E0RGa+qQXe9a1S1JFJ17ynvv/9+q+kXX3yRSy65hFdeeYW/+7u/63C9pqYm7r77bj766COysrJoaGjg8OHDka2sMcZ0IJK/bL8U2K+qBwFEZDVwIxAeJDcCD7vjLwOrxHlO643AalVtAA6JyH53exsiUtO198OJ7ed3m0OnweceO2ORhIQEqqurAThw4ADV1dX85Cc/4cc//vEZg6SqqopAIEBKSgoAMTExTJgw4bxV3RhjzkYkT21lAkfDpvPdee2WUdUAUAGkdLKuAn8WkU0icndHby4id4tInojkFRcXd2tHesKLL75Ibm4u8+bNY+/evRQVFXVYdsiQIdxwww2MGjWK3NxcfvOb37Tq6NEYY3pSJFsk0s68tv2xdFTmTOteoarHRCQdeENE9qjqu6cVVn0WeBacLlLOWNNOWg49YfXq1bzyyit4PB6+/OUv81//9V8sW7asw/K/+MUv2L59O2+++SZPPvkkb7zxBs8//3zPVdgYY1yRDJJ8YETYdBbQ9mEbzWXyRcQHJANlZ1pXVZuHRSLyCs4pr9OCpD/Ztm0bn3zyCQsWLACgsbGRMWPGnDFIAKZNm8a0adO47bbbyM7OtiAxxvSKSJ7a2giME5FsEYnGuXi+pk2ZNcAd7vgtwDp1epFcAyxx7+rKBsYBH4lIvIgkAohIPLAQ2BHBfegRL774Ig8//DCHDx/m8OHDHDt2jIKCAj799NN2y1dXV/P222+3TG/ZsoVRo0b1UG2NMaa1iLVIVDUgIsuB1wEv8Jyq7hSRFUCeqq4Bfgm84F5ML8MJG9xyL+FcmA8Ay1Q1KCIZwCvO9Xh8wG9V9bVI7UNPWb16NWvXrm0176abbmL16tXMmTOHt956i6ysrJZlL774Ik888QTf+MY3iI2NJT4+3lojxpheY93Im4iw42tM/2fdyBtjjOkRFiTGGGO6xYLEGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEh6kdfrZcaMGUydOpWvfOUr1NbWnpftFhYWsmjRopYu5j//+c8DkJ2dzd69e1uVvffee3niiSd4++23SU5OZubMmUyYMIH58+fz6quvnpf6GGMubBYkvSg2NpYtW7awY8cOoqOjeeaZZ7q8biAQ6HDZgw8+yIIFC9i6dSu7du3iscecvsSWLFnC6tWrW8qFQiFefvllFi9eDMC8efPYvHkze/fuZeXKlSxfvpy33nrrHPfOGDNQRLKvrX7j8Y8eZ0/ZnvO6zYlDJvLPl/5zl8vPmzePbdu2cfjwYRYtWsSOHU7PL08++STV1dU8/PDDXH311cydO5f33nuPG264gdtvv51vfvObHDlyBICf/vSnXHHFFRw/fpyFCxe2bHv69OkA5ObmsnjxYh566CEA3n33XUaPHs2oUaM4dOhQq/rMmDGDBx98kFWrVnHttdd261gYYy5s1iLpAwKBAGvXrmXatGmdli0vL+edd97hu9/9Lvfccw/33XcfGzdu5Pe//z133XUXAMuWLePOO+/kmmuu4Uc/+hHHjjl9ZU6fPh2Px8PWrVsBp2uW3NzcDt9r1qxZ7NlzfgPWGHPhsRYJnFXL4Xyqq6tjxowZgNMiufPOO1s+9DvSfBoK4M0332TXrlPPCausrKSqqorrrruOgwcP8tprr7F27VpmzpzJjh07SEtLIzc3l9WrVzNlyhT+8Ic/sGLFig7fayB0n2OM6T4Lkl7UfI0knM/na/WQqvr6+lbL4+PjW8ZDoRAbNmwgNjb2tG0PGTKEpUuXsnTpUhYtWsS7777LzTffTG5uLgsXLuSqq65i+vTppKend1i/zZs3W39ZxphO2amtPiYjI4OioiJKS0tpaGg4451TCxcuZNWqVS3TzaG0bt26ljvAqqqqOHDgACNHjgRg7NixpKSkcP/995/xtNa2bdt45JFHOn0mijHGWIukj4mKiuLBBx9kzpw5ZGdnM3HixA7Lrly5kmXLljF9+nQCgQDz58/nmWeeYdOmTSxfvryldXPXXXdxySWXtKyXm5vLAw88wE033dRqe+vXr2fmzJnU1taSnp7OypUr7UK7MaZT1o28iQg7vsb0f9aNvDHGmB5hQWKMMaZbBnSQDITTer3BjqsxA8uADRK/309paal96J1nqkppaSl+v7+3q2KM6SED9q6trKws8vPzKS4u7u2qXHD8fj9ZWVm9XQ1jTA8ZsEESFRVFdnZ2b1fDGGP6vYie2hKR60Vkr4jsF5H721keIyK/c5d/KCKjw5Y94M7fKyLXdXWbxhhjelbEgkREvMDTwOeAyUCuiExuU+xO4KSqXgQ8BTzurjsZWAJMAa4H/q+IeLu4TWOMMT0oki2SS4H9qnpQVRuB1cCNbcrcCPzKHX8ZuFZExJ2/WlUbVPUQsN/dXle2aYwxpgdF8hpJJnA0bDofmNNRGVUNiEgFkOLO/6DNupnueGfbBEBE7gbudierRWRve+XOk1SgJILbj6T+XHew+vc2q3/vinT9R3WlUCSDRNqZ1/Ze247KdDS/vRZUu/fvquqzwLNnquD5IiJ5XelGoC/qz3UHq39vs/r3rr5S/0ie2soHRoRNZwFtH7bRUkZEfEAyUHaGdbuyTWOMMT0okkGyERgnItkiEo1z8XxNmzJrgDvc8VuAder8QnANsMS9qysbGAd81MVtGmOM6UERO7XlXvNYDrwOeIHnVHWniKwA8lR1DfBL4AUR2Y/TElnirrtTRF4CdgEBYJmqBgHa22ak9uEs9MgptAjpz3UHq39vs/r3rj5R/wHRjbwxxpjIGbB9bRljjDk/LEiMMcZ0iwXJWRCRESLyFxHZLSI7ReQed/4QEXlDRD5xh4N7u65n4vYSsFlEXnWns90uaj5xu6yJ7u06dkREBonIyyKyx/13uLy/HH8Ruc/9u9khIi+KiL+vH3sReU5EikRkR9i8do+3OFa63RdtE5FZvVfzDuv+E/dvZ5uIvCIig8KWtdstU29pr/5hy74nIioiqe50rx57C5KzEwC+q6qTgMuAZW4XLfcDb6nqOOAtd7ovuwfYHTb9OPCUW/+TOF3X9FU/A15T1YnAxTj70eePv4hkAt8BclR1Ks7NIkvo+8f+eZxuisJ1dLw/h3OH5TicHwP/Ww/VsSPPc3rd3wCmqup0YB/wAHTcLVPPVbVdz3N6/RGREcAC4EjY7N499qpqr3N8AX9w/0H3AsPcecOAvb1dtzPUOQvnP/9ngFdxfvxZAvjc5ZcDr/d2PTuoexJwCPcmkbD5ff74c6oXhyE4d0u+ClzXH449MBrY0dnxBn4O5LZXrq/Uvc2ym4DfuOMPAA+ELXsduLyvHXt33ss4X6IOA6l94dhbi+QcuT0VzwQ+BDJU9TiAO0zvvZp16qfAPwEhdzoFKFfVgDsd3h1NXzMGKAb+wz019wsRiacfHH9VLQCexPkWeRyoADbRf459uI6Od3vdIvXl/fkasNYd7xd1F5EbgAJV3dpmUa/W34LkHIhIAvB74F5Vrezt+nSViCwCilR1U/jsdor21XvCfcAs4N9UdSZQQx88jdUe9zrCjUA2MByIxzkd0VZfPfZd0W/+lkTkf+Ocqv5N86x2ivWpuotIHPC/gQfbW9zOvB6rvwXJWRKRKJwQ+Y2q/rc7u1BEhrnLhwFFvVW/TlwB3CAih3F6Tv4MTgtlkNtFDfTtbmfygXxV/dCdfhknWPrD8f8scEhVi1W1CfhvYC7959iH6+h494sujETkDmARcKu654HoH3Ufi/NFZKv7fzgL+FhEhtLL9bcgOQsiIji/xt+tqv8atii8q5c7cK6d9Dmq+oCqZqnqaJwLi+tU9VbgLzhd1EDfrv8J4KiITHBnXYvT+0F/OP5HgMtEJM79O2que7849m10dLzXALe7dxBdBlQ0nwLrK0TkeuCfgRtUtTZsUUfdMvUZqrpdVdNVdbT7fzgfmOX+v+jdY9/bF5P60wu4Eqe5uA3Y4r4+j3Od4S3gE3c4pLfr2oV9uZr/197dhVhRxnEc//4Q2Y3YIMheCFxJkcUoVsjKaksJiopAKYoKyV4uIkgUhS4Eca+KuopeDI2UNhLxIsEgqGxly5fwZTc1MDDYyLskC4mSXv5dPP9lj+ueo2cn2O30+8Cwz87MM/PMLGf+Z57Z+T/wUZavo3xoTgDbgbbJbl+DdncDB/NvsAO4/L9y/oFe4DhwDOgD2qb6uQe2Up7p/EG5cD1T73xTulfeBL4DjlL+Q22qtf0E5VnCyOf37Zr112bbvwXum4rnfszyYUYftk/quXeKFDMzq8RdW2ZmVokDiZmZVeJAYmZmlTiQmJlZJQ4kZmZWiQOJtSRJL0laJGmJpKbefpc0IzPyDkrqGbNst6SbLlB/Zb6FPCHZ7tvqLGuT9JmkIUmPNrndWZIen2i7zOpxILFWdQslD9pdwBdN1r0bOB4R8yOi2boAK4EJBxLKOz7jBhJKfrfpEdEdEdua3O4soKlAUvPWvVldDiTWUnK8iSPAAmAf8CywQdJ5+YkkdUraleM37JI0U1I38Apwf37rv6TBvjZIOqgyxkhvzltByaXVL6k/590jaZ+kw5K2Z642JA1L6s35RyV1ZTLQ54BVuf+emv1dCbwPdOey2ZLWSTqgMsbJxnxrHklz8s7l69z+bOBloCfrrlIZD2Vz7ntQ0uKsuzzbuRP4RNI1kgay3rGxd2lmk/7mrCdP//YE3Ay8DkwH9jRYbyfwZJafBrxQoqgAAAJhSURBVHZkeTnwRp06u8m3hhl9o3tazr8xfx9m9I3jK4AB4NL8/UVgXc16L2T5eeCdLK8H1tTZ/yIyI0FtG7LcBzyY5a+ApVlup9whja27Gtic5S5KGpf2PP6TNce3Glhbc6wdk/039jS1Jt+RWCuaT0l/0UXJZ1XPQuCDLPdRUuA04xFJh4FByoBI88ZZ59acv0fSECU3VWfN8pHEn4coXU/NWpzPc45SknBeL6kDuDYiPgSIiN/j3LxSI+6gHDcRcRz4Hpibyz6NiJ+yfAB4StJ64IaIODOBdloLc/+ntYzsltpCyXx6ivItXHkBXxgRv11gExedLygT+60BFkTEaUlbKN/mz1uVclF+rM6mzubPv2jy8yipHXiLcof0Q17o2xk/pfi4m2iw7NeRQkQMSLoTeADok/RqRLzXTFuttfmOxFpGRAxFRDdlCNV5wOfAvVEeTI8XRPZSsiADPAF82cTuLqNcbH+RdBXnji1yBujI8n7gdklzoIwpIWkujdXWb2QkcJ3K5y4PA0QZI+ekpCW5z7b8L7Kx2x2gHDfZppmUhIXnkNRJGcdmEyX79aSOxW5TjwOJtRRJM4DTEfE30BURjbq2VlC6bI4Ayyhj2V+UKCPUDQLfAO8Ce2oWbwQ+ltQfET9Snjlszf3sp3S5NbITWDr2Yfs4bfgZ2ETJ9rqD0gU1YhmwIve5F7iakjH5z3wAv4pyNzMtu8W2Acsj4iznWwQMSRoEHgJeu0D77X/G2X/NzKwS35GYmVklDiRmZlaJA4mZmVXiQGJmZpU4kJiZWSUOJGZmVokDiZmZVfIP0+s0buDScsEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_rank_scores([ssvd_rank_scores,\n", + " ials_rank_scores,\n", + " psvd_rank_scores]);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It can be seen that scaling has a significant impact on the quality of recommendations. This is, however, a preliminary result, which is yet to be verified via cross-validation." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Models comparison" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The results above were computed only with a single split into train-test corresponding to a single fold. In order to verify the obtained results, perform a full CV with optimal parameters fixed. It can be achieved with the built-in `run_cv_experiment` function from Polara's evaluation engine as shown below." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## cross-validation experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.evaluation import evaluation_engine as ee" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Fixing optimal configurations:" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [], + "source": [ + "set_config(psvd, {'rank': psvd_best_rank})\n", + "set_config(ssvd, ssvd_best_config)\n", + "set_config(ials, ials_best_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Performing 5-fold CV:" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "5/5\n", + "[04:15<00:51, 50.93s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 5/5 [04:15<00:51, 50.93s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "models = [psvd, ssvd, ials]\n", + "metrics = ['ranking', 'relevance', 'experience']\n", + "\n", + "# run experiments silently\n", + "data_model.verbose = False\n", + "for model in models:\n", + " model.verbose = False\n", + "\n", + "# perform cross-validation on models, report scores according to metrics\n", + "cv_results = ee.run_cv_experiment(models,\n", + " metrics=metrics,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output contains results for all folds:" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
typerelevancerankingexperience
metrichrmrrcoverage
foldmodel
1PureSVD0.0768570.0291010.085902
PureSVD-s0.0847290.0322210.148946
iALS0.0737800.0272740.089461
2PureSVD0.0798680.0293850.089836
PureSVD-s0.0869530.0340590.150539
\n", + "
" + ], + "text/plain": [ + "type relevance ranking experience\n", + "metric hr mrr coverage\n", + "fold model \n", + "1 PureSVD 0.076857 0.029101 0.085902\n", + " PureSVD-s 0.084729 0.032221 0.148946\n", + " iALS 0.073780 0.027274 0.089461\n", + "2 PureSVD 0.079868 0.029385 0.089836\n", + " PureSVD-s 0.086953 0.034059 0.150539" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cv_results.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## plotting results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will plot average scores and confidence intervals for them. The following function will do this based on raw input from CV:" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_cv_results(scores, subplot_size=(6, 3.5)):\n", + " scores_mean = scores.mean(level='model')\n", + " scores_errs = ee.sample_ci(scores, level='model')\n", + " # remove top-level columns with classes of metrics (for convenience)\n", + " scores_mean.columns = scores_mean.columns.get_level_values(1)\n", + " scores_errs.columns = scores_errs.columns.get_level_values(1)\n", + " # plot results\n", + " n = len(scores_mean.columns)\n", + " return scores_mean.plot.bar(yerr=scores_errs, rot=0,\n", + " subplots=True, layout=(1, n),\n", + " figsize=(subplot_size[0]*n, subplot_size[1]),\n", + " legend=False);" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X203WV95/33x0RAUUFjsMpTYKC24UEsR2jHh6ZSMKyKsSOUAFPgvrlXtMo4q962hZlCI2M70DVTagfaMRUQcRAoLdO0RNGK+DQWEwQJgXI3xgAxWsJDEVDAwPf+Y/9O3GzOydnJ2efsffZ5v9Y6K799/a7ftb+/vc+52Hz39ZCqQpIkSZIkqVde1O8AJEmSJEnScDHZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIOGRpKNSX6133FIkiRJ0mxnskGSJEmSJPWUyQbNOknm9jsGSZrNxuqH7ZslaXolmdPvGDTcTDZo2ByR5M4kjyW5NsluSRYl2ZTk95L8ALii30FK0jBopq/9TtPvPpnksiSvSfLZJI8n+Yckr0yyIEklOSvJ/cDNY5X1+34kabol2TfJ3yTZkuThJJckeVGS309yX5IHk3wqyR5N/c8lObujjW8n+XfN8c8l+UKSR5Lcm+Q32up9MslfJFmV5EngV5L8WpLbk/wwyQNJlne0fXoTx8NJzmufttzEeU6S7zTnr0vyqql+zTRzmGzQsPkNYDFwAHA4cGZT/jPAq4D9gWV9iUyShtN7gGOBnwVOAD4L/Cfg1bQ+Z3ywre4vAz8PvGOCMkkaes3Igr8H7gMWAHsD19D6/Hom8CvAgcDLgEuay64GTmlrYyGtz7c3Jtkd+EJTZ6+m3p8nOaTtaU8F/hB4OfA14EngdGBP4NeA30ry7ra2/xw4DXgtsEcT46gPAu+m1Y+/DngUuHQSL4mGjMkGDZs/q6rNVfUI8HfAEU35c8AfVNXTVfXj/oUnSUPnf1TVv1TV94CvArdW1e1V9TRwA/DGtrrLq+rJjn54rDJJmg2OovU/6b/T9INPVdXXaP3P/Z9U1YaqegI4F1jaTDe7gdZI3v2bNk4D/qbpc98JbKyqK6pqa1V9C/hr4MS25/zbqvp6VT3XPN8tVbW2eXwn8BlayQOa6/6uqr5WVc8A5wPV1tZ7gf9cVZua518OnOi0OI0y2aBh84O24x/RygQDbKmqp/oQjyQNu39pO/7xGI9f1vb4gTGuH6tMkmaDfYH7qmprR/nraI12GHUfMBd4TVU9DtwILG3OLQX+V3O8P3B0kn8d/aGVjPiZtrae1+cmOTrJl5ppHI8B76M1Mm00jm31q+pHwMNtl+8P3ND2XPcAzwKv6foV0FAz2aDZoiauIkmaYmP1xfbPkmarB4D9xhgJsJnW/8iP2g/Yyk+TuZ8BTknyS8BLgC+1tfflqtqz7edlVfVbbW119rlXAyuBfatqD+B/AmnOfR/YZ7RikpcA8zriP77j+XZrRrpJJhskSZIkqQ++Set/6C9MsnuzsPmbaSUTfjvJAUleBvwRcG3bCIhVtJIRFzTlzzXlfw/8bJLfTPLi5udNSX5+OzG8HHikqp5KchStNR1GXQ+ckOTfJtkF+Ag/TURAKzHxh6NTOpLMT7JkEq+HhozJBkmSJEmaZlX1LK2FdQ8C7gc2AScDlwNXAV8Bvgs8BfyHtuueBv4G+FVaIxNGyx8HjqM1tWIzrenFFwG7bieM9wMXJHmc1poM17W1t6553mtoJUUeBx4Enm6qfIzWqIjPN9f/I3D0Dr8QGlqpcvSiJEmSJGl8zSiLfwUOrqrv9jseDT5HNkiSJEmSXiDJCUle2myr+d+AtcDG/kalmcJkgyRJkiRpLEtoTcnYDBwMLC2HxqtLTqOQJEmSJEk95cgGSZIkSZLUUyYbJEmSJElST83tdwCdXv3qV9eCBQv6HYYkvcBtt932UFXN73cc08G+WNIgsh+WpP7rti8euGTDggULWLNmTb/DkKQXSHJfv2OYLvbFkgaR/bAk9V+3fbHTKCRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGDa1FixaxaNGifochSbOafbEk9Zf9sPrFZIMkDZAki5Pcm2R9knPGOL9rkmub87cmWdCUH5Xkjubn20l+ve2ajUnWNufWTN/dSJIkabaa2+8AJEktSeYAlwLHApuA1UlWVtXdbdXOAh6tqoOSLAUuAk4G7gJGqmprktcC307yd1W1tbnuV6rqoem7G0mSJM1mJhskaXAcBayvqg0ASa4BlgDtyYYlwPLm+HrgkiSpqh+11dkNqKkPV5IkTdZhVx42pe1v+MGGaXmetWesndL2NfM4jUKSBsfewANtjzc1ZWPWaUYtPAbMA0hydJJ1wFrgfW2jGgr4fJLbkiwb78mTLEuyJsmaLVu29OSGJEmSNDs5skF9s+CcG6e0/R9seHhanmfjhb82pe1rVskYZZ0jFMatU1W3Aock+XngyiSfraqngDdX1eYkewFfSPJPVfWVFzRStQJYATAyMuLICEmSJO00RzZI0uDYBOzb9ngfYPN4dZLMBfYAHmmvUFX3AE8ChzaPNzf/PgjcQGu6hiRJkjRlTDZI0uBYDRyc5IAkuwBLgZUddVYCZzTHJwI3V1U118wFSLI/8HpgY5Ldk7y8Kd8dOI7WYpKSJGkWOPDcAznw3AP7HYZmIadRSNKAaHaSOBu4CZgDXF5V65JcAKypqpXAZcBVSdbTGtGwtLn8LcA5SX4CPAe8v6oeSnIgcEMSaPX5V1fV56b3ziRJkjTbmGzQ0PqZUy/sdwjSDquqVcCqjrLz246fAk4a47qrgKvGKN8AvKH3kWpoLN9jatvf+OTUP8/yx6aubalNksXAx2glhD9RVRd2nH8b8KfA4cDSqrq+4/wrgHuAG6rq7OmJWpL6w2kUkiRJ0gSSzAEuBY4HFgKnJFnYUe1+4Ezg6nGa+S/Al6cqRkkaJCYbJEmSpIkdBayvqg1V9QxwDbCkvUJVbayqO2lNZ3ueJEcCrwE+Px3BSlK/mWyQJEmSJrY38EDb401N2YSSvAj478DvTFBvWZI1SdZs2bJlpwOVpEFgskGSJEmaWMYoqy6vfT+wqqoe2F6lqlpRVSNVNTJ//vwdDlCSBokLREqSJEkT2wTs2/Z4H2Bzl9f+EvDWJO8HXgbskuSJqjqnxzFK0sDoamRDksVJ7k2yPskLOsUkuya5tjl/a5IFTfmLk1yZZG2Se5Kc29vwB9OiRYtYtGhRv8OQJElS76wGDk5yQJJdaG09vLKbC6vqtKrar6oWAB8GPmWiQdKwmzDZ0OXKu2cBj1bVQcDFwEVN+UnArlV1GHAk8N7RRIQkSZI0U1TVVuBs4CZa21deV1XrklyQ5F0ASd6UZBOtz8AfT7KufxFLUn91M41i28q7AElGV969u63OEmB5c3w9cEmS0JrHtnuSucBLgGeAH/YmdEmSJGn6VNUqYFVH2fltx6tpTa/YXhufBD45BeFJ0kDpJtkw1sq7R49Xp6q2JnkMmEcr8bAE+D7wUuC3q+qRyQYtaWYZnVZ0yy239DUOSdPvljN373cIkiSpD7pZs6GblXfHq3MU8CzwOuAA4P9NcuALnsBtfiRJkiRJGhrdJBu6WXl3W51mysQewCPAqcDnquonVfUg8HVgpPMJ3OZHkiRJkqTh0c00im0r7wLfo7Xy7qkddVYCZwDfAE4Ebq6qSnI/8PYkn6Y1jeIXgT/tVfA7a8E5N05p+z/Y8PCUP8/GC39tytqWpJnOqTuSJEn9NeHIhm5W3gUuA+YlWQ98CBjdyudSWnsJ30UraXFFVd3Z43uQJEmSJEkDpJuRDd2svPsUrS1+Oq97YqxySZIkSZI0vLpZs0GSJEmSJKlrXY1skDTcXMdEkoab65hIkqabIxskaYAkWZzk3iTrk5wzxvldk1zbnL81yYKm/KgkdzQ/307y6922KUmSJPWaIxumwM+cemG/Q5A0AyWZQ2th3WNpbSm8OsnKqrq7rdpZwKNVdVCSpcBFwMm0FuIdqaqtSV4LfDvJ3wHVRZuSJElSTzmyQZIGx1HA+qraUFXPANcASzrqLAGubI6vB45Jkqr6UbN7EMButJIM3bYpSZIk9ZQjGyRpcOwNPND2eBNw9Hh1mlEMjwHzgIeSHA1cDuwP/GZzvps2AUiyDFgGsN9++03+brZn+R5T2/7GJ6fneZY/NrXtS5IkzVCObJCkwZExyqrbOlV1a1UdArwJODfJbl22SXP9iqoaqaqR+fPn70DYkiRJ0vM5skHSlHMdk65tAvZte7wPsHmcOpuSzAX2AB5pr1BV9yR5Eji0yzYlSZKknnJkgyQNjtXAwUkOSLILsBRY2VFnJXBGc3wicHNVVXPNXIAk+wOvBzZ22aYkSZLUUyYbJGlANAs8ng3cBNwDXFdV65JckORdTbXLgHlJ1gMfAka3snwLrR0o7gBuAN5fVQ+N1+b03ZUkDY8utid+W5JvJdma5MS28iOSfCPJuiR3Jjl5eiOXpOnnNApJGiBVtQpY1VF2ftvxU8BJY1x3FXBVt21KGjAumjrwutye+H7gTODDHZf/CDi9qv45yeuA25LcVFX/Og2hS1JfmGyQJEmSJrZtK2GAJKNbCW9LNlTVxubcc+0XVtX/13a8OcmDwHzAZIOkoWWyQZI0dG45c/d+hyBp+HS9lfD2JDkK2AX4zhjnpm8LYkmaYq7ZIEmSJE2s662Ex20geS2tKW//V1U913neLYglDROTDZIkSdLEJrWVcJJXADcCv19V/9jj2CRp4JhskCRJkia201sJN/VvAD5VVX81hTFK0sAw2SBJkiRNoJvtiZO8KckmWrsGfTzJ6FbDvwG8DTgzyR3NzxF9uA1JmjYuEClJkiR1oYvtiVfTml7Red2ngU9PeYCSNEAc2SBJkiRJknrKkQ2SJElDzu1gJUnTzZENkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkDZAki5Pcm2R9knPGOL9rkmub87cmWdCUH5vktiRrm3/f3nbNLU2bdzQ/e03fHUmSJGk2mtvvACRJLUnmAJcCxwKbgNVJVlbV3W3VzgIeraqDkiwFLgJOBh4CTqiqzUkOBW4C9m677rSqWjMtNyJJkqRZz5ENkjQ4jgLWV9WGqnoGuAZY0lFnCXBlc3w9cEySVNXtVbW5KV8H7JZk12mJWpIkSepgskGSBsfewANtjzfx/NEJz6tTVVuBx4B5HXXeA9xeVU+3lV3RTKE4L0l6G7YkSZL0fCYbJGlwjJUEqB2pk+QQWlMr3tt2/rSqOgx4a/Pzm2M+ebIsyZoka7Zs2bJDgUuSJEntuko27OyCZc25w5N8I8m6ZuGy3XoXviQNlU3Avm2P9wE2j1cnyVxgD+CR5vE+wA3A6VX1ndELqup7zb+PA1fTmq7xAlW1oqpGqmpk/vz5PbkhSZIkzU4TJhvaFiw7HlgInJJkYUe1bQuWARfT+lZt9IPwp4H3VdUhwCLgJz2LXpKGy2rg4CQHJNkFWAqs7KizEjijOT4RuLmqKsmewI3AuVX19dHKSeYmeXVz/GLgncBdU3wfkjSUuvgC7m1JvpVka5ITO86dkeSfm58zOq+VpGHTzciGnV6wDDgOuLOqvg1QVQ9X1bO9CV2ShkuzBsPZtHaSuAe4rqrWJbkgybuaapcB85KsBz4EjH7YPRs4CDivY4vLXYGbktwJ3AF8D/jL6bsrSRoOXX4Bdz9wJq1RZO3Xvgr4A+BoWp+t/yDJK6c6Zknqp262vhxrwbKjx6tTVVuTjC5Y9rNAJbkJmA9cU1V/3PkESZYBywD222+/Hb0HSRoaVbUKWNVRdn7b8VPASWNc91Hgo+M0e2QvY5SkWWrbF3AASUa/gNu2PXFVbWzOPddx7TuAL1TV6LS3LwCLgc9MfdiS1B/djGyYzIJlc4G3AKc1//56kmNeUNF5wpIkSRps3ewYNKlrXahX0jDpJtkwmQXLNgFfrqqHqupHtL6t+4XJBi1JkiRNs26+gJvUtX4BJ2mYdJNs2OkFy2jNOz48yUubJMQv0zbUTJIkSZohuvkCbiqulaQZacJkw2QWLKuqR4E/oZWwuAP4VlXd2PvbkCRJkqZUN1/Ajecm4Lgkr2wWhjyuKZOkodXNApE7vWBZc+7TtLa/lCRJkmakZhH00S/g5gCXj34BB6ypqpVJ3gTcALwSOCHJR6rqkKp6JMl/oZWwALhgdLFISRpWXSUbJEmSpNmuiy/gVtOaIjHWtZcDl09pgJI0QLpZs0GSJEmSJO2kRYsWsWjRon6HMa1MNkiSJEmSpJ4y2SBJkiRJknrKNRskSZIkSbPeYVceNmVtb/jBhil/DoC1Z6yd0vZ3hCMbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZRrNkiSJEnqudFt/m655Za+xiENggPPPbDfIUw7RzZIkiRJkqSecmSDJEmSNAtN9ar407H6/iCtvC/p+RzZIEmSJEmSespkgyQNkCSLk9ybZH2Sc8Y4v2uSa5vztyZZ0JQfm+S2JGubf9/eds2RTfn6JH+WJNN3R5IkSZqNnEYhSQMiyRzgUuBYYBOwOsnKqrq7rdpZwKNVdVCSpcBFwMnAQ8AJVbU5yaHATcDezTV/ASwD/hFYBSwGPjsd9yRJmr1m44J4kn7KkQ2SNDiOAtZX1Yaqega4BljSUWcJcGVzfD1wTJJU1e1VtbkpXwfs1oyCeC3wiqr6RlUV8Cng3VN/K5IkSZrNTDZI0uDYG3ig7fEmfjo64QV1qmor8Bgwr6POe4Dbq+rppv6mCdoEIMmyJGuSrNmyZctO34QkSZJkskGSBsdYaynUjtRJcgitqRXv3YE2W4VVK6pqpKpG5s+f30W4kjS7TGJdnRcnubJZP+eeJOdOd+ySNN1MNkjS4NgE7Nv2eB9g83h1kswF9gAeaR7vA9wAnF5V32mrv88EbUqSJtC2rs7xwELglCQLO6ptW1cHuJhW8hfgJGDXqjoMOBJ472giQpKGlckGSRocq4GDkxyQZBdgKbCyo85K4Izm+ETg5qqqJHsCNwLnVtXXRytX1feBx5P8YrMLxenA3071jUjSENrpdXVojSjbvUkSvwR4Bvjh9IQtSf1hskGSBkSzBsPZtHaSuAe4rqrWJbkgybuaapcB85KsBz4EjA7jPRs4CDgvyR3Nz17Nud8CPgGsB76DO1FI0s6YzLo61wNPAt8H7gf+W1U90vkErp0jaZi49aUkDZCqWkVre8r2svPbjp+iNRy387qPAh8dp801wKG9jVSSZp3JrKtzFPAs8DrglcBXk/xDVW14XsWqFcAKgJGRkTHX15GkmcKRDZIkSdLEJrOuzqnA56rqJ1X1IPB1YGTKI5akPjLZIEmSJE1sp9fVoTV14u1p2R34ReCfpiluSeoLkw2SJEnSBCa5rs6lwMuAu2glLa6oqjun9QYkaZq5ZoMkSZLUhUmsq/PEWOWSNMwc2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqqa6SDUkWJ7k3yfok54xxftck1zbnb02yoOP8fkmeSPLh3oQtSZIkSZIG1YTJhiRzgEuB44GFwClJFnZUOwt4tKoOAi4GLuo4fzHw2cmHK0mSJEmSBl03IxuOAtZX1Yaqega4BljSUWcJcGVzfD1wTJIAJHk3sAFY15uQJUmSJEnSIOsm2bA38EDb401N2Zh1qmor8BgwL8nuwO8BH5l8qJIkSZIkaSboJtmQMcqqyzofAS6uqie2+wTJsiRrkqzZsmVLFyFJkiRJkqRBNbeLOpuAfdse7wNsHqfOpiRzgT2AR4CjgROT/DGwJ/Bckqeq6pL2i6tqBbACYGRkpDORIUmSJEmSZpBuRjasBg5OckCSXYClwMqOOiuBM5rjE4Gbq+WtVbWgqhYAfwr8UWeiQZL0Uzu7+0+SeUm+1Oz8c0nHNbc0bd7R/Ow1PXcjSZKk2WrCkQ1VtTXJ2cBNwBzg8qpal+QCYE1VrQQuA65Ksp7WiIalUxm0JA2jtt1/jqU1Ymx1kpVVdXdbtW27/yRZSmv3n5OBp4DzgEObn06nVdWaKb0BSZIkqdHNNAqqahWwqqPs/Lbjp4CTJmhj+U7EJ0mzybbdfwCSjO7+055sWAIsb46vBy5Jkqp6EvhakoOmMV5JkiRpTN1Mo5AkTY+d3v2ni7avaKZQnDe6NXEnF+uVpO3b2aluzbnDk3wjyboka5PsNp2xS9J0M9kgSYNjMrv/bM9pVXUY8Nbm5zfHqlRVK6pqpKpG5s+fP2GwkjSbtE11Ox5YCJySZGFHtW1T3YCLaU11o1lA/dPA+6rqEGAR8JNpCl2S+sJkgyQNjh3Z/Wf0w+vo7j/jqqrvNf8+DlxNa7qGJGnHbJvqVlXPAKNT3dotAa5sjq8HjmlGkx0H3FlV3waoqoer6tlpiluS+sJkgyQNjp3e/We8BpPMTfLq5vjFwDuBu3oeuSQNv8lMdftZoJLclORbSX53rCdwOpukYdLVApGSpKk32d1/kmwEXgHskuTdtL5Juw+4qUk0zAH+AfjLabwtSRoWk5nqNhd4C/Am4EfAF5PcVlVffF7FqhXACoCRkZGJpshJ0kAz2SBJA2Qyu/9U1YJxmj2yV/FJ0iy2I1PdNnVMddsEfLmqHgJIsgr4BeCLSNKQchqFJEmSNLHJTHW7CTg8yUubJMQv8/xtjSVp6DiyQZIkSZrAZKa6VdWjSf6EVsKigFVVdWNfbkSSponJBkmSJKkLk5zq9mla219K0qzgNApJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskaYAkWZzk3iTrk5wzxvldk1zbnL81yYKmfF6SLyV5IsklHdccmWRtc82fJcn03I0kSZJmK5MNkjQgkswBLgWOBxYCpyRZ2FHtLODRqjoIuBi4qCl/CjgP+PAYTf8FsAw4uPlZ3PvoJUmSpJ8y2SBJg+MoYH1VbaiqZ4BrgCUddZYAVzbH1wPHJElVPVlVX6OVdNgmyWuBV1TVN6qqgE8B757Su5CkIbWzo8/azu/XjEAbKzEsSUPFZIMkDY69gQfaHm9qysasU1VbgceAeRO0uWmCNgFIsizJmiRrtmzZsoOhS9Jwm+Tos1EXA5+d6lglaRCYbJCkwTHWWgq1E3V2qn5VraiqkaoamT9//naalKRZaadHnwEkeTewAVg3TfFKUl+ZbJCkwbEJ2Lft8T7A5vHqJJkL7AE8MkGb+0zQpiRpYjs9+izJ7sDvAR+ZhjglaSCYbJCkwbEaODjJAUl2AZYCKzvqrATOaI5PBG5u1mIYU1V9H3g8yS82366dDvxt70OXpKE3mdFnHwEurqontvsETmeTNETm9jsASVJLVW1NcjZwEzAHuLyq1iW5AFhTVSuBy4CrkqynNaJh6ej1STYCrwB2aYbrHldVdwO/BXwSeAmtucLOF5akHbcjo882dYw+Oxo4MckfA3sCzyV5qqqet1VxVa0AVgCMjIxsb4qcJA08kw2SNECqahWwqqPs/Lbjp4CTxrl2wTjla4BDexelJM1K20afAd+jlew9taPO6Oizb/D80WdvHa2QZDnwRGeiQZKGjckGSZIkaQKTHX0mSbNNV8mGJIuBj9HqWD9RVRd2nN+V1t7tRwIPAydX1cYkxwIXArsAzwC/U1U39zB+SZIkaVpMZvRZW53lUxKcJA2YCReInOSewg8BJ1TVYbSGlF3Vq8AlSZIkSdJg6mY3ip3eU7iqbq+q0YVz1gG7NaMgJEmSJEnSkOom2bDTewp31HkPcHtVPd35BG7zI0mSJEnS8Ogm2TCZPYVbJ5NDaE2teO9YT1BVK6pqpKpG5s+f30VIkiRJkiRpUHWTbNiRPYXp2FOYJPsANwCnV9V3JhuwJEmSJEkabN0kG7btKZxkF1pb+KzsqDO6pzC07SmcZE/gRuDcqvp6r4KWJEmSJEmDa8JkQ7MGw+iewvcA143uKZzkXU21y4B5zZ7CHwLOacrPBg4CzktyR/OzV8/vQpIkSZIkDYy53VTa2T2Fq+qjwEcnGaMkSZIkSZpBuplGIUmSJEmS1DWTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZI0gBJsjjJvUnWJzlnjPO7Jrm2OX9rkgVt585tyu9N8o628o1J1ia5I8ma6bkTSZIkzWYmGyRpQCSZA1wKHA8sBE5JsrCj2lnAo1V1EHAxcFFz7UJgKXAIsBj486a9Ub9SVUdU1cgU34YkDa2dTQgnOTbJbU3i97Ykb5/u2CVpuplskKTBcRSwvqo2VNUzwDXAko46S4Arm+PrgWOSpCm/pqqerqrvAuub9iRJPTCZhDDwEHBCVR0GnAFcNT1RS1L/mGyQpMGxN/BA2+NNTdmYdapqK/AYMG+Cawv4fPNt2rIpiFuSZoOdTghX1e1VtbkpXwfslmTXaYlakvrEZIMkDY6MUVZd1tnetW+uql+g9W3cB5K8bcwnT5YlWZNkzZYtW7qNWZJmi8kkhNu9B7i9qp6eojglaSCYbJCkwbEJ2Lft8T7A5vHqJJkL7AE8sr1rR79Nq6oHgRsYZ3pFVa2oqpGqGpk/f/6kb0aShsxkEsKtk8khtKZWvHfMJzDpK2mImGyQpMGxGjg4yQFJdqG14OPKjjorac33BTgRuLmqqilf2ixOdgBwMPDNJLsneTlAkt2B44C7puFeJGnYTCYhTJJ9aCV8T6+q74z1BCZ9JQ2Tuf0OQJLUUlVbk5wN3ATMAS6vqnVJLgDWVNVK4DLgqiTraX2AXdpcuy7JdcAg4wkmAAAKIElEQVTdwFbgA1X1bJLXADe01pBkLnB1VX1u2m9Okma+bQlh4Hu0+t9TO+qMJoS/QVtCOMmewI3AuVX19WmMWZL6xmSDJA2QqloFrOooO7/t+CngpHGu/UPgDzvKNgBv6H2kkjS7TCYhDJwNHAScl+S8puy4ZnqbJA0lkw2SJElSF3Y2IVxVHwU+OuUBStIAcc0GSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPdVVsiHJ4iT3Jlmf5Jwxzu+a5Nrm/K1JFrSdO7cpvzfJO3oXuiQNn6nobydqU5LUHT8TS1L3Jkw2JJkDXAocDywETkmysKPaWcCjVXUQcDFwUXPtQmApcAiwGPjzpj1JUoep6G+7bFOSNAE/E0vSjulmZMNRwPqq2lBVzwDXAEs66iwBrmyOrweOSZKm/Jqqerqqvgusb9qTJL3QVPS33bQpSZqYn4klaQd0k2zYG3ig7fGmpmzMOlW1FXgMmNfltZKklqnob+2HJak3/EwsSTtgbhd1MkZZdVmnm2tJsgxY1jx8Ism9XcQ16F4NPDRVjeeiqWp56Ezp+wC+FztgGP4m9p/i9qeivx0rqfyCfhiGsi+e8r9/PjLWy64xTO174fvQrWH4m5jqfnh7/Ey8c6b2v/9n+vffpan/TOx70Y1heR+66ou7STZsAvZte7wPsHmcOpuSzAX2AB7p8lqqagWwopuAZ4oka6pqpN9xzHa+D4PD96IrU9XfTtgPw/D1xf7ODQ7fi8Hg+zBpfibeCf7eDQbfh8Ew296HbqZRrAYOTnJAkl1oLW6zsqPOSuCM5vhE4OaqqqZ8abMy7wHAwcA3exO6JA2dqehvu2lTkjQxPxNL0g6YcGRDVW1NcjZwEzAHuLyq1iW5AFhTVSuBy4Crkqynlb1d2ly7Lsl1wN3AVuADVfXsFN2LJM1oU9XfjtXmdN+bJM10fiaWpB2TVrJVvZZkWTMUTn3k+zA4fC803fydGxy+F4PB90H94O/dYPB9GAyz7X0w2SBJkiRJknqqmzUbJEmSJEmSumayAUjybJI7ktyV5K+SvLRH7b4myd8n+XaSu5Osasq/m+T1HXX/NMnvJlmU5LEktye5N8lXkryzF/EMGl/34ZDk/3Q8/u0kTyXZo61sUZK/H+Padzav+eh79d7piFmDx/6gP3zdh4P9sHrFPqE/fN2Hg33xC5lsaPlxVR1RVYcCzwDv6/bCtLY1Gs8FwBeq6g1VtRA4pym/hmbBoKaNF9FasfjapuirVfXGqno98EHgkiTHdH87M4av+xCoqn/bUXQKrRW7f3171yV5Ma3tvU6oqjcAbwRumYoYNSPYH/SHr/sQsB9WD9kn9Iev+xCwL34hkw0v9FXgoCQLktw1Wpjkw0mWN8e3JPmjJF8G/mOS+Un+Osnq5ufNzWWvpbWvMgBVdWdz+Bna/sCBtwEbq+q+zmCq6g5aHcXZPbzHQTRjXvckc5J8ssk+r03y25O68xkuyRNtx/8GeBnw+7Q62O15Oa0dcR4GqKqnq+reqYpTM8qM6Q+GzIx53e2Hn89+WFNkxvQJQ2bGvO72xc9nX/xCJhvaNJnB44G1XVTfs6p+uar+O/Ax4OKqehPwHuATTZ1LgcuSfCnJf07yOtj2h/5ckjc09ZbS+qMfz7eAn9vxO5oZZuDrfgSwd1UdWlWHAVd0EfdscQqt1/SrwOuT7DVexap6hNa+4/cl+UyS05rMumaxGdgfDIUZ+LrbD4/PfliTNgP7hKEwA193++Lx2RdjsmHUS5LcAawB7qe1R/JErm07/lVaw4vuoPWL8ookL6+qm4ADgb+k9Qd6e5L5zTWfAZY2ncoS4K+281zZobuZOWbq674BODDJ/0iyGPhhF3HPFkuBa6rqOeBvgJO2V7mq/h/gGOCbwIeBy6c8Qg2qmdofzHQz9XW3Hx6f/bAmY6b2CTPdTH3d7YvHZ19Ma7iGmnlS7QVJtvL8ZMxuHdc82Xb8IuCXqurHnQ03maqrgavTWgzkbcBf0/oD/zzwZeDOqnpwO/G9Ebiny3uZSWbE655kDnBbU7ayqs5vMsHvAD4A/Abwf2/3TmeBJIcDBwNfSAKwC63/CF26veuqai2wNslVwHeBM6c2Ug2oGdEfdHkvM8mMeN3th7tjP6wemBF9Qpf3MpPMiNfdvrg79sU/5ciG8f0LsFeSeUl2Bba3CuvnaZvHlOSI5t+3p1lNNsnLgX9DK1tJVX2H1rycC9nOsKXml/U8JvjlHCID97pX1bPNoj1HNJ3qq4EXVdVfN3V+Yedvd6icAiyvqgXNz+uAvZPsP1blJC9Lsqit6AjgBXMFNasNXH8wqbuZOQbudbcf7pr9sKbCwPUJk7qbmWPgXnf74q7ZFzcc2TCOqvpJkguAW2lllv5pO9U/CFya5E5ar+lXaK0ieyStIU2jmclPVNXqtus+A/xX4IaO9t6a5HbgpcCDwAer6os9uK2BN0Ne972BK9rmUp27I/c4xJbSmmfY7oam/FbgmCSb2s6dAvxuko8DP6aVoT9zGuLUDDFD+oOhM0Ned/vhsdkPq+dmSJ8wdGbI625fPDb74kaqqt8xSJIkSZKkIeI0CkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhs0qyTZ2OwJPKk6kqSdZ18sSf1lP6zpYLJBkiRJkiT1lMkGDbwkC5L8U5JPJLkryf9K8qtJvp7kn5McleRVSf53kjuT/GOSw5tr5yX5fJLbk3wcSFu7/z7JN5PckeTjSeb07SYlacDZF0tSf9kPa6Yx2aCZ4iDgY8DhwM8BpwJvAT4M/CfgI8DtVXV48/hTzXV/AHytqt4IrAT2A0jy88DJwJur6gjgWeC0absbSZqZ7Islqb/shzVjzO13AFKXvltVawGSrAO+WFWVZC2wANgfeA9AVd3cZG/3AN4G/Lum/MYkjzbtHQMcCaxOAvAS4MFpvB9JmonsiyWpv+yHNWOYbNBM8XTb8XNtj5+j9Xu8dYxrquPfdgGurKpzexahJA0/+2JJ6i/7Yc0YTqPQsPgKzZCvJIuAh6rqhx3lxwOvbOp/ETgxyV7NuVcl2X+6g5akIWNfLEn9ZT+sgeHIBg2L5cAVSe4EfgSc0ZR/BPhMkm8BXwbuB6iqu5P8PvD5JC8CfgJ8ALhvugOXpCGyHPtiSeqn5dgPa0CkaqzRNJIkSZIkSTvHaRSSJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6qn/H/5Eue+p2MGUAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_cv_results(cv_results);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The difference between PureSVD and iALS is not significant. In contrast, the advantage of the scaled version of PureSVD denoted as `PureSVD-s` over the other models is much more pronounced making it a clear favorite. Interestingly, the difference is especially pronounced in terms of the `coverage` metric, which is defined as the ratio of unique recommendations generated for all test users to the total number of items in the training data. This indicates that generated recommendations are not only more relevant but also are significantly more diverse. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## comparing training time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another important practical aspect is how long does it take to compute a model? Sometimes the best model in terms of quality of recommendations can be the slowest to compute. You can check each model's training time by accessing the `training_time` list attribute. It holds the history of trainings, hence, if you have just performed 5-fold CV experiment, the last 5 entries in this list will correspond to the training time on each fold. This information can be used to get average time with some error bounds as shown below." + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "timings = {}\n", + "for model in models:\n", + " timings[f'{model.method} rank {model.rank}'] = model.training_time[-5:]" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHNhJREFUeJzt3XuYHFWd//H3BxJQIYCS4ZKQkBWzcdGFgHkiLsqOgiw3xXVRE1wgKxpvrOJlFV1XWRCN/lS8RMWAWeAnBBY1muUisCjLRUAHNoSAIDEGEwIkXAQiICR8949zBiqd7plOd89MJufzep5+prrqVJ1TVd2frjpd1aOIwMzMyrHFUDfAzMwGl4PfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn7rGEmvk3TXINX1TklXDFJd+0u6W9IaSW8ZjDo3Rm7XSwdguTMkXdfp5fZT586SrpH0uKSvSvq0pLMGsw0lcPAPAklHS+rJb9D7JF0m6bVD3a4qSd2SVmzkPCHpZb3PI+LaiJg0AG2bkOsaUanrvIg4uNN1NXAKMDsito2InwxSnXVJulrSu6vjcruWDlWbOmwm8CCwXUR8LCK+EBHv7m8m2zgO/gEm6aPA14EvADsD44HvAEcOZbtso+wO3N7KjNUPK2vK7sAd4TtLB1ZE+DFAD2B7YA3wtj7KbE36YFiZH18Hts7TuoEVwCeAVcB9wFuAw4DfAg8Dn64s62Tgh8CFwOPALcDelekBvKzy/Gzg88A2wJPAs7m9a4AxwFTgBuCPue7ZwFZ53mvy8v6Uy7+jt72V5f8VcHWe/3bgzTV1fxu4JLf1JmCPBtvoD7mu3ra9BpgBXFezbh8A7s7LOxXYI7f/MeA/e9ueyx8BLMxt+yWwV4O6f5e3y5O57q3ztlmQt/8S4D119sEPcr3vbvC6OBdYDdwDfAbYIk+bAVwPfAt4FLgTODBPOw1YBzyV2zK7dr/m7fod4LJc5npgF9Lr6pG8vH0qbTkpr+PjwB3A31emrbeN66zHa/O2+yOwHJjR5PpdB3wlt+f3wKGVtj8DPJ3bflDenj+o1HlsXuZDwL8By4CDmnw/fhK4N6/rXb3btcTHkDdgc34AhwBrgRF9lDkFuBHYCejKb6RT87TuPP9ngZHAe/Kb6XxgFPCKHAIvzeVPzm+co3L5j+c31sg8vW7wV+paUdO2VwH7ASOACcBvgBMr02uX99wycv1LgE8DWwFvyG+4SZW6HyZ9uIwAzgMuaLCNJuS6RlTGzWDD4F8AbJe3y5+Bq4CXkoLoDuC4XHZf0gfpq4EtgeNygGzdoP71wgX4H1K4vgCYnPfJgTX74C2kM+oX1lneucBP8z6cQPoQP76yXmuBj+Rt+A7SB8BL8vSrqfkwYcPgfzDvuxcAP8+vgWPzun4e+EVl3reRPsi2yHX9Cdi13jauqXN83p/Tczt3BCY3uX7PkF7LWwLvJx3wqPY1WdmeP8jDe5I+EF5Lek19JS+r3+AHJpE+nMZUXlN1DzRKeLirZ2DtCDwYEWv7KPNO4JSIWBURq4F/B46pTH8GOC0ingEuAEYD34iIxyPidtKR9F6V8jdHxA9z+a+R3vz7tdL4iLg5Im6MiLURsQz4HvC3Tc6+H7AtMCsino6InwMXk4Ki148j4ld5+5xHCtF2fCkiHsvbZTFwRUQsjYhHSUfA++Ry7wG+FxE3RcS6iDiH9EHR73aSNI4UPJ+MiKciYiFwFuvvsxsi4icR8WxEPFkz/5akgP1U3ofLgK/WzL8K+HpEPBMRF5KOTg/fiO0wP++7p4D5wFMRcW5ErCOdDfZuByLioohYmdt6IemMaWoTdbwT+O+ImJfb+VBELGxy/e6JiDNze84BdiV1g/bnKOC/IuK6iHiadEDUbJfQOtLZ2p6SRkbEsoj4XZPzbnYc/APrIWB0P/28Y0inrr3uyeOeW0Z+g0DqbgB4oDL9SVLA9lreOxARz5K6iqrLa5qkv5R0saT7JT1G+p5idJOzjwGW5zb0ugcYW3l+f2X4CdZfj1bUbpdG22l34GOS/tj7AMbR3HYaAzwcEY9XxtWu13IaG006Wq3d59X57418WFqZvjH7sNntgKRjJS2sbIdX0tw+HkfqIqrVzPo9t98j4ok82My+H8P6r+8nSO+xfkXEEuBE0hnEKkkXSGrpfbE5cPAPrBtIXTF9XQK4khREvcbnca0a1zsgaQtgt8ryngBeVCm7S2W43pHTd0l9whMjYjtSt42abMdKYFxuQ6/xpD7WjdXpL/qWk86idqg8XhQR85qYdyXwEkmjKuNq16uv9j5IOour3efV+cdKUs303n3YsW0haXfgTOAEYMeI2IF0ptTMPl5O+g6lVjPr16r7SK9nACS9kHRW3ZSIOD8iXpvbFsCXOtCmYcnBP4ByF8NngW9LeoukF0kaKelQSV/OxeYBn5HUJWl0Lv+DNqp9laS35rOME0ldGDfmaQuBoyVtKekQ1u+2eQDYUdL2lXGjSF9QrpH0clJ/LDXzNLp+/CZSf/En8jp3A28idVdtrNWkL1g7da36mcD7JL1ayTaSDq8J87oiYjnpe5gvSnqBpL2A40ldVf3KZ2//CZwmaVQO34+y/j7fCfhQ3m5vI31Jfmme1tc231jbkAJwNYCkfyId8TfjPOAgSW+XNELSjpImN7l+rfoh8CZJfyNpK1K36HMfUvmS5LofjJImSXqDpK1JB2NPkrp/iuTgH2AR8TXSC/8zpDfYctIRVu/14J8HeoBFwG2kK3E+30aVPyX1sT5C6ld9a+7vB/gwKXz/SOqjfe6a9Ii4k/QhtDSf9o8hfTl8NOlLvDNJ/cNVJwPn5PJvr1nvp4E3A4eSjgK/Axyb69ko+ZT+NOD6XFdL31lUltdD6uefTdpOS0hfOjZrOunLwZWkPvTPRcSVGzH/P5M+FJeSrnA5H5hbmX4TMJG03U4DjoqI3i6NbwBHSXpE0jc3os4NRMQdpP73G0gfKH9NugqomXn/QLq67GOkL+kXAnvnyf2tX6vtvT0v+wLS0f/jpO9D/pyLjMvrUs/WwCzSNr2f9OH66XbbNFxp/a5EG84knUy6uuMfh7ot1hpJM0hX7WxSN/htiiRtSzqImRgRv893+F4UEZcPcdM2eb65xMyGDUlvIl2mK9LlnLeRLrclfIdv09zVY2bDyZE8f7PjRGBauNtio7mrx8ysMD7iNzMrzCbZxz969OiYMGHCUDfDzGzYuPnmmx+MiK5mym6SwT9hwgR6enqGuhlmZsOGpHv6L5W4q8fMrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjMrQnd3N93d3UPdjE2Cg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDCb5P/cNbMyTDjpkkGr6/6lDw1qnctmHT4o9bTCR/xmZoVx8JuZFabfrh5Jc4EjgFUR8co87kJgUi6yA/DHiJhcZ95lwOPAOmBtREzpULvNzKxFzfTxnw3MBs7tHRER7+gdlvRV4NE+5n99RDzYagPNzKyz+g3+iLhG0oR60yQJeDvwhs42y8zMBkq7ffyvAx6IiLsbTA/gCkk3S5rZ14IkzZTUI6ln9erVbTbLzMwaaTf4pwPz+pi+f0TsCxwKfFDSAY0KRsSciJgSEVO6urrabJaZmTXScvBLGgG8FbiwUZmIWJn/rgLmA1Nbrc/MzDqjnSP+g4A7I2JFvYmStpE0qncYOBhY3EZ9ZmbWAf0Gv6R5wA3AJEkrJB2fJ02jpptH0hhJl+anOwPXSboV+BVwSUT8rHNNNzOzVjRzVc/0BuNn1Bm3EjgsDy8F9m6zfWZmHbHL0bOGugmbDN+5a2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhmvmfu3MlrZK0uDLuZEn3SlqYH4c1mPcQSXdJWiLppE423MzMWtPMEf/ZwCF1xp8eEZPz49LaiZK2BL4NHArsCUyXtGc7jTUzs/b1G/wRcQ3wcAvLngosiYilEfE0cAFwZAvLMTOzDmqnj/8ESYtyV9CL60wfCyyvPF+Rx9UlaaakHkk9q1evbqNZZmbWl1aD/7vAHsBk4D7gq3XKqM64aLTAiJgTEVMiYkpXV1eLzTIzs/60FPwR8UBErIuIZ4EzSd06tVYA4yrPdwNWtlKfmZl1TkvBL2nXytO/BxbXKfZrYKKkv5C0FTANWNBKfWZm1jkj+isgaR7QDYyWtAL4HNAtaTKp62YZ8N5cdgxwVkQcFhFrJZ0AXA5sCcyNiNsHZC3MzKxp/QZ/REyvM/r7DcquBA6rPL8U2OBSTzMzGzq+c9fMrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwvQb/JLmSlolaXFl3P+TdKekRZLmS9qhwbzLJN0maaGknk423MzMWtPMEf/ZwCE1464EXhkRewG/BT7Vx/yvj4jJETGltSaamVkn9Rv8EXEN8HDNuCsiYm1+eiOw2wC0zczMBkAn+vjfBVzWYFoAV0i6WdLMDtRlZmZtGtHOzJL+FVgLnNegyP4RsVLSTsCVku7MZxD1ljUTmAkwfvz4dpplZmZ9aPmIX9JxwBHAOyMi6pWJiJX57ypgPjC10fIiYk5ETImIKV1dXa02y8zM+tFS8Es6BPgk8OaIeKJBmW0kjeodBg4GFtcra2Zmg6eZyznnATcAkyStkHQ8MBsYReq+WSjpjFx2jKRL86w7A9dJuhX4FXBJRPxsQNbCzMya1m8ff0RMrzP6+w3KrgQOy8NLgb3bap2ZmXWc79w1MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjQV/JLmSlolaXFl3EskXSnp7vz3xQ3mPS6XuVvScZ1quJmZtabZI/6zgUNqxp0EXBURE4Gr8vP1SHoJ8Dng1cBU4HONPiDMzGxwNBX8EXEN8HDN6COBc/LwOcBb6sz6d8CVEfFwRDwCXMmGHyBmZjaI2unj3zki7gPIf3eqU2YssLzyfEUetwFJMyX1SOpZvXp1G80yM7O+DPSXu6ozLuoVjIg5ETElIqZ0dXUNcLPMzMrVTvA/IGlXgPx3VZ0yK4Bxlee7ASvbqNPMzNrUTvAvAHqv0jkO+GmdMpcDB0t6cf5S9+A8zszMhkizl3POA24AJklaIel4YBbwRkl3A2/Mz5E0RdJZABHxMHAq8Ov8OCWPMzOzITKimUIRMb3BpAPrlO0B3l15PheY21LrzMys43znrplZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv1mTuru76e7uHupmmLXNwW9mVhgHv5lZYZr6rR6zTdWEky4ZtLruX/rQoNe5bNbhg1aXlcNH/GZmhXHwm5kVxsFvZlYY9/GbNWmXo2cNdRPMOsJH/GZmhXHwm5kVpuXglzRJ0sLK4zFJJ9aU6Zb0aKXMZ9tvspmZtaPlPv6IuAuYDCBpS+BeYH6dotdGxBGt1mNmZp3Vqa6eA4HfRcQ9HVqemZkNkE4F/zRgXoNpr5F0q6TLJL2i0QIkzZTUI6ln9erVHWqWmZnVajv4JW0FvBm4qM7kW4DdI2Jv4FvATxotJyLmRMSUiJjS1dXVbrPMzKyBThzxHwrcEhEP1E6IiMciYk0evhQYKWl0B+o0M7MWdSL4p9Ogm0fSLpKUh6fm+h7qQJ1mZtaitu7clfQi4I3Aeyvj3gcQEWcARwHvl7QWeBKYFhHRTp1mZtaetoI/Ip4AdqwZd0ZleDYwu506zMyss3znrplZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFaTv4JS2TdJukhZJ66kyXpG9KWiJpkaR9263TzMxa19b/3K14fUQ82GDaocDE/Hg18N3818zMhsBgdPUcCZwbyY3ADpJ2HYR6zcysjk4EfwBXSLpZ0sw608cCyyvPV+Rx65E0U1KPpJ7Vq1d3oFlmZlZPJ4J//4jYl9Sl80FJB9RMV515YoMREXMiYkpETOnq6upAs8zMrJ62gz8iVua/q4D5wNSaIiuAcZXnuwEr263XzMxa01bwS9pG0qjeYeBgYHFNsQXAsfnqnv2ARyPivnbqNTOz1rV7Vc/OwHxJvcs6PyJ+Jul9ABFxBnApcBiwBHgC+Kc26zQzsza0FfwRsRTYu874MyrDAXywnXrMzKxzfOeumVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBP4i6u7vp7u4e6maYWeHa/Q9cw96Eky4ZtLruX/rQoNe5bNbhg1aXmQ0PPuI3MytMy0f8ksYB5wK7AM8CcyLiGzVluoGfAr/Po34cEae0Wudwt8vRs4a6CWZmbXX1rAU+FhG3SBoF3Czpyoi4o6bctRFxRBv1mJlZB7Xc1RMR90XELXn4ceA3wNhONczMzAZGR/r4JU0A9gFuqjP5NZJulXSZpFd0oj4zM2td21f1SNoW+BFwYkQ8VjP5FmD3iFgj6TDgJ8DEBsuZCcwEGD9+fLvNMjOzBto64pc0khT650XEj2unR8RjEbEmD18KjJQ0ut6yImJOREyJiCldXV3tNMvMzPrQcvBLEvB94DcR8bUGZXbJ5ZA0Ndf3UKt1mplZ+9rp6tkfOAa4TdLCPO7TwHiAiDgDOAp4v6S1wJPAtIiINuo0M7M2tRz8EXEdoH7KzAZmt1qHmZl1nu/cNTMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8K0FfySDpF0l6Qlkk6qM31rSRfm6TdJmtBOfWZm1r6Wg1/SlsC3gUOBPYHpkvasKXY88EhEvAw4HfhSq/WZmVlntHPEPxVYEhFLI+Jp4ALgyJoyRwLn5OEfAgdKUht1mplZm0a0Me9YYHnl+Qrg1Y3KRMRaSY8COwIP1i5M0kxgZn66RtJdbbRtUzaaOus/UORzrE7z/hveBm3/DcG+273Zgu0Ef70j92ihTBoZMQeY00Z7hgVJPRExZajbYa3x/hvevP+Sdrp6VgDjKs93A1Y2KiNpBLA98HAbdZqZWZvaCf5fAxMl/YWkrYBpwIKaMguA4/LwUcDPI6LuEb+ZmQ2Olrt6cp/9CcDlwJbA3Ii4XdIpQE9ELAC+D/x/SUtIR/rTOtHoYW6z787azHn/DW/ef4B8AG5mVhbfuWtmVhgHv5lZYTar4Je0TtJCSYslXSTpRR1a7s6SLpZ0q6Q7JF2ax/9e0qSasl+X9AlJ3ZIelfS/+WctrpF0RCfa06CNyySN7qfM1bktC/Njpzx+UH5ao+T90wxJa5ooM1fSKkmLa8afKmlR3r5XSBqTx0vSN/O+XSRp34Fqf7Mk/bLm+UckPSVp+8q4bkkX15n3iLzPevf1eweojc28n7aSNEfSbyXdKekf8vhN/6dqImKzeQBrKsPnAR/diHlH9DHte8CHK8/3yn+/CHyuMn4L0iWsuwPdwMWVaZOBZcCBrbajn/mWAaP7KXM1MKXO+A8AZ+ThacCF3j9trWer+3BNE2UOAPYFFteM364y/KHK/jwMuIx0T81+wE0DsW/b3F6/Aq4FZlTGrbd/8riRpEvGd8vPtwYmDdC+aOb99O/A5yuvrdF5eFDeT+08Nqsj/hrXAi+TNKF6dCTp45JOzsNXS/qCpP8BPiypS9KPJP06P/bPs+1KCgwAImJRHpzH+lcqHQAsi4h7ahsTEQuBU4ATaqdJOjkfOVwBnJvbfK2kW/Ljb3K57tzmH+YjjPOk9X8CQ9ILJf1M0ns2YlsNxU9rDKf9s6Wks/OZym2SPlKnzNmSvibpF8CXJE2V9Mt8dPrL3jMPSTMk/Tjvo7slfbnOskZLukHS4XXaeQ117oWJiMcqT7fh+RsljwTOjeRGYAdJu9bOP5iqZzaS9gC2BT4DTO9n1lGkKxEfAoiIP0fEBnf4D+L76V2kgwsi4tmI6L0jeJP/qZp27tzdZCndLHYo8LMmiu8QEX+b5zsfOD0irpM0nnSp6l+RfozuQqXLV/8b+I+IWBkRiyQ9K2nviLiVFDLz+qjrFuBfGkx7FfDaiHhSqQvkjRHxlKSJeZm9dxvuA7yCdORzPbA/cF2eti3pN5POjYhzG9TzH5LWAT8iHa0EG/HTGp0wDPfPZGBsRLwyt2OHBvP/JXBQRKyTtB1wQN6eBwFfAP6hsrx9gD8Dd0n6VkQsz8vemXT/y2ci4somts9zJJ0GHAs8Crw+j6730ypjgfs2ZtkDaDppn1wLTJK0U0SsqlcwIh6WtAC4R9JVwMXAvIh4tk7xAX0/VV4Dp0rqBn4HnBARDzDI76dWbG5H/C+UtBDoAf5Auo+gPxdWhg8CZudlLAC2kzQqIi4HXgqcCbwc+F9JXXmeecC0HGZHAhf1UVdfn/oLIuLJPDwSOFPSbXl51V89/VVErMgv9oXAhMq0n5JCr1HovzMi/hp4XX4c00e7BuI63+G6f5YCL5X0LUmHAI81KHdRRKzLw9sDF+WzmdNJ4dLrqoh4NCKeAu7g+d9YGQlcBXxiY0MfICL+NSLGkbrRes9cBmvftmoacEF+Pf8YeFtfhSPi3cCBpO6hjwNzGxQd6PfTCNKvFVwfEfsCNwBfydM29W2+2QX/kxExOT/+OdKvhq5l/fV8Qc08f6oMbwG8prKMsRHxOKSjjYg4PyKOId21fECeZx7wdlIoLWp0tJLtA/ymwbRqOz4CPADsTToy2aoy7c+V4XWsf9Z2PXBoo9PKiLg3/30cOJ/0C6sweD+tMSz2T+7a6f0C/JSIeIS0L64GPgic1WD+altPBX6RzxLeVLNejfbhWuBm4O/6aGMzzuf5s4tmflplSEjaC5gIXClpGelDoL/uHiLitog4HXgjz69nrYF+Pz0EPAHMz88vIn33AsPgp2o2t+Cv5wFgJ0k7Stoa6OvKjSuo9PFKmpz/viGfLiJpFLAH6YiViPgd6UUwiz66EfKL/N9I3RL92R64Lx+FHEO6M7oZn81t+U6d+kcoX6UgaSRpO/T2rQ/lT2tscvsnItZVPlw+m7fbFhHxo1ymmStjtgfuzcMzmigP6ajwXcDLVecfG/Uld2H0ejNwZx5eAByrZD/g0YjYlLp5To6ICfkxBhgrqe6vTEraNner9JoMbPB9TR0dfz/l98d/kb6EhnQWckce3uR/qmazD/6IeIb0pd1NpD7BO/so/iFgitJlb3cA78vjXwX0SFpEOqU7KyJ+XZlvHqmLYT7re13+cu8uUuB/KCKuaqLZ3wGOk3Qjqd/4T/2UrzoReEGdLw23Bi7P67CQFEpn5mnfB3ZU+mmNjwIbFTrtGCb7Zyxwde5iOhv4VBOr9mXgi5Kup/mgIXcVTQNeL+kDtdMlzSOt4yRJKyQdnyfNUvryeRFwMPDhPP5SUlfVEtL+3mCZQ2gaG+6T+Tz/hfyBeR1XSFpBOiP7hPIlyaSramY0Uc9AvJ8APgmcnLf5McDH8vghez81yz/ZYGZWmM3+iN/MzNbn4DczK4yD38ysMA5+M7PCOPjNzArj4DczK4yD38ysMP8HCFh3sSdTyLEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "time_df = pd.DataFrame(timings)\n", + "time_df.mean().plot.bar(yerr=time_df.std(), rot=0, title='Computation time for optimal config, s');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`PureSVD-s` compares favoribly to the iALS, even though it requires higher rank value, which results in a longer training time comparing to `PureSVD`. Another interesting measure is what time does it take to achieve approximately the same quality by all models. \n", + "Note that all models give approximately the same quality at the optimal rank of iALS. Let's compare training time for this value of rank." + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF7RJREFUeJzt3X24XVVh5/HvjxcRJIKYC0ggpgpNBxgTbIwv1DbKi7yKjoxNxrGo+ESd8ogdHYpFAamttFNfWmBKI0TUkcAgDYMlCCkVgVGRAIGAQOUlSAiSQCSAvBn4zR97XTkczsm9Oeckl2T9Ps9zn7vP2mvvtfbZyW/vvfY+58o2ERFRj83GugMREbFhJfgjIiqT4I+IqEyCPyKiMgn+iIjKJPgjIiqT4I91JulkSf97rPuxqZJ0q6QZG6itxyW9bkO0FS8dCf5NlKSlkp4s/7F/KekcSduOUV/eKOmq0pcHJR3bMm+SpB9IekLS7ZL2H4s+Dkp530e9DWW/fLG1zPZetq9cD327UtJH29ra1vbdg24rXtoS/Ju2w21vC0wF9gE+u6E7IGk88H3gn4BXA7sDl7dUmQfcWOadAHxX0tCG7mdETRL8FbD9S+AymgMAAJIOlXSjpEcl3Sfp5JZ5kyRZ0lGSfiHpIUkndFq3pC0lzZN0oaSXdajy34HLbH/H9tO2H7N9W1n2d4E3AifZftL2hcAS4H1d2tpa0pcl3StptaRrJG1d5r27DJE8Us5s/0PLcksl/Q9JN0v6taSzJe0k6VJJj0n6V0mvatv22ZKWS3pA0qdb1vWCM3RJMyQtK9PfBiYC3ytXN8eV8gvKVdfqcuWzVymfDXwAOK7U/15Lf/cv01tJ+lrpy/IyvVVr25I+LWlF6euHu7x3fwW8HTi9tHV6Kbek3Vu27X+V9+VxSf9P0s6lzV+VK7J9Wta5S9nvKyXdI+mTndru0p8/l3R/ef/vkLTfaJeN/iX4KyBpV+Bg4M6W4l8DfwJsDxwKfELSe9oW/QNgMrAfcGJrmJb1bg1cBDwNvN/2Mx2afwuwStKPSjh9T9LEMm8v4G7bj7XUv6mUd/J3wO8DbwN2AI4DnisHkHnAp4AhYAFN+LYeiN4HHAD8LnA4cCnwF8B4mv8H7aH1DmAP4EDg+NEM39j+IPALypWW7b8tsy4t69oRuAH4Tqk/p0z/bal/eIfVnkDzHk4FpgDTgc+1zN8Z2A6YABwNnDF8EGvr2wnA1cAxpa1jumzG+8v6x9Ps1x+XPo8Hvgt8BUDSZsD3aPbXBJp/I5+S9K61vUdl2cnAMcCbbI8D3gUsHWm5GJwE/6btIkmPAfcBK4CThmfYvtL2EtvP2b6ZJjj/qG35L5Qz8Zto/oNPaZn3SpohnLuAD9t+tksfdgWOAo6lORu+p7QFsC2wuq3+amBc+0pK0HwEONb2/baftf0j208DfwxcYnuh7d/QHCC2pjlADDvN9oO276cJwGtt31iWn08zFNa+7b+2vQT4BjCry/aNyPbccqXzNHAyMEXSdqNc/APAKbZX2F4JfAH4YMv835T5v7G9AHic5mDdq/m2r7f9FM378pTtb5X9ez7Pv09vAoZsn2L7mXKf4OvAzFG08SywFbCnpC1tL7V9Vx99jnWU4N+0vaecUc0Afo/mrA0ASW9Wc1N1paTVwMdb5xe/bJl+giaoh70FeANwqtf+TX9P0oTJdSVMvgC8rQTf4zQHkFavBB7jxcYDL6c50LTbBbh3+IXt52gOdhNa6jzY1qf21+03vu9rmb63tLHOJG0u6VRJd0l6lOfPbNvf625esG0d+vKw7TUtr9v307oa7fv0WmCXMrT2iKRHaK6gdhqpAdt30lydnQyskHSepJ7e3+hNgr8Ctn8InENzJjzsXOBiYDfb2wFnAlqH1V4OfAm4QtLa/rPfDLQeGIanBdwKvE5S6xn+lFLe7iHgKeD1HeYtpwmiZsWSgN2A+0fYhrXZrWV6YmkDmiGybVrm7dy2XPtB8L8ARwD70wzJTBruZpf67V6wbW19WVeD/Cre+4B7bG/f8jPO9iGj6oh9ru0/oNk2A38zwL7FCBL89fgacICk4Ru844BVtp+SNJ0moNZJGcM+lyb8u53BfgN4r6SpkrYEPg9cY/sR2/8OLAZOkvRySe+luYq4sENbzwFzga+Um4qbS3prudH5f4BDJe1X2vg0zfj0j9Z1m1p8XtI25Ubsh2mGOSj9PUTSDpJ2pjlzbfUg0Ppc/LjSl4dpDhh/PUL9dvOAz0kaKu/xiUCvn6EYqa118VPg0XKTduuyP/aW9Cb47Y3njgcaSZMlvbPsu6doriS6DRXGepDgr0QZH/4WTfAC/DfglHIP4ESa8OxlvX9Jc4P3XyXt0GH+v9EMAVxCc59hd154kJkJTAN+BZwKHFn62slnaJ76uQ5YRXOWuJntO4D/CpxGc2VwOM0N1k43m0frhzQ3w68A/s728COo36a537GU5qrn/LblvkQT1I9I+gzNe34vzdXHz4CftNU/m2as+xFJF3XoxxeBRTRXTktobrR+sUO90fh74MjyhM4/9LgOAMqY/+E0N53voXnfz6K5qoHmiunHXRbfimZfP0QznLgjzb+R2ECUP8QS8TxJk2iCbMu2sfNYB5LOAi6wfdlY9yVeLMEf0SLBHzXIUE9ERGVyxh8RUZmc8UdEVGaLse5AJ+PHj/ekSZPGuhsRERuN66+//iHbo/qCw5dk8E+aNIlFixaNdTciIjYaku4duVYjQz0REZVJ8EdEVCbBHxFRmQR/RERlEvwREZUZ8akeSXOBw4AVtvcuZefz/B972B54xPbUDssupflu9WeBNbanDajfERHRo9E8znkOcDrNtwwCYPuPh6clfZkX/xWlVu+w/VCvHYyIiMEaMfhtX1W+uOpFyh+8eD/wzsF2KyIi1pd+x/jfDjxo++dd5hu4XNL1kmavbUWSZktaJGnRypXdvo49IiL61e8nd2fx/B/O7mRf28sl7QgslHS77as6VbQ9B5gDMG3atHxzXEQFJh1/yVh3Yb1ZeuqhY92Frno+45e0BfCfePFfIPot28vL7xXAfGB6r+1FRMRg9DPUsz9wu+1lnWZKesXwH9GW9ArgQOCWPtqLiIgBGDH4Jc2j+duZkyUtk3R0mTWTtmGe8kewF5SXOwHXSLqJ5g8zX2L7+4PrekRE9GI0T/XM6lL+oQ5ly4FDyvTdwJQ++xcREQOWT+5GRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFRmxOCXNFfSCkm3tJSdLOl+SYvLzyFdlj1I0h2S7pR0/CA7HhERvRnNGf85wEEdyr9qe2r5WdA+U9LmwBnAwcCewCxJe/bT2YiI6N+IwW/7KmBVD+ueDtxp+27bzwDnAUf0sJ6IiBigfsb4j5F0cxkKelWH+ROA+1peLytlHUmaLWmRpEUrV67so1sREbE2vQb/PwKvB6YCDwBf7lBHHcrcbYW259ieZnva0NBQj92KiIiR9BT8th+0/azt54Cv0wzrtFsG7NbyeldgeS/tRUTE4PQU/JJe0/LyvcAtHapdB+wh6XckvQyYCVzcS3sRETE4W4xUQdI8YAYwXtIy4CRghqSpNEM3S4GPlbq7AGfZPsT2GknHAJcBmwNzbd+6XrYiIiJGbcTgtz2rQ/HZXeouBw5peb0AeNGjnhERMXbyyd2IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyIwa/pLmSVki6paXsf0q6XdLNkuZL2r7LskslLZG0WNKiQXY8IiJ6M5oz/nOAg9rKFgJ7234D8O/AZ9ey/DtsT7U9rbcuRkTEII0Y/LavAla1lV1ue015+RNg1/XQt4iIWA8GMcb/EeDSLvMMXC7pekmzB9BWRET0aYt+FpZ0ArAG+E6XKvvaXi5pR2ChpNvLFUSndc0GZgNMnDixn25FRMRa9HzGL+ko4DDgA7bdqY7t5eX3CmA+ML3b+mzPsT3N9rShoaFeuxURESPoKfglHQT8OfBu2090qfMKSeOGp4EDgVs61Y2IiA1nNI9zzgN+DEyWtEzS0cDpwDia4ZvFks4sdXeRtKAsuhNwjaSbgJ8Cl9j+/nrZioiIGLURx/htz+pQfHaXusuBQ8r03cCUvnoXEREDl0/uRkRUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUZlTBL2mupBWSbmkp20HSQkk/L79f1WXZo0qdn0s6alAdj4iI3oz2jP8c4KC2suOBK2zvAVxRXr+ApB2Ak4A3A9OBk7odICIiYsMYVfDbvgpY1VZ8BPDNMv1N4D0dFn0XsND2Ktu/Ahby4gNIRERsQP2M8e9k+wGA8nvHDnUmAPe1vF5Wyl5E0mxJiyQtWrlyZR/dioiItVnfN3fVocydKtqeY3ua7WlDQ0PruVsREfXqJ/gflPQagPJ7RYc6y4DdWl7vCizvo82IiOhTP8F/MTD8lM5RwP/tUOcy4EBJryo3dQ8sZRERMUZG+zjnPODHwGRJyyQdDZwKHCDp58AB5TWSpkk6C8D2KuAvgevKzymlLCIixsgWo6lke1aXWft1qLsI+GjL67nA3J56FxERA5dP7kZEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVKbn4Jc0WdLilp9HJX2qrc4MSatb6pzYf5cjIqIfW/S6oO07gKkAkjYH7gfmd6h6te3Dem0nIiIGa1BDPfsBd9m+d0Dri4iI9WRQwT8TmNdl3lsl3STpUkl7dVuBpNmSFklatHLlygF1KyIi2vUd/JJeBrwbuKDD7BuA19qeApwGXNRtPbbn2J5me9rQ0FC/3YqIiC4GccZ/MHCD7QfbZ9h+1PbjZXoBsKWk8QNoMyIiejSI4J9Fl2EeSTtLUpmeXtp7eABtRkREj3p+qgdA0jbAAcDHWso+DmD7TOBI4BOS1gBPAjNtu582IyKiP30Fv+0ngFe3lZ3ZMn06cHo/bURExGDlk7sREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERl+g5+SUslLZG0WNKiDvMl6R8k3SnpZklv7LfNiIjo3RYDWs87bD/UZd7BwB7l583AP5bfERExBjbEUM8RwLfc+AmwvaTXbIB2IyKig0EEv4HLJV0vaXaH+ROA+1peLytlERExBgYx1LOv7eWSdgQWSrrd9lUt89VhGbcXlIPGbICJEycOoFsREdFJ32f8tpeX3yuA+cD0tirLgN1aXu8KLO+wnjm2p9meNjQ01G+3IiKii76CX9IrJI0bngYOBG5pq3Yx8Cfl6Z63AKttP9BPuxER0bt+h3p2AuZLGl7Xuba/L+njALbPBBYAhwB3Ak8AH+6zzYiI6ENfwW/7bmBKh/IzW6YN/Gk/7URExODkk7sREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlBvU3dzdak46/ZKy7sF4tPfXQse5CRLzEVB/8sXHLgTti3WWoJyKiMgn+iIjKJPgjIiqT4I+IqEyCPyKiMgn+iIjKJPgjIirTc/BL2k3SDyTdJulWScd2qDND0mpJi8vPif11NyIi+tXPB7jWAJ+2fYOkccD1khba/llbvattH9ZHOxERMUA9n/HbfsD2DWX6MeA2YMKgOhYREevHQMb4JU0C9gGu7TD7rZJuknSppL3Wso7ZkhZJWrRy5cpBdCsiIjroO/glbQtcCHzK9qNts28AXmt7CnAacFG39dieY3ua7WlDQ0P9disiIrroK/glbUkT+t+x/c/t820/avvxMr0A2FLS+H7ajIiI/vTzVI+As4HbbH+lS52dSz0kTS/tPdxrmxER0b9+nurZF/ggsETS4lL2F8BEANtnAkcCn5C0BngSmGnbfbQZERF96jn4bV8DaIQ6pwOn99pGREQMXj65GxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmb6CX9JBku6QdKek4zvM30rS+WX+tZIm9dNeRET0r+fgl7Q5cAZwMLAnMEvSnm3VjgZ+ZXt34KvA3/TaXkREDEY/Z/zTgTtt3237GeA84Ii2OkcA3yzT3wX2k6Q+2oyIiD5t0ceyE4D7Wl4vA97crY7tNZJWA68GHmpfmaTZwOzy8nFJd/TRt5ey8XTY/vVFucYatOy/jdsG239jsO9eO9qK/QR/pzN391CnKbTnAHP66M9GQdIi29PGuh/Rm+y/jVv2X6OfoZ5lwG4tr3cFlnerI2kLYDtgVR9tRkREn/oJ/uuAPST9jqSXATOBi9vqXAwcVaaPBP7Ndscz/oiI2DB6HuopY/bHAJcBmwNzbd8q6RRgke2LgbOBb0u6k+ZMf+YgOr2R2+SHszZx2X8bt+w/QDkBj4ioSz65GxFRmQR/RERlEvwjkPSspMWSbpF0gaRtBrTenST9i6SbJP1M0oJSfo+kyW11vybpOEkzJK2WdGP5qoyrJB02iP5sCrKv6iXpR22v/0zSU5K2aymbIelfOix7WNlPw/v3Yxuiz2MpwT+yJ21Ptb038Azw8dEuWB5h7eYUYKHtKbb3BIa/6+g8Wm6CS9qM5omo80vR1bb3sT0Z+CRwuqT9Rr85m7Tsq0rZfltb0SyaJw/fu7blJG1Jc8P3cNtTgH2AK9dHH19KEvzr5mpgd0mTJN0yXCjpM5JOLtNXSvprST8EjpU0JOlCSdeVn33LYq+h+ZwDALZvLpPzeOHTT38ILLV9b3tnbC+mCaVjBriNm4qNZl9J2lzSOeVKZYmkP+tryysk6fGW6dcD2wKfozkArM04mqcbHwaw/bTtTfVbA34rwT9K5YzwYGDJKKpvb/uPbH8Z+Hvgq7bfBLwPOKvUOQM4W9IPJJ0gaRf4bag8J2lKqTeTJmC6uQH4vXXfok3XRrivpgITbO9t+z8C3xhFv6O7WTT74WpgsqQdu1W0vYrm80b3Spon6QPlym2Ttslv4ABsLWkxsAj4Bc1nE0Zyfsv0/jSX+Itp/oG9UtI425cBrwO+ThMGN0oaKsvMA2aWADsCuGAtbeVL7563se6ru4HXSTpN0kHAo6Pod3Q3EzjP9nPAPwP/eW2VbX8U2A/4KfAZYO567+EY6+e7emrxpO2prQWS1vDCg+bL25b5dcv0ZsBbbT/ZvuJytnEucG656fSHwIU0YXI58EPgZtsr1tK/fYDbRrktm7qNYl+p+Urz60vZxbZPLFcN7wL+FHg/8JG1bml0JOkNwB7AQjVfBPwymgPrGWtbzvYSYImkbwP3AB9avz0dWznj782DwI6SXi1pK2BtT2tcTsu4rqSp5fc7h586kTQOeD3NWSq276IZczyVtQwdlH/kn2eEf9SVe8ntK9vPlpvQU0vojwc2s31hqfPG3je3erOAk21PKj+7ABMkdfzmSknbSprRUjQVeNE9mk1Nzvh7YPs3ar6a4lqas4Pb11L9k8AZkm6meb+vonna5PdphhWGz0jPsn1dy3LzgC8B89vW93ZJNwLbACuAT9q+YgCbtUnaSPbVBOAbLWPLn12XbYwXmElzf6fV/FJ+Lc3fBFnWMm8WcJykfwKepLkC/NAG6OeYylc2RERUJkM9ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZn/D4l8dOyzlPZ0AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fixed_rank_timings = {}\n", + "for model in models:\n", + " model.rank = ials_best_config['rank']\n", + " model.build()\n", + " fixed_rank_timings[model.method] = model.training_time[-1]\n", + "\n", + "pd.Series(fixed_rank_timings).plot.bar(rot=0, title=f'Rank {ials.rank} computation time, s')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By all means computing SVD on this dataset is much faster than ALS. This may, however, vary on other datasets due to a different sparsity structure. Nevertheless, you can still expect, that SVD-based models will be perfroming well due the usage of to highly optimized BLAS and LAPACK routines." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Bonus: scaling for iALS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You may reasonably question whether that scaling trick also works for non SVD-based models. Let's verify its applicability for iALS. We will reuse Polara's built-in scaling functions in order to create a new class of the *scaled iALS-based model*." + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.models import ScaledMatrixMixin" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [], + "source": [ + "class ScaledIALS(ScaledMatrixMixin, ImplicitALS): pass # similarly to how PureSVD is extended to its scaled version" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "sals = ScaledIALS(data_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to save time, we will utilize the optimal configuration for scaling, found by tuning scaled version of PureSVD. Alternatively, you could include scaling parameters into the grid search step by extending `als_param_grid` and `als_param_names` variables. However, taking configuration of PureSVD-s should be a good enough approximation at least for verifying the effect of scaling. The tuning itself has to be repeated from the beginning." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### hyper-parameter tuning" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "60/60\n", + "[22:42<00:24, 22.70s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 60/60 [22:42<00:24, 22.70s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sals_best_config, sals_param_scores = find_optimal_config(sals,\n", + " als_param_grid,\n", + " als_param_names,\n", + " target_metric,\n", + " init_config=[init_config,\n", + " ssvd_best_config], # the rank value will be overriden\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "15/15\n", + "[07:03<00:39, 28.18s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 15/15 [07:03<00:39, 28.18s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sals_best_rank, sals_rank_scores = find_optimal_config(sals,\n", + " rank_grid,\n", + " 'rank',\n", + " target_metric,\n", + " init_config=[init_config,\n", + " ssvd_best_config,\n", + " sals_best_config],\n", + " return_scores=True,\n", + " iterator=track)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### visualizing rank tuning results" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XmcVNWd///Xp/bqfe8GGuim2ZcWsN3ABWPAJUTjkgF01BgdkxlNJs6q80uMMd9Ekzjf5OsXv2OcxCwmkSRmnBATYlAHSVwB2TeBBrsb6H3vrr3O7497u6luummgKZrl83w86lF3OffWuQVd7zrn3jpXjDEopZRSJ8sx0hVQSil1dtMgUUopNSwaJEoppYZFg0QppdSwaJAopZQaFg0SpZRSw6JBotRpICJrROS+k9x2nIh0iojzVNdrJIjIARH5uD39byLyg5GukxoeDRJ1StgfDgH7A69WRH4sImkjXa+zUeIHLYAxpsoYk2aMiY1kvZLBGPNNY8x9ACJSIiJGRFwjXS91YjRI1Kn0SWNMGjAbmAM8MsL1UUqdBhok6pQzxtQCr2IFCgAi4hWRp0SkSkTqRORZEfEnrL9JRDaJSLuI7BOR6+zlo0VkpYg0i8heEfmbhG0eE5Ffi8jPRKRDRLaKyGQReURE6kWkWkQWJZRfIyL/S0TetltOvxORXBH5uf2660SkJKH8VBFZbb/2bhH5q4R1PxaRZ0Tk9/ZrvyciZQnrF4rILhFpE5HlgCSsKxORN0SkSUQa7dfPste9AIwDfmfX8V/6f1M/jvfkVyLyU7te20WkImH9v4rIQXvdbhG5ZqB/Q/t9WWm/L++LyNdF5C/2uqNaDoldd8c6vgFe5zER+Zk9u9Z+brWP/Sr7GGcllC+wW775A+1PjQwNEnXKiUgxcD2wN2Hxt4DJWOEyERgDPGqXvxj4KfDPQBZwJXDA3u5FoAYYDdwGfLPfh98ngReAbGAjVoA57P0/Dny/X/WWAnfa68uAd4AfATnATuCrdp1SgdXAL4ACYBnw/0RkRsK+lgFfs197L/ANe9s84DfAl4E8YB8wP/EtAp6wj2kaMBZ4DMAYcydQhd26M8Z8m6MN9Z7cCKyw38uVwHK7XlOAB4GLjDHpwLUceZ/7ewYIAqOAz9qP4zXo8Q3hSvs5yz72N+3j+OuEMsuA14wxDSdQH5Vsxhh96GPYD6wPpE6gAzDA61gfCGB9sHQBZQnlLwP229PfB747wD7HAjEgPWHZE8CP7enHgNUJ6z5p18Fpz6fbdempxxrg/0so/+/Aqn7bb7KnlwB/7lef7wNftad/DPwgYd0NwC57+i7g3YR1gvXBf98g792ngI393suPJ8yX2MfhOs735LWEddOBgD09EagHPg64j/Fv6QQiwNSEZd8E/tK/Pgnr15zM8dn1/dkx9nsJUA047Pn1wF+N9P93ffR9aItEnUqfMtY33QXAVKxv4wD5QAqwQURaRaQV+KO9HKwPx30D7G800GyM6UhY9hFWa6JHXcJ0AGg0R05KB+zntGOU7z/fU3Y8cElPfe063wEUJZSvTZjuTth2NNaHHwDG+gTsnbe7Z1bYXUztwM848l4N5Xjek/718omIyxizF/gS1od3vV2H0QO8Rj5WaFUnLPvoOOs33OPrwxjzHtaXkKtEZCpWGK48mX2p5NEgUaecsbokfgw8ZS9qxPqQnmGMybIfmcY6MQ/WB1bZ0XviEJAjIukJy8YBB5NT8z6qgTcT6tvT3fK3x7HtYaxwBEBEJHEeqwVhgHJjTAZW140krD/WkNzDek+MMb8wxlyOFZQGq8uxvwYg2q/O4xKmu+znlIRliQE71PENWr1Blv/E3sedwEvGmOBx7EudRhokKlm+BywUkdnGmDjwn8B3RaQAQETGiMi1dtkfAveIyDUi4rDXTTXGVANvA0+IiE9EyoF7gZ+fhvq/AkwWkTtFxG0/LhKRacex7e+BGSJyi31C+ov0/aBNx+qCaxWRMVjnhhLVARMG2vFw3hMRmSIiHxMRL9b5jwBWN1n/14gB/wU8JiIpIjIduDthfQNWcP21iDhF5LP0/SIw1PENpgGIc/SxvwDcjBUmPz3OfanTSINEJYX9YfNT4Cv2on/FOiH9rt3d8RowxS77PnAP8F2gDXgT6xszWCdXS7C+ib+MdY5i9WmofwewCOvk/CGs7qJvAd7j2LYR+DTwJNAETALeSijyNWAu1rH+HutDO9ETwJftLrV/GuAlTvY98dp1arSPpwD4t0HKPojVVVeL1br8Ub/1f4MVEE3ADKxw6zHU8Q3IGNONdcHCW/axX2ovrwE+wGqx/Pl49qVOL7G6b5VSanAi8hmsk+mXj9DrPw8cMsZ8eSReXx2b/oJUKXVGE+u3Pbdg/chVnYG0a0spdcYSka8D24DvGGP2j3R91MC0a0sppdSwaItEKaXUsJwX50jy8vJMSUnJSFdDKaXOKhs2bGg0xgw5rtl5ESQlJSWsX79+pKuhlFJnFRE5rhENzosgUUqp08kYQ3NXGBHB73bidTlwOI7nx/1nJw0SpZQ6QfG4ob4jxMHWbmpaAgmPbg62BDjYGiAUjffZxuty4HM78bud+D1WuPg9Tnwua97vduJ1O6z1bqdVNqFc7zK7XIrHRW6qh/x0Lz73yN48U4NEKaX6icUNde1BaloCVlg020HRagXFodYg4VjfoMhN9VCc7WfaqAw+Pr2QUZk+HCIEIjEC4RjBaIxgOEYwEreWRWIE7UdDR7S3XChqPQciMeLHeVFths9FQYaP/DQvBRleCtK95Kd7KUj3UZBuLctP85Hhd2EN/XZqaZAopc55xhjCsTjhaJxQ9Mhzb1jYrYmesDjcGiTa71M8P93LmCw/M8dkct3MUYzJ9lOc7Wdstp/RWX5SPKf249QYQyRmCERihOzgscInbgdNlMaOMA2dIerbg9R3hGjoCLGxqpX6jiDBSPyofXpdDvJ7Q+ZI0OSn9wSQj/x0L7mpHlzO47+oN6lBItZd7v4P1v0NfmCMebLfei/WeEwXYo3Zs8QYc8C+0dFzPcWAx4wxL9vbHMC650UMiBpjKlBKnfVicUNjZ4jDbUFq2wIcbgvSGYwSjvX98A9FY4SjfUPBKjPA8mickB0gxyICBeleirNTmDsumzHlfoqzUyjO9jMm28+YLP9p7z4SETwuweNygN99QtsaY+gIRWnoCFHfHqK+I0iDHTQ9gbO/sYv39jfT2h05anuHQE7qkMPK9UpakIiIE+suawuxbuqzTkRWGmN2JBS7F2gxxkwUkaVYg+Itwfola4UxJioio4DNIvI7Y0zU3u5qe2A8pdRZIBqLU9/RExJBDrcFrOf2IIdbrem6jhCxAfpyHAIelwOP04HX7bSeXQ48riPPPreDTL8bj7Pvck9vOetcg7UPR2+5gnQfxdl+RmX58LpG9jzDqSQiZPjcZPjclOWnHbNsKBqjsTNMfXuwN2issAmy4ThfL5ktkouBvcaYSgARWQHcBCQGyU0cuQXnS8ByERF7FNAePo59fwal1AgK211Ete3BPq0JKzCs0GjoCB3V3+9zOxid6aco08dlZXmMyvRRlOnrfS7K8JHpd59QF4s6cV6XkzFZVqurvycHKD+QZAbJGPreYa0G67aZA5axWx9tQC7QKCKXAM9jDSd+Z0JrxAB/EhEDfN8Y8xxKqaQKRmIcaOqisqGL/Y1d7GvopLKhi4OtARo7Q/QfaSnV42RUlp9RmT4mFeTb4WB98x+V6WNUhj9pJ37V6ZfMIBnof0j/lsWgZexbbM6wbyT0ExFZZd8Zbb4x5pB9g6TVIrLLGLP2qBcXuR+4H2DcuHH9Vyul+onHDbXtQSobuqhs7LSfu6hs6ORga6BPWBRl+JiQn8rHphT0hkNRphUcozJ9pPtOrE9fnd2SGSQ19L1VZzHWjXgGKlNj30kuE2hOLGCM2SkiXcBMYL0x5pC9vF5EXsbqQjsqSOyWynMAFRUV2jWmlK0jGGF/o9W6qGzoZJ89faCxi0DkyA0TUz1OSvNTmTsum9suLGZCfhoT8lIpzUsl1asXfKojkvm/YR0wSURKsW7LuRS4vV+ZlVi38HwHuA14wxhj7G2q7e6u8Vh30jsgIqmAwxjTYU8vAh5P4jEodVbqCEaobQtS1dxtd0VZoVHZ2EVDR6i3nENgbE4KpXmpXDYhlwn5qUzIT6UsP42CdK92PanjkrQgsUPgQeBVrMt/nzfGbBeRx7FaFiux7tX9gojsxWqJLLU3vxx4WEQiWPdw/jtjTKOITABetv9zu4BfGGP+mKxjUOpM03OJbG2bdXK7zj7BXWfP17Zb013hvrdiz05xMyE/jQWT8ynNT2VCXhpl+amMy005p65WUiPjvLgfSUVFhdFBG9WZLhCOWWHQFqS2PUBtW8i6GiohNOoHuETW5RAKM3wUZngpyvRRmGFd8VSUaV3aOiEvjexUzwgdlTqbiciG4/mtnnZ0KnUaNXWG+LCukz31Heyp6+Sj5m5q7d9UtAejR5VP97ootC+FLbMvke2ZL8rwUZjpJS/Ve04PCKjOfBokSiVBS1eYD+s6+LC+kz11HXxYZwVHU1e4t0y610VJXirjc1O5dEJun5ZEof2cpie11VlA/5cqNQyt3WE+rOu0g6LDbm100th55IR2mtfFxII0rplWwOTCdCYVpjO5MI2iDJ+ezFbnBA0SpY5DW3eEPfUdR0LDnk68AirV42RiYTpXT8m3AyONyYXpjMrUwFDnNg0SpfoxxrCrtoPVO+p4f38zH9Z1UJ8QGCkeJ5MK0rhyUj6T7bCYVJjGmCy/BoY6L2mQKIU1qOD6j1r40/Y6Vu+spbo5gAhMK8rg8kl5TLa7oyYVpDMmy68nt5VKoEGizlvd4ShrP2zkTztqeWNXPa3dETwuB/PLcvm7BRO5ZloBBem+ka6mUmc8DRJ1XmnsDPH6zjr+tL2Ov+xtJBSNk+l387GpBSyaXsiVk/N1+A+lTpD+xahzXmVDJ6t31LF6Rx0bqlowBsZk+Vl28TgWzSjkopIc3DpUuVInTYNEnXPiccOmmtbe8Nhb3wnAjNEZ/P01k1g4vZDpozL0xLhSp4gGiTonhKIx3t7XxJ+21/HazjoaOkI4HcKlE3L460vG8fHphRRnp4x0NZU6J2mQqLNSNNBOw4fvU11Tza+aSli1N0RXOEaqx8mCKQUsnF7I1VMKyEzR+2IolWwaJOqMZYyhoSNEZUMHTfu3YmrWkd60ieKuHZTEPmKUGEYBc3Hwd+kXEJt8PeMuuxVvwcSRrrpS5xUNEjXi2oMRDjR29d43o7G2Gl/9RoratzIjvodyRyXpEgCgg1Sq/NN4L+djxEZfyKjCQspa32XCh6tg4zetR/5UmHI9TLkBxlwIDh0mXalk0mHk1WkRisaobu7uved3zx36DjY0U9j9IXMce5nj2Mtsx16KpRGAGE5aMyYTKpyLr/RisibOw5E3ERyDXGHVcgB2/xF2/wE+egviUUjNh8nXWqEy4Wrw6HkSpY7X8Q4jr0GiTrl43LDjcDtvftjAugPN7G/sorq5m7gxlEots2Uvl3krudBZSUm0EifWTZgi6cU4iitwjr0Iiitg1AXg9p9cJQItsPd1K1T2rIZQO7h8VphMuR4mXwfphafwqJU692iQJNAgSb6GjhB/3l3L5ne30bBxG/kNVUxqq6Es1IA3BdJSQqR5WvF6A7j9MVzpPlyl03BNvgjHhEthTEXyPtijYah6G3avgl1/gLYqa/mYiiNdYAXTQC8HVqoPDZIEGiSnXrC9g81rN7D/3U107dhJzuEDlLQfxheLWAUc4M2I4k6NEAs7iYZ9RLvARONH7cuRno6roABXQT6u/ALIyyaWm0kkO41gTgqBTD+dGW66JExXpIvuSDddUevZKU7K88uZUzCHXH/u0BU3Bup3WC2V3aswNRvAQDxtHKbkGszYK4nnzcREY8SDQRypqXgnTkQG605T6hymd0hUp4QxhmhdHcEdO6nduJX6jVth3x6yWurJwHABEPL6cBamkT3eRZqnHl92BBk3it2Tr2Jv7ng60/LoMlG6w12E21qhsQlHYxuulnY8zV34WwOktB0kveoAGTti5HSAy84bB5BqPzp84EqHWJrQnQbxDDdtvjhrI3HejRrynJmMcuVS4Mwi15GOP+bEhEKYUIh4KIQJBo9M9y4rhngciAKv2o++nHl5pM67jLT580mdNw9Xfv7pevuVOitoi0T1MuEwof37Ce7cSWjXbrp27KB75y6cHe29ZQ6l5lKXV0zu+Awmjuqg2LUdd/QjDrpdbB41ha05Y9gqUXZ1fEQkHumzf6/TS6o7lRRXCqnuVGvanTDds9yZQkZISG+PkNoawt8axNvSjaulHWdjG9LUQryhiWhjI8Ss8ytGIOp2EHQawi5D2AUxjwuPLxV/agZp6bmkpWbj9PkRrxfxeXF4fYjXi8PnRbw+xAWOtv1I03akfjOOWDvRkJuuznF0VYWJtXdbxzFlCqnz55M6fx4pF16Iw6cDO6pzk3ZtJdAgOZoxhtCePXS/+y7BHTsJ7t5NaM8eiFr3DY843ezPKGJfxmhqcorJmV7KlRO7mWs+IFb9BtsIssWXwtbsQrY54rTErMtz/S4/03OnU55Xzqz8WUzNmUqmN5MUVwoux6ltAJtYjHgggMPjAbcbESFu4uxv288H9R/wQd0HbKzfyMHOg711uyD/AuYWzmVuwVxm5c0ixT3IVVzxGNSsg52/g+3/jWmrIdSeQmd4Ol11XgK7azCRCOL1klJRYQfLfLyTJ+nQK+qccUYEiYhcB/wfwAn8wBjzZL/1XuCnwIVAE7DEGHNARC4GnuspBjxmjHn5ePY5EA0SS7y7m65336Nz7Zt0rl1L9NBhAKKZ2RzKG8tmbwHbUwqpzBxN1qQJXF8qfNy1gXDja2xr3sZWt5Mt/hQOuKzzBYIwIXMCs/JnUZ5fTnleOWVZZac8MIartquWTfWb2FC3gY31G/mw5UMMBqc4mZYzrTdYZhfMHvg8SzwOB9fD9pdh+39DxyHixku362K6Wgvo3FlLuHI/AK78fFLnzSP18stJnXcZrtzjOG+j1BlqxINERJzAh8BCoAZYBywzxuxIKPN3QLkx5vMishS42RizRERSgLAxJioio4DNwGjADLXPgZzPQRKurqZzjRUc3e+9hwmHkZQUYnMqeCNzEj+NjaLBn0VemofLy3K5PH8P7q43+LBxA1ujbezweAjZJ5pz3OmUF86lPP8CZuXPYkbuDNI96SN8hCeuI9zBpvpNbKzfyAf1H7C1YSvheBiAkowS5hbOZU7BHOYWzGVs+ti+LYx4HGreh23/BTt+C5214PITKVxAV6CMrsoOut55n1hbGwDe6dOscyvz5+OfO9dqPSl1ljgTguQyrJbEtfb8IwDGmCcSyrxql3lHRFxALZBvEiolIqXAu8AY4KKh9jmQ8ylITDhM94YNdL65ls433yS83/qm7Bk/ntSrrqJm8hyWN6WxZn8b6T64dk4HBe711LRtZGuwjkb7zn9eA9P8BcwqupjysVcyq6Cc0amjz8lum3AszI6mHXxQ/wEb66xwaQ9b54WyvdmUZZVRllXGhMwJvdO5vlzExKHqXaulsuO30FUP7hTMxEUE/ZfQVROn65336d64EaJRxOcj5eKLeoPFU1Z2Tr6f6txxJly1NQaoTpivAS4ZrIzd+mgDcoFGEbkEeB4YD9xprz+efQIgIvcD9wOMGzdu+EdzBovU19O1di2db66l6+23iXd1IW43KRdfTPaypaRecQV/CabwzP/s5YMNDZTmruNjMzawM76fV1uty6NKIlEu8+Yxq+hCyqfeyuTRl+J2nh8DHnqcHmYXzGZ2wWyYCXETp7K1kg/qP2B703YqWyv5Q+Uf6Ih09G6T6c2kLLOMCVkTKCudzYQLPsXEzmby97yB7Pwd/u7/xu9OJe+m64k99FW6mzPoencdXW+9Rd0TVm+sq7AQ/+zZuIuKcBUW4i4qxFVYiKuwCHdBPnKGtF5MPE6spYVoXR2R2jqiBw8QrTlA5PAhHG7BXZSHZ3Q+nqI83EV5OLwJ9T4qKPvND7XekwYp2eDPAW/G4KManEmMgUgAgm3gyzwvRlNIZpAM9FWrf/Nn0DLGmPeAGSIyDfiJiKw6zn1ib/8c9nmWioqKc+qKAhOLEdiyhc61VqsjtGMnAK6iIjI+8QnSFlxF6iWXYPwp/H7rYZ79700Q+D1pWVspmNxCowPCkRjXRIRrcmcxd+ptZE6+4bz4D388HOJgYvZEJmYfGfzRGENDoIF9rfuobKtkX+s+9rXuY/VHq3kp9FJvuXR3OhNmXEqZM5UJ7Q1MrH6Tsp0vU+hMIX36DfDpLxDxT6XzvfV0/eUtQrt20bl2LSYQOKoeztxc3IV2uBQV2tNFuAsLcBUV4SooxJmWOqxjjQeDRGsPE63aS6S6kujBj4jW1hJpaCDa2EK0pYtIewji/f6ExODyxonHhHik74e7yx/DkxbFnRbFkxbDkx7FnWYtc3qG8acoTvBnWaGSkpPwnG09EpYZbxaxkINIW4hoY4sVgLWHidTWEak9TLS2jmhdHbhcODMzcWZk4MzIwJGViTPDnk9PxeF34/Q7cHoFpzuG0x3F4QrjlG4k3A6BVmsUhWBr3+lY+Ei9fZmQPgrSiyB9tP08CjJGHVmeVghn8Re3ZAZJDTA2Yb4YODRImRq7aysTaE4sYIzZKSJdwMzj3Oc5KdbaSudf3qLzzTfp+vOfibW2gsOBf84c8v/hH0i76kq8kycjIoSiMV76859Zu+mHtHh2UJ8VIpQjZMdi3CCZLBo1n4tm3Ym7qFx/zX2cRISClAIKUgq4bPRlvcuNMTQFm6hsrWRf277egFnTtpv/CjZDjh9yxpCCk7KmvzBh9WrK4k7KiuZSdu/NjCv7MtF4nFhHG+HaWiL19UTq6onVNxCtbyTa0Ei4ah/xDeugvfOoepkUHyY3E5OXSTwnnVhuBrGcNKL2wx2Pk9bQQmpjK56GZmhsIdrUTrS1i2hHhFjw6A92ccVx++O4UuKk5HtwTUrBlZOBKz/PCrMxY3GNGo9kFIInlVh7J+HD9YQP1hE5VE/4UB3hww10Hqojtr+tz76dGWm4RxfgGVWAZ1S+NT06H8/oApyZ6f26+gyEOiHQbH1AdzdDoBnT2USsuYFI9QGizZuJtHQT7YgRCTiJdDuJdjuJBpyYeN//2+IEV7oLd5Yff34a7qllmFiMWHsnsa464nVVRPZHCAZjxIIGEzv234bDbXD6BIfPiTPVgzPVhzN1PM6MmTgyM3BmZuHN8+DLjuIIN0D7YWhca51Xi0f7v+vWuHDpRZCREDbpCWGTMdoKyjOwVZbMcyQurBPj1wAHsU6M326M2Z5Q5gFgVsLJ9luMMX9lnxeptruzxgPvAOVA61D7HMjZeI4k3tVFYNt2Ahs30rl2LYFNmyAex5mdTeoVl5N21VWkzZ+PMyvL2iASpGH3n/jV+z/hg+BuNvkh7BByY4aPpxSzqPQG5pbficufPbIHdh5pDjZT2VpJZVsle1v3Utm6l31Nu2hM6CI7Ee6IIacTcjogp8OQm/Cc3fPcCY5B/qTjQHsqtKdDV5qDUIaLaKYXslNx5mXjKSwkZfR4MgtKyMoaT1bGWLJ82aS4Uk76XE68q4twTQ3hqioiVVWEq6oJV31EpKqayOHD9o9BLZKSgmfsWDzjxuEeNxbP2HGYaHTAloSJ9P2NkrjduArycOdl4cpJx53px5Xpxp0muPxR3L4wTulAgi0JodQCLq/VmvFl2a2dI9PGnU4s5iMW9RCLuoiFHMRDhlgwTqw7TKy9g3h7G7G2dmJtbcTared4W1vf+ongKBmHe/pU/DNmkDZzFqklRThibdBRC+2HrOeOwwmPWuhqOPoNdbgTQsZuyaQVQlq+9ZxaAGn2w+U9qX+zPu/rSJ9stytxA/A9rEt1nzfGfENEHgfWG2NWiogPeAGYg9USWWqMqRSRO4GHgQjW///HjTH/Pdg+h6rHmR4kJholtGcPgS1bCWzZTHDLVkL79vX+kfmmTydtwVWkXXklvlmzEKc9LHpzJS07f8dre3/H64Eq3vN5iIqQE3MwL2Uyt82+nTmTb8Shw6ifUdpCbVQ27WbvvlW0tO7HIU6cIjjFgRMHDnHgFPvZnnfY6x3iwCXOI+sc/baJC86OEM7WAI6WABGHg47cTFry0mnM8tFKmJZQC62hVlqDrdZzqJW2UBtm4F5i3A432d5sMn2Z1rPXes7yZZHjyyHHl0OuL5dcfy65vlwyvBk4ZOhvzSYcJnzwIJHqasIfVRGuriLyURXh6moi1dW9H8biduMqKrJaQ6NGWeeSiorsc0tFuEcV4czOPvFhbIw5JS3ySDzC3pa9bGvaxrbGbWxr2Ep14z5Su2KMrzdMqDVMqIXSWuuLAFgfaofzHdSM9lI7NoWGcRm0jM3ClZqGz+XD7/Ljd3jwG4MvFsUfDeGLBPGHu/AFO0kJtuHvbqG4s5nRXS0Ddy35MvuFyyCBk5o/aLfaGREkZ4qkBUlnPbz2GESDR77R9D5nHr3Mk44RIXroEIEtW+zg2EJw+3ZMMAiAMzMT3wXl+GeV4y+fha+8HFe23YoId8OBv9D44e95o2YNq+linc9HTITMiJtxTOOWCz/LLbOvPq4/ZKV6xOIxOsIdA4ZMS6ilz3zP+rZwG3Fz9NhpLnGR7cvuDZYcX07vdK6/73y2L3vA3x2ZWIxofT3i8ZxcSCRJ3MSpaq86EhqN29jVvItQLARYF2HMzJvJzNyZTM+djsfpIRgNEogGCEQDROvrce+pxrf3IGmVdWTub8TfZv3txwWaCnzUjPHy0SgXlUWwJz9GmyNI1PTvCjvC5XAxPnU0Jb58St2ZlIiH0rhQEg6T0d0MnQ3QWWd9XoUHaQ2n5PYNl7RCSCtALv+SBkmPpARJaxX89FPQftDquwy0WldpmFifYrGwEGj2EGxyE2j2EGj2EgtY34LEKfhGpeArycVfNgr/lFLcY4sRf/aRAHI44cBb1O1ZxWtNW1jt9/CBz4sRIS3sJ9Y+kxlFn+Sfr/4Y00dnntpjVOoY4iaTd3BdAAAgAElEQVROW6iNpkATTcEmmgJNNAebe6ebgk00B47M9/xWp78sb1afFk2Ov28LJ8eX07vM5zq9w9HUddX1CY3tjdt7r97zu/xMy5nGzLyZzMqbxYy8GRSnFZ9wN2Ckrp7g9u3WY9s2Aju2E2uw7smDw4FnQine6dNwTJ2EmVpGpKyYkNv6PVR1RzUH2g+wv20/B9oPUN1e3Sd0cn25lGSWUJpZSklGCaWpoyl1pTE6Lji7Gq2A6UoIms76I9PRAPK1dg2SHqc8SBr3WCES7oDbfw3jrCuQTShEcOtGAhvXEdy6jcD23YQP1vVu5inMwD82A98oD/588GUGkXCbFUD2lR5BERqcDpqcThqdTqrcLt5ISWGzz+rvzDE5NDZdQLRzNjfPrODzV5VRkje8K3eUSjZjDJ2RTito+gfPAEHUFekacD8prpTeFk1vt1pPKyehxZPjyyHTm3lCLfO2UBvbG7f3CY6GgHWewiUuJmVPslob9mNC5oSkjeLQGy7bthHcvn3AcPFNnYb4vBCLQzyOiceJxyIEwt10BTvoDnfSHe4iGO4iEO4mFoviMNY5NKcR/A4vPocXv3jwOjx4ceMRJw4jmHgMYjEmvvqqBkmPUxokhzfDC7eACOaO39CxtY7udesIbN1CaMfO3n5dZ14e/vJy/OXleGfNJDR5LM2uII3djTQGG2nobqAx0EhToImGgDXdGGigc4A/oJKU8cQDF7JzbwleM4o7LhnHfVdMoChTBwtU56ZgNNjbomkONvdp6fRMNwebaQ400xJqGbCLzSlOq4utX6umJ2iyfdlUd1T3hkZVR1XvtiUZJX1CY0r2lNPeGurPCpdtBLfZrZc9H1oh4nBYXX89z04nOARxOMHptFpITicx4oRMhKAJE4iFCJgg3bEggXiQmFgDn8YFXG4PfncqKd40rnvhTxokPU5ZkHz0Dvzir8CbQey2Fzn8nR/QsXo1xuclPGksbWX51JVkcGCMmyp/Nw3BRhoDjTQHmgfs40xxpZDnz+t95Kfkk+HOJh7JIBBMoaXDy56DTtZVhsn0u7l7Xgn3zCshO/XM+KGaUmeCWDxGW7ittxutN3gSQydhXSDa9zc7BSkFzMqb1Rsa03Onk+HJGKGjOf0isQjVndUcaDvSRba/bT/72/bz9u1va5D0OCVBsuc1+OVfQ+YYwlf/P6of/l+E9u3jhY85+EMFxO2hRRziIMeXQ54/j1x/Lvn+/L5h4c8n15dLNJLG4RbDvobOI4/6Lmrbg70v6RAoyUtl6UVjuf2S8aR5z6zBEJU6G3VHumkONtMSbKEwtZCClIKRrtIZyRiDw+EY8SFSzh3bX4bf/A0UTKVz4iPU3PMFAvEg31kijFpwLcsnfqq3RZHlzertNw1GYuxv7LJC4mAX7zdagVHZUE93+MhJ+XSviwkFacybmEtZfhpl+amU5acxLjcFr0sv3VXqVEpxp5DiTqE4vXikq3JGO5GLBjRIhvLBC/C7L2LGXERz7Cbqv/gvHC5w8eTNwh0f/xfunHYnTV0R9jV08sHeTvbV11th0dhJTUuAxAbfmCw/ZQVpXFSSYweGFRr56V4dvE8pddbSIDmWd56BV/+N+LgFHN5aSvuq5bw/zcXPbsnkmwv/nbr6MVz6xBvUd4R6N/G7nUzIT2X22GxunVvcGxilean4Pdq6UEqdezRIBmIMrHkC3vwWkVHXUr0yTHD3q7x4lYPKT87imUu/xfLVjazcvJELxmbxtwvKrMAoSGNUhg+HQ1sXSqnzhwZJf/E4vPoIvPcsXRmfoPpHlQSDnfzv2xxMumEpn83+LHc+t5OmzjD/uHAyf7ugDJfzzPjVrVJKjQQNkkSxKKz8AmbTL2iJXkftDzdTl+3g3z/j5q7rvsLWXZO597ebmFSQxg/vvoiZY/SX5EoppUHSIxqClz5LfPsr1B68ira3t/DBJAe/WlLEZ+d8nf/7SpADTR9x3+Wl/NO1U/C59XyHUkqBBokl1Am/vIPItrVUb5tDqHIPv54vVH96Hhebz/FPP6tjVKafX9x3KZeV5Y50bZVS6oyiQRJogZ9/mu5NW6haN4FgsIGnb3Ew7vp7OLjxIlYfquO2C4v56ienk+47e+9gppRSyXJ+B0lHHfzsFlre/YjD6/NpyAjy9D2pTJryEL94NYt0b5jv33kh184oGumaKqXUGev8DZLWKsyPbqT2f9pp/TCdzRMM/3XHBGJd97HiTTcLp+fzxC2zyEsb/l3GlFLqXHZ+BknDh0S/fxNVq2OE6rz89lJh0ycXsHPrdWC8fOe26dx24YnfV0Appc5H51+QHN5M4Lu3cuB1F+Gwi/+4yUnN9KVsW3cBl5Tm8tSnL2BsTspI11Ippc4a51eQfPQ2bV+/g5p3fDSnCsvvzuTD+F107Snhy5+Ywmfnl+qv0pVS6gSdN0Fidr1K7cOfp3WXjx3jhB/eXMbug8uYXjCW7949m8mF6SNdRaWUOiudF0FiuprZ9/m/I1Lr4w8VwopLLqep+hM8uGAqX/jYJDwuHeJEKaVOVlI/QUXkOhHZLSJ7ReThAdZ7ReSX9vr3RKTEXr5QRDaIyFb7+WMJ26yx97nJfgx5V5pQ1SG6Gzz8xyfc/MeMpaSFbuelz1/JPy6aoiGilFLDlLQWiYg4gWeAhUANsE5EVhpjdiQUuxdoMcZMFJGlwLeAJUAj8EljzCERmQm8CoxJ2O4OY8xx3/IwivC127PZGP4Mfz19Hg9fP5UUz3nRGFNKqaRL5qfpxcBeY0wlgIisAG4CEoPkJuAxe/olYLmIiDFmY0KZ7YBPRLzGmBAnoSbbS6bzYX58x2VcOTn/ZHahlFJqEMns1xkDVCfM19C3VdGnjDEmCrQB/QezuhXY2C9EfmR3a31FBvmxh4jcLyLrRWS90+Typy9eryGilFJJkMwgGegD3pxIGRGZgdXd9bmE9XcYY2YBV9iPOwd6cWPMc8aYCmNMxaRR2WSm6DhZSimVDMkMkhpgbMJ8MXBosDIi4gIygWZ7vhh4GbjLGLOvZwNjzEH7uQP4BVYXmlJKqRGSzCBZB0wSkVIR8QBLgZX9yqwE7ranbwPeMMYYEckCfg88Yox5q6ewiLhEJM+edgOLgW1JPAallFJDSFqQ2Oc8HsS64mon8CtjzHYReVxEbrSL/RDIFZG9wD8APZcIPwhMBL7S7zJfL/CqiGwBNgEHgf9M1jEopZQamhjT/7TFuaeiosKsX3/cVwsrpZQCRGSDMaZiqHL6azyllFLDokGilFJqWDRIlFJKDYsGiVJKqWHRIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOiQaKUUmpYNEiUUkoNiwaJUkqpYdEgUUopNSwaJEoppYZFg0QppdSwaJAopZQaFg0SpZRSw6JBopRSalg0SJRSSg2LBolSSqlh0SBRSik1LEkNEhG5TkR2i8heEXl4gPVeEfmlvf49ESmxly8UkQ0istV+/ljCNhfay/eKyNMiIsk8BqWUUseWtCARESfwDHA9MB1YJiLT+xW7F2gxxkwEvgt8y17eCHzSGDMLuBt4IWGb/wDuBybZj+uSdQxKKaWGlswWycXAXmNMpTEmDKwAbupX5ibgJ/b0S8A1IiLGmI3GmEP28u2Az269jAIyjDHvGGMM8FPgU0k8BqWUUkNIZpCMAaoT5mvsZQOWMcZEgTYgt1+ZW4GNxpiQXb5miH0CICL3i8h6EVnf0NBw0gehlFLq2JIZJAOduzAnUkZEZmB1d33uBPZpLTTmOWNMhTGmIj8//ziqq5RS6mQMGSQi4hSRvIR5j/1tf+cQm9YAYxPmi4FDg5UREReQCTTb88XAy8Bdxph9CeWLh9inUkqp0+iYQSIiS7E+2LeIyJsicjVQiXUC/Y4h9r0OmCQipSLiAZYCK/uVWYl1Mh3gNuANY4wRkSzg98Ajxpi3egobYw4DHSJyqX211l3Ab4/nQJVSSiWHa4j1XwYuNMbsFZG5wDvAUmPMy0Pt2BgTFZEHgVcBJ/C8MWa7iDwOrDfGrAR+CLwgInuxAmupvfmDwETgKyLyFXvZImNMPfC3wI8BP7DKfiillBohYl38NMhKkQ+MMXMT5ncZY6aelpqdQhUVFWb9+vUjXQ2llDqriMgGY0zFUOWGapEUiMg/JMynJc4bY/73yVZQKaXUuWGoIPlPIP0Y80oppc5zxwwSY8zXBlsnIqmnvjpKKaXONsdz+e8YEamwr7xCRApE5JvAnqTXTiml1BlvqMt/vwRsAv4v8K6I3A3sxLpi6sLkV08ppdSZbqhzJPcDU4wxzSIyDtgLXGmMeTf5VVNKKXU2GKprK2iMaQYwxlQBH2qIKKWUSjRUi6RYRJ5OmC9InDfGfDE51VJKKXW2GCpI/rnf/IZkVUQppdTZaajLf39yrPVKKaXUMYNERPoPstiHMebGU1sdpZRSZ5uhurYuw7rx1IvAewx8PxCllFLnsaGCpAhYCCwDbsca2v1FY8z2ZFdMKaXU2eGYl/8aY2LGmD8aY+4GLsX6HckaEfnCaamdUkqpM95QLRJExAt8AqtVUgI8DfxXcqullFLqbDHUyfafADOxbh71NWPMttNSK6WUUmeNoVokdwJdwGTgi9bdbQHrpLsxxmQksW5KKaXOAkP9jmTI0YGVUkqd3zQolFJKDYsGiVJKqWFJapCIyHUisltE9orIwwOs94rIL+3174lIib08V0T+R0Q6RWR5v23W2PvcZD8KknkMSimljm3Iy39Plog4gWewftBYA6wTkZXGmB0Jxe4FWowxE0VkKfAtYAkQBL6CdcXYzAF2f4cxZn2y6q6UUur4JbNFcjGw1xhTaYwJAyuAm/qVuQnoGRjyJeAaERFjTJcx5i9YgaKUUuoMlswgGYM1TlePGnvZgGWMMVGgDcg9jn3/yO7W+ookXJOslFLq9EtmkAz0AW9Ookx/dxhjZgFX2I87B3xxkftFZL2IrG9oaBiyskoppU5OMoOkBhibMF8MHBqsjIi4gEyg+Vg7NcYctJ87gF9gdaENVO45Y0yFMaYiPz//pA5AKaXU0JIZJOuASSJSKiIeYCnQ//4mK4G77enbgDeMMYO2SETEJSJ59rQbWAzosC1KKTWCknbVljEmKiIPAq8CTuB5Y8x2EXkcWG+MWQn8EHhBRPZitUSW9mwvIgeADMAjIp8CFgEfAa/aIeIEXgP+M1nHoJRSamhyjAbAOaOiosKsX69XCyul1IkQkQ3GmIqhyukv25VSSg2LBolSSqlh0SBRSik1LBokSimlhkWDRCml1LBokCillBoWDRKllFLDokGilFJqWDRIlFJKDYsGiVJKqWHRIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOiQaKUUmpYknaHxDNdJBKhpqaGYDA40lU55/h8PoqLi3G73SNdFaXUaXDeBklNTQ3p6emUlJQgIiNdnXOGMYampiZqamooLS0d6eoopU6D87ZrKxgMkpubqyFyiokIubm52tJT6jxy3gYJoCGSJPq+KnV+Oa+DRCml1PAlNUhE5DoR2S0ie0Xk4QHWe0Xkl/b690SkxF6eKyL/IyKdIrK83zYXishWe5un5Sz++ut0Opk9ezYzZ87k05/+NN3d3adkv3V1dSxevJgLLriA6dOnc8MNNwBQWlrK7t27+5T90pe+xLe//W3WrFlDZmYmc+bMYcqUKVx55ZW88sorp6Q+SqlzW9KCREScwDPA9cB0YJmITO9X7F6gxRgzEfgu8C17eRD4CvBPA+z6P4D7gUn247pTX/vTw+/3s2nTJrZt24bH4+HZZ5897m2j0eig6x599FEWLlzI5s2b2bFjB08++SQAS5cuZcWKFb3l4vE4L730EkuWLAHgiiuuYOPGjezevZunn36aBx98kNdff/0kj04pdb5I5lVbFwN7jTGVACKyArgJ2JFQ5ibgMXv6JWC5iIgxpgv4i4hMTNyhiIwCMowx79jzPwU+BawaTkW/9rvt7DjUPpxdHGX66Ay++skZx13+iiuuYMuWLRw4cIDFixezbds2AJ566ik6Ozt57LHHWLBgAfPmzeOtt97ixhtv5K677uLzn/88VVVVAHzve99j/vz5HD58mEWLFvXuu7y8HIBly5axZMkSvvrVrwKwdu1aSkpKGD9+PPv37+9Tn9mzZ/Poo4+yfPlyrrnmmj7rYrEY9957L+vXr0dE+OxnP8tDDz104m+SUuqckMwgGQNUJ8zXAJcMVsYYExWRNiAXaDzGPmv67XPMQAVF5H6slgvjxo070bqfVtFolFWrVnHddUM3rlpbW3nzzTcBuP3223nooYe4/PLLqaqq4tprr2Xnzp088MADLFmyhOXLl/Pxj3+ce+65h9GjR1NeXo7D4WDz5s1ccMEFrFixgmXLlg36WnPnzuU73/nOUcs3bdrEwYMHe8OutbX1JI9cKXUuSGaQDHTuwpxEmZMqb4x5DngOoKKi4lj7PKGWw6kUCASYPXs2YLVI7r33Xg4dOnTMbXq6oQBee+01duw40sBrb2+no6ODa6+9lsrKSv74xz+yatUq5syZw7Zt28jPz2fZsmWsWLGCGTNm8Nvf/pbHH3980NcyZuC3bcKECVRWVvKFL3yBT3ziE31aP0qp808yg6QGGJswXwz0/5TsKVMjIi4gE2geYp/FQ+zzrNFzjiSRy+UiHo/3zvf/PUZqamrvdDwe55133sHv9x+175ycHG6//XZuv/12Fi9ezNq1a7n11ltZtmwZixYt4qqrrqK8vJyCgoJB67dx40amTZtGLBbjwgsvBODGG2/k8ccfZ/Pmzbz66qs888wz/OpXv+L5558/qfdAKXX2S+ZVW+uASSJSKiIeYCmwsl+ZlcDd9vRtwBtmsK/BgDHmMNAhIpfaV2vdBfz21Fd95BQWFlJfX09TUxOhUOiYV04tWrSI5cuPXNTWE0pvvPFG7xVgHR0d7Nu3r7d7r6ysjNzcXB5++OFjdmtt2bKFr3/96zzwwAM4nU42bdrEpk2bePzxx2lsbCQej3Prrbfy9a9/nQ8++OBUHLpS6iyVtBaJfc7jQeBVwAk8b4zZLiKPA+uNMSuBHwIviMherJbI0p7tReQAkAF4RORTwCJjzA7gb4EfA36sk+zDOtF+pnG73Tz66KNccskllJaWMnXq1EHLPv300zzwwAOUl5cTjUa58sorefbZZ9mwYQMPPvhgb+vmvvvu46KLLurdbtmyZTzyyCPcfPPNffb35z//mTlz5tDd3U1BQQFPP/30USfaAQ4ePMg999zT23J64oknTtHRK6XORnKMBsA5o6Kiwqxfv77Psp07dzJt2rQRqtG5T99fpc5+IrLBGFMxVDn9ZbtSSqlh0SBRSik1LBokSimlhkWDRCml1LBokCillBoWDRKllFLDokEygubNm9dn/rvf/S4+n4+2trbeZWvWrGHx4sVHbfvKK68wZ86c3qHiv//97ye9vkopNZDz9p7tZ4K33367z/yLL77IRRddxMsvv8xnPvOZQbeLRCLcf//9vP/++xQXFxMKhThw4EByK6uUUoPQIAFY9TDUbj21+yyaBdc/ecwiaWlpdHZ2ArBv3z46Ozv5zne+wze/+c1jBklHRwfRaJTc3FwAvF4vU6ZMOaqcDveulDodNEjOEC+++CLLli3jiiuuYPfu3dTX1w86oGJOTg433ngj48eP55prrmHx4sUsW7YMh6NvT6UO966UOh00SGDIlsPpsGLFCl5++WUcDge33HILv/71r3nggQcGLf+DH/yArVu38tprr/HUU0+xevVqfvzjH/cpo8O9K6VOBz3ZfgbYsmULe/bsYeHChZSUlLBixQpefPHFIbebNWsWDz30EKtXr+Y3v/kNsViM2bNn997dMDs7m82bN7NgwQKeeeYZ7rvvvtNwNEqp8422SM4AL774Io899hiPPPJI77LS0lI++uijAct3dnayfv16FixYAFhdWOPHj+8d7r1HY2MjHo+HW2+9lbKysmOed1FKqZOlQXIGWLFiBatW9R0N/+abb2bFihVccsklvP766xQXH7mf14svvsi3v/1tPve5z+H3+0lNTT2qWwt0uHel1Omhw8irpND3V6mznw4jr5RS6rTQIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOS1CARketEZLeI7BWRhwdY7xWRX9rr3xORkoR1j9jLd4vItQnLD4jIVhHZJCLr++/zbKLDyCulzgVJ+0GiiDiBZ4CFQA2wTkRWGmN2JBS7F2gxxkwUkaXAt4AlIjIdWArMAEYDr4nIZGNMzN7uamNMY7LqfrroMPJKqXNBMn/ZfjGw1xhTCSAiK4CbgMQguQl4zJ5+CVguImIvX2GMCQH7RWSvvb93klHRb73/LXY17zql+5yaM5V/vfhfj1km2cPIK6XU6ZDMrq0xQHXCfI29bMAyxpgo0AbkDrGtAf4kIhtE5P7BXlxE7heR9SKyvqGhYVgHcjoMNIz8YBKHkV+2bBk///nPe4dBUUqp0y2ZLRIZYFn/8VgGK3OsbecbYw6JSAGwWkR2GWPWHlXYmOeA58AaIuVYFR2q5XA6JGMYeaWUOh2SGSQ1wNiE+WLg0CBlakTEBWQCzcfa1hjT81wvIi9jdXkdFSRnk8Rh5AHC4TATJkw4ZpCANYz8rFmzuPPOOyktLdUgUUqNiGR2ba0DJolIqYh4sE6er+xXZiVwtz19G/CGsUaRXAksta/qKgUmAe+LSKqIpAOISCqwCNiWxGM4LXqGkT9w4AAHDhzg0KFDHDx48JjDyK9Zs6Z3vmcYeaWUGglJa5EYY6Ii8iDwKuAEnjfGbBeRx4H1xpiVwA+BF+yT6c1YYYNd7ldYJ+ajwAPGmJiIFAIvW+fjcQG/MMb8MVnHcLokaxh5pZQ6HXQYeZUU+v4qdfbTYeSVUkqdFhokSimlhkWDRCml1LBokCillBoWDRKllFLDokGilFJqWDRIRpDT6WT27NnMnDmTT3/603R3d5+S/dbV1bF48eLeIeZvuOEGAEpLS9m9e3efsl/60pf49re/zZo1a8jMzGTOnDlMmTKFK6+8kldeeeWU1EcpdW7TIBlBfr+fTZs2sW3bNjweD88+++xxbxuNRgdd9+ijj7Jw4UI2b97Mjh07ePLJJwFYunQpK1as6C0Xj8d56aWXWLJkCQBXXHEFGzduZPfu3Tz99NM8+OCDvP766yd5dEqp80Uyx9o6a9R+85uEdp7aYeS906ZS9G//dtzlr7jiCrZs2cKBAwdYvHgx27ZZI7889dRTdHZ28thjj7FgwQLmzZvHW2+9xY033shdd93F5z//eaqqqgD43ve+x/z58zl8+DCLFi3q3Xd5eTkAy5YtY8mSJXz1q18FYO3atZSUlDB+/Hj279/fpz6zZ8/m0UcfZfny5VxzzTXDei+UUuc2bZGcAaLRKKtWrWLWrFlDlm1tbeXNN9/kH//xH/n7v/97HnroIdatW8dvfvMb7rvvPgAeeOAB7r33Xq6++mq+8Y1vcOiQNVZmeXk5DoeDzZs3A9bQLMuWLRv0tebOncuuXac2YJVS5x5tkcAJtRxOpUAgwOzZswGrRXLvvff2fugPpqcbCuC1115jx44j9wlrb2+no6ODa6+9lsrKSv74xz+yatUq5syZw7Zt28jPz+f/b+/+g6wq6ziOvz8DK4ukKeAisQIrYhvYBo4I+ANQC39EKoNTLI5hyjhNEMroJDvMMIp/lNlkEgqhKUUmJqkgM2YGOJAiCS4ukJCQpOtYwIpEEuTKtz+eZ+Gy3Ls/uOyee3e/r5k7e87znHPu9z57733uec4531NeXs6iRYsYOHAgS5YsYdasWRmfqz2kz3HOZc87kgTVHSNJ1bFjx6NuUnXgwIGj6rt06XJ4+tChQ6xZs4bOnTsfs+2uXbsyYcIEJkyYwJgxY1i1ahXjxo2jvLyc0aNHM3LkSMrKyigqKsoYX2VlpefLcs41yoe2ckyPHj3YuXMnNTU1HDx4sMEzp0aPHs2cOXMOz9d1SitWrDh8Bti+ffvYvn07vXv3BqBfv35069aN6dOnNzisVVVVxX333dfoPVGcc873SHJMQUEBM2fOZOjQoZSUlFBaWppx2dmzZzN58mTKysqora1lxIgRzJs3j/Xr1zNlypTDezeTJk1iyJAhh9crLy+noqKCsWPHHrW91atXM3jwYPbv309RURGzZ8/2A+3OuUZ5GnnXIrx9nct/nkbeOedcq/COxDnnXFbadUfSHob1kuDt6lz70m47ksLCQmpqavxL7wQzM2pqaigsLEw6FOdcK2m3Z20VFxdTXV3Nrl27kg6lzSksLKS4uDjpMJxzraTddiQFBQWUlJQkHYZzzuW9Fh3aknSVpK2Stkmanqa+k6SnY/1aSX1T6ipi+VZJVzZ1m84551pXi3UkkjoADwNXAwOAckkD6i12K7DHzM4BHgTuj+sOAMYDA4GrgEckdWjiNp1zzrWiltwjuRDYZmZ/N7P/AYuA6+otcx3wqzi9GLhCkmL5IjM7aGbvAtvi9pqyTeecc62oJY+R9ALeT5mvBoZmWsbMaiXtBbrF8tfrrdsrTje2TQAk3QbcFmf/I2lruuVOkO7A7hbcfkvK59jB40+ax5+slo6/T1MWasmORGnK6p9rm2mZTOXp9qDSnr9rZvOB+Q0FeKJIWteUNAK5KJ9jB48/aR5/snIl/pYc2qoGzkqZLwbq32zj8DKSOgKfBz5qYN2mbNM551wrasmO5A2gv6QSSScRDp4vrbfMUmBinL4BWGHhCsGlwPh4VlcJ0B/4SxO36ZxzrhW12NBWPOYxBXgJ6AA8bmabJc0C1pnZUuCXwEJJ2wh7IuPjupsl/Q74K1ALTDazzwDSbbOlXkMztMoQWgvJ59jB40+ax5+snIi/XaSRd84513Laba4t55xzJ4Z3JIP7m+4AAAeLSURBVM4557LiHUkzSDpL0kpJb0vaLOn2WN5V0suS3ol/T0861obELAGVkpbF+ZKYouadmLLmpKRjzETSaZIWS9oS/w/D86X9JU2L75tNkp6SVJjrbS/pcUk7JW1KKUvb3gpmx/RFVZLOTy7yjLE/EN87VZKek3RaSl3atExJSRd/St1dkkxS9zifaNt7R9I8tcCdZvYlYBgwOaZomQ4sN7P+wPI4n8tuB95Omb8feDDGv4eQuiZXPQT8wcxKga8QXkfOt7+kXsBU4AIzO49wssh4cr/tFxDSFKXK1N5XE86w7E+4GHhuK8WYyQKOjf1l4DwzKwP+BlRA5rRMrRdqWgs4Nn4knQV8DXgvpTjZtjczfxznA1gS/6FbgZ6xrCewNenYGoi5mPDhvxxYRrj4czfQMdYPB15KOs4MsZ8KvEs8SSSlPOfbnyNZHLoSzpZcBlyZD20P9AU2NdbewC+A8nTL5Urs9erGAk/G6QqgIqXuJWB4rrV9LFtM+BG1A+ieC23veyTHKWYqHgysBXqY2YcA8W9RcpE16mfAD4BDcb4b8LGZ1cb51HQ0ueZsYBfwRByae0xSF/Kg/c3sA+AnhF+RHwJ7gfXkT9unytTe6dIi5fLruQV4MU7nReySrgU+MLO36lUlGr93JMdB0ueA3wN3mNm/k46nqSSNAXaa2frU4jSL5uo54R2B84G5ZjYY+IQcHMZKJx5HuA4oAb4AdCEMR9SXq23fFHnzXpI0gzBU/WRdUZrFcip2SScDM4CZ6arTlLVa/N6RNJOkAkIn8qSZPRuL/yWpZ6zvCexMKr5GXAxcK2kHIXPy5YQ9lNNiihrI7bQz1UC1ma2N84sJHUs+tP9XgXfNbJeZfQo8C1xE/rR9qkztnRcpjCRNBMYAN1ocByI/Yu9H+CHyVvwMFwNvSjqThOP3jqQZJIlwNf7bZvbTlKrUVC8TCcdOco6ZVZhZsZn1JRxYXGFmNwIrCSlqILfj/yfwvqQvxqIrCNkP8qH93wOGSTo5vo/qYs+Ltq8nU3svBb4dzyAaBuytGwLLFZKuAu4GrjWz/SlVmdIy5Qwz22hmRWbWN36Gq4Hz4+ci2bZP+mBSPj2ASwi7i1XAhvi4hnCcYTnwTvzbNelYm/BaRgHL4vTZhA/NNuAZoFPS8TUQ9yBgXfwfPA+cni/tD9wLbAE2AQuBTrne9sBThGM6nxK+uG7N1N6E4ZWHge3ARsIZarkW+zbCsYS6z++8lOVnxNi3AlfnYtvXq9/BkYPtiba9p0hxzjmXFR/acs45lxXvSJxzzmXFOxLnnHNZ8Y7EOedcVrwjcc45lxXvSFybJOmHkkZJul5Ss65+l3RGzMhbKenSenWvSLqgkfXviFchH5cY90UZ6jpJ+pOkDZK+1czt9pU04Xjjci4T70hcWzWUkAdtJLC6meteAWwxs8Fm1tx1Ae4AjrsjIVzjk7YjIeR3KzCzQWb2dDO32xdoVkeSctW9cxl5R+LalHi/iSpgCLAGmATMlXRMfiJJfSQtj/dvWC6pt6RBwI+Ba+Kv/s4NPNdcSesU7jFybyybSsiltVLSylg2WtIaSW9KeibmakPSDkn3xvKNkkpjMtDvAtPi81+a8nxFwG+AQbGun6SZkt5QuMfJ/HjVPJLOiXsub8Xt9wN+BFwa152mcD+UJ+JzV0q6LK57c4zzBeCPknpKWhXX21R/L825xK+c9Yc/TvQDuBD4OVAAvNrAci8AE+P0LcDzcfpmYE6GdV4hXjXMkSu6O8Tysji/gyNXHHcHVgFd4vzdwMyU5b4fp78HPBan7wHuyvD8o4gZCVJjiNMLgW/E6bXA2DhdSNhDqr/uncATcbqUkMalML7+6pTXdycwI+W1npL0/9gfufXwPRLXFg0mpL8oJeSzymQ48Ns4vZCQAqc5vinpTaCScEOkAWmWGRbLX5W0gZCbqk9KfV3iz/WEoafmuiwez9lISMI5UNIpQC8zew7AzA7Y0Xml6lxCeN2Y2RbgH8C5se5lM/soTr8BfEfSPcCXzWzfccTp2jAf/3RtRhyWWkDIfLqb8Ctc8Qt8uJn9t5FNNDlfUEzsdxcwxMz2SFpA+DV/zKKEL+XyDJs6GP9+RjM/j5IKgUcIe0jvxy/6QtKnFE+7iQbqPqmbMLNVkkYAXwcWSnrAzH7dnFhd2+Z7JK7NMLMNZjaIcAvVAcAK4EoLB6bTdSKvEbIgA9wI/LkZT3cq4ct2r6QeHH1vkX3AKXH6deBiSedAuKeEpHNpWOr6DanruHbH4y43AFi4R061pOvjc3aKZ5HV3+4qwusmxtSbkLDwKJL6EO5j8ygh+3Wi92J3ucc7EtemSDoD2GNmh4BSM2toaGsqYcimCriJcC/7JrFwh7pKYDPwOPBqSvV84EVJK81sF+GYw1PxeV4nDLk15AVgbP2D7Wli+Bh4lJDt9XnCEFSdm4Cp8TlfA84kZEyujQfgpxH2ZjrEYbGngZvN7CDHGgVskFQJjAMeaiR+18549l/nnHNZ8T0S55xzWfGOxDnnXFa8I3HOOZcV70icc85lxTsS55xzWfGOxDnnXFa8I3HOOZeV/wNfw82jPPH+zQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_rank_scores([ssvd_rank_scores,\n", + " sals_rank_scores,\n", + " ials_rank_scores,\n", + " psvd_rank_scores]);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There seem to be no difference between the original and scaled versions of iALS. Let's verify this with CV experiment." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### cross-validation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You only need to perform CV computations for the new model. Configuration of data will be the same as previously, as the `data_model` instance ensures reproducible data state." + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'alpha': 0.3,\n", + " 'epsilon': 0.1,\n", + " 'weight_func': ,\n", + " 'regularization': 1.0,\n", + " 'rank': 60}" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sals_best_config.update(sals_best_rank)\n", + "sals_best_config" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "set_config(sals, sals_best_config)" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "5/5\n", + "[03:00<00:36, 36.06s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 5/5 [03:00<00:36, 36.06s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sals.verbose = False\n", + "sals_cv_results = ee.run_cv_experiment([sals],\n", + " metrics=metrics,\n", + " iterator=track)" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+0nVV97/v3x0RAUVExWuVX4EBtg1BattBeq02lKIyqsUesQW6Fe7kjWuX2jHptC+cUGji2RzrOKbUH2msqIGIVLC23qUSjFfHXtZggSAyU2xijxGgJQlFQwMD3/rGejYuVneyV7Gevtfba79cYa+xnzWfOueYz2Xuy893zR6oKSZIkSZKktjxl2A2QJEmSJEnjxWCDJEmSJElqlcEGSZIkSZLUKoMNkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhls0NhIsiXJrw27HZIkSZI03xlskCRJkiRJrTLYoHknycJht0GS5rOpxmHHZkkarCQLht0GjTeDDRo3xyW5PckDSa5Nsl+SpUm2JvmDJN8Frhx2IyVpHDTL136vGXcfSnJ5khck+XiSHyT5pyTPSbI4SSU5O8m3gBunShv280jSoCU5JMnfJ9me5HtJLk3ylCR/mOSbSe5J8sEkBzT5P5HknJ46vprkPzbXP5PkU0nuS3JXkt/syveBJH+VZE2Sh4BfTfLrSW5N8v0kdydZ2VP3W5p2fC/J+d3Llpt2npvk6839jyZ57mz3meYOgw0aN78JnAIcDhwLnNWk/xTwXOAwYMVQWiZJ4+kNwMnATwOvBT4O/GfgeXR+z/idrry/Avws8Opp0iRp7DUzCz4GfBNYDBwEXEPn99ezgF8FjgCeAVzaFPswcHpXHUvo/H57Q5L9gU81eZ7f5PvLJEd3feybgT8Gngl8AXgIeAvwbODXgd9O8vquuv8SOAN4IXBA08ZJvwO8ns44/iLgfuCyGXSJxozBBo2bv6iqbVV1H/CPwHFN+uPAH1XVI1X1o+E1T5LGzv+sqn+rqm8Dnwdurqpbq+oR4Hrg57vyrqyqh3rG4anSJGk+OIHOP9J/rxkHH66qL9D5x/2fVdXmqnoQOA9Y3iw3u57OTN7DmjrOAP6+GXNfA2ypqiurakdVfQX4O+C0rs/8h6r6YlU93nzeTVW1oXl/O/AROsEDmnL/WFVfqKpHgQuA6qrrrcB/qaqtzeevBE5zWZwmGWzQuPlu1/UP6USCAbZX1cNDaI8kjbt/67r+0RTvn9H1/u4pyk+VJknzwSHAN6tqR0/6i+jMdpj0TWAh8IKq+gFwA7C8ubcc+Jvm+jDgxCT/PvmiE4z4qa66njTmJjkxyWeaZRwPAG+jMzNtsh1P5K+qHwLf6yp+GHB912fdCTwGvKDvHtBYM9ig+aKmzyJJmmVTjcWOz5Lmq7uBQ6eYCbCNzj/kJx0K7OAnwdyPAKcn+SXgacBnuur7bFU9u+v1jKr67a66esfcDwOrgUOq6gDg/wbS3PsOcPBkxiRPAw7saf+pPZ+3XzPTTTLYIEmSJElD8GU6/6B/T5L9m43NX0YnmPC7SQ5P8gzgT4Bru2ZArKETjLioSX+8Sf8Y8NNJfivJU5vXS5P87G7a8Ezgvqp6OMkJdPZ0mHQd8Nok/0uSfYAL+UkgAjqBiT+eXNKRZFGSZTPoD40Zgw2SJEmSNGBV9RidjXWPBL4FbAXeBFwBXA18DvgG8DDwf3aVewT4e+DX6MxMmEz/AfAqOksrttFZXnwxsO9umvF24KIkP6CzJ8NHu+rb2HzuNXSCIj8A7gEeabK8l86siE825f8ZOHGPO0JjK1XOXpQkSZIk7Vozy+LfgaOq6hvDbo9GnzMbJEmSJEk7SfLaJE9vjtX878AGYMtwW6W5wmCDJEmSJGkqy+gsydgGHAUsL6fGq08uo5AkSZIkSa1yZoMkSZIkSWqVwQZJkiRJktSqhcNuQK/nPe95tXjx4mE3Q5J2csstt9xbVYuG3Y5BcCyWNIochyVp+Podi0cu2LB48WLWr18/7GZI0k6SfHPYbRgUx2JJo8hxWJKGr9+x2GUUkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBrVi6dKlLF26dNjNkKR5zbFYkobLcVj6CYMNkjRCkpyS5K4km5KcO8X9fZNc29y/OcniJv2EJLc1r68m+Y2uMluSbGjurR/c00iSJGm+WjjsBkiSOpIsAC4DTga2AuuSrK6qO7qynQ3cX1VHJlkOXAy8CfgaMFFVO5K8EPhqkn+sqh1NuV+tqnsH9zSSJEmazww2SNLoOAHYVFWbAZJcAywDuoMNy4CVzfV1wKVJUlU/7MqzH1Cz31xJksbHMVcdM+M6Nn93c2t1AWw4c0Mr9UjD4DIKSRodBwF3d73f2qRNmaeZtfAAcCBAkhOTbAQ2AG/rmtVQwCeT3JJkxa4+PMmKJOuTrN++fXsrDyRJkqT5yZkNYvG5N8y4ju9u/l5rdQFsec+vt1KPNMdkirTeGQq7zFNVNwNHJ/lZ4KokH6+qh4GXVdW2JM8HPpXkX6rqcztVUrUKWAUwMTHhzAhJkiTtNWc2SNLo2Aoc0vX+YGDbrvIkWQgcANzXnaGq7gQeAl7SvN/WfL0HuJ7Ocg1JkiRp1hhskKTRsQ44KsnhSfYBlgOre/KsBs5srk8DbqyqasosBEhyGPBiYEuS/ZM8s0nfH3gVnc0kJUlSy4447wiOOO+IYTdDGgkuo5CkEdGcJHEOsBZYAFxRVRuTXASsr6rVwOXA1Uk20ZnRsLwp/svAuUl+DDwOvL2q7k1yBHB9EuiM+R+uqk8M9skkSZI03xhsUCt+6s3vGXYTpLFQVWuANT1pF3RdPwy8cYpyVwNXT5G+Gfi59luqVq08oJ16tjzUXn0rH5h5HdKYSXIK8F46AeH3V9V7eu6/Avhz4FhgeVVd13P/WcCdwPVVdc5gWi1Jw+EyCkmSJGkaSRYAlwGnAkuA05Ms6cn2LeAs4MO7qOa/Ap+drTZK0igx2CBJkiRN7wRgU1VtrqpHgWuAZd0ZqmpLVd1OZznbkyQ5HngB8MlBNFaShs1ggyRJkjS9g4C7u95vbdKmleQpwP8Afm+afCuSrE+yfvv27XvdUEkaBQYbJEmSpOllirTqs+zbgTVVdffuMlXVqqqaqKqJRYsW7XEDJWmUuEGkJEmSNL2twCFd7w8GtvVZ9peAlyd5O/AMYJ8kD1bVuS23UZJGRl8zG5KckuSuJJuS7DQoJtk3ybXN/ZuTLG7Sn5rkqiQbktyZ5Lx2mz/7li5dytKlS4fdDEmSJA3XOuCoJIcn2YfO0cOr+ylYVWdU1aFVtRh4F/BBAw2Sxt20wYY+d949G7i/qo4ELgEubtLfCOxbVccAxwNvnQxESJIkSXNFVe0AzgHW0jm+8qNVtTHJRUleB5DkpUm20vkd+H1JNg6vxZI0XP0so3hi512AJJM7797RlWcZsLK5vg64NEnorGPbP8lC4GnAo8D322m6pHE1OZvopptuGmo7JEnqVlVrgDU9aRd0Xa+js7xid3V8APjALDRPkkZKP8GGqXbePXFXeapqR5IHgAPpBB6WAd8Bng78blXdN9NGS3OV/4iWNJtuOmv/YTdBkiQJ6G/Phn523t1VnhOAx4AXAYcD/1eSI3b6AI/5kSRJkiRpbPQTbOhn590n8jRLJg4A7gPeDHyiqn5cVfcAXwQmej/AY34kSZIkSRof/SyjeGLnXeDbdHbefXNPntXAmcCXgNOAG6uqknwLeGWSD9FZRvGLwJ+31fjpLD73hhnX8d3N32utLoAt7/n1VuqRpPnEJUiSJElzy7TBhmYPhsmddxcAV0zuvAusr6rVwOXA1Uk20ZnRsLwpfhlwJfA1Okstrqyq22fhOSSNgLaCcm0G+QzwSZIkSYPXz8yGfnbefZjOET+95R6cKl2SJEmjy9lEkqSZ6mfPBkmSJEmSpL71NbNBmu9cHiBJ48G/2EuSNBjObJCkEZLklCR3JdmU5Nwp7u+b5Nrm/s1JFjfpJyS5rXl9Nclv9FunJEmS1DZnNkzjp978nmE3QdI8kWQBnY11T6ZzpPC6JKur6o6ubGcD91fVkUmWAxcDb6KzEe9Es6nvC4GvJvlHoPqoU5IkSWqVwQZJI2ceB/lOADZV1WaAJNcAy4DuwMAyYGVzfR1waZJU1Q+78uxHJ8jQb52SJElSqww2SNLoOAi4u+v9VuDEXeVpZjE8ABwI3JvkROAK4DDgt5r7/dQJQJIVwAqAQw89dOZPA7DygHbq2fJQe/WtfGDmdUiSJGm33LNBkkZHpkirfvNU1c1VdTTwUuC8JPv1WSdN+VVVNVFVE4sWLdqDZkuSJElP5swGaYDm8fIA9WcrcEjX+4OBbbvIszXJQuAA4L7uDFV1Z5KHgJf0WaekcdLGDKA2ZxOBM4okaR5yZoMkjY51wFFJDk+yD7AcWN2TZzVwZnN9GnBjVVVTZiFAksOAFwNb+qxTkiRJapXBBkkaEVW1AzgHWAvcCXy0qjYmuSjJ65pslwMHJtkEvBOYPMryl+mcQHEbcD3w9qq6d1d1Du6pJGl89HE88SuSfCXJjiSndaUfl+RLSTYmuT3JmwbbckkaPJdRSNIIqao1wJqetAu6rh8G3jhFuauBq/utU5qTXB6gIerzeOJvAWcB7+op/kPgLVX1r0leBNySZG1V/fsAmi5JQ2GwQZIkSZretEcJV9WW5t7j3QWr6v/rut6W5B5gEWCwQdLYMtggSRp5N521/7CbIEl9HyW8O0lOAPYBvj7FvfaPIJakIXHPBkmSJGl6fR8lvMsKkhfSWfL2v1XV4733PYJY0jgx2CBJkiRNb0ZHCSd5FnAD8IdV9c8tt02SRo7BBkmSJGl6e32UcJP/euCDVfW3s9hGSRoZ7tkgSZKkJ3GflJ1V1Y4kk0cJLwCumDyeGFhfVauTvJROUOE5wGuTXFhVRwO/CbyCztHFZzVVnlVVtw3+SSRpMAw2SJIkSX3o43jidXSWV/SW+xDwoVlvoCSNEJdRSJIkSZKkVjmzQZIkzRsuD5AkaTCc2SBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWyQpBGS5JQkdyXZlOTcKe7vm+Ta5v7NSRY36ScnuSXJhubrK7vK3NTUeVvzev7gnkiSJEnz0cJhN0CS1JFkAXAZcDKwFViXZHVV3dGV7Wzg/qo6Msly4GLgTcC9wGuraluSlwBrgYO6yp1RVesH8iCSJEma95zZIEmj4wRgU1VtrqpHgWuAZT15lgFXNdfXASclSVXdWlXbmvSNwH5J9h1IqyVJkqQeBhskaXQcBNzd9X4rT56d8KQ8VbUDeAA4sCfPG4Bbq+qRrrQrmyUU5ydJu82WJEmSnsxggySNjqmCALUneZIcTWdpxVu77p9RVccAL29evzXlhycrkqxPsn779u171HBJkiS1a+nSpSxdunTYzdhrfQUb9nbDsubesUm+lGRjs3HZfu01X5LGylbgkK73BwPbdpUnyULgAOC+5v3BwPXAW6rq65MFqurbzdcfAB+ms1xjJ1W1qqomqmpi0aJFrTyQJEmS5qdpgw1dG5adCiwBTk+ypCfbExuWAZfQ+ava5C/CHwLeVlVHA0uBH7fWekkaL+uAo5IcnmQfYDmwuifPauDM5vo04MaqqiTPBm4AzquqL05mTrIwyfOa66cCrwG+NsvPIUljqY8/wL0iyVeS7EhyWs+9M5P8a/M6s7esJI2bfk6jeGLDMoAkkxuWde+OvgxY2VxfB1zarAl+FXB7VX0VoKq+11K7JWnsVNWOJOfQOUliAXBFVW1MchGwvqpWA5cDVyfZRGdGw/Km+DnAkcD5Sc5v0l4FPASsbQINC4B/Av56YA8lSWOizxODvgWcBbyrp+xzgT8CJugsfbulKXv/INouabCOueqYVurZ/N3NrdW34cwNM65jT/UTbJhqw7ITd5Wn+WV5csOynwYqyVpgEXBNVf1p7wckWQGsADj00EP39BkkaWxU1RpgTU/aBV3XDwNvnKLcu4F376La49tsoyTNU9P+Aa6qtjT3Hu8p+2rgU1U1ueztU8ApwEdmv9mSNBz97Nkwkw3LFgK/DJzRfP2NJCftlNF1wpIkSRpt/ZwYNKOybtQraZz0M7NhTzYs29qzYdlW4LNVdS9AkjXALwCfnmG7JUmSpEHq5w9wMypbVauAVQATExP91i1pTB1x3hHDbsKM9DOzYa83LKOz7vjYJE9vghC/wpP3epAkSZLmgn7+ADcbZSVpTpo22FBVO+hsPLYWuBP46OSGZUle12S7HDiw2bDsncC5Tdn7gT+jE7C4DfhKVd3Q/mNIkiRJs6qfP8DtylrgVUmek+Q5dDbwXTtL7ZSkkdDPMoq93rCsufchOsdfSpIkSXNSPycGJXkpcD3wHOC1SS6sqqOr6r4k/5VOwALgosnNIiVpXPUVbJAkSZLmuz7+ALeOzhKJqcpeAVwxqw2UpBHSz54NkiRJkiS1YunSpSxdunTYzdAsM9ggSZIkSZJaZbBBkiRJkiS1yj0bJEmSJEnTOuaqY1qpZ/N3N7dW34YzN8y4Ds0OZzZIkiRJkqRWGWyQJEmSJEmtMtggSZIkSZJa5Z4NkiRJknZr8pjCm266aajt0Hg44rwjht0EDYAzGyRJkiRJUquc2SBJkiSNsTZ2/G/z9ADwBAFpPnBmgyRJkiRJapXBBkkaIUlOSXJXkk1Jzp3i/r5Jrm3u35xkcZN+cpJbkmxovr6yq8zxTfqmJH+RJIN7IkmSJM1HLqOQpBGRZAFwGXAysBVYl2R1Vd3Rle1s4P6qOjLJcuBi4E3AvcBrq2pbkpcAa4GDmjJ/BawA/hlYA5wCfHwQzyRJGg9u6CdpTzmzQZJGxwnApqraXFWPAtcAy3ryLAOuaq6vA05Kkqq6taq2Nekbgf2aWRAvBJ5VVV+qqgI+CLx+9h9FkiRJ85nBBkkaHQcBd3e938pPZifslKeqdgAPAAf25HkDcGtVPdLk3zpNnQAkWZFkfZL127dv3+uHkCRJkgw2SNLomGovhdqTPEmOprO04q17UGcnsWpVVU1U1cSiRYv6aK4kzS8z2FfnqUmuavbPuTPJeYNuuyQNmsEGSRodW4FDut4fDGzbVZ4kC4EDgPua9wcD1wNvqaqvd+U/eJo6JUnT6NpX51RgCXB6kiU92Z7YVwe4hE7wF+CNwL5VdQxwPPDWyUCEJI0rgw2SNDrWAUclOTzJPsByYHVPntXAmc31acCNVVVJng3cAJxXVV+czFxV3wF+kOQXm1Mo3gL8w2w/iCSNob3eV4fOjLL9myDx04BHge8PptmSNBwGGyRpRDR7MJxD5ySJO4GPVtXGJBcleV2T7XLgwCSbgHcCk9N4zwGOBM5Pclvzen5z77eB9wObgK/jSRSStDdmsq/OdcBDwHeAbwH/varu6/0A986RNE48+lKSRkhVraFzPGV32gVd1w/TmY7bW+7dwLt3Ued64CXttlSS5p2Z7KtzAvAY8CLgOcDnk/xTVW1+UsaqVcAqgImJiSn315GkucKZDZIkSdL0ZrKvzpuBT1TVj6vqHuCLwMSst1iShshggyRJkjS9vd5Xh87SiVemY3/gF4F/GVC7JWkoDDZIkiRJ05jhvjqXAc8AvkYnaHFlVd0+0AeQpAFzzwZJkiSpDzPYV+fBqdIlaZw5s0GSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRW9RVsSHJKkruSbEpy7hT3901ybXP/5iSLe+4fmuTBJO9qp9mSJEmSJGlUTRtsSLIAuAw4FVgCnJ5kSU+2s4H7q+pI4BLg4p77lwAfn3lzJUmSJEnSqOtnZsMJwKaq2lxVjwLXAMt68iwDrmqurwNOShKAJK8HNgMb22myJEmSJEkaZf0EGw4C7u56v7VJmzJPVe0AHgAOTLI/8AfAhTNvqiRJkiRJmgv6CTZkirTqM8+FwCVV9eBuPyBZkWR9kvXbt2/vo0mSJEmSJGlULewjz1bgkK73BwPbdpFna5KFwAHAfcCJwGlJ/hR4NvB4koer6tLuwlW1ClgFMDEx0RvIkCRJkiRJc0g/MxvWAUclOTzJPsByYHVPntXAmc31acCN1fHyqlpcVYuBPwf+pDfQIEn6ib09/SfJgUk+05z8c2lPmZuaOm9rXs8fzNNIkiRpvpp2ZkNV7UhyDrAWWABcUVUbk1wErK+q1cDlwNVJNtGZ0bB8NhstSeOo6/Sfk+nMGFuXZHVV3dGV7YnTf5Isp3P6z5uAh4HzgZc0r15nVNX6WX0ASZIkqdHPMgqqag2wpiftgq7rh4E3TlPHyr1onyTNJ0+c/gOQZPL0n+5gwzJgZXN9HXBpklTVQ8AXkhw5wPZKkiRJU+pnGYUkaTD2+vSfPuq+sllCcf7k0cS93KxXknZvb5e6NfeOTfKlJBuTbEiy3yDbLkmDZrBBkkbHTE7/2Z0zquoY4OXN67emylRVq6pqoqomFi1aNG1jJWk+6VrqdiqwBDg9yZKebE8sdQMuobPUjWYD9Q8Bb6uqo4GlwI8H1HRJGgqDDZI0Ovbk9J/JX14nT//Zpar6dvP1B8CH6SzXkCTtmSeWulXVo8DkUrduy4CrmuvrgJOa2WSvAm6vqq8CVNX3quqxAbVbkobCYIMkjY69Pv1nVxUmWZjkec31U4HXAF9rveWSNP5mstTtp4FKsjbJV5L8/lQf4HI2SeOkrw0iJUmzb6an/yTZAjwL2CfJ6+n8Je2bwNom0LAA+Cfgrwf4WJI0Lmay1G0h8MvAS4EfAp9OcktVffpJGatWAasAJiYmplsiJ0kjzWCDJI2QmZz+U1WLd1Ht8W21T5LmsT1Z6ra1Z6nbVuCzVXUvQJI1wC8An0aSxpTLKCRJkqTpzWSp21rg2CRPb4IQv8KTjzWWpLHjzAZJkiRpGjNZ6lZV9yf5MzoBiwLWVNUNQ3kQSRoQgw2SJElSH2a41O1DdI6/lKR5wWUUkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSRkiSU5LclWRTknOnuL9vkmub+zcnWdykH5jkM0keTHJpT5njk2xoyvxFkgzmaSRJkjRfGWyQpBGRZAFwGXAqsAQ4PcmSnmxnA/dX1ZHAJcDFTfrDwPnAu6ao+q+AFcBRzeuU9lsvSZIk/YTBBkkaHScAm6pqc1U9ClwDLOvJswy4qrm+DjgpSarqoar6Ap2gwxOSvBB4VlV9qaoK+CDw+ll9CkkaU3s7+6zr/qHNDLSpAsOSNFYMNkjS6DgIuLvr/dYmbco8VbUDeAA4cJo6t05TJwBJViRZn2T99u3b97DpkjTeZjj7bNIlwMdnu62SNAoMNkjS6JhqL4Xaizx7lb+qVlXVRFVNLFq0aDdVStK8tNezzwCSvB7YDGwcUHslaagMNkjS6NgKHNL1/mBg267yJFkIHADcN02dB09TpyRpens9+yzJ/sAfABcOoJ2SNBIMNkjS6FgHHJXk8CT7AMuB1T15VgNnNtenATc2ezFMqaq+A/wgyS82f117C/AP7TddksbeTGafXQhcUlUP7vYDXM4maYwsHHYDJEkdVbUjyTnAWmABcEVVbUxyEbC+qlYDlwNXJ9lEZ0bD8snySbYAzwL2aabrvqqq7gB+G/gA8DQ6a4VdLyxJe25PZp9t7Zl9diJwWpI/BZ4NPJ7k4ap60lHFVbUKWAUwMTGxuyVykjTyDDZI0gipqjXAmp60C7quHwbeuIuyi3eRvh54SXutlKR56YnZZ8C36QR739yTZ3L22Zd48uyzl09mSLISeLA30CBJ48ZggyRJkjSNmc4+k6T5pq9gQ5JTgPfSGVjfX1Xv6bm/L52z248Hvge8qaq2JDkZeA+wD/Ao8HtVdWOL7ZckSZIGYiazz7ryrJyVxknSiJl2g8gZnil8L/DaqjqGzpSyq9tquCRJkiRJGk39nEax12cKV9WtVTW5cc5GYL9mFoQkSZIkSRpT/QQb9vpM4Z48bwBurapHej/AY34kSZIkSRof/QQbZnKmcOdmcjSdpRVvneoDqmpVVU1U1cSiRYv6aJIkSZIkSRpV/QQb9uRMYXrOFCbJwcD1wFuq6uszbbAkSZIkSRpt/QQbnjhTOMk+dI7wWd2TZ/JMYeg6UzjJs4EbgPOq6ottNVqSJEmSJI2uaYMNzR4Mk2cK3wl8dPJM4SSva7JdDhzYnCn8TuDcJv0c4Ejg/CS3Na/nt/4UkiRJkiRpZCzsJ9PenilcVe8G3j3DNkqSJEmSpDmkn2UUkiRJkiRJfTPYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRphCQ5JcldSTYlOXeK+/smuba5f3OSxV33zmvS70ry6q70LUk2JLktyfrBPIkkSZLmM4MNkjQikiwALgNOBZYApydZ0pPtbOD+qjoSuAS4uCm7BFgOHA2cAvxlU9+kX62q46pqYpYfQ5LG1t4GhJOcnOSWJvB7S5JXDrrtkjRoBhskaXScAGyqqs1V9ShwDbCsJ88y4Krm+jrgpCRp0q+pqkeq6hvApqY+SVILZhIQBu4FXltVxwBnAlcPptWSNDwGGyRpdBwE3N31fmuTNmWeqtoBPAAcOE3ZAj7Z/DVtxSy0W5Lmg70OCFfVrVW1rUnfCOyXZN+BtFqShsRggySNjkyRVn3m2V3Zl1XVL9D5a9w7krxiyg9PViRZn2T99u3b+22zJM0XMwkId3sDcGtVPTJL7ZSkkWCwQZJGx1bgkK73BwPbdpUnyULgAOC+3ZWd/GtaVd0DXM8ulldU1aqqmqiqiUWLFs34YSRpzMwkINy5mRxNZ2nFW6dMJgyOAAALAklEQVT8AIO+ksaIwQZJGh3rgKOSHJ5kHzobPq7uybOaznpfgNOAG6uqmvTlzeZkhwNHAV9Osn+SZwIk2R94FfC1ATyLJI2bmQSESXIwnYDvW6rq61N9gEFfSeNk4bAbIEnqqKodSc4B1gILgCuqamOSi4D1VbUauBy4OskmOr/ALm/KbkzyUeAOYAfwjqp6LMkLgOs7e0iyEPhwVX1i4A8nSXPfEwFh4Nt0xt839+SZDAh/ia6AcJJnAzcA51XVFwfYZkkaGoMNkjRCqmoNsKYn7YKu64eBN+6i7B8Df9yTthn4ufZbKknzy0wCwsA5wJHA+UnOb9Je1Sxvk6SxZLBBkiRJ6sPeBoSr6t3Au2e9gZI0QtyzQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWyQJEmSJEmt6ivYkOSUJHcl2ZTk3Cnu75vk2ub+zUkWd907r0m/K8mr22u6JI2f2Rhvp6tTktQffyeWpP5NG2xIsgC4DDgVWAKcnmRJT7azgfur6kjgEuDipuwSYDlwNHAK8JdNfZKkHrMx3vZZpyRpGv5OLEl7pp+ZDScAm6pqc1U9ClwDLOvJswy4qrm+DjgpSZr0a6rqkar6BrCpqU+StLPZGG/7qVOSND1/J5akPdBPsOEg4O6u91ubtCnzVNUO4AHgwD7LSpI6ZmO8dRyWpHb4O7Ek7YGFfeTJFGnVZ55+ypJkBbCieftgkrv6aNcgPQ+4t42KcnEbtYws+6k/rfTTmPcRjGY/HdZqbTubjfF2qqDyTuMwzKOx+MKpumqs2E/Ta+3/V/ZTn9rrp9keh3fH34nb/F3vLH92+mE/TW/M+whGs5/6Gov7CTZsBQ7pen8wsG0XebYmWQgcANzXZ1mqahWwqp8GD0OS9VU1Mex2jDr7qT/2U3/maT/N1ng77TgMjsXjwn6ann3UH/tpJ/5O7PdEX+yn/thP/ZnL/dTPMop1wFFJDk+yD53NbVb35FkNnNlcnwbcWFXVpC9vduY9HDgK+HI7TZeksTMb420/dUqSpufvxJK0B6ad2VBVO5KcA6wFFgBXVNXGJBcB66tqNXA5cHWSTXSit8ubshuTfBS4A9gBvKOqHpulZ5GkOW22xtup6hz0s0nSXOfvxJK0Z9IJtmp3kqxoprVpN+yn/thP/bGf1Mvvif7YT9Ozj/pjP6mX3xP9sZ/6Yz/1Zy73k8EGSZIkSZLUqn72bJAkSZIkSerbWAYbkjyW5LYkX0vyt0me3lK9L0jysSRfTXJHkjVN+jeSvLgn758n+f0kS5M8kOTWJHcl+VyS17TRnj1su30yy5L8vz3vfzfJw0kO6EpbmuRjU5R9TdMfk/341kG0eRDsl/nJMWfKttsnA+CYszP7ZH5yzNll++2XWeaYM7V52S9VNXYv4MGu678B3rkHZRfu5t77gP/U9f7Y5ut/A/6oK/0pdI44OgxYCnys695xwBbgJPtkuH0ygD7/MvB54KyutCc9e5P2VDrHXx3cvN8XePGw22+/+Jrhf2fHHPtkJF6OOfbJfH055tgvo/JyzJm//TKWMxt6fB44MsniJF+bTEzyriQrm+ubkvxJks8C/ynJoiR/l2Rd83pZU+yFdAYHAKrq9ubyIzS7DTdeAWypqm/2NqaqbgMuAs5p8Rn31JzpkyQLknygiT5vSPK7M3ryWZTkwa7r/wA8A/hD4PRpij6Tzskw3wOoqkeq6q7ZauegzXa/zKXvkXlszow5AzRn+mSu/Yw5Fu/McVjMoTFnwOZMv8ylnzPH4anNx7F4rIMNSRYCpwIb+sj+7Kr6lar6H8B7gUuq6qXAG4D3N3kuAy5P8pkk/yXJi+CJweTxJD/X5FtOZ2DZla8AP7PnTzRzc7BPjgMOqqqXVNUxwJV9tHsUnE7neT8PvDjJ83eVsaruo3P+9jeTfCTJGUnG9WdzNvplrn6PzAtzcMyZdXOwT+byz5hj8c4ch+eZOTjmDMQc7Je5+nPmODy1eTEWj+t/vKcluQ1YD3yLzpnH07m26/rXgEubOlYDz0ryzKpaCxwB/DWdQeDWJIuaMh8BljcD1zLgb3fzWdmjp2nHXO2TzcARSf5nklOA7/fR7lGwHLimqh4H/h544+4yV9X/AZxEZzrVu4ArZr2FwzEb/TJXv0fG3Vwdc2bTXO2Tufwz5li8M8fh+WOujjmzba72y1z9OXMcntq8GIsXDrsBs+RHVXVcd0KSHTw5uLJfT5mHuq6fAvxSVf2ot+ImsvRh4MPpbN7xCuDv6AwinwQ+C9xeVffspn0/D9zZ57O0ZU70SZIFwC1N2uqquqCJBL8aeAfwm8D/vtsnHbIkxwJHAZ9KArAPnR/+y3ZXrqo2ABuSXA18Azhrdls6WG31S5KzmePfI/PEnBhz+nyWtsyJPhmHcRgci6fiODzvzIkxp89nadOc6JdxGIsdh6c2n8bicZ3ZMJV/A56f5MAk+wK72+n1k3StlUpyXPP1lWl2rE3yTOA/0ImIUlVfp7OO5j3sZmpU8811PtN8Mw3IyPVJVT1WVcc1rwuSPA94SlX9XZPnF/b+cQfmdGBlVS1uXi8CDkpy2FSZkzwjydKupOOAndbxjYFW+mVMvkfmq5Ebc2b0NO0YuT4Zo58xx+KdOQ5r5MacGT1Ne0auX8bk58xxeGrzZiwe15kNO6mqHye5CLiZToTsX3aT/XeAy5LcTqePPge8DTiezrSpyejn+6tqXVe5j9DZcfb6nvpenuRW4OnAPcDvVNWnW3isGZkjfXIQcGV+si7pvD15xiFZTmcNYLfrm/SbgZOSbO26dzrw+0neB/yITvT8rAG0c9Bmq1/m4vfIvDRHxpyBmiN9Mld/xhyLd+Y4PM/NkTFn4OZIv8zFnzPH4anNm7E41Tk+Q5IkSZIkqRXzaRmFJEmSJEkaAIMNkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYoHklyZbmDNoZ5ZEk7T3HYkkaLsdhDYLBBkmSJEmS1CqDDRp5SRYn+Zck70/ytSR/k+TXknwxyb8mOSHJc5P8P0luT/LPSY5tyh6Y5JNJbk3yPiBd9f6vSb6c5LYk70uyYGgPKUkjzrFYkobLcVhzjcEGzRVHAu8FjgV+Bngz8MvAu4D/DFwI3FpVxzbvP9iU+yPgC1X188Bq4FCAJD8LvAl4WVUdBzwGnDGwp5GkucmxWJKGy3FYc8bCYTdA6tM3qmoDQJKNwKerqpJsABYDhwFvAKiqG5vo7QHAK4D/2KTfkOT+pr6TgOOBdUkAngbcM8DnkaS5yLFYkobLcVhzhsEGzRWPdF0/3vX+cTrfxzumKFM9X7sFuKqqzmuthZI0/hyLJWm4HIc1Z7iMQuPiczRTvpIsBe6tqu/3pJ8KPKfJ/2ngtCTPb+49N8lhg260JI0Zx2JJGi7HYY0MZzZoXKwErkxyO/BD4Mwm/ULgI0m+AnwW+BZAVd2R5A+BTyZ5CvBj4B3ANwfdcEkaIytxLJakYVqJ47BGRKqmmk0jSZIkSZK0d1xGIUmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa36/wG5EhUVaMdJbgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_cv_results(cv_results.append(sals_cv_results));" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Surprisingly, the iALS model remains largely insensitive to the scaling trick. At least in the current settings and for the current dataset. \n", + "**Remark**: You may want to repeat all experiments in a different setting with `random_holdout` set to `True`. My own results indicate that in this case iALS becomes more responsive to scaling and gives the same result as the scaled version of SVD. However, SVD is still much easier to compute and tune." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conclusion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With a proper tuning the quality of recommendations of one of the simplest SVD-based models can be substantially improved. Despite common beliefs, it turns out that PureSVD with simple scaling trick compares favorably to a much more popular iALS algorithm. It not only generates more relevant recommendations, but also makes them more diverse and potentially more interesting. In addition to that it has a number of unique advantages and merely requires to simply do `from scipy.sparse.linalg import svds` to start using it. Of course, the obtained results may not necessarily hold on all other datasets and require further verification. However, in the view of all its features, the scaled SVD-based model can be certainly considered as at least one of the default baseline candidates. The result obtained here resembles situation in the Natural Language Processing field, where simple SVD-based model with proper tuning [turns out to be competitive](http://www.aclweb.org/anthology/Q15-1016) on a variety of downstream tasks even in comparison with Neural Network-based models. In the end it's all about fair comparison.\n", + "\n", + "As a final remark, Polara is designed to support opennes and reproducibility of research. It allows to perform thorough experiments with minimal efforts. It can be used to quickly test known ideas or implement something new by providing controlled environment and rich functionality with high level abstractions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + }, + "toc": { + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 24d36cc0552e5a87e35bac40a2b73db8f40b6687 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 2 May 2019 09:16:43 +0300 Subject: [PATCH 06/82] allow single-valued parameter configuration in find_optimal_config --- polara/evaluation/pipelines.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/polara/evaluation/pipelines.py b/polara/evaluation/pipelines.py index 569e341..417d2df 100644 --- a/polara/evaluation/pipelines.py +++ b/polara/evaluation/pipelines.py @@ -160,6 +160,13 @@ def find_optimal_tucker_ranks(model, tucker_ranks, target_metric, return_scores= return best_mlrank +def params_to_dict(names, params): + try: + return dict(zip(names, params)) + except TypeError: # encountered single value + return {names: params} + + def find_optimal_config(model, param_grid, param_names, target_metric, return_scores=False, init_config=None, reset_config=None, verbose=False, force_build=True, evaluator=None, iterator=lambda x: x, **kwargs): @@ -175,8 +182,8 @@ def find_optimal_config(model, param_grid, param_names, target_metric, return_sc model.verbose = verbose grid_results = {} for params in iterator(param_grid): + param_config = params_to_dict(param_names, params) try: - param_config = dict(zip(param_names, params)) set_config(model, param_config) if not model._is_ready or force_build: model.build() @@ -195,10 +202,7 @@ def find_optimal_config(model, param_grid, param_names, target_metric, return_sc scores = pd.Series(**dict(zip(('index', 'data'), (zip(*grid_results.items()))))) best_params = scores.idxmax() - try: - best_config = dict(zip(param_names, best_params)) - except TypeError: - best_config = {param_names: best_params} + best_config = params_to_dict(param_names, best_params) if return_scores: try: From c38dc7851061717e51424e0df4437d4dda50dee2 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 2 May 2019 10:42:45 +0300 Subject: [PATCH 07/82] update tutorial on tuning and CV --- ...ing_and_cross_validation_experiments.ipynb | 162 +++++++++--------- 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb b/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb index f4792c8..fc7bde2 100644 --- a/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb +++ b/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb @@ -13,16 +13,16 @@ "source": [ "In this tutorial we will go through a full cycle of model tuning and evaluation with Polara. This will include 2 phases: grid-search for finding (almost) optimal values of hyper-parameters and verification of results via 5-fold cross-validation.\n", "\n", - "
We will focus on performing a fair comparison of popular ALS-based matrix factorization (MF) model called Weighted Regularized Matrix Factorization (WRMF a.k.a. iALS) [Hu2008] with less popular model called PureSVD [Cremonesi2010] based on standard SVD.
\n", + "
We will focus on performing a fair comparison of popular ALS-based matrix factorization (MF) model called Weighted Regularized Matrix Factorization (WRMF a.k.a. iALS) [Hu, 2008] with less popular model called PureSVD [Cremonesi, 2010] based on standard SVD.
\n", "\n", "We will use standard *Scipy*'s implementation for the latter and a great library called [*implicit*](https://github.com/benfred/implicit) for iALS. Both are wrapped by Polara and can be accessed via the corresponding classes. Due to its practicality the *implicit* library is often recommended to beginners and sometimes even serves as a default tool in production. On the other hand, there are some important yet often overlooked features, which make SVD-based models stand out. Ignoring them in my opinion leads to certain misconceptions and myths, not to say that it also overcomplicates things quite a bit.\n", "\n", "Note that by saying SVD I really mean *Singular Value Decomposition*, not just an arbitrary matrix factorization. In that sense, **methods like FunkSVD, SVD++, SVDFeature, etc., are not SVD-based at all**, even though historically they use SVD acronym in their names and are often referenced as if they are real substitutes for SVD. These methods utilize another optimization algorithm, typically based on stochastic gradient descent, and do not preserve the algebraic properties of SVD. This is really an important distinction, especially in the view of the following remarks:\n", "\n", "1. **SVD-based approach has a number of unique and beneficial properties**. To name a few, it produces stable and determenistic output with global guarantees. It admits the same prediction formula for both known and previously unseen users (as long as at least one user rating is known). It can take a hybrid form to include side information via the generalized formulation (see Chapter 6 of [my thesis](https://www.skoltech.ru/en/2018/09/phd-thesis-defense-evgeny-frolov)). Even without hybridization it can be quite successfully applied in the cold start regime (paper on this is coming). It requires minimal tuning and allows to compute and store a single latent feature matrix - either for users or for items - instead of computing and storing both of them. This luxury is not available in the majority of other matrix factorization approaches, definitely not in popular ones. \n", - "2. Computational complexity of truncated SVD **scales linearly with the number of known observations** and quadratically with the rank of decomposition (thanks to Lanczos procedure). There are [open source implementations](https://github.com/criteo/rsvd), allowing to handle in a distributed manner nearly *billion-scale problems* with its [efficient randomized version](https://medium.com/criteo-labs/sparkrsvd-open-sourced-by-criteo-for-large-scale-recommendation-engines-6695b649f519). \n", - "3. At least **in some cases PureSVD outperforms other more sophisticated matrix factorization methods** [Cremonesi2010].\n", - "4. Moreover, **PureSVD can be quite easily tuned to perform even better** [Nikolakopoulos2019].\n", + "2. Computational complexity of truncated SVD **scales linearly with the number of known observations** and quadratically with the rank of decomposition (thanks to Lanczos procedure). There are [open source implementations](https://github.com/criteo/rsvd), allowing to handle nearly *billion-scale problems* with its [efficient randomized version](https://medium.com/criteo-labs/sparkrsvd-open-sourced-by-criteo-for-large-scale-recommendation-engines-6695b649f519). \n", + "3. At least **in some cases PureSVD outperforms other more sophisticated matrix factorization methods** [Cremonesi, 2010].\n", + "4. Moreover, **PureSVD can be quite easily tuned to perform even better** [Nikolakopoulos, 2019].\n", "\n", "Despite that impresisve list, PureSVD technique (and especially its modifications) rarely gets into the list of baseline models to compare with. Hence, this tutorial also aims at performing a thorough assessment of the default choice of many practitioners to see whether it really provides the celebrated advantages over the simpler approach." ] @@ -38,9 +38,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "* [Hu 2008], Hu Y., Koren, Y. and Volinsky, C., 2008, December. *Collaborative Filtering for Implicit Feedback Datasets*. In ICDM (Vol. 8, pp. 263-272). [Link](http://yifanhu.net/PUB/cf.pdf). \n", - "* [Cremonesi2010] Cremonesi, P., Koren, Y. and Turrin, R., 2010, September. *Performance of recommender algorithms on top-n recommendation tasks*. In Proceedings of the fourth ACM conference on Recommender systems (pp. 39-46). ACM. [Link](https://www.researchgate.net/publication/221141030_Performance_of_recommender_algorithms_on_top-N_recommendation_tasks). \n", - "* [Nikolakopoulos2019] Nikolakopoulos, A.N., Kalantzis, V., Gallopoulos, E. and Garofalakis, J.D., 2019. *EigenRec: generalizing PureSVD for effective and efficient top-N recommendations*. Knowledge and Information Systems, 58(1), pp.59-81. [Link](https://arxiv.org/abs/1511.06033)." + "* [Hu, 2008] Hu Y., Koren, Y. and Volinsky, C., 2008, December. *Collaborative Filtering for Implicit Feedback Datasets*. In ICDM (Vol. 8, pp. 263-272). [Link](http://yifanhu.net/PUB/cf.pdf). \n", + "* [Cremonesi, 2010] Cremonesi, P., Koren, Y. and Turrin, R., 2010, September. *Performance of recommender algorithms on top-n recommendation tasks*. In Proceedings of the fourth ACM conference on Recommender systems (pp. 39-46). ACM. [Link](https://www.researchgate.net/publication/221141030_Performance_of_recommender_algorithms_on_top-N_recommendation_tasks). \n", + "* [Nikolakopoulos, 2019] Nikolakopoulos, A.N., Kalantzis, V., Gallopoulos, E. and Garofalakis, J.D., 2019. *EigenRec: generalizing PureSVD for effective and efficient top-N recommendations*. Knowledge and Information Systems, 58(1), pp.59-81. [Link](https://arxiv.org/abs/1511.06033)." ] }, { @@ -285,14 +285,14 @@ { "data": { "text/plain": [ - "{'shuffle_data': False,\n", - " 'test_fold': 5,\n", - " 'test_ratio': 0.2,\n", - " 'permute_tops': False,\n", - " 'warm_start': True,\n", - " 'random_holdout': False,\n", + "{'test_ratio': 0.2,\n", " 'negative_prediction': False,\n", + " 'warm_start': True,\n", + " 'permute_tops': False,\n", " 'test_sample': None,\n", + " 'random_holdout': False,\n", + " 'test_fold': 5,\n", + " 'shuffle_data': False,\n", " 'holdout_size': 3}" ] }, @@ -520,11 +520,11 @@ "\n", "100%\n", "15/15\n", - "[00:57<00:04, 3.79s/it]" + "[01:03<00:04, 4.21s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 15/15 [00:57<00:04, 3.79s/it]" + " [████████████████████████████████████████████████████████████] 15/15 [01:03<00:04, 4.21s/it]" ] }, "metadata": {}, @@ -556,7 +556,7 @@ { "data": { "text/plain": [ - "[9.0811659]" + "[9.6594064]" ] }, "execution_count": 16, @@ -608,8 +608,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We will employ a simple scaling trick over the rating matrix $R$ that was proposed by the authors of [EIGENREC model](https://arxiv.org/abs/1511.06033) [Nikolakopoulos2019]: $R \\rightarrow RD^{f-1},$\n", - "where $D$ is a diagonal scaling matrix with elements corresponding to the norm of the matrix columns (or square root of the number of nonzero elements in each column for the binary case). Parameter $f$ controls the effect of scaling and typically lies in the range [0, 1]. Finding the optimal value is an experimental task and will be performed via grid-search. We will use built-in support for such model in Polara." + "We will employ a simple scaling trick over the rating matrix $R$ that was proposed by the authors of the [EIGENREC model](https://arxiv.org/abs/1511.06033) [Nikolakopoulos2019]: $R \\rightarrow RD^{f-1},$\n", + "where $D$ is a diagonal scaling matrix with elements corresponding to the norm of the matrix columns (or square root of the number of nonzero elements in each column for the binary case). Parameter $f$ controls the effect of scaling and typically lies in the range [0, 1]. Finding the optimal value is an experimental task and will be performed via grid-search. We will use built-in support for such model in Polara. If you're interested in technical aspects of this implementation, see [Reproducing EIGENREC results](./Reproducing_EIGENREC_results.ipynb) tutorial." ] }, { @@ -675,11 +675,11 @@ "\n", "100%\n", "60/60\n", - "[04:17<00:03, 4.29s/it]" + "[05:03<00:04, 5.05s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 60/60 [04:17<00:03, 4.29s/it]" + " [████████████████████████████████████████████████████████████] 60/60 [05:03<00:04, 5.05s/it]" ] }, "metadata": {}, @@ -865,11 +865,11 @@ "\n", "100%\n", "60/60\n", - "[24:14<00:24, 24.23s/it]" + "[24:18<00:26, 24.30s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 60/60 [24:14<00:24, 24.23s/it]" + " [████████████████████████████████████████████████████████████] 60/60 [24:18<00:26, 24.30s/it]" ] }, "metadata": {}, @@ -912,11 +912,11 @@ "\n", "100%\n", "15/15\n", - "[06:23<00:32, 25.56s/it]" + "[07:56<00:45, 31.71s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 15/15 [06:23<00:32, 25.56s/it]" + " [████████████████████████████████████████████████████████████] 15/15 [07:56<00:45, 31.71s/it]" ] }, "metadata": {}, @@ -950,10 +950,10 @@ { "data": { "text/plain": [ - "{'alpha': 1.0,\n", - " 'epsilon': 0.03,\n", - " 'weight_func': ,\n", - " 'regularization': 0.003,\n", + "{'alpha': 0.3,\n", + " 'epsilon': 0.3,\n", + " 'weight_func': ,\n", + " 'regularization': 0.03,\n", " 'rank': 60}" ] }, @@ -983,7 +983,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -1001,12 +1001,12 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 32, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xl8lfWd9//X55yT5GQHsgEJS0D2RZYoioJaC9qWaq12IDjqtFrbDrTqtDOj992qxWld6m9sufEe67SOrdNKHTtOqf2hValKFZUg+yqbkABZyb6ecz73H9eVcBISEggnC/k8H4/zuLbvdZ3vdRHO+3yv6zrfS1QVY4wx5lx5ersCxhhj+jcLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbrEgMcYY0y0WJMb0ABF5W0TuOsd1R4pItYh4z3e9eoOIHBaRz7rj/0tEftHbdTLdY0Fizgv3w6HO/cA7ISLPi0hCb9erPwr/oAVQ1SOqmqCqwd6sVySo6o9V9S4AERktIioivt6ulzk7FiTmfPqiqiYAM4CZwAO9XB9jTA+wIDHnnaqeAF7HCRQARCRGRJ4UkSMiUigiz4hIbNjyG0Vki4hUisgBEbnenT9cRNaISJmI7BeRr4et87CI/JeI/KeIVInIdhEZLyIPiEiRiBwVkYVh5d8WkX8RkffdltMfRSRFRH7jvu9GERkdVn6iiLzhvvdeEfmbsGXPi8jTIvIn970/FJGxYcsXiMgeEakQkVWAhC0bKyLrRKRURErc9x/kLnsBGAn80a3jP7X9pt6FY/KSiPzarddOEckJW/7PIlLgLtsrIte292/oHpc17nH5SEQeEZG/ustOazmEn7o70/618z4Pi8h/upPvusNyd9+vcvdxWlj5dLflm9be9kzvsCAx552IZAGfA/aHzX4cGI8TLhcBmcCDbvlLgV8D/wgMAuYDh931XgTygeHALcCP23z4fRF4ARgMbMYJMI+7/RXAz9tUbwlwm7t8LLAB+A9gCLAbeMitUzzwBvBbIB3IBf6viEwJ21Yu8EP3vfcDP3LXTQV+D3wfSAUOAFeEHyLgUXefJgEjgIcBVPU24Ahu605Vn+B0nR2TG4DV7rFcA6xy6zUBWA5coqqJwHWcOs5tPQ3UA8OAr7mvrupw/zox3x0Ocvf9HXc//jasTC7wpqoWn0V9TKSpqr3s1e0XzgdSNVAFKPAWzgcCOB8sNcDYsPKXA4fc8Z8DT7WzzRFAEEgMm/co8Lw7/jDwRtiyL7p18LrTiW5dmuvxNvC/w8r/f8DaNutvcccXA+vb1OfnwEPu+PPAL8KWfR7Y447fDnwQtkxwPvjv6uDYfQnY3OZYfjZserS7H74uHpM3w5ZNBurc8YuAIuCzQNQZ/i29QBMwMWzej4G/tq1P2PK3z2X/3Pr+5xm2Owc4Cnjc6Tzgb3r7791erV/WIjHn05fU+aZ7NTAR59s4QBoQB2wSkXIRKQdec+eD8+F4oJ3tDQfKVLUqbN6nOK2JZoVh43VAiZ66KF3nDhPOUL7tdHPZUcCc5vq6db4VGBpW/kTYeG3YusNxPvwAUOcTsGXaPT2z2j3FVAn8J6eOVWe6ckza1ssvIj5V3Q/ci/PhXeTWYXg775GGE1pHw+Z92sX6dXf/WlHVD3G+hFwlIhNxwnDNuWzLRI4FiTnv1Dkl8TzwpDurBOdDeoqqDnJfyepcmAfnA2vs6VviGDBERBLD5o0ECiJT81aOAu+E1bf5dMu3urDucZxwBEBEJHwapwWhwHRVTcI5dSNhy8/UJXe3jomq/lZVr8QJSsU55dhWMRBoU+eRYeM17jAubF54wHa2fx1Wr4P5v3K3cRvwsqrWd2FbpgdZkJhI+SmwQERmqGoI+HfgKRFJBxCRTBG5zi37S+CrInKtiHjcZRNV9SjwPvCoiPhFZDpwJ/CbHqj/q8B4EblNRKLc1yUiMqkL6/4JmCIiX3YvSH+H1h+0iTin4MpFJBPn2lC4QmBMexvuzjERkQki8hkRicG5/lGHc5qs7XsEgf8GHhaROBGZDNwRtrwYJ7j+VkS8IvI1Wn8R6Gz/OlIMhDh9318AbsIJk193cVumB1mQmIhwP2x+DfzAnfXPOBekP3BPd7wJTHDLfgR8FXgKqADewfnGDM7F1dE438RfwblG8UYP1L8KWIhzcf4Yzumix4GYLqxbAnwFeAwoBcYB74UV+SEwC2df/4TzoR3uUeD77im177XzFud6TGLcOpW4+5MO/K8Oyi7HOVV3Aqd1+R9tln8dJyBKgSk44dass/1rl6rW4tyw8J6775e58/OBj3FaLOu7si3Ts8Q5fWuMMR0Tkb/DuZh+ZS+9/3PAMVX9fm+8vzkz+wWpMaZPE+e3PV/G+ZGr6YPs1JYxps8SkUeAHcBPVPVQb9fHtM9ObRljjOkWa5EYY4zplgFxjSQ1NVVHjx7d29Uwxph+ZdOmTSWq2mm/ZgMiSEaPHk1eXl5vV8MYY/oVEelSjwYDIkiMMaYnqSplNY2ICLFRXmJ8Hjyervy4v3+yIDHGmLMUCilFVQ0UlNeSf7Iu7FVLwck6CsrraAiEWq0T4/Pgj/ISG+UlNtoJl9hoL36fMx0b5SUmyuMsj/I6ZcPKtcxzy8VF+0iJjyYtMQZ/VO8+PNOCxBhj2giGlMLKevJP1jlhUeYGRbkTFMfK62kMtg6KlPhosgbHMmlYEp+dnMGwZD8eEeqagtQ1BqkPBKlvDFLfFHLmNQWpd1/FVYGWcg0BZ1jXFCTUxZtqk/w+0pP8pCXEkJ4UQ3piDGmJMaQn+klPdOalJfhJivXhdP12flmQGGMueKpKYzBEYyBEQ+DUsCUs3NZEc1gcL68n0OZTPC0xhsxBsUzNTOb6qcPIHBxL1uBYRgyOZfigWOKiz+/HqarSFFTqmoI0uMHjhE/IDZoAJVWNFFc3UFRZT1FVA8VVDWw+Uk5RVT31TaHTthnj85DWEjKngiYtsTmA/KQlxpASH43P2/WbeiMaJOI85e5nOM83+IWqPtZmeQxOf0yzcfrsWayqh90HHT3bXAx4WFVfcdc5jPPMiyAQUNUcjDH9XjCklFQ3cLyinhMVdRyvqKe6PkBjsPWHf0MgSGOgdSg4ZdqZHwjR4AbImYhAemIMWYPjmDVyMJnTY8kaHEfW4FgyB8eSOSi2x08fiQjRPiHa54HYqLNaV1WpaghQXNVAUWUDRVX1FLtB0xw4h0pq+PBQGeW1Taet7xEYEt9pt3ItIhYkIuLFecraApyH+mwUkTWquius2J3ASVW9SESW4HSKtxjnl6w5qhoQkWHAVhH5o6oG3PWucTvGM8b0A4FgiKKq5pCo53hFnTOsrOd4uTNeWNVAsJ1zOR6BaJ+HaK+HmCivM/R5iPadGvqjPCTHRhHtbT0/uqWcc63B2YanpVx6op+swbEMG+Qnxte71xnOJxEhyR9Fkj+KsWkJZyzbEAhSUt1IUWV9S9A4YVPPpi6+XyRbJJcC+1X1IICIrAZuBMKD5EZOPYLzZWCViIjbC2gzP2d+PoMxphc1uqeITlTWt2pNOIHhhEZxVcNp5/v9UR6GJ8cyNNnP5WNTGZbsZ2iyv2U4NMlPcmzUWZ1iMWcvxuclc5DT6mrrsXbKtyeSQZJJ6yes5eM8NrPdMm7rowJIAUpEZA7wHE534reFtUYU+LOIKPBzVX0WY0xE1TcFOVxaw8HiGg6V1HCguJqDxTUUlNdRUt1A256W4qO9DBsUy7BkP+PS09xwcL75D0v2MywpNmIXfk3Pi2SQtPcX0rZl0WEZ9xGbU9wHCf1KRNa6T0a7QlWPuQ9IekNE9qjqu6e9ucjdwN0AI0eObLvYGNNGKKScqKznYHENB0uq3WENB4urKSivaxUWQ5P8jEmL5zMT0lvCYWiyExzDkv0k+s/unL7p3yIZJPm0flRnFs6DeNork+8+SS4ZKAsvoKq7RaQGmArkqeoxd36RiLyCcwrttCBxWyrPAuTk5NipMWNcVfVNHCpxWhcHi6s54I4fLqmhrunUAxPjo71kp8Uza+RgbpmdxZi0BMakxpOdGk98jN3waU6J5F/DRmCciGTjPJZzCbC0TZk1OI/w3ADcAqxTVXXXOeqe7hqF8yS9wyISD3hUtcodXwisiOA+GNMvVdU3caKiniNlte6pKCc0DpbUUFzV0FLOIzBiSBzZqfFcPiaFMWnxjEmLZ2xaAumJMXbqyXRJxILEDYHlwOs4t/8+p6o7RWQFTstiDc6zul8Qkf04LZEl7upXAveLSBPOM5z/XlVLRGQM8Ir7x+0Dfquqr0VqH4zpa5pvkT1R4VzcLnQvcBe60ycqnfGaxtaPYh8cF8WYtASuHp9Gdlo8Y1ITGJsWz8iUuAvqbiXTOwbE80hycnLUOm00fV1dY9AJg4p6TlTWcaKiwbkbKiw0itq5RdbnETKS/GQkxTA02U9GknPH09Bk59bWMakJDI6P7qW9Mv2ZiGzqym/17ESnMT2otLqBfYXVfFJUxSeF1XxaVssJ9zcVlfWB08onxvjIcG+FHeveIts8PTTJT0ZyDKnxMRd0h4Cm77MgMSYCTtY0sq+win1F1XxSWMW+Qic4SmsaW8okxvgYnRrPqJR4LhuT0qolkeEOE+yitukH7K/UmG4or21kX2G1GxRVbmujmpLqUxe0E2J8XJSewLWT0hmfkci4jETGZyQwNMlvF7PNBcGCxJguqKht4pOiqlOh4Y6H3wEVH+3looxErpmQ5gZGAuMzEhmWbIFhLmwWJMa0oarsOVHFG7sK+ehQGfsKqygKC4y4aC/j0hOYPy6N8W5YjMtIIHNQrAWGGZAsSIzB6VQw79OT/HlnIW/sPsHRsjpEYNLQJK4cl8p493TUuPREMgfF2sVtY8JYkJgBq7YxwLv7SvjzrhOs21NEeW0T0T4PV4xN4e+vvohrJ6WTnujv7Woa0+dZkJgBpaS6gbd2F/LnnYX8dX8JDYEQybFRfGZiOgsnZzB/fJp1/2HMWbL/MeaCd7C4mjd2FfLGrkI2HTmJKmQOiiX30pEsnJLBJaOHEGVdlRtzzixIzAUnFFK25Je3hMf+omoApgxP4p5rx7FgcgaThyXZhXFjzhMLEnNBaAgEef9AKX/eWcibuwsprmrA6xEuGzOEv50zks9OziBrcFxvV9OYC5IFiemXAsEQ+Sfr2HK0nD/vOsE7e4upaQwSH+3l6gnpLJicwTUT0kmOs+diGBNpFiSmz1JViqsa3Icr1XCopNp5jkZJDUdKawm4nRemJcZw48xMFkzOYO7YFOvN1pgeZkFiel1lfROHS2panptxqMQNjeKaVt2hR/s8ZKfEMz49keunDCU7NZ4JQxOZOjzZftdhTC+yIDE9oiEQ5GhZbcszv1ue0FdS06pfKhHIGhxLdmoCOaOGMCbNeSJfdmo8w5Pth4DG9EUWJOa8C4WUXccreWdfMRsPl3GopIajZbWEP0YjNSGa7NR4PjMxjTFpCWSnxjMmNZ4RQ+LwR9mpKWP6EwsSc14UVzXwzr4TvL53JxuP76CWo3j9x/H7K4nPSGbayFSGJWQwetAwxqdmMmrQENJj00mNSyXWF9vb1TfGdIMFiTkn5XXV/HHPJv5ycAu7SvdQGfoUT8wJxNMEqRCLl9HJYxiVNJGKhgqKag+Td3Ij75U0wP7W20qMSiQtLo20uDTSY9NIixlMelQiab540r1+UokiTbzENDVAYxU0VENjNTTWgHgg6xIYeTkkpJ31fqgqAQ3QEGigPlhPY7Dx1DBQT3xUPGMHjcUj9oNFYzpiQWLOSFUprC1kT9kePszfwUfHdvBp1X7qKULEOVfliY5jhH8MF6dfzuUjpjEpZSJj4jOJLt4LRbugoQpiq9GGSirryyluOElRYyXFgWqKA7UUNdZQXF1Bkexjk0CRz0ugnR8LJgeDpAWDpAfcoXpICgZo3Pk89R6hMXYw9YkZNMSn0hA7iAZvFA3BhpZXeEA0B0ZDsIGQhs54DFL8KVw+/HLmDp/L5cMvJzU2NSLH2pj+yp7Zblo0BZs4WHGQvSf3srdsLztLdrO7bC+1gcqWMqHGFGJCWYxNHs8VI6ezaOJsxg7KRMoPQ36e8yrIgxPbIdjY+g18fohOgJgEiE50hwlhw0SITiAUHU+F10eRRynWJopDjRQF6ygO1FDUVElxQzlF9aWU1pUSVOeuLgH8CDGhINGhEH5VYsRLTFQCMf5k/HGpRMem4PfFEu2Nxu/zE+ONaXn5fX5nvjdsvi+GkroSNhzbwAfHP6CsvgyA8YPHt4TKrPRZ+H3WsaO5MHX1me0WJAOUqrK/fD8fHv+Q3WW72XdyH/vL9xMIOc8NF40iWJ9BoH4YvkAmU9Mm8dmxM1gwcRSj4xuhYBPkb3JCo2AT1JY6G46Kg+EzIXM2ZOXA0OkQO9gJCu/5bQAHQ0HqAnXEeGPweXxOlyehEJTsgyMbTr3Kj7h1i4cRl8DIuTDyMqd+0fFdeq+Qhthbtpf3j73PhmMb+LjoY5pCTcR4Y5idMbslWMYNGmddr5gLRp8IEhG5HvgZ4AV+oaqPtVkeA/wamA2UAotV9bCIXAo821wMeFhVX+nKNttjQeKobarloxMfsT5/PesL1nO85jgA8b7BxASzKC9Po6Y6nVD9cCamZnPV+KFcNXYQs/3HiDq+yQ2PPCj9xN2iQNoEyMxxPpSzciBt0nkPjG6rKICjH8CnG+DIB1C4A1AQLwy7GEa5wTLisi5fZ6ltqmVT4aaWYDlQcQCAtNi0ltNglw27jJTYlAjumDGR1etBIiJeYB+wAMgHNgK5qrorrMzfA9NV9ZsisgS4SVUXi0gc0KiqAREZBmwFhgPa2TbbM5CD5GjVUd7Nf5f1BevZeHwjjaFGYn2xTBmcw8mSsWz7ZCihpkGkJkQz76JUrhvRxNyYwySVbnVC4/gWCNQ7G4tPcy5sN7c2hs8Ef3Kv7t85qa+Aox+5LZYPnP0Mur9lSbnIuXA/8nInXIaMcX7c0okTNSfYcGyDEyzHN1DRUAHApCGTWoJlZvpMor3RkdwzY86rvhAkl+O0JK5zpx8AUNVHw8q87pbZICI+4ASQpmGVEpFs4AMgE7iks222ZyAFSVOwiU1Fm1ifv55389/lcOVhAEYljWJe5jzSvDN4a0s8731SwRA//NPkk1wd/ykZlTuQgk1QXehsyOd3vq1n5kDWbGc4aGSXPlT7nUADHNtyKliObID6cmdZXIrTykqbAGkTTw0T0js8FsFQkD1le3j/2Pu8f+x9thRtIaAB/F4/OUNzmDt8LnOHz2VM8hg7DWb6tK4GSSTPQWQCR8Om84E5HZVxWx8VQApQIiJzgOeAUcBt7vKubBMAEbkbuBtg5MiR3d+bPqy4tpj1BetZn7+eDcc3UNNUQ5QnikuGXsLiCYu5MvNKDhyL5em/7GfnkSK+EPcxr4/azvjy9cgu90J6ykUw5hqnpZE5GzKmgm+AfHv2xcDIOc4L3Osse51AKfjYueay/WVwWxmAc90nPFjSJjiBkzgUr8fLlNQpTEmdwtenf52aphryTuS1BMsTG58AID0unYvTLmZo/FAy4jLIiMsgPS6djPgM0mPTifL2jQ4nQxriZP1JimqLKKotorC2sGU82hvNiMQRjEwcyYjEEWQlZtnNBwNQJIOkva9abZs/HZZR1Q+BKSIyCfiViKzt4jZx138W9zpLTk7OBXVHQTAUZHvJ9pbw2F22G4CMuAw+l/055mfOZ86wOcR4Y/nT9uPc89wOhhW/x7f8eVwd/zFRwVqoGASTvggTFzmncOKG9PJe9SEeD6RPcl45X3PmqULVCSjeA8V7Tw13/QHqnj+1bkyyGypuwKRPJD5tIldlzeeqEVcBcKz6GBuObeC9Y++x7+Q+/lrwV+oCdadVY4h/SEvAZMS7IRMWNhlxGcRHde1mgY7UB+opri1uFQ7h40W1RRTVFbXchNFyiMRDij+F+mA9VY1VrZalx6W3CpcRSSOcYeIIkqKTulXfrgppiLL6MgprCjlRe6L1sOZEyz76PD6SopNIikkiKTqJ5OjkU+Mxyc6y8PEYp0xidCJej/XA0CySQZIPjAibzgKOdVAm3z21lQyUhRdQ1d0iUgNM7eI2L0gVDRW8V/Ae7xa8y3sF71HeUI5HPMxIm8E9s+5hXuY8xg8ej4jQEAiy5qN97H7nJXJq/8rvvFvwRzei/hRk4ldg8o2QPR/6yDfefkEEkoY5r7HXnJqvCjXFrQOmaA/sXQubXzhVLjqhJVyGp03g5rRJ3HzxPZA4DA0FqGqooLDmBEW1hRTWFFJYV0RhbTFF9cUcqzzMlsJNlDdVnVatBG8s6TGDyIgZRHp0MhnRyWREJZIRnUx6VALiiaJIQhSqcwt1UX0ZhXWnQqIivJXlivXFtgTY7IzZpMeltwqx9Lh0UmJT8Hmcj4+KhgqOVh3lSOURZ1h1hPyqfNYXrKekrqTVtgfFDGJk4kiyErOcsEka2RIyKf6ULp3qawmJWjcUOgiJplBTq/WiPFFkxGUwNH4oM9JnkB6XTigUoqKxgsqGSioaKzhafZTK0koqGyvbDfdwiVGJLaHTMgwLneSYZMYkj2HikInERV3Yz8KJ5DUSH86F8WuBApwL40tVdWdYmWXAtLCL7V9W1b9xr4scdU9njQI2ANOB8s622Z7+eI2ktqmWnaU72VK0hfUF69lavJWQhhgcM5grMq9gftZ85g6fS3LMqYvdtZWlfPTaf+LZ/UfmhLYQI03U+9OImXojMuVG57bXvnZH1YWspiSs9bLnVNg0X4c6S/UiFHu9nPB5KfJ6KfR5KfL63KEzv8TrJdTBh7GokhIKkR7ykC5RZPhiSY9KJD16MOlxaWTEDyU9aQQJCcOQuBTn+lDcECcEz/FaTm1TLUerjpJflc+RqtZBc7zmeKsfg8b6Yk87TRYIBc4qJDLiM1pOFbYdDvEPOatrUk3BppaQqWx0XhUNFc64GzzNy5rnNw/D6ycIYxKymDxkApNTpjA57WImpk7pF+HS6xfb3Up8Hvgpzq26z6nqj0RkBZCnqmtExA+8AMzEaYksUdWDInIbcD/QBISAFar6Px1ts7N69PUgCYQC7C/fz/aS7Wwv3s72ku0crDjY8p9s0pBJzM+az7yseUxNmdq6SV1TSu32NRR9+BKZJz8kiiClnlTqxy1i+NzFyIjLnFM1pu+oLXOuuxTthtoS5zZkjxc8vlPj4nGH3jZDj1PutHnOdAAoDdRQ1FhJYWM5GmwkXb1kBIOkNDUQVVfuvH9dmfPbn9pSd/okHZwlBm+0EyqxQ5xgiRvihkwKxKVCfKpz80F8ujP0D+rS31xTsImC6oJW4dIcNvlV+S0fxmcKiYz4DIbGDWWwf3DvdWMTbHJ6cCj4GAo2occ+pr54Lyc9yr6oaHbFuK/oKIp9zhc5UWVMIMTkoDBZfUz2xDLRm0BcVILzW6zoOIiKdX77FBXrToe9ouOc30ANGuW8IvQFsU8ESV/Rl4JEVTlec5xtJdvYUbyD7SXb2VW6i/qgc4ttckwy01KnMS11GlNTpzItdRqD/YNbb6S6CHb/kcbt/4PvyHt4CHIklMauwdcwal4uk2ZdbeFhzk4o6NwWHR4utaVtAudk63l1J6G97mU8PidgEtKccIlPOzWekO4ET/N4XGq7H4LBUJDiumKiPFG9GxJthUJQdhCOOaFBwcdwYtupW+RjBzs3qwyfBcNngDcGmmpbXkW1xeyqKWBX3Ql2NZSys6mcEnV6gPAA2epzwqUpyOT6BibW1xLXWANtrlG14omClLHODTOp4yF1nDNMuQhiB3Vrdy1IwvRmkFQ2VrKjZAfbi7c7w5LtlNY7vwKP9kQzMWUi01OnMzV1KtNTp5OVmNV+87vyGOz+I+z6A/rp+wjKIR3K/x+cQ83YL/DF665n0vB++JsO03+FQk6Y1BQ5X25qip1XdZE7r7j1vGBD+9uJHeIGTNqpYct487Tb6onq4Z6iK4+1tDSc8Nh86u69qDgYNgMyZzmv4bNg8OizPg1YVFvErtJd7Crdxc7Snewq3dVybckjHrKTspk8ZCKTk8cyJSmbCXHDiNMQ1FfCyUNQ8onzKv3ECbnw0IlPd4NlHKS4AZN6kdOK6cLNAhYkYXoqSJqCTew9uZdtxdtaQqP5dxwA2cnZLa2NaanTGD94/Om3eDbVOXcHVRc559LLDsKeP0H+RwAcj8nmpdpZvBGaw7RZl/ONq8YyOrV7d+4YE3Gq0FDpXDdqCZoiZ/q0ICp2enluT3TCqRZNeMA0j8eHBVHs4LNrmdeddEKjOTAKNkH1CWeZxwcZU5ywyJzltDpSJ0TslFJzuDQHS3vhMmHIBPw+P8FQEEUJapBQMECosZpQfQWhxiqCDdXOdFMNoWATQREUCIoHjfIT9PkJ+fyEfDGEfNEEPT5CIoQ0REhDvPrlVy1ImkUqSEIa4q0jb5F3Io8dJTvYXba75bxuij+FaWnTnNZGyhSmxg8nsaEmLCTCwqKq0BlWFzr/2dqoGzKZ17mM/3N8Esd8I7l1zkjumjeGocl2v765QDXVhQVNc8smbDp8WW1J+6fYxBsWLqlhrRx3PC4Fyg6dOk1VdvDUuikXnTpFlTkbhk7t+dZQG0W1Rews2cmuMidYPjn5CUEN4hEPXvEiCF6PF4948ODB4wmbL148GsQTaMQTaMATqMPbVI+nqRZPUx0eVTyARxWvLwaJTsAbk8iTS9dZkDSLRJDUNNXw/b9+nzePvEmsJ5rJ8cOZFjWEaRLLtECQoTUVSHXhqW9f7Z3jjE5w/rAThjrDxKE0+lMp1MEcbUxgf108fz0RxZ8/DZIcG8Udc0fz1bmjGRw/QH4oaExXhILuKbbisJaN29JpHg+f31TTev3E4adOT2XOdk5XdfPaQr8SaISTh51TYyX7oGS/O9yHPHDEgqTZ+Q6So5VH+c5fvs3B8oP8Q9lJbq2oPPWDHPGc+uaTkNEqJJpDQxPSORFM5kAFHCiuPvUqquFEZX3L+3gERqfGs+SSESydM4qEGLt115hua6xxQ6UUkoY7vw0yp1NFPJ5e7yLlgvR+wft8753v4Wmq5ZnjJ7h8zOdg0d+ealnEpbScN60L/yAVAAAgAElEQVRvCnKopKYlJA4ecgLjYPEn1DYGW7aZGONjTHoCcy9KYWxaAmPT4hmblsDIlDhifPbrWWPOq+h45zV4dG/XpG87i5sGLEi6SFX51c5f8dSmpxgbVH52/DgjrnkQvWwZJTVNTlgcqeZA0T4nLEqqyT9ZR3iDL3NQLGPTE7hk9BA3MJzQSEuMsc77jDH9lgVJF9QF6njo/YdYe2gtC2sbeKQqSNzS/+bVqrGseHQdRVWnbmuMjfIyJi2eGSMGc/OsrJbAyE6NJzbaWhfGmAuPBUknjlUf495197Dn5B7uKSvnzoRxVN7xHN9Zd5I1Wzdz8YhBfOvqsU5gpCcwLMmPx2OtC2PMwGFBcgYbT2zku3+5j6bGSladKGL+lKWsH/tdvvfLPZRWN/LdBeP51tVj8Xn7yK9ujTGmF1iQtENVeXHPizyx8XFGNgX5WVEpwxf8hB/kz+KFX21lXHoCv7zjEqZm2i/JjTHGgqSNhmAD//LBv/A/+/+Hq2vrebTez9HrXuL6dUEOl37KXVdm873rJuCPsusdxhgDFiStFNYU8g9/uZdtpTv45skKvjF4Fs+MeIB//a8ShiXH8tu7LuPysSm9XU1jjOlTLEhcW4q2cN+671Bbf5KfFhUzY+LXuenQtWzfXcIts7N46IuTSfTbg6CMMaYtCxLg9/t+z7988AjDmpp4tqyaQ2N/xOUfDScxpomf3zab66YM7e0qGmNMnzWgg6Qp2MTjHz3G7/a9xBV19fyocRCPxD3M/2yMY8HkNB798jRSE2J6u5rGGNOnDdggKakr4bt/uZePi7fy1fJKboqdw/Wlt1KHn5/cMplbZnfwXBBjjDGtDMgg2Vmyk3veWkZFXSmPl5RRFX0b1xy8kjnZKTz5lYsZMaTvP0vZGGP6igEXJH888Ed++N6DDGlq4BdljTxd9z3eLp/I978wga9dkW2/SjfGmLM0YIIkEArwrxuf5IU9v+GSunruqUzhG2V/T8rwbF5dPIPxGYm9XUVjjOmXBkSQBDXIN1+7kw+LP+bWiirGVlzK4tqlfP0zk/j2Z8YR7bMuTowx5lxFNEhE5HrgZ4AX+IWqPtZmeQzwa2A2UAosVtXDIrIAeAyIBhqBf1TVde46bwPDgDp3MwtVtehM9ThYfgBPURMPl1byfmkurw/5HL+942JmjRx83vbVGGMGqogFiYh4gaeBBUA+sFFE1qjqrrBidwInVfUiEVkCPA4sBkqAL6rqMRGZCrwOZIatd6uqdvmRhxps4snjjTxZ+c/Mvuwq/vS5icRFD4jGmDHGRFwkP00vBfar6kEAEVkN3AiEB8mNwMPu+MvAKhERVd0cVmYn4BeRGFVt4BxkNHp5NPQE3//qFcwfn3YumzDGGNOBSAZJJnA0bDofmNNRGVUNiEgFkILTIml2M7C5TYj8h4gEgd8D/6LtPHheRO4G7gZIHj6GvPu+QHKcdXFijDHnWySvMrd3H23bD/wzlhGRKTinu74RtvxWVZ0GzHNft7X35qr6rKrmqGrORcMGW4gYY0yERDJI8oERYdNZwLGOyoiID0gGytzpLOAV4HZVPdC8gqoWuMMq4Lc4p9CMMcb0kkgGyUZgnIhki0g0sARY06bMGuAOd/wWYJ2qqogMAv4EPKCq7zUXFhGfiKS641HAImBHBPfBGGNMJyIWJKoaAJbj3HG1G3hJVXeKyAoRucEt9ksgRUT2A/8A3O/OXw5cBPxARLa4r3QgBnhdRLYBW4AC4N8jtQ/GGGM6J+1cp77g5OTkaF5el+8WNsYYA4jIJlXN6ayc/aTbGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbrEgMcYY0y0WJMYYY7rFgsQYY0y3WJAYY4zpFgsSY4wx3WJBYowxplssSIwxxnSLBYkxxphusSAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdYkFijDGmWyxIjDHGdEtEg0RErheRvSKyX0Tub2d5jIj8zl3+oYiMducvEJFNIrLdHX4mbJ3Z7vz9IrJSRCSS+2CMMebMIhYkIuIFngY+B0wGckVkcptidwInVfUi4CngcXd+CfBFVZ0G3AG8ELbOvwF3A+Pc1/WR2gdjjDGdi2SL5FJgv6oeVNVGYDVwY5syNwK/csdfBq4VEVHVzap6zJ2/E/C7rZdhQJKqblBVBX4NfCmC+2CMMaYTkQySTOBo2HS+O6/dMqoaACqAlDZlbgY2q2qDWz6/k20CICJ3i0ieiOQVFxef804YY4w5s0gGSXvXLvRsyojIFJzTXd84i206M1WfVdUcVc1JS0vrQnWNMcaci06DRES8IpIaNh3tftvf3cmq+cCIsOks4FhHZUTEByQDZe50FvAKcLuqHggrn9XJNo0xxvSgMwaJiCzB+WDfJiLviMg1wEGcC+i3drLtjcA4EckWkWhgCbCmTZk1OBfTAW4B1qmqisgg4E/AA6r6XnNhVT0OVInIZe7dWrcDf+jKjhpjjIkMXyfLvw/MVtX9IjIL2AAsUdVXOtuwqgZEZDnwOuAFnlPVnSKyAshT1TXAL4EXRGQ/TmAtcVdfDlwE/EBEfuDOW6iqRcC3gOeBWGCt+zLGGNNLxLn5qYOFIh+r6qyw6T2qOrFHanYe5eTkaF5eXm9Xwxhj+hUR2aSqOZ2V66xFki4i/xA2nRA+rar/eq4VNMYYc2HoLEj+HUg8w7QxxpgB7oxBoqo/7GiZiMSf/+oYY4zpb7py+2+miOS4d14hIuki8mPgk4jXzhhjTJ/X2e2/9wJbgP8DfCAidwC7ce6Ymh356hljjOnrOrtGcjcwQVXLRGQksB+Yr6ofRL5qxhhj+oPOTm3Vq2oZgKoeAfZZiBhjjAnXWYskS0RWhk2nh0+r6nciUy1jjDH9RWdB8o9tpjdFqiLGGGP6p85u//3VmZYbY4wxZwwSEWnbyWIrqnrD+a2OMcaY/qazU1uX4zx46kXgQ9p/HogxxpgBrLMgGQosAHKBpThdu7+oqjsjXTFjjDH9wxlv/1XVoKq+pqp3AJfh/I7kbRH5do/UzhhjTJ/XWYsEEYkBvoDTKhkNrAT+O7LVMsYY0190drH9V8BUnIdH/VBVd/RIrYwxxvQbnbVIbgNqgPHAd5yn2wLORXdV1aQI1s0YY0w/0NnvSDrtHdgYY8zAZkFhjDGmWyxIjDHGdEtEg0RErheRvSKyX0Tub2d5jIj8zl3+oYiMdueniMhfRKRaRFa1Wedtd5tb3Fd6JPfBGGPMmXV6+++5EhEv8DTODxrzgY0iskZVd4UVuxM4qaoXicgS4HFgMVAP/ADnjrGp7Wz+VlXNi1TdjTHGdF0kWySXAvtV9aCqNgKrgRvblLkRaO4Y8mXgWhERVa1R1b/iBIoxxpg+LJJBkonTT1ezfHdeu2VUNQBUACld2PZ/uKe1fiBh9yQbY4zpeZEMkvY+4PUcyrR1q6pOA+a5r9vafXORu0UkT0TyiouLO62sMcaYcxPJIMkHRoRNZwHHOiojIj4gGSg700ZVtcAdVgG/xTmF1l65Z1U1R1Vz0tLSzmkHjDHGdC6SQbIRGCci2SISDSwB2j7fZA1whzt+C7BOVTtskYiIT0RS3fEoYBFg3bYYY0wvithdW6oaEJHlwOuAF3hOVXeKyAogT1XXAL8EXhCR/TgtkSXN64vIYSAJiBaRLwELgU+B190Q8QJvAv8eqX0wxhjTOTlDA+CCkZOTo3l5drewMcacDRHZpKo5nZWzX7YbY4zpFgsSY4wx3WJBYowxplssSIwxxnSLBYkxxphusSAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdYkFijDGmWyxIjDHGdIsFiTHGmG6xIDHGGNMtFiTGGGO6xYLEGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEiMMcZ0iwWJMcaYbolokIjI9SKyV0T2i8j97SyPEZHfucs/FJHR7vwUEfmLiFSLyKo268wWke3uOitFRCK5D8YYY84sYkEiIl7gaeBzwGQgV0Qmtyl2J3BSVS8CngIed+fXAz8AvtfOpv8NuBsY576uP/+1N8YY01WRbJFcCuxX1YOq2gisBm5sU+ZG4Ffu+MvAtSIiqlqjqn/FCZQWIjIMSFLVDaqqwK+BL0VwH4wxxnQikkGSCRwNm85357VbRlUDQAWQ0sk28zvZJgAicreI5IlIXnFx8VlW3RhjTFdFMkjau3ah51DmnMqr6rOqmqOqOWlpaWfYpDHGmO6IZJDkAyPCprOAYx2VEREfkAyUdbLNrE62aYwxpgdFMkg2AuNEJFtEooElwJo2ZdYAd7jjtwDr3Gsf7VLV40CViFzm3q11O/CH8191Y4wxXeWL1IZVNSAiy4HXAS/wnKruFJEVQJ6qrgF+CbwgIvtxWiJLmtcXkcNAEhAtIl8CFqrqLuBbwPNALLDWfRljjOklcoYGwAUjJydH8/LyersaxhjTr4jIJlXN6ayc/bLdGGNMt0Ts1FZf19TURH5+PvX19Z0XNmfF7/eTlZVFVFRUb1fFGNMDBmyQ5Ofnk5iYyOjRo7FeVs4fVaW0tJT8/Hyys7N7uzrGmB4wYE9t1dfXk5KSYiFynokIKSkp1tIzZgAZsEECWIhEiB1XYwaWAR0kxhhjus+CpBd5vV5mzJjB1KlT+cpXvkJtbe152W5hYSGLFi3i4osvZvLkyXz+858HIDs7m71797Yqe++99/LEE0/w9ttvk5yczMyZM5kwYQLz58/n1VdfPS/1McZc2CxIelFsbCxbtmxhx44dREdH88wzz3R53UAg0OGyBx98kAULFrB161Z27drFY489BsCSJUtYvXp1S7lQKMTLL7/M4sWLAZg3bx6bN29m7969rFy5kuXLl/PWW2+d494ZYwaKAXvXVrgf/nEnu45VntdtTh6exENfnNLl8vPmzWPbtm0cPnyYRYsWsWPHDgCefPJJqqurefjhh7n66quZO3cu7733HjfccAO333473/zmNzly5AgAP/3pT7niiis4fvw4CxcubNn29OnTAcjNzWXx4sU89NBDALz77ruMHj2aUaNGcejQoVb1mTFjBg8++CCrVq3i2muvbbUsGAxy5513kpeXh4jwta99jfvuu+/sD5Ix5oJgQdIHBAIB1q5dy/XXd/6MrvLyct555x0Ali5dyn333ceVV17JkSNHuO6669i9ezfLli1j8eLFrFq1is9+9rN89atfZfjw4UyfPh2Px8PWrVu5+OKLWb16Nbm5uR2+16xZs/jJT35y2vwtW7ZQUFDQEnbl5eXnuOfGmAuBBQmcVcvhfKqrq2PGjBmA0yK58847OXbszJ0ZN5+GAnjzzTfZtWtXy3RlZSVVVVVcd911HDx4kNdee421a9cyc+ZMduzYQVpaGrm5uaxevZopU6bwhz/8gRUrVnT4Xh11nzNmzBgOHjzIt7/9bb7whS+0av0YYwYeC5Je1HyNJJzP5yMUCrVMt/09Rnx8fMt4KBRiw4YNxMbGnrbtIUOGsHTpUpYuXcqiRYt49913ufnmm8nNzWXhwoVcddVVTJ8+nfT09A7rt3nzZiZNmkQwGGT27NkA3HDDDaxYsYKtW7fy+uuv8/TTT/PSSy/x3HPPndMxMMb0f3axvY/JyMigqKiI0tJSGhoaznjn1MKFC1m1alXLdHMorVu3ruUOsKqqKg4cOMDIkSMBGDt2LCkpKdx///1nPK21bds2HnnkEZYtW4bX62XLli1s2bKFFStWUFJSQigU4uabb+aRRx7h448/Ph+7bozpp6xF0sdERUXx4IMPMmfOHLKzs5k4cWKHZVeuXMmyZcuYPn06gUCA+fPn88wzz7Bp0yaWL1/e0rq56667uOSSS1rWy83N5YEHHuCmm25qtb3169czc+ZMamtrSU9PZ+XKladdaAcoKCjgq1/9akvL6dFHHz1Pe2+M6Y8GbDfyu3fvZtKkSb1UowufHV9j+j/rRt4YY0yPsCAxxhjTLRYkxhhjusWCxBhjTLdYkBhjjOkWCxJjjDHdEtEgEZHrRWSviOwXkfvbWR4jIr9zl38oIqPDlj3gzt8rIteFzT8sIttFZIuI5LXdZn8yd+7cVtNPPfUUfr+fioqKlnlvv/02ixYtOm3dV199lZkzZ7Z0Ff/zn/884vU1xpj2ROwHiSLiBZ4GFgD5wEYRWaOqu8KK3QmcVNWLRGQJ8DiwWEQmA0uAKcBw4E0RGa+qQXe9a1S1JFJ17ynvv/9+q+kXX3yRSy65hFdeeYW/+7u/63C9pqYm7r77bj766COysrJoaGjg8OHDka2sMcZ0IJK/bL8U2K+qBwFEZDVwIxAeJDcCD7vjLwOrxHlO643AalVtAA6JyH53exsiUtO198OJ7ed3m0OnweceO2ORhIQEqqurAThw4ADV1dX85Cc/4cc//vEZg6SqqopAIEBKSgoAMTExTJgw4bxV3RhjzkYkT21lAkfDpvPdee2WUdUAUAGkdLKuAn8WkU0icndHby4id4tInojkFRcXd2tHesKLL75Ibm4u8+bNY+/evRQVFXVYdsiQIdxwww2MGjWK3NxcfvOb37Tq6NEYY3pSJFsk0s68tv2xdFTmTOteoarHRCQdeENE9qjqu6cVVn0WeBacLlLOWNNOWg49YfXq1bzyyit4PB6+/OUv81//9V8sW7asw/K/+MUv2L59O2+++SZPPvkkb7zxBs8//3zPVdgYY1yRDJJ8YETYdBbQ9mEbzWXyRcQHJANlZ1pXVZuHRSLyCs4pr9OCpD/Ztm0bn3zyCQsWLACgsbGRMWPGnDFIAKZNm8a0adO47bbbyM7OtiAxxvSKSJ7a2giME5FsEYnGuXi+pk2ZNcAd7vgtwDp1epFcAyxx7+rKBsYBH4lIvIgkAohIPLAQ2BHBfegRL774Ig8//DCHDx/m8OHDHDt2jIKCAj799NN2y1dXV/P222+3TG/ZsoVRo0b1UG2NMaa1iLVIVDUgIsuB1wEv8Jyq7hSRFUCeqq4Bfgm84F5ML8MJG9xyL+FcmA8Ay1Q1KCIZwCvO9Xh8wG9V9bVI7UNPWb16NWvXrm0176abbmL16tXMmTOHt956i6ysrJZlL774Ik888QTf+MY3iI2NJT4+3lojxpheY93Im4iw42tM/2fdyBtjjOkRFiTGGGO6xYLEGGNMt1iQGGOM6RYLEmOMMd1iQWKMMaZbLEh6kdfrZcaMGUydOpWvfOUr1NbWnpftFhYWsmjRopYu5j//+c8DkJ2dzd69e1uVvffee3niiSd4++23SU5OZubMmUyYMIH58+fz6quvnpf6GGMubBYkvSg2NpYtW7awY8cOoqOjeeaZZ7q8biAQ6HDZgw8+yIIFC9i6dSu7du3iscecvsSWLFnC6tWrW8qFQiFefvllFi9eDMC8efPYvHkze/fuZeXKlSxfvpy33nrrHPfOGDNQRLKvrX7j8Y8eZ0/ZnvO6zYlDJvLPl/5zl8vPmzePbdu2cfjwYRYtWsSOHU7PL08++STV1dU8/PDDXH311cydO5f33nuPG264gdtvv51vfvObHDlyBICf/vSnXHHFFRw/fpyFCxe2bHv69OkA5ObmsnjxYh566CEA3n33XUaPHs2oUaM4dOhQq/rMmDGDBx98kFWrVnHttdd261gYYy5s1iLpAwKBAGvXrmXatGmdli0vL+edd97hu9/9Lvfccw/33XcfGzdu5Pe//z133XUXAMuWLePOO+/kmmuu4Uc/+hHHjjl9ZU6fPh2Px8PWrVsBp2uW3NzcDt9r1qxZ7NlzfgPWGHPhsRYJnFXL4Xyqq6tjxowZgNMiufPOO1s+9DvSfBoK4M0332TXrlPPCausrKSqqorrrruOgwcP8tprr7F27VpmzpzJjh07SEtLIzc3l9WrVzNlyhT+8Ic/sGLFig7fayB0n2OM6T4Lkl7UfI0knM/na/WQqvr6+lbL4+PjW8ZDoRAbNmwgNjb2tG0PGTKEpUuXsnTpUhYtWsS7777LzTffTG5uLgsXLuSqq65i+vTppKend1i/zZs3W39ZxphO2amtPiYjI4OioiJKS0tpaGg4451TCxcuZNWqVS3TzaG0bt26ljvAqqqqOHDgACNHjgRg7NixpKSkcP/995/xtNa2bdt45JFHOn0mijHGWIukj4mKiuLBBx9kzpw5ZGdnM3HixA7Lrly5kmXLljF9+nQCgQDz58/nmWeeYdOmTSxfvryldXPXXXdxySWXtKyXm5vLAw88wE033dRqe+vXr2fmzJnU1taSnp7OypUr7UK7MaZT1o28iQg7vsb0f9aNvDHGmB5hQWKMMaZbBnSQDITTer3BjqsxA8uADRK/309paal96J1nqkppaSl+v7+3q2KM6SED9q6trKws8vPzKS4u7u2qXHD8fj9ZWVm9XQ1jTA8ZsEESFRVFdnZ2b1fDGGP6vYie2hKR60Vkr4jsF5H721keIyK/c5d/KCKjw5Y94M7fKyLXdXWbxhhjelbEgkREvMDTwOeAyUCuiExuU+xO4KSqXgQ8BTzurjsZWAJMAa4H/q+IeLu4TWOMMT0oki2SS4H9qnpQVRuB1cCNbcrcCPzKHX8ZuFZExJ2/WlUbVPUQsN/dXle2aYwxpgdF8hpJJnA0bDofmNNRGVUNiEgFkOLO/6DNupnueGfbBEBE7gbudierRWRve+XOk1SgJILbj6T+XHew+vc2q3/vinT9R3WlUCSDRNqZ1/Ze247KdDS/vRZUu/fvquqzwLNnquD5IiJ5XelGoC/qz3UHq39vs/r3rr5S/0ie2soHRoRNZwFtH7bRUkZEfEAyUHaGdbuyTWOMMT0okkGyERgnItkiEo1z8XxNmzJrgDvc8VuAder8QnANsMS9qysbGAd81MVtGmOM6UERO7XlXvNYDrwOeIHnVHWniKwA8lR1DfBL4AUR2Y/TElnirrtTRF4CdgEBYJmqBgHa22ak9uEs9MgptAjpz3UHq39vs/r3rj5R/wHRjbwxxpjIGbB9bRljjDk/LEiMMcZ0iwXJWRCRESLyFxHZLSI7ReQed/4QEXlDRD5xh4N7u65n4vYSsFlEXnWns90uaj5xu6yJ7u06dkREBonIyyKyx/13uLy/HH8Ruc/9u9khIi+KiL+vH3sReU5EikRkR9i8do+3OFa63RdtE5FZvVfzDuv+E/dvZ5uIvCIig8KWtdstU29pr/5hy74nIioiqe50rx57C5KzEwC+q6qTgMuAZW4XLfcDb6nqOOAtd7ovuwfYHTb9OPCUW/+TOF3X9FU/A15T1YnAxTj70eePv4hkAt8BclR1Ks7NIkvo+8f+eZxuisJ1dLw/h3OH5TicHwP/Ww/VsSPPc3rd3wCmqup0YB/wAHTcLVPPVbVdz3N6/RGREcAC4EjY7N499qpqr3N8AX9w/0H3AsPcecOAvb1dtzPUOQvnP/9ngFdxfvxZAvjc5ZcDr/d2PTuoexJwCPcmkbD5ff74c6oXhyE4d0u+ClzXH449MBrY0dnxBn4O5LZXrq/Uvc2ym4DfuOMPAA+ELXsduLyvHXt33ss4X6IOA6l94dhbi+QcuT0VzwQ+BDJU9TiAO0zvvZp16qfAPwEhdzoFKFfVgDsd3h1NXzMGKAb+wz019wsRiacfHH9VLQCexPkWeRyoADbRf459uI6Od3vdIvXl/fkasNYd7xd1F5EbgAJV3dpmUa/W34LkHIhIAvB74F5Vrezt+nSViCwCilR1U/jsdor21XvCfcAs4N9UdSZQQx88jdUe9zrCjUA2MByIxzkd0VZfPfZd0W/+lkTkf+Ocqv5N86x2ivWpuotIHPC/gQfbW9zOvB6rvwXJWRKRKJwQ+Y2q/rc7u1BEhrnLhwFFvVW/TlwB3CAih3F6Tv4MTgtlkNtFDfTtbmfygXxV/dCdfhknWPrD8f8scEhVi1W1CfhvYC7959iH6+h494sujETkDmARcKu654HoH3Ufi/NFZKv7fzgL+FhEhtLL9bcgOQsiIji/xt+tqv8atii8q5c7cK6d9Dmq+oCqZqnqaJwLi+tU9VbgLzhd1EDfrv8J4KiITHBnXYvT+0F/OP5HgMtEJM79O2que7849m10dLzXALe7dxBdBlQ0nwLrK0TkeuCfgRtUtTZsUUfdMvUZqrpdVdNVdbT7fzgfmOX+v+jdY9/bF5P60wu4Eqe5uA3Y4r4+j3Od4S3gE3c4pLfr2oV9uZr/197dhVhRxnEc//4Q2Y3YIMheCFxJkcUoVsjKaksJiopAKYoKyV4uIkgUhS4Eca+KuopeDI2UNhLxIsEgqGxly5fwZTc1MDDYyLskC4mSXv5dPP9lj+ueo2cn2O30+8Cwz87MM/PMLGf+Z57Z+T/wUZavo3xoTgDbgbbJbl+DdncDB/NvsAO4/L9y/oFe4DhwDOgD2qb6uQe2Up7p/EG5cD1T73xTulfeBL4DjlL+Q22qtf0E5VnCyOf37Zr112bbvwXum4rnfszyYUYftk/quXeKFDMzq8RdW2ZmVokDiZmZVeJAYmZmlTiQmJlZJQ4kZmZWiQOJtSRJL0laJGmJpKbefpc0IzPyDkrqGbNst6SbLlB/Zb6FPCHZ7tvqLGuT9JmkIUmPNrndWZIen2i7zOpxILFWdQslD9pdwBdN1r0bOB4R8yOi2boAK4EJBxLKOz7jBhJKfrfpEdEdEdua3O4soKlAUvPWvVldDiTWUnK8iSPAAmAf8CywQdJ5+YkkdUraleM37JI0U1I38Apwf37rv6TBvjZIOqgyxkhvzltByaXVL6k/590jaZ+kw5K2Z642JA1L6s35RyV1ZTLQ54BVuf+emv1dCbwPdOey2ZLWSTqgMsbJxnxrHklz8s7l69z+bOBloCfrrlIZD2Vz7ntQ0uKsuzzbuRP4RNI1kgay3rGxd2lmk/7mrCdP//YE3Ay8DkwH9jRYbyfwZJafBrxQoqgAAAJhSURBVHZkeTnwRp06u8m3hhl9o3tazr8xfx9m9I3jK4AB4NL8/UVgXc16L2T5eeCdLK8H1tTZ/yIyI0FtG7LcBzyY5a+ApVlup9whja27Gtic5S5KGpf2PP6TNce3Glhbc6wdk/039jS1Jt+RWCuaT0l/0UXJZ1XPQuCDLPdRUuA04xFJh4FByoBI88ZZ59acv0fSECU3VWfN8pHEn4coXU/NWpzPc45SknBeL6kDuDYiPgSIiN/j3LxSI+6gHDcRcRz4Hpibyz6NiJ+yfAB4StJ64IaIODOBdloLc/+ntYzsltpCyXx6ivItXHkBXxgRv11gExedLygT+60BFkTEaUlbKN/mz1uVclF+rM6mzubPv2jy8yipHXiLcof0Q17o2xk/pfi4m2iw7NeRQkQMSLoTeADok/RqRLzXTFuttfmOxFpGRAxFRDdlCNV5wOfAvVEeTI8XRPZSsiADPAF82cTuLqNcbH+RdBXnji1yBujI8n7gdklzoIwpIWkujdXWb2QkcJ3K5y4PA0QZI+ekpCW5z7b8L7Kx2x2gHDfZppmUhIXnkNRJGcdmEyX79aSOxW5TjwOJtRRJM4DTEfE30BURjbq2VlC6bI4Ayyhj2V+UKCPUDQLfAO8Ce2oWbwQ+ltQfET9Snjlszf3sp3S5NbITWDr2Yfs4bfgZ2ETJ9rqD0gU1YhmwIve5F7iakjH5z3wAv4pyNzMtu8W2Acsj4iznWwQMSRoEHgJeu0D77X/G2X/NzKwS35GYmVklDiRmZlaJA4mZmVXiQGJmZpU4kJiZWSUOJGZmVokDiZmZVfIP0+s0buDScsEAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xl8leWd///X55wk52QHQhKWsARkkU2QKIqCWgWtQ7VWOxAddVr82kXaatv5js60anHaqnWq5adT67TW1mmlHTtOqf1R1yJUUQnKDmEJCAGyk30953y+f9x3wknIBuFkIZ/n43E893Ldd677lpx3ruu+z3WLqmKMMcacKU9fV8AYY8zAZkFijDGmRyxIjDHG9IgFiTHGmB6xIDHGGNMjFiTGGGN6xILEmF4gIutE5K4z3HasiFSLiPds16sviMghEbnGnf4XEfl5X9fJ9IwFiTkr3A+HOvcDr0BEXhCRhL6u10AU/kELoKqHVTVBVYN9Wa9IUNUfqOpdACIyXkRURKL6ul7m9FiQmLPpM6qaAMwG5gAP9HF9jDG9wILEnHWqWgC8hhMoAIiIT0SeEJHDIlIoIs+KSGzY+htFZIuIVIrIARG5zl0+SkTWiEiZiOwXkf8Tts3DIvLfIvJfIlIlIttFZLKIPCAiRSJyREQWh5VfJyL/JiLvuS2nP4lIioj8xv25m0RkfFj5qSLyhvuzc0Xk78PWvSAiz4jIn92f/YGITAxbv0hE9ohIhYg8DUjYuoki8raIlIpIifvzh7jrXgTGAn9y6/h/2/6l3o1z8nsR+bVbr50ikhW2/p9F5Ki7LldErm7v/6F7Xta45+VDEXlERP7mrjul5RDeddfZ8bXzcx4Wkf9yZ9e77+XusV/hHuPMsPJpbss3tb39mb5hQWLOOhHJAD4N7A9b/BgwGSdczgNGAw+65S8Gfg38EzAEWAgccrd7CcgHRgG3AD9o8+H3GeBFYCjwMU6Aedz9rwR+1qZ6y4Db3fUTgY3AL4FhwG7gIbdO8cAbwG+BNCAb+A8RmR62r2zge+7P3g983912OPAH4DvAcOAAcFn4KQJ+6B7T+cAY4GEAVb0dOIzbulPVxzlVV+fkBmC1ey7XAE+79ZoCrAAuUtVE4FpOnue2ngHqgZHAF91Xd3V4fF1Y6L4PcY/9Hfc4/iGsTDbwpqoWn0Z9TKSpqr3s1eMXzgdSNVAFKPAWzgcCOB8sNcDEsPKXAgfd6Z8BT7azzzFAEEgMW/ZD4AV3+mHgjbB1n3Hr4HXnE926NNdjHfCvYeX/HVjbZvst7vRSYEOb+vwMeMidfgH4edi664E97vQdwPth6wTng/+uDs7dZ4GP25zLa8Lmx7vHEdXNc/Jm2LppQJ07fR5QBFwDRHfy/9ILNAFTw5b9APhb2/qErV93Jsfn1ve/OtnvPOAI4HHnc4C/7+t/7/Zq/bIWiTmbPqvOX7pXAlNx/hoHSAXigM0iUi4i5cBf3OXgfDgeaGd/o4AyVa0KW/YJTmuiWWHYdB1QoicvSte57wmdlG8731x2HDCvub5unW8DRoSVLwibrg3bdhTOhx8A6nwCtsy73TOr3S6mSuC/OHmuutKdc9K2Xn4RiVLV/cC9OB/eRW4dRrXzM1JxQutI2LJPulm/nh5fK6r6Ac4fIVeIyFScMFxzJvsykWNBYs46dbokXgCecBeV4HxIT1fVIe4rWZ0L8+B8YE08dU8cA4aJSGLYsrHA0cjUvJUjwDth9W3ubvlKN7Y9jhOOAIiIhM/jtCAUmKWqSThdNxK2vrMhuXt0TlT1t6p6OU5QKk6XY1vFQKBNnceGTde473Fhy8IDtqvj67B6HSz/lbuP24GXVbW+G/syvciCxETKU8AiEZmtqiHgP4EnRSQNQERGi8i1btlfAF8QkatFxOOum6qqR4D3gB+KiF9EZgHLgd/0Qv1fBSaLyO0iEu2+LhKR87ux7Z+B6SLyOfeC9Ndp/UGbiNMFVy4io3GuDYUrBCa0t+OenBMRmSIinxIRH871jzqcbrK2PyMI/A/wsIjEicg04M6w9cU4wfUPIuIVkS/S+g+Bro6vI8VAiFOP/UXgJpww+XU392V6kQWJiQj3w+bXwHfdRf+Mc0H6fbe7401gilv2Q+ALwJNABfAOzl/M4FxcHY/zl/grONco3uiF+lcBi3Euzh/D6S56DPB1Y9sS4PPAo0ApMAl4N6zI94ALcY71zzgf2uF+CHzH7VL7djs/4kzPic+tU4l7PGnAv3RQdgVOV10BTuvyl23W/x+cgCgFpuOEW7Oujq9dqlqLc8PCu+6xX+Iuzwc+wmmxbOjOvkzvEqf71hhjOiYi/4hzMf3yPvr5zwPHVPU7ffHzTefsG6TGmH5NnO/2fA7nS66mH7KuLWNMvyUijwA7gB+p6sG+ro9pn3VtGWOM6RFrkRhjjOmRQXGNZPjw4Tp+/Pi+roYxxgwomzdvLlHVLsc1GxRBMn78eHJycvq6GsYYM6CISLdGNBgUQWKMMb1JVSmraUREiI324ovy4PF058v9A5MFiTHGnKZQSCmqauBoeS35J+rCXrUcPVHH0fI6GgKhVtv4ojz4o73ERnuJjXHCJTbGiz/KmY+N9uKL9jjro71O2bByLcvccnExUaTEx5Ca6MMf3bcPz7QgMcaYNoIhpbCynvwTdU5YlLlBUe4ExbHyehqDrYMiJT6GjKGxnD8yiWumpTMy2Y9HhLqmIHWNQeoDQeobg9Q3hZxlTUHq3VdxVaClXEPAea9rChLq5k21Sf4o0pL8pCb4SEvykZboIzXRR1qin7REZ1lqgp+k2Cicod/OLgsSY8w5T1VpDIZoDIRoCJx8bwkLtzXRHBbHy+sJtPkUT030MXpILDNGJ3PdjJGMHhpLxtBYxgyNZdSQWOJizu7HqarSFFTqmoI0uMHjhE/IDZoAJVWNFFc3UFRZT1FVA8VVDXx8uJyiqnrqm0Kn7NMX5SG1JWROBk1qYnMA+UlN9JESH0OUt/s39UY0SMR5yt1PcJ5v8HNVfbTNeh/OeExzccbsWaqqh9wHHT3XXAx4WFVfcbc5hPPMiyAQUNUsjDEDXjCklFQ3cLyinoKKOo5X1FNdH6Ax2PrDvyEQpDHQOhScMu0sD4RocAOkMyKQlugjY2gcF44dyuhZsWQMjSNjaCyjh8Yyekhsr3cfiQgxUUJMlAdio09rW1WlqiFAcVUDRZUNFFXVU+wGTXPgHCyp4YODZZTXNp2yvUdgWHyXw8q1iFiQiIgX5ylri3Ae6rNJRNao6q6wYsuBE6p6nogswxkUbynON1mzVDUgIiOBrSLyJ1UNuNtd5Q6MZ4wZAALBEEVVzSFRz/GKOue9sp7j5c50YVUDwXb6cjwCMVEeYrwefNFe5z3KQ0zUyXd/tIfk2GhivK2Xx7SUc641OPvwtJRLS/STMTSWkUP8+KL69jrD2SQiJPmjSfJHMzE1odOyDYEgJdWNFFXWtwSNEzb1bO7mz4tki+RiYL+q5gGIyGrgRiA8SG7k5CM4XwaeFhFxRwFt5qfz5zMYY/pQo9tFVFBZ36o14QSGExrFVQ2n9Pf7oz2MSo5lRLKfSycOZ2SynxHJ/pb3EUl+kmOjT6uLxZw+X5SX0UOcVldbj7ZTvj2RDJLRtH7CWj7OYzPbLeO2PiqAFKBEROYBz+MMJ357WGtEgddFRIGfqepzGGMiqr4pyKHSGvKKazhYUsOB4mryims4Wl5HSXUDbUdaio/xMnJILCOT/UxKS3XDwfnLf2Syn5FJsRG78Gt6XySDpL1/IW1bFh2WcR+xOd19kNCvRGSt+2S0y1T1mPuApDdEZI+qrj/lh4vcDdwNMHbs2LarjTFthEJKQWU9ecU15JVUu+815BVXc7S8rlVYjEjyMyE1nk9NSWsJhxHJTnCMTPaT6D+9Pn0zsEUySPJp/ajODJwH8bRXJt99klwyUBZeQFV3i0gNMAPIUdVj7vIiEXkFpwvtlCBxWyrPAWRlZVnXmDGuqvomDpY4rYu84moOuNOHSmqoazr5wMT4GC+ZqfFcOHYot8zNYEJqAhOGx5M5PJ54n93waU6K5L+GTcAkEcnEeSznMuDWNmXW4DzCcyNwC/C2qqq7zRG3u2sczpP0DolIPOBR1Sp3ejGwMoLHYMyAVFXfREFFPYfLat2uKCc08kpqKK5qaCnnERgzLI7M4fFcOiGFCanxTEiNZ2JqAmmJPut6Mt0SsSBxQ2AF8BrO7b/Pq+pOEVmJ07JYg/Os7hdFZD9OS2SZu/nlwP0i0oTzDOevqmqJiEwAXnH/cUcBv1XVv0TqGIzpb5pvkS2ocC5uF7oXuAvd+YJKZ7qmsfWj2IfGRTMhNYErJ6eSmRrPhOEJTEyNZ2xK3Dl1t5LpG4PieSRZWVlqgzaa/q6uMeiEQUU9BZV1FFQ0OHdDhYVGUTu3yEZ5hPQkP+lJPkYk+0lPcu54GpHs3No6YXgCQ+Nj+uiozEAmIpu781096+g0pheVVjewt7CafUVV7Cus5pOyWgrc71RU1gdOKZ/oiyLdvRV2onuLbPP8iCQ/6ck+hsf7zukBAU3/Z0FiTAScqGlkb2EVe4uq2VdYxd5CJzhKaxpbyiT6ohg/PJ5xKfFcMiGlVUsi3X1PsIvaZgCwf6XG9EB5bSN7C6vdoKhyWxvVlFSfvKCd4IvivLQErj4/jcnpiUxKT2RyegIjkvx2MducEyxIjOmGitom9hVVnQwNdzr8Dqj4GC/npSdy1ZRUNzASmJyeyMhkCwxzbrMgMaYNVWVPQRVv7Crkw4Nl7C2soigsMOJivExKS2DhpFQmu2ExKT2B0UNiLTDMoGRBYgzOoII5n5zg9Z2FvLG7gCNldYjA+SOSuHzScCa73VGT0hIZPSTWLm4bE8aCxAxatY0B1u8t4fVdBby9p4jy2iZiojxcNjGFr155Hlefn0Zaor+vq2lMv2dBYgaVkuoG3tpdyOs7C/nb/hIaAiGSY6P51NQ0Fk9LZ+HkVBv+w5jTZL8x5pyXV1zNG7sKeWNXIZsPn0AVRg+JJfvisSyens5F44cRbUOVG3PGLEjMOScUUrbkl7eEx/6iagCmj0riG1dPYtG0dKaNTLIL48acJRYk5pzQEAjy3oFSXt9ZyJu7CymuasDrES6ZMIx/mDeWa6alkzE0rq+racw5yYLEDEiBYIj8E3VsOVLO67sKeCe3mJrGIPExXq6cksaiaelcNSWN5Dh7LoYxkWZBYvotVaW4qsF9uFINB0uqnedolNRwuLSWgDt4YWqijxvnjGbRtHTmT0yx0WyN6WUWJKbPVdY3caikpuW5GQdL3NAormk1HHpMlIfMlHgmpyVy3fQRZA6PZ8qIRGaMSrbvdRjThyxITK9oCAQ5Ulbb8szvlif0ldS0GpdKBDKGxpI5PIGsccOYkOo8kS9zeDyjku2LgMb0RxYk5qwLhZRdxyt5Z28xmw6VcbCkhiNltYQ/RmN4QgyZw+P51NRUJqQmkDk8ngnD4xkzLA5/tHVNGTOQWJCYs6K4qoF39hbwWu5ONh3fQS1HiPIfI8FfRmJqAlmjhjAybhhjElKYmJzK2IRhpPmU4TFeYr0h0AoIAEUKCu5/QNWZbnkAm55c1kw8YS8B8bZZFvbydLC8o1eUH7z2a2JMZ+w3xJyR8rpq/rRnM3/N28Ku0j1Uhj7B4ytAPE0wHOIUJgYCjG5opMLroSjgJa/RS0OlB4613ldiMERqMEhqMEhaIEhqMEBaMEhqIEhaMMhwd50vAg/zVJz8ahChXoRGj/vuzsfjYWJyJp606ZA+DdKmQ/p0SM5wQmswC4Xc4B7k58FYkJjOqSqFtYXsKdvDB/k7+PDYDj6p2k89RYg4n+zRMT6mE8fsJmF6RSlTGxuZEJNCzKRFMP5y5696EVSVymADxU1VFDVVUdxYeXK6qZKixko2N1ZR1FRJQIOn1CU5Ko7UmGTSYpKdd18ySd5YGjVAfbCRxlAT9cFGGkKNNASbnPdQEw2hAA2hJupDTU7ZUBONoQD1GqAhFCBE5wmVQj2Xln3A/EP/P5fW1TM8GAJfMqSd74bLNEif4czHDonI/4c+V1MKhTugaJfzXrgTivaALwEyF0LmFc77sMy+rqnpA/bMdtOiKdhEXkUeuSdyyS3LZWfJbnaX5VIbqGwpE2pMISE0gotifHyKE1xSspORNSWIeCDjIpi0CCZdCyNmnvFfqiENUdFQQVFtEcV1xRTXFrdMF9UWOfN1RZTWlRJ0A0cQ/FF+fF4fMd4Y/F4/vigfPo8PX5QPv9ffsjzGG9NStvnljzq5vmV5lI+SuhI2HtvI+8ffp6y+DIDJvuHM98RzaXU1Fxbm4W+oOFn5pIzW4ZI+DVImQdQAeWZ6oAGKc1sHRuEuqC44WSYuxQ3OaVBbAgfXQ3Whs27IWDdU3GBJTO+b4zBnRXef2W5BMkipKvvL9/PB8Q/YXbabvSf2sr98P4GQ89xw0WiC9ekE6kcSFRjNp5ITWRZbytyazfgLNoOGIHaYGxyLYeKnIG5Yrx5DMBSkLlCHz+sjyhMV0SFPQhoityyX9469x8ZjG/mo6COaQk34vD7mpkxnfuxoLg16mXTiGFK0G0r2QqjJ2dgTBcMnu+Hido2lTevb7jFVqMg/NTBK9kJza9AbA6lTT9Y5fbrTtZeQ1rreqk74HHzHCZVDG6DeDdfUqU6oTLgCxl127rbYzlH9IkhE5DrgJ4AX+LmqPtpmvQ/4NTAXKAWWquohEbkYeK65GPCwqr7SnX22x4LEUdtUy4cFH7IhfwMbjm7geM1xAOKjhuILZlBenkpNdRqh+lHMSUnnttTDXBbaTFrhBqTKKcvI2U5wTFoMoy8Ez+C8w6q2qZbNhZtbguVAxQEAUmNTuXTUpcxPv5hLfGmkVBx1P6h3OR/aFUdO7sSXDEPHQkwCRMdBTLzzio6DmLiw5XEQ7a5rmW5nfUc3BdRXQtFuKNp5MjAKd0J4Syp5rBsW004GRsp5Z3ajQSgIx7eeDJZPNkKgzrl5YeRsp6Uy4QoYc4lTd9Nv9XmQiIgX2AssAvKBTUC2qu4KK/NVYJaqfllElgE3qepSEYkDGlU1ICIjga3AKJxro53usz2DOUiOVB1hff56NhzdwKbjm2gMNRIbFcv0oVmcKJnItn0jCDUNYXh8NDePq2VJ7A6mVL1PTP77zl/UviSYeJXTXXXeNdZV0YGCmgI2HtvoBMvxjVS4H9LnDzvfCZZR85mTNoeYpjrnQ705XCqPQmMNNNU674210FTjTAfqT68SXt+poVNbBuWfnCwTk9g6MJqv7fiTz+LZaCPQAPmbnFDJeweO5kAo4LR4Mi52QiVzIYyeC14b0qY/6Q9BcilOS+Jad/4BAFX9YViZ19wyG0UkCigAUjWsUiKSCbwPjAYu6mqf7RlMQdIUbGJz0WY25G9gff56DlUeAmBc0jgWjF5Aqnc2b22J5919FYz0N/LtqaVc6dnGsGPrkOYPnLRpJ7usxsyzX+7TFAwF2VO2h/eOvcd7x95jS9EWAhrA7/WTNSKL+aPmM3/UfCYkT+i8Oy4UdAOmFhqr259uDp2Opn2JJwMjfTokj+n7u6waquHwRqfFkvcOFGwH1AnAcfPdYLnCqbPHhvfvS/0hSG4BrlPVu9z524F5qroirMwOt0y+O3/ALVMiIvOA54FxwO2q+kp39hm277uBuwHGjh0795NPPmlb5JxRXFvMhqMb2JC/gY3HN1LTVEO0J5qLRlzEgtELuHz05Rw46uf3b72H//gmLvcd4Kq4PFJq9iOo0z2SecXJ8Bgypq8P6ZxS01RDTkFOS7A0h3taXBoXpF7AiPgRpMelkx6XTlpcGunx6aTFphHdTwI8pCFO1J+gqLaIotoiCmsLW6ZjvDGMSRzD2MSxjEkcQ0ZiBv6o03yqZG2Zc12lucVSus9ZHjsMUqdA/HCITw17tZn3D+k/gRMKQkOVc42osdpp0SekQZSvr2t2RrobJJG8/be9P3vaplaHZVT1A2C6iJwP/EpE1nZzn7jbP4d7nSUrK+ucuqMgGAqyvWR7S3jsLtsNQHpcOp/O/DQLRy9kXtpcfCX72fXB6xx+5atMb9zFz+QExIBGJyDpWTDmZhh7CYy9FKLtkbKREh8dzxVjruCKMVcAcKz6GBuPbeTdY++y98Re/nb0b9QF6k7Zbph/WEvApMe7IRMWNulx6cRHx/eobvWBeopri1uFQ/h0UW0RRXVFLTdhNPOIhxR/CvXBeqoaq1qtS4tLaxUuY5LGOO+JY0iKSTq1EnHDYNqNzgug4qgbLBucbrmSffDJe07gtPfrLt6wcHHeQ3HDKfMnUBgdQ4FXKCRIQaiewqYqCupKWo4xyhNFUkwSSb4kkmKSSI6KJ8nrJ8kTQ7JEkYSHJIXkUIikYJCkQAPJjQ0kNtbhbah0rj81v9dXQJtz0SJ2KCSMcEIl0X1PGNF6OiHN6WLs6xbjGej3XVtuub8C/wREd7XP9pwLXVsVDRW8e/Rd1h9dz7tH36W8oRyPeJidOpsFGQtYkDqHyZVlSP4HhD55n1B+DlFB58OpSIbTMOoiRs28Eu+4S50LqfZt7X5DValqqqKwprDlg7ywtrDVfFFtEeUN5adsmxCd0G7ANM+LyCnhED5dEX7B3RUbFdtqH82v8PmU2BSiPM6/oYqGCo5UHeFw5WHnveow+VX5HK46TEldSat9D/ENYWziWDISM5ywSRrbEjIp/pTOu/qCAagrI1RdSFn5QQorDlFQeYTC2kInHJoqKQjUUqhNFHmUpjb7ilYlPRBgRAjSiSbNG0tIg1SEGqnUJioIUekRKj0eKj0e6rpo5SSqOEHjiSbJ4ycpyk9SVDzJMYkk+ZJJ9iUzwRPL1KAQV1Pq3CJdXQhV7nuw4dSdRvnDQiYdEtLbBJC7LD61V36H+0PXVhTOhfGrgaM4F8ZvVdWdYWXuAWaGXWz/nKr+vXtd5Ih7sX0csBGYBZR3tc/2DMQgqW2qZWfpTrYUbWHD0Q1sLd5KSEMM9Q3lstGXsXDo+cxvCJF8fCsc/sC5IwglhIe9Mo73myZRPHQ2Fy24noVZs22ww3NAc+uhoLagVSA0B05BbQEldSWENNTu9oKQEpvSbjCEzydEJ5y1W6lrm2o5UnWkJVjCg+Z4zfFWdY2Nij2lmywQClBQW0BhTSEFNQUtx9zUfGu1K9oT3dJya+4qHOEbRrrXzwiiSA8pwxrqkNpSqCmGmhLn3RvtdD/5k8Gf1Gq6KTqeiqgoJ1xQKglREaynsqmKyoZKKhorqGyopLKxkoqGilbv4fUThAnJE5iWMq3lNXXoFOKCjVBdBFUFznt1Qevp5nX1p/4BAeKEy9hLne7oCN0I0+dB4lbieuApnFt1n1fV74vISiBHVdeIiB94EZgDlAHLVDXPvfZxP9AEhICVqvq/He2zq3r09yAJhALsL9/P9pLtbC/ezvaS7eRV5LX8kp0/bCoLkyezIOBlRvFBvEc+PPkFsZhEmkZlsTk0mV8eSWdD3XgumDCar141kcvPG26Pkx1kAqEApXWlLUGjaEtIpMSmEO3pH9ddwLkx5Gj10VNaMc3B0/xh3G5INF9Xik9nRNwIhvqH4pH+cZ1EVakP1nOi/gR7T+xlV+mulldxXTHQQbgMm0pcdDu3QzfVQ03RyZZMdYEzXX4Y8tad/CwYeUHYrflzz8qt+f0iSPqL/hQkqsrxmuNsK9nGjuIdbC/Zzq7SXdQHnVs9k33JzBwymZneRGbU1zOz9DBDj25x7tIB537/sfNgzDzKUi7kuVwfL76fT01jkGvOT+MrV57H3HFD+/AIjem5YChIcV0x0Z7ofhUSPVVUW9QqWHaW7mzp/vOIh8ykzO6FSzNV5663fa/Dvjcg/0P3y8JDYeLVbmvlaufa0RmwIAnTl0FS2VjJjpIdbC/e7ryXbKe0vhSAGE8MU4dOYpYvlRmBELPKC8k4tuPkl/880TBihnML7ph5zoXxpFEcKavlZ+sP8PucfALBEEtmjeIrV07k/JHtXMg0xvRr4eGys3Qnu0p3dRgu04dPZ8rQKR2HS20Z5P0V9r0J+99wuu8Q58vDkxbDeYtg1Jxu3+VmQRKmt4KkKdhE7olcthVvawmN5ls9ATKTxjMzPoOZoShmVp9gckEu0SX7aLkTZdhEp0makeW8p89odTfVvsIqfrruAH/cegyvCDfPzeBLCycwfnjP7twxxvQvzeHSHCzthcuUYVPwR/kJhoIoSlCDhEIhQoQIaYhQKEiw7gShmiJCNcWE6isIiqDeaIKxQ1H/EIL+RELiccpryNmHOx3SEK9+7lULkmaRCpKQhnjr8FvkFOSwo2QHu8t2t/TrpvhTmJk8kVmeeGbU1TCj+CCJx3ecvFMjPhVGu4Ex+kLnr4QOxqraeqSc/1i3n9d2FhIb7eW2eWO5a8EERiTbLbvGDBZFtUXsLNnJrjInWPad2EdQg3jEg1e8CILX48UjHjx48HjClosXjwbxNFThqavAU1+ON9iEB/D4kvHED8cTn4rHPxSvx4uIs80TVz5hQdIsEkFS01TDd/72Hd48/CaxUbFMGzKJmdFDmdkYYGZZPiOObUea77aIjnOCYvSFbnBkdThgX01DgLziGg4UV3OguJqcQyfYmFdKcmw0d84fzxfmj2do/AAZSdYY0z+FgnD0I6f7a9/rcOxjZ3lCunMH2KRFMOEqJG6oBUmzsx0kRyqP8PW/fp288jy+6RvLbccPEVVx2FkpXmeIkdEXnuyiGj6l1T3fqsrxivpWgXGguJoDRTUUVJ4cX8kjMH54PMsuGsOt88aR4LPvfhhjIqC6CPa/5YTKgbecL1eKF3n4RJ9/s/2c9N7R9/j2+m/jCYV4tqKRS6tyYPK1MO9LTmiMnOUMmAfUNwU5WFLDgZ1FHCiqIa/ECYy84hpqG08+uCnRF8WEtATmn5fCxNQEJqbGMzE1gbEpcfiiBufousaYXpSQBrOznVcw4Aysue914KFubW5B0k2qyq92/oonP3qSib5h/OTgbsbEpaHLX6ckfrLToiio5sC2T5ywKKkm/0Qd4Q2+0UNimZiWwEXjh7mB4YRGaqLPvu9hjOkfvFHu0EmXYEFyFtUF6njovYfHNbmNAAAgAElEQVRYe3Ati2PSeWRPDnGZV/LatB/w3V8co6jqcEvZ2GgvE1LjmT1mKDdfmNESGJnD44mNsdaFMebcY0HShWPVx7j3r/eyp2wP3wglsTx3Ew0Xr+De8s/yvy8f5IIxQ/jKlROdwEhLYGSS34YjMcYMKhYkndhUsIlvrfsWTYF6nq4IsLDyALvnP8U/5oyhtLqYby2azFeunEiU99z41q0xxpwJC5J2qCov7XmJxzc9ztiYZH5y5Ajj/MN5ZsJ/8KO3fUxKi+YXd17EjNERfKqcMcYMEBYkbTQEG/i39/+N/93/v1wZk8YPczcTGnEpN1V+hW3bvdx1eSbfvnYK/mi73mGMMWBB0kphTSHfXPdNtpVs48uaxFdyc/ho5K1kH/o0ackJ/PauC7h0YkpfV9MYY/oVCxLXlqIt3LfuPmobq3mqKsRVJ/bx7/Hf4pmDc7llbgYPfWYaif7+MwS3Mcb0FxYkwB/2/oF/++DfGBmVyHP5+YzQJD7X8DBHmMTPbp/JtdNH9HUVjTGm3xrUQdIUbOKxTY/xu9zfcVlMKo/t+4j8qJlcUflV5k6bxM8/N5PhCb6+rqYxxvRrgzZISupK+Na6b/FR0Ud8QRP5Ru5mfqvX8+/Bf+Bfb5nFLXMz7NvmxhjTDYMySHaW7OQbf/0GFfUneLQyxOKyvXy78SscG3cjf/r8BYwZ1skTyYwxxrQy6ILkTwf+xPc2fo9hHj8v5B8npcHP0sDDXP/p6/n3yzLtW+nGGHOaBk2QBEIBfrz5x7y460XmRg/jx/u3sS8wme8P+w6PZl/J5PTEvq6iMcYMSIMiSIIa5MtvfpkPjn/ALYEE/uXgFn4bXEzZZQ/zwjXnExNlQ5wYY8yZimiQiMh1wE8AL/BzVX20zXof8GtgLlAKLFXVQyKyCHgUiAEagX9S1bfdbdYBI4E6dzeLVbWos3rklefhKRQeOBHg5hN7+bH/ayy67VtcOHboWTtWY4wZrCIWJCLiBZ4BFgH5wCYRWaOqu8KKLQdOqOp5IrIMeAxYCpQAn1HVYyIyA3gNGB223W2q2u1HHoY0yHNHChjV4OOFKT/la7d8jriYQdEYM8aYiItkn87FwH5VzVPVRmA1cGObMjcCv3KnXwauFhFR1Y9V9Zi7fCfgd1svZ2RiQz3eprEc+tyf+dKtf28hYowxZ1Ekg2Q0cCRsPp/WrYpWZVQ1AFQAbQezuhn4WFUbwpb9UkS2iMh3pYMve4jI3SKSIyI5NZpA5jff4tILpvXkeIwxxrQjkkHS3ge8nk4ZEZmO0931pbD1t6nqTGCB+7q9vR+uqs+papaqZiWPOo/kxPjTqrwxxpjuiWSQ5ANjwuYzgGMdlRGRKCAZKHPnM4BXgDtU9UDzBqp61H2vAn6L04VmjDGmj0QySDYBk0QkU0RigGXAmjZl1gB3utO3AG+rqorIEODPwAOq+m5zYRGJEpHh7nQ0sATYEcFjMMYY04WIBYl7zWMFzh1Xu4Hfq+pOEVkpIje4xX4BpIjIfuCbwP3u8hXAecB33WshW0QkDfABr4nINmALcBT4z0gdgzHGmK6JatvLFueerKwszcnp9t3CxhhjABHZrKpZXZWzr3QbY4zpEQsSY4wxPWJBYowxpkcsSIwxxvSIBYkxxpgesSAxxhjTIxYkxhhjesSCxBhjTI9YkBhjjOkRCxJjjDE9YkFijDGmRyxIjDHG9IgFiTHGmB6xIDHGGNMjFiTGGGN6xILEGGNMj1iQGGOM6RELEmOMMT1iQWKMMaZHLEiMMcb0iAWJMcaYHolokIjIdSKSKyL7ReT+dtb7ROR37voPRGS8u3yRiGwWke3u+6fCtpnrLt8vIqtERCJ5DMYYYzoXsSARES/wDPBpYBqQLSLT2hRbDpxQ1fOAJ4HH3OUlwGdUdSZwJ/Bi2DY/Be4GJrmv6yJ1DMYYY7oWyRbJxcB+Vc1T1UZgNXBjmzI3Ar9yp18GrhYRUdWPVfWYu3wn4HdbLyOBJFXdqKoK/Br4bASPwRhjTBciGSSjgSNh8/nusnbLqGoAqABS2pS5GfhYVRvc8vld7BMAEblbRHJEJKe4uPiMD8IYY0znIhkk7V270NMpIyLTcbq7vnQa+3QWqj6nqlmqmpWamtqN6hpjjDkTXQaJiHhFZHjYfIz71/7uLjbNB8aEzWcAxzoqIyJRQDJQ5s5nAK8Ad6jqgbDyGV3s0xhjTC/qNEhEZBnOB/s2EXlHRK4C8nAuoN/Wxb43AZNEJFNEYoBlwJo2ZdbgXEwHuAV4W1VVRIYAfwYeUNV3mwur6nGgSkQuce/WugP4Y3cO1BhjTGREdbH+O8BcVd0vIhcCG4FlqvpKVztW1YCIrABeA7zA86q6U0RWAjmqugb4BfCiiOzHCaxl7uYrgPOA74rId91li1W1CPgK8AIQC6x1X8YYY/qIODc/dbBS5CNVvTBsfo+qTu2Vmp1FWVlZmpOT09fVMMaYAUVENqtqVlflumqRpInIN8PmE8LnVfXHZ1pBY4wx54auguQ/gcRO5o0xxgxynQaJqn6vo3UiEn/2q2OMMWag6c7tv6NFJMu98woRSRORHwD7Il47Y4wx/V5Xt//eC2wB/j/gfRG5E9iNc8fU3MhXzxhjTH/X1TWSu4EpqlomImOB/cBCVX0/8lUzxhgzEHTVtVWvqmUAqnoY2GshYowxJlxXLZIMEVkVNp8WPq+qX49MtYwxxgwUXQXJP7WZ3xypihhjjBmYurr991edrTfGGGM6DRIRaTvIYiuqesPZrY4xxpiBpquurUtxHjz1EvAB7T8PxBhjzCDWVZCMABYB2cCtOEO7v6SqOyNdMWOMMQNDp7f/qmpQVf+iqncCl+B8j2SdiHytV2pnjDGm3+uqRYKI+IC/w2mVjAdWAf8T2WoZY4wZKLq62P4rYAbOw6O+p6o7eqVWxhhjBoyuWiS3AzXAZODrztNtAeeiu6pqUgTrZowxZgDo6nskXY4ObIwxZnCzoDDGGNMjFiTGGGN6JKJBIiLXiUiuiOwXkfvbWe8Tkd+56z8QkfHu8hQR+auIVIvI0222Wefuc4v7SovkMRhjjOlcl7f/nikR8QLP4HyhMR/YJCJrVHVXWLHlwAlVPU9ElgGPAUuBeuC7OHeMzWhn97epak6k6m6MMab7ItkiuRjYr6p5qtoIrAZubFPmRqB5YMiXgatFRFS1RlX/hhMoxhhj+rFIBslonHG6muW7y9oto6oBoAJI6ca+f+l2a31Xwu5JNsYY0/siGSTtfcDrGZRp6zZVnQkscF+3t/vDRe4WkRwRySkuLu6yssYYY85MJIMkHxgTNp8BHOuojIhEAclAWWc7VdWj7nsV8FucLrT2yj2nqlmqmpWamnpGB2CMMaZrkQySTcAkEckUkRhgGdD2+SZrgDvd6VuAt1W1wxaJiESJyHB3OhpYAtiwLcYY04cidteWqgZEZAXwGuAFnlfVnSKyEshR1TXAL4AXRWQ/TktkWfP2InIISAJiROSzwGLgE+A1N0S8wJvAf0bqGIwxxnRNOmkAnDOysrI0J8fuFjbGmNMhIptVNaurcvbNdmOMMT1iQWKMMaZHLEiMMcb0iAWJMcaYHrEgMcYY0yMWJMYYY3rEgsQYY0yPWJAYY4zpEQsSY4wxPWJBYowxpkcsSIwxxvSIBYkxxpgesSAxxhjTIxYkxhhjesSCxBhjTI9YkBhjjOkRCxJjjDE9YkFijDGmRyxIjDHG9IgFiTHGmB6xIDHGGNMjEQ0SEblORHJFZL+I3N/Oep+I/M5d/4GIjHeXp4jIX0WkWkSebrPNXBHZ7m6zSkQkksdgjDGmcxELEhHxAs8AnwamAdkiMq1NseXACVU9D3gSeMxdXg98F/h2O7v+KXA3MMl9XXf2a2+MMaa7ItkiuRjYr6p5qtoIrAZubFPmRuBX7vTLwNUiIqpao6p/wwmUFiIyEkhS1Y2qqsCvgc9G8BiMMcZ0IZJBMho4Ejaf7y5rt4yqBoAKIKWLfeZ3sU8ARORuEckRkZzi4uLTrLoxxpjuimSQtHftQs+gzBmVV9XnVDVLVbNSU1M72aUxxpieiGSQ5ANjwuYzgGMdlRGRKCAZKOtinxld7NMYY0wvimSQbAImiUimiMQAy4A1bcqsAe50p28B3navfbRLVY8DVSJyiXu31h3AH89+1Y0xxnRXVKR2rKoBEVkBvAZ4gedVdaeIrARyVHUN8AvgRRHZj9MSWda8vYgcApKAGBH5LLBYVXcBXwFeAGKBte7LGGNMH5FOGgDnjKysLM3JyenrahhjzIAiIptVNaurcvbNdmOMMT0Ssa6t/q6pqYn8/Hzq6+u7LmxOi9/vJyMjg+jo6L6uijGmFwzaIMnPzycxMZHx48djo6ycPapKaWkp+fn5ZGZm9nV1jDG9YNB2bdXX15OSkmIhcpaJCCkpKdbSM2YQGbRBAliIRIidV2MGl0EdJMYYY3rOgqQPeb1eZs+ezYwZM/j85z9PbW3tWdlvYWEhS5Ys4YILLmDatGlcf/31AGRmZpKbm9uq7L333svjjz/OunXrSE5OZs6cOUyZMoWFCxfy6quvnpX6GGPObRYkfSg2NpYtW7awY8cOYmJiePbZZ7u9bSAQ6HDdgw8+yKJFi9i6dSu7du3i0UcfBWDZsmWsXr26pVwoFOLll19m6dKlACxYsICPP/6Y3NxcVq1axYoVK3jrrbfO8OiMMYPFoL1rK9z3/rSTXccqz+o+p41K4qHPTO92+QULFrBt2zYOHTrEkiVL2LFjBwBPPPEE1dXVPPzww1x55ZXMnz+fd999lxtuuIE77riDL3/5yxw+fBiAp556issuu4zjx4+zePHiln3PmjULgOzsbJYuXcpDDz0EwPr16xk/fjzjxo3j4MGDreoze/ZsHnzwQZ5++mmuvvrqVuuCwSDLly8nJycHEeGLX/wi99133+mfJGPMOcGCpB8IBAKsXbuW667r+hld5eXlvPPOOwDceuut3HfffVx++eUcPnyYa6+9lt27d3PPPfewdOlSnn76aa655hq+8IUvMGrUKGbNmoXH42Hr1q1ccMEFrF69muzs7A5/1oUXXsiPfvSjU5Zv2bKFo0ePtoRdeXn5GR65MeZcYEECp9VyOJvq6uqYPXs24LRIli9fzrFjnQ9m3NwNBfDmm2+ya9eulvnKykqqqqq49tprycvL4y9/+Qtr165lzpw57Nixg9TUVLKzs1m9ejXTp0/nj3/8IytXruzwZ3U0fM6ECRPIy8vja1/7Gn/3d3/XqvVjjBl8LEj6UPM1knBRUVGEQqGW+bbfx4iPj2+ZDoVCbNy4kdjY2FP2PWzYMG699VZuvfVWlixZwvr167n55pvJzs5m8eLFXHHFFcyaNYu0tLQO6/fxxx9z/vnnEwwGmTt3LgA33HADK1euZOvWrbz22ms888wz/P73v+f5558/o3NgjBn47GJ7P5Oenk5RURGlpaU0NDR0eufU4sWLefrpp1vmm0Pp7bffbrkDrKqqigMHDjB27FgAJk6cSEpKCvfff3+n3Vrbtm3jkUce4Z577sHr9bJlyxa2bNnCypUrKSkpIRQKcfPNN/PII4/w0UcfnY1DN8YMUNYi6Weio6N58MEHmTdvHpmZmUydOrXDsqtWreKee+5h1qxZBAIBFi5cyLPPPsvmzZtZsWJFS+vmrrvu4qKLLmrZLjs7mwceeICbbrqp1f42bNjAnDlzqK2tJS0tjVWrVp1yoR3g6NGjfOELX2hpOf3whz88S0dvjBmIBu0w8rt37+b888/voxqd++z8GjPw2TDyxhhjeoUFiTHGmB6xIDHGGNMjFiTGGGN6xILEGGNMj1iQGGOM6ZGIBomIXCciuSKyX0Tub2e9T0R+567/QETGh617wF2eKyLXhi0/JCLbRWSLiOS03edAMn/+/FbzTz75JH6/n4qKipZl69atY8mSJads++qrrzJnzpyWoeJ/9rOfRby+xhjTnoh9IVFEvMAzwCIgH9gkImtUdVdYseXACVU9T0SWAY8BS0VkGrAMmA6MAt4UkcmqGnS3u0pVSyJV997y3nvvtZp/6aWXuOiii3jllVf4x3/8xw63a2pq4u677+bDDz8kIyODhoYGDh06FNnKGmNMByL5zfaLgf2qmgcgIquBG4HwILkReNidfhl4WpzntN4IrFbVBuCgiOx397cxIjVdez8UbD+7+xwxEz79aKdFEhISqK6uBuDAgQNUV1fzox/9iB/84AedBklVVRWBQICUlBQAfD4fU6ZMOWtVN8aY0xHJrq3RwJGw+Xx3WbtlVDUAVAApXWyrwOsisllE7u7oh4vI3SKSIyI5xcXFPTqQ3vDSSy+RnZ3NggULyM3NpaioqMOyw4YN44YbbmDcuHFkZ2fzm9/8ptVAj8YY05si2SKRdpa1HY+lozKdbXuZqh4TkTTgDRHZo6rrTyms+hzwHDhDpHRa0y5aDr1h9erVvPLKK3g8Hj73uc/x3//939xzzz0dlv/5z3/O9u3befPNN3niiSd44403eOGFF3qvwsYY44pkkOQDY8LmM4C2D9toLpMvIlFAMlDW2baq2vxeJCKv4HR5nRIkA8m2bdvYt28fixYtAqCxsZEJEyZ0GiQAM2fOZObMmdx+++1kZmZakBhj+kQku7Y2AZNEJFNEYnAunq9pU2YNcKc7fQvwtjqjSK4Blrl3dWUCk4APRSReRBIBRCQeWAzsiOAx9IqXXnqJhx9+mEOHDnHo0CGOHTvG0aNH+eSTT9otX11dzbp161rmt2zZwrhx43qptsYY01rEWiSqGhCRFcBrgBd4XlV3ishKIEdV1wC/AF50L6aX4YQNbrnf41yYDwD3qGpQRNKBV5zr8UQBv1XVv0TqGHrL6tWrWbt2batlN910E6tXr2bevHm89dZbZGRktKx76aWXePzxx/nSl75EbGws8fHx1hoxxvQZG0beRISdX2MGPhtG3hhjTK+wIDHGGNMjFiTGGGN6xILEGGNMj1iQGGOM6RELEmOMMT1iQdKHvF4vs2fPZsaMGXz+85+ntrb2rOy3sLCQJUuWtAwxf/311wOQmZlJbm5uq7L33nsvjz/+OOvWrSM5OZk5c+YwZcoUFi5cyKuvvnpW6mOMObdZkPSh2NhYtmzZwo4dO4iJieHZZ5/t9raBQKDDdQ8++CCLFi1i69at7Nq1i0cfdcYSW7ZsGatXr24pFwqFePnll1m6dCkACxYs4OOPPyY3N5dVq1axYsUK3nrrrTM8OmPMYBHJsbYGjMc+fIw9ZXvO6j6nDpvKP1/8z90uv2DBArZt28ahQ4dYsmQJO3Y4I7888cQTVFdX8/DDD3PllVcyf/583n33XW644QbuuOMOvvzlL3P48GEAnnrqKS677DKOHz/O4sWLW/Y9a9YsALKzs1m6dCkPPfQQAOvXr2f8+PGMGzeOgwcPtqrP7NmzefDBB3n66ae5+uqre3QujDHnNmuR9AOBQIC1a9cyc+bMLsuWl5fzzjvv8K1vfYtvfOMb3HfffWzatIk//OEP3HXXXQDcc889LF++nKuuuorvf//7HDvmjJU5a9YsPB4PW7duBZyhWbKzszv8WRdeeCF79pzdgDXGnHusRQKn1XI4m+rq6pg9ezbgtEiWL1/e8qHfkeZuKIA333yTXbtOPiessrKSqqoqrr32WvLy8vjLX/7C2rVrmTNnDjt27CA1NZXs7GxWr17N9OnT+eMf/8jKlSs7/FmDYfgcY0zPWZD0oeZrJOGioqJaPaSqvr6+1fr4+PiW6VAoxMaNG4mNjT1l38OGDePWW2/l1ltvZcmSJaxfv56bb76Z7OxsFi9ezBVXXMGsWbNIS0vrsH4ff/yxjZdljOmSdW31M+np6RQVFVFaWkpDQ0Ond04tXryYp59+umW+OZTefvvtljvAqqqqOHDgAGPHjgVg4sSJpKSkcP/993farbVt2zYeeeSRLp+JYowx1iLpZ6Kjo3nwwQeZN28emZmZTJ06tcOyq1at4p577mHWrFkEAgEWLlzIs88+y+bNm1mxYkVL6+auu+7ioosuatkuOzubBx54gJtuuqnV/jZs2MCcOXOora0lLS2NVatW2YV2Y0yXbBh5ExF2fo0Z+GwYeWOMMb3CgsQYY0yPDOogGQzden3Bzqsxg8ugDRK/309paal96J1lqkppaSl+v7+vq2KM6SWD9q6tjIwM8vPzKS4u7uuqnHP8fj8ZGRl9XQ1jTC8ZtEESHR1NZmZmX1fDGGMGvIh2bYnIdSKSKyL7ReT+dtb7ROR37voPRGR82LoH3OW5InJtd/dpjDGmd0UsSETECzwDfBqYBmSLyLQ2xZYDJ1T1POBJ4DF322nAMmA6cB3wHyLi7eY+jTHG9KJItkguBvarap6qNgKrgRvblLkR+JU7/TJwtYiIu3y1qjao6kFgv7u/7uzTGGNML4rkNZLRwJGw+XxgXkdlVDUgIhVAirv8/Tbbjnanu9onACJyN3C3O1stIrntlTtLhgMlEdx/JA3kuoPVv69Z/ftWpOs/rjuFIhkk0s6ytvfadlSmo+XttaDavX9XVZ8DnuusgmeLiOR0ZxiB/mgg1x2s/n3N6t+3+kv9I9m1lQ+MCZvPANo+bKOljIhEAclAWSfbdmefxhhjelEkg2QTMElEMkUkBufi+Zo2ZdYAd7rTtwBvq/MNwTXAMveurkxgEvBhN/dpjDGmF0Wsa8u95rECeA3wAs+r6k4RWQnkqOoa4BfAiyKyH6clsszddqeI/B7YBQSAe1Q1CNDePiN1DKehV7rQImQg1x2s/n3N6t+3+kX9B8Uw8sYYYyJn0I61ZYwx5uywIDHGGNMjFiSnQUTGiMhfRWS3iOwUkW+4y4eJyBsiss99H9rXde2MO0rAxyLyqjuf6Q5Rs88dsiamr+vYEREZIiIvi8ge9//DpQPl/IvIfe6/mx0i8pKI+Pv7uReR50WkSER2hC1r93yLY5U7fNE2Ebmw72reYd1/5P7b2SYir4jIkLB17Q7L1Ffaq3/Yum+LiIrIcHe+T8+9BcnpCQDfUtXzgUuAe9whWu4H3lLVScBb7nx/9g1gd9j8Y8CTbv1P4Axd01/9BPiLqk4FLsA5jn5//kVkNPB1IEtVZ+DcLLKM/n/uX8AZpihcR+f70zh3WE7C+TLwT3upjh15gVPr/gYwQ1VnAXuBB6DjYZl6r6rteoFT64+IjAEWAYfDFvftuVdVe53hC/ij+z80FxjpLhsJ5PZ13TqpcwbOL/+ngFdxvvxZAkS56y8FXuvrenZQ9yTgIO5NImHL+/355+QoDsNw7pZ8Fbh2IJx7YDywo6vzDfwMyG6vXH+pe5t1NwG/cacfAB4IW/cacGl/O/fuspdx/og6BAzvD+feWiRnyB2peA7wAZCuqscB3Pe0vqtZl54C/i8QcudTgHJVDbjz4cPR9DcTgGLgl27X3M9FJJ4BcP5V9SjwBM5fkceBCmAzA+fch+vofLc3LFJ/Pp4vAmvd6QFRdxG5ATiqqlvbrOrT+luQnAERSQD+ANyrqpV9XZ/uEpElQJGqbg5f3E7R/npPeBRwIfBTVZ0D1NAPu7Ha415HuBHIBEYB8TjdEW3113PfHQPm35KI/CtOV/Vvmhe1U6xf1V1E4oB/BR5sb3U7y3qt/hYkp0lEonFC5Deq+j/u4kIRGemuHwkU9VX9unAZcIOIHMIZOflTOC2UIe4QNdC/h53JB/JV9QN3/mWcYBkI5/8a4KCqFqtqE/A/wHwGzrkP19H5HhBDGInIncAS4DZ1+4EYGHWfiPOHyFb3dzgD+EhERtDH9bcgOQ0iIjjfxt+tqj8OWxU+1MudONdO+h1VfUBVM1R1PM6FxbdV9TbgrzhD1ED/rn8BcEREpriLrsYZ/WAgnP/DwCUiEuf+O2qu+4A49210dL7XAHe4dxBdAlQ0d4H1FyJyHfDPwA2qWhu2qqNhmfoNVd2uqmmqOt79Hc4HLnR/L/r23Pf1xaSB9AIux2kubgO2uK/rca4zvAXsc9+H9XVdu3EsVwKvutMTcH5p9gP/Dfj6un6d1Hs2kOP+P/hfYOhAOf/A94A9wA7gRcDX38898BLONZ0mnA+u5R2db5zulWeAA8B2nDvU+lvd9+NcS2j+/X02rPy/unXPBT7dH899m/WHOHmxvU/PvQ2RYowxpkesa8sYY0yPWJAYY4zpEQsSY4wxPWJBYowxpkcsSIwxxvSIBYk5J4nID0XkShH5rIj8v/bOLsSqKorjvz8iTsT0lH0QOIIig1KMkJXllBIUJEFiFBnSJD5I4OCg0IMg+ib1FEWCRkoTivSgMA9BaiNT4weWd/IDfFT0LdFCpIRq9bDWxTN37r3jmRuM3NYPNrPu2Wd/cZmzzt777v8qdfpd0uxQ5K1I6q3JOy7p6UnKb4pTyFMi+v18g7xZko5KGpP0dsl650paM9V+JUkj0pEk7cqzuA7aS8APJcu+DFwys8VmVrYswCZgyo4EP+NT15Hg+m4zzazHzA6WrHcuUMqRFE7dJ0lD0pEkbUXEmzgHLAFOAuuBXZIm6BNJ6pJ0LOI3HJM0R1IP8BHwWrz1P9CkrV2SfpLHGNkR1/pxLa1hScNx7RVJJyWdlfRNaLUh6bKkHXH9vKTuEAPdAAxE+72F9h4BvgZ6Im+epG2SzshjnOyOU/NImh8zl1+i/nnATqA3yg7I46HsjbYrklZE2b7o5xDwnaTHJY1EuQu1s7QkmfaTs5ky/dcJeAb4FJgJjDa5bwh4L+x1wOGw+4DPGpQ5Tpwa5u6J7hlx/an4fJm7J44fBkaAB+Pzh8C2wn0bw/4A+CLs7cCWBu0vJxQJin0IexB4PezTwKqwO/AZUm3ZzcDesLtxGZeOGP+1wvg2A1sLY+2c7u840/2VckaStCOLcfmLblzPqhFLgf1hD+ISOGV4S9JZoIIHRFpY557n4vqopDFcmwfiBq0AAAHSSURBVKqrkF8V/vwZX3oqy4rYzzmPi3AuktQJPGFmhwDM7E8brytVZRk+bszsEnAFWBB5R8zsRthngPclbQeeNLNbU+hn0sbk+mfSNsSy1D5c+fQ6/haueIAvNbM/JqninvWCQthvC7DEzG5K2oe/zU+4FX8ov9Ogqjvx929K/j9K6gA+x2dIV+NB30F9SfG6VTTJu101zGxE0ovASmBQ0sdm9lWZvibtTc5IkrbBzMbMrAcPoboQ+B541Xxjup4TOYGrIAO8C/xYormH8Ift75IeZXxskVtAZ9ingBckzQePKSFpAc0plm9G1XFdj32XNwHMY+Rck/RGtDkrfkVWW+8IPm6iT3NwwcJxSOrC49jswdWvpzUWe3L/kY4kaSskzQZumtk/QLeZNVva6seXbM4Ba/FY9veEeYS6CnAR+BIYLWTvBr6VNGxmv+J7DgeinVP4klszhoBVtZvtdfrwG7AHV3s9jC9BVVkL9EebJ4DHcMXkv2IDfgCfzcyIZbGDQJ+Z3WEiy4ExSRVgNfDJJP1P/mek+m+SJEnSEjkjSZIkSVoiHUmSJEnSEulIkiRJkpZIR5IkSZK0RDqSJEmSpCXSkSRJkiQtkY4kSZIkaYl/AQP+YqNq3KFfAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -1025,7 +1025,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "It can be seen that scaling has a significant impact on the quality of recommendations. This is, however, a preliminary result, which is yet to be verified via cross-validation." + "It can be seen that scaling (`PureSVD-s` line) has a significant impact on the quality of recommendations. This is, however, a preliminary result, which is yet to be verified via cross-validation." ] }, { @@ -1051,7 +1051,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -1067,7 +1067,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -1085,7 +1085,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -1095,11 +1095,11 @@ "\n", "100%\n", "5/5\n", - "[04:15<00:51, 50.93s/it]" + "[05:23<01:07, 64.52s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 5/5 [04:15<00:51, 50.93s/it]" + " [████████████████████████████████████████████████████████████] 5/5 [05:23<01:07, 64.52s/it]" ] }, "metadata": {}, @@ -1130,7 +1130,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -1194,9 +1194,9 @@ " \n", " \n", " iALS\n", - " 0.073780\n", - " 0.027274\n", - " 0.089461\n", + " 0.076428\n", + " 0.028240\n", + " 0.093489\n", " \n", " \n", " 2\n", @@ -1221,12 +1221,12 @@ "fold model \n", "1 PureSVD 0.076857 0.029101 0.085902\n", " PureSVD-s 0.084729 0.032221 0.148946\n", - " iALS 0.073780 0.027274 0.089461\n", + " iALS 0.076428 0.028240 0.093489\n", "2 PureSVD 0.079868 0.029385 0.089836\n", " PureSVD-s 0.086953 0.034059 0.150539" ] }, - "execution_count": 55, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -1251,7 +1251,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -1271,12 +1271,12 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 38, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X203WV95/33x0RAUUFjsMpTYKC24UEsR2jHh6ZSMKyKsSOUAFPgvrlXtMo4q962hZlCI2M70DVTagfaMRUQcRAoLdO0RNGK+DQWEwQJgXI3xgAxWsJDEVDAwPf+Y/9O3GzOydnJ2efsffZ5v9Y6K799/a7ftb+/vc+52Hz39ZCqQpIkSZIkqVde1O8AJEmSJEnScDHZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIOGRpKNSX6133FIkiRJ0mxnskGSJEmSJPWUyQbNOknm9jsGSZrNxuqH7ZslaXolmdPvGDTcTDZo2ByR5M4kjyW5NsluSRYl2ZTk95L8ALii30FK0jBopq/9TtPvPpnksiSvSfLZJI8n+Yckr0yyIEklOSvJ/cDNY5X1+34kabol2TfJ3yTZkuThJJckeVGS309yX5IHk3wqyR5N/c8lObujjW8n+XfN8c8l+UKSR5Lcm+Q32up9MslfJFmV5EngV5L8WpLbk/wwyQNJlne0fXoTx8NJzmufttzEeU6S7zTnr0vyqql+zTRzmGzQsPkNYDFwAHA4cGZT/jPAq4D9gWV9iUyShtN7gGOBnwVOAD4L/Cfg1bQ+Z3ywre4vAz8PvGOCMkkaes3Igr8H7gMWAHsD19D6/Hom8CvAgcDLgEuay64GTmlrYyGtz7c3Jtkd+EJTZ6+m3p8nOaTtaU8F/hB4OfA14EngdGBP4NeA30ry7ra2/xw4DXgtsEcT46gPAu+m1Y+/DngUuHQSL4mGjMkGDZs/q6rNVfUI8HfAEU35c8AfVNXTVfXj/oUnSUPnf1TVv1TV94CvArdW1e1V9TRwA/DGtrrLq+rJjn54rDJJmg2OovU/6b/T9INPVdXXaP3P/Z9U1YaqegI4F1jaTDe7gdZI3v2bNk4D/qbpc98JbKyqK6pqa1V9C/hr4MS25/zbqvp6VT3XPN8tVbW2eXwn8BlayQOa6/6uqr5WVc8A5wPV1tZ7gf9cVZua518OnOi0OI0y2aBh84O24x/RygQDbKmqp/oQjyQNu39pO/7xGI9f1vb4gTGuH6tMkmaDfYH7qmprR/nraI12GHUfMBd4TVU9DtwILG3OLQX+V3O8P3B0kn8d/aGVjPiZtrae1+cmOTrJl5ppHI8B76M1Mm00jm31q+pHwMNtl+8P3ND2XPcAzwKv6foV0FAz2aDZoiauIkmaYmP1xfbPkmarB4D9xhgJsJnW/8iP2g/Yyk+TuZ8BTknyS8BLgC+1tfflqtqz7edlVfVbbW119rlXAyuBfatqD+B/AmnOfR/YZ7RikpcA8zriP77j+XZrRrpJJhskSZIkqQ++Set/6C9MsnuzsPmbaSUTfjvJAUleBvwRcG3bCIhVtJIRFzTlzzXlfw/8bJLfTPLi5udNSX5+OzG8HHikqp5KchStNR1GXQ+ckOTfJtkF+Ag/TURAKzHxh6NTOpLMT7JkEq+HhozJBkmSJEmaZlX1LK2FdQ8C7gc2AScDlwNXAV8Bvgs8BfyHtuueBv4G+FVaIxNGyx8HjqM1tWIzrenFFwG7bieM9wMXJHmc1poM17W1t6553mtoJUUeBx4Enm6qfIzWqIjPN9f/I3D0Dr8QGlqpcvSiJEmSJGl8zSiLfwUOrqrv9jseDT5HNkiSJEmSXiDJCUle2myr+d+AtcDG/kalmcJkgyRJkiRpLEtoTcnYDBwMLC2HxqtLTqOQJEmSJEk95cgGSZIkSZLUUyYbJEmSJElST83tdwCdXv3qV9eCBQv6HYYkvcBtt932UFXN73cc08G+WNIgsh+WpP7rti8euGTDggULWLNmTb/DkKQXSHJfv2OYLvbFkgaR/bAk9V+3fbHTKCRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGDa1FixaxaNGifochSbOafbEk9Zf9sPrFZIMkDZAki5Pcm2R9knPGOL9rkmub87cmWdCUH5Xkjubn20l+ve2ajUnWNufWTN/dSJIkabaa2+8AJEktSeYAlwLHApuA1UlWVtXdbdXOAh6tqoOSLAUuAk4G7gJGqmprktcC307yd1W1tbnuV6rqoem7G0mSJM1mJhskaXAcBayvqg0ASa4BlgDtyYYlwPLm+HrgkiSpqh+11dkNqKkPV5IkTdZhVx42pe1v+MGGaXmetWesndL2NfM4jUKSBsfewANtjzc1ZWPWaUYtPAbMA0hydJJ1wFrgfW2jGgr4fJLbkiwb78mTLEuyJsmaLVu29OSGJEmSNDs5skF9s+CcG6e0/R9seHhanmfjhb82pe1rVskYZZ0jFMatU1W3Aock+XngyiSfraqngDdX1eYkewFfSPJPVfWVFzRStQJYATAyMuLICEmSJO00RzZI0uDYBOzb9ngfYPN4dZLMBfYAHmmvUFX3AE8ChzaPNzf/PgjcQGu6hiRJkjRlTDZI0uBYDRyc5IAkuwBLgZUddVYCZzTHJwI3V1U118wFSLI/8HpgY5Ldk7y8Kd8dOI7WYpKSJGkWOPDcAznw3AP7HYZmIadRSNKAaHaSOBu4CZgDXF5V65JcAKypqpXAZcBVSdbTGtGwtLn8LcA5SX4CPAe8v6oeSnIgcEMSaPX5V1fV56b3ziRJkjTbmGzQ0PqZUy/sdwjSDquqVcCqjrLz246fAk4a47qrgKvGKN8AvKH3kWpoLN9jatvf+OTUP8/yx6aubalNksXAx2glhD9RVRd2nH8b8KfA4cDSqrq+4/wrgHuAG6rq7OmJWpL6w2kUkiRJ0gSSzAEuBY4HFgKnJFnYUe1+4Ezg6nGa+S/Al6cqRkkaJCYbJEmSpIkdBayvqg1V9QxwDbCkvUJVbayqO2lNZ3ueJEcCrwE+Px3BSlK/mWyQJEmSJrY38EDb401N2YSSvAj478DvTFBvWZI1SdZs2bJlpwOVpEFgskGSJEmaWMYoqy6vfT+wqqoe2F6lqlpRVSNVNTJ//vwdDlCSBokLREqSJEkT2wTs2/Z4H2Bzl9f+EvDWJO8HXgbskuSJqjqnxzFK0sDoamRDksVJ7k2yPskLOsUkuya5tjl/a5IFTfmLk1yZZG2Se5Kc29vwB9OiRYtYtGhRv8OQJElS76wGDk5yQJJdaG09vLKbC6vqtKrar6oWAB8GPmWiQdKwmzDZ0OXKu2cBj1bVQcDFwEVN+UnArlV1GHAk8N7RRIQkSZI0U1TVVuBs4CZa21deV1XrklyQ5F0ASd6UZBOtz8AfT7KufxFLUn91M41i28q7AElGV969u63OEmB5c3w9cEmS0JrHtnuSucBLgGeAH/YmdEmSJGn6VNUqYFVH2fltx6tpTa/YXhufBD45BeFJ0kDpJtkw1sq7R49Xp6q2JnkMmEcr8bAE+D7wUuC3q+qRyQYtaWYZnVZ0yy239DUOSdPvljN373cIkiSpD7pZs6GblXfHq3MU8CzwOuAA4P9NcuALnsBtfiRJkiRJGhrdJBu6WXl3W51mysQewCPAqcDnquonVfUg8HVgpPMJ3OZHkiRJkqTh0c00im0r7wLfo7Xy7qkddVYCZwDfAE4Ebq6qSnI/8PYkn6Y1jeIXgT/tVfA7a8E5N05p+z/Y8PCUP8/GC39tytqWpJnOqTuSJEn9NeHIhm5W3gUuA+YlWQ98CBjdyudSWnsJ30UraXFFVd3Z43uQJEmSJEkDpJuRDd2svPsUrS1+Oq97YqxySZIkSZI0vLpZs0GSJEmSJKlrXY1skDTcXMdEkoab65hIkqabIxskaYAkWZzk3iTrk5wzxvldk1zbnL81yYKm/KgkdzQ/307y6922KUmSJPWaIxumwM+cemG/Q5A0AyWZQ2th3WNpbSm8OsnKqrq7rdpZwKNVdVCSpcBFwMm0FuIdqaqtSV4LfDvJ3wHVRZuSJElSTzmyQZIGx1HA+qraUFXPANcASzrqLAGubI6vB45Jkqr6UbN7EMButJIM3bYpSZIk9ZQjGyRpcOwNPND2eBNw9Hh1mlEMjwHzgIeSHA1cDuwP/GZzvps2AUiyDFgGsN9++03+brZn+R5T2/7GJ6fneZY/NrXtS5IkzVCObJCkwZExyqrbOlV1a1UdArwJODfJbl22SXP9iqoaqaqR+fPn70DYkiRJ0vM5skHSlHMdk65tAvZte7wPsHmcOpuSzAX2AB5pr1BV9yR5Eji0yzYlSZKknnJkgyQNjtXAwUkOSLILsBRY2VFnJXBGc3wicHNVVXPNXIAk+wOvBzZ22aYkSZLUUyYbJGlANAs8ng3cBNwDXFdV65JckORdTbXLgHlJ1gMfAka3snwLrR0o7gBuAN5fVQ+N1+b03ZUkDY8utid+W5JvJdma5MS28iOSfCPJuiR3Jjl5eiOXpOnnNApJGiBVtQpY1VF2ftvxU8BJY1x3FXBVt21KGjAumjrwutye+H7gTODDHZf/CDi9qv45yeuA25LcVFX/Og2hS1JfmGyQJEmSJrZtK2GAJKNbCW9LNlTVxubcc+0XVtX/13a8OcmDwHzAZIOkoWWyQZI0dG45c/d+hyBp+HS9lfD2JDkK2AX4zhjnpm8LYkmaYq7ZIEmSJE2s662Ex20geS2tKW//V1U913neLYglDROTDZIkSdLEJrWVcJJXADcCv19V/9jj2CRp4JhskCRJkia201sJN/VvAD5VVX81hTFK0sAw2SBJkiRNoJvtiZO8KckmWrsGfTzJ6FbDvwG8DTgzyR3NzxF9uA1JmjYuEClJkiR1oYvtiVfTml7Red2ngU9PeYCSNEAc2SBJkiRJknrKkQ2SJElDzu1gJUnTzZENkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkDZAki5Pcm2R9knPGOL9rkmub87cmWdCUH5vktiRrm3/f3nbNLU2bdzQ/e03fHUmSJGk2mtvvACRJLUnmAJcCxwKbgNVJVlbV3W3VzgIeraqDkiwFLgJOBh4CTqiqzUkOBW4C9m677rSqWjMtNyJJkqRZz5ENkjQ4jgLWV9WGqnoGuAZY0lFnCXBlc3w9cEySVNXtVbW5KV8H7JZk12mJWpIkSepgskGSBsfewANtjzfx/NEJz6tTVVuBx4B5HXXeA9xeVU+3lV3RTKE4L0l6G7YkSZL0fCYbJGlwjJUEqB2pk+QQWlMr3tt2/rSqOgx4a/Pzm2M+ebIsyZoka7Zs2bJDgUuSJEntuko27OyCZc25w5N8I8m6ZuGy3XoXviQNlU3Avm2P9wE2j1cnyVxgD+CR5vE+wA3A6VX1ndELqup7zb+PA1fTmq7xAlW1oqpGqmpk/vz5PbkhSZIkzU4TJhvaFiw7HlgInJJkYUe1bQuWARfT+lZt9IPwp4H3VdUhwCLgJz2LXpKGy2rg4CQHJNkFWAqs7KizEjijOT4RuLmqKsmewI3AuVX19dHKSeYmeXVz/GLgncBdU3wfkjSUuvgC7m1JvpVka5ITO86dkeSfm58zOq+VpGHTzciGnV6wDDgOuLOqvg1QVQ9X1bO9CV2ShkuzBsPZtHaSuAe4rqrWJbkgybuaapcB85KsBz4EjH7YPRs4CDivY4vLXYGbktwJ3AF8D/jL6bsrSRoOXX4Bdz9wJq1RZO3Xvgr4A+BoWp+t/yDJK6c6Zknqp262vhxrwbKjx6tTVVuTjC5Y9rNAJbkJmA9cU1V/3PkESZYBywD222+/Hb0HSRoaVbUKWNVRdn7b8VPASWNc91Hgo+M0e2QvY5SkWWrbF3AASUa/gNu2PXFVbWzOPddx7TuAL1TV6LS3LwCLgc9MfdiS1B/djGyYzIJlc4G3AKc1//56kmNeUNF5wpIkSRps3ewYNKlrXahX0jDpJtkwmQXLNgFfrqqHqupHtL6t+4XJBi1JkiRNs26+gJvUtX4BJ2mYdJNs2OkFy2jNOz48yUubJMQv0zbUTJIkSZohuvkCbiqulaQZacJkw2QWLKuqR4E/oZWwuAP4VlXd2PvbkCRJkqZUN1/Ajecm4Lgkr2wWhjyuKZOkodXNApE7vWBZc+7TtLa/lCRJkmakZhH00S/g5gCXj34BB6ypqpVJ3gTcALwSOCHJR6rqkKp6JMl/oZWwALhgdLFISRpWXSUbJEmSpNmuiy/gVtOaIjHWtZcDl09pgJI0QLpZs0GSJEmSJO2kRYsWsWjRon6HMa1MNkiSJEmSpJ4y2SBJkiRJknrKNRskSZIkSbPeYVceNmVtb/jBhil/DoC1Z6yd0vZ3hCMbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZRrNkiSJEnqudFt/m655Za+xiENggPPPbDfIUw7RzZIkiRJkqSecmSDJEmSNAtN9ar407H6/iCtvC/p+RzZIEmSJEmSespkgyQNkCSLk9ybZH2Sc8Y4v2uSa5vztyZZ0JQfm+S2JGubf9/eds2RTfn6JH+WJNN3R5IkSZqNnEYhSQMiyRzgUuBYYBOwOsnKqrq7rdpZwKNVdVCSpcBFwMnAQ8AJVbU5yaHATcDezTV/ASwD/hFYBSwGPjsd9yRJmr1m44J4kn7KkQ2SNDiOAtZX1Yaqega4BljSUWcJcGVzfD1wTJJU1e1VtbkpXwfs1oyCeC3wiqr6RlUV8Cng3VN/K5IkSZrNTDZI0uDYG3ig7fEmfjo64QV1qmor8Bgwr6POe4Dbq+rppv6mCdoEIMmyJGuSrNmyZctO34QkSZJkskGSBsdYaynUjtRJcgitqRXv3YE2W4VVK6pqpKpG5s+f30W4kjS7TGJdnRcnubJZP+eeJOdOd+ySNN1MNkjS4NgE7Nv2eB9g83h1kswF9gAeaR7vA9wAnF5V32mrv88EbUqSJtC2rs7xwELglCQLO6ptW1cHuJhW8hfgJGDXqjoMOBJ472giQpKGlckGSRocq4GDkxyQZBdgKbCyo85K4Izm+ETg5qqqJHsCNwLnVtXXRytX1feBx5P8YrMLxenA3071jUjSENrpdXVojSjbvUkSvwR4Bvjh9IQtSf1hskGSBkSzBsPZtHaSuAe4rqrWJbkgybuaapcB85KsBz4EjA7jPRs4CDgvyR3Nz17Nud8CPgGsB76DO1FI0s6YzLo61wNPAt8H7gf+W1U90vkErp0jaZi49aUkDZCqWkVre8r2svPbjp+iNRy387qPAh8dp801wKG9jVSSZp3JrKtzFPAs8DrglcBXk/xDVW14XsWqFcAKgJGRkTHX15GkmcKRDZIkSdLEJrOuzqnA56rqJ1X1IPB1YGTKI5akPjLZIEmSJE1sp9fVoTV14u1p2R34ReCfpiluSeoLkw2SJEnSBCa5rs6lwMuAu2glLa6oqjun9QYkaZq5ZoMkSZLUhUmsq/PEWOWSNMwc2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqqa6SDUkWJ7k3yfok54xxftck1zbnb02yoOP8fkmeSPLh3oQtSZIkSZIG1YTJhiRzgEuB44GFwClJFnZUOwt4tKoOAi4GLuo4fzHw2cmHK0mSJEmSBl03IxuOAtZX1Yaqega4BljSUWcJcGVzfD1wTJIAJHk3sAFY15uQJUmSJEnSIOsm2bA38EDb401N2Zh1qmor8BgwL8nuwO8BH5l8qJIkSZIkaSboJtmQMcqqyzofAS6uqie2+wTJsiRrkqzZsmVLFyFJkiRJkqRBNbeLOpuAfdse7wNsHqfOpiRzgT2AR4CjgROT/DGwJ/Bckqeq6pL2i6tqBbACYGRkpDORIUmSJEmSZpBuRjasBg5OckCSXYClwMqOOiuBM5rjE4Gbq+WtVbWgqhYAfwr8UWeiQZL0Uzu7+0+SeUm+1Oz8c0nHNbc0bd7R/Ow1PXcjSZKk2WrCkQ1VtTXJ2cBNwBzg8qpal+QCYE1VrQQuA65Ksp7WiIalUxm0JA2jtt1/jqU1Ymx1kpVVdXdbtW27/yRZSmv3n5OBp4DzgEObn06nVdWaKb0BSZIkqdHNNAqqahWwqqPs/Lbjp4CTJmhj+U7EJ0mzybbdfwCSjO7+055sWAIsb46vBy5Jkqp6EvhakoOmMV5JkiRpTN1Mo5AkTY+d3v2ni7avaKZQnDe6NXEnF+uVpO3b2aluzbnDk3wjyboka5PsNp2xS9J0M9kgSYNjMrv/bM9pVXUY8Nbm5zfHqlRVK6pqpKpG5s+fP2GwkjSbtE11Ox5YCJySZGFHtW1T3YCLaU11o1lA/dPA+6rqEGAR8JNpCl2S+sJkgyQNjh3Z/Wf0w+vo7j/jqqrvNf8+DlxNa7qGJGnHbJvqVlXPAKNT3dotAa5sjq8HjmlGkx0H3FlV3waoqoer6tlpiluS+sJkgyQNjp3e/We8BpPMTfLq5vjFwDuBu3oeuSQNv8lMdftZoJLclORbSX53rCdwOpukYdLVApGSpKk32d1/kmwEXgHskuTdtL5Juw+4qUk0zAH+AfjLabwtSRoWk5nqNhd4C/Am4EfAF5PcVlVffF7FqhXACoCRkZGJpshJ0kAz2SBJA2Qyu/9U1YJxmj2yV/FJ0iy2I1PdNnVMddsEfLmqHgJIsgr4BeCLSNKQchqFJEmSNLHJTHW7CTg8yUubJMQv8/xtjSVp6DiyQZIkSZrAZKa6VdWjSf6EVsKigFVVdWNfbkSSponJBkmSJKkLk5zq9mla219K0qzgNApJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskaYAkWZzk3iTrk5wzxvldk1zbnL81yYKmfF6SLyV5IsklHdccmWRtc82fJcn03I0kSZJmK5MNkjQgkswBLgWOBxYCpyRZ2FHtLODRqjoIuBi4qCl/CjgP+PAYTf8FsAw4uPlZ3PvoJUmSpJ8y2SBJg+MoYH1VbaiqZ4BrgCUddZYAVzbH1wPHJElVPVlVX6OVdNgmyWuBV1TVN6qqgE8B757Su5CkIbWzo8/azu/XjEAbKzEsSUPFZIMkDY69gQfaHm9qysasU1VbgceAeRO0uWmCNgFIsizJmiRrtmzZsoOhS9Jwm+Tos1EXA5+d6lglaRCYbJCkwTHWWgq1E3V2qn5VraiqkaoamT9//naalKRZaadHnwEkeTewAVg3TfFKUl+ZbJCkwbEJ2Lft8T7A5vHqJJkL7AE8MkGb+0zQpiRpYjs9+izJ7sDvAR+ZhjglaSCYbJCkwbEaODjJAUl2AZYCKzvqrATOaI5PBG5u1mIYU1V9H3g8yS82366dDvxt70OXpKE3mdFnHwEurqontvsETmeTNETm9jsASVJLVW1NcjZwEzAHuLyq1iW5AFhTVSuBy4CrkqynNaJh6ej1STYCrwB2aYbrHldVdwO/BXwSeAmtucLOF5akHbcjo882dYw+Oxo4MckfA3sCzyV5qqqet1VxVa0AVgCMjIxsb4qcJA08kw2SNECqahWwqqPs/Lbjp4CTxrl2wTjla4BDexelJM1K20afAd+jlew9taPO6Oizb/D80WdvHa2QZDnwRGeiQZKGjckGSZIkaQKTHX0mSbNNV8mGJIuBj9HqWD9RVRd2nN+V1t7tRwIPAydX1cYkxwIXArsAzwC/U1U39zB+SZIkaVpMZvRZW53lUxKcJA2YCReInOSewg8BJ1TVYbSGlF3Vq8AlSZIkSdJg6mY3ip3eU7iqbq+q0YVz1gG7NaMgJEmSJEnSkOom2bDTewp31HkPcHtVPd35BG7zI0mSJEnS8Ogm2TCZPYVbJ5NDaE2teO9YT1BVK6pqpKpG5s+f30VIkiRJkiRpUHWTbNiRPYXp2FOYJPsANwCnV9V3JhuwJEmSJEkabN0kG7btKZxkF1pb+KzsqDO6pzC07SmcZE/gRuDcqvp6r4KWJEmSJEmDa8JkQ7MGw+iewvcA143uKZzkXU21y4B5zZ7CHwLOacrPBg4CzktyR/OzV8/vQpIkSZIkDYy53VTa2T2Fq+qjwEcnGaMkSZIkSZpBuplGIUmSJEmS1DWTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZI0gBJsjjJvUnWJzlnjPO7Jrm2OX9rkgVt585tyu9N8o628o1J1ia5I8ma6bkTSZIkzWYmGyRpQCSZA1wKHA8sBE5JsrCj2lnAo1V1EHAxcFFz7UJgKXAIsBj486a9Ub9SVUdU1cgU34YkDa2dTQgnOTbJbU3i97Ykb5/u2CVpuplskKTBcRSwvqo2VNUzwDXAko46S4Arm+PrgWOSpCm/pqqerqrvAuub9iRJPTCZhDDwEHBCVR0GnAFcNT1RS1L/mGyQpMGxN/BA2+NNTdmYdapqK/AYMG+Cawv4fPNt2rIpiFuSZoOdTghX1e1VtbkpXwfslmTXaYlakvrEZIMkDY6MUVZd1tnetW+uql+g9W3cB5K8bcwnT5YlWZNkzZYtW7qNWZJmi8kkhNu9B7i9qp6eojglaSCYbJCkwbEJ2Lft8T7A5vHqJJkL7AE8sr1rR79Nq6oHgRsYZ3pFVa2oqpGqGpk/f/6kb0aShsxkEsKtk8khtKZWvHfMJzDpK2mImGyQpMGxGjg4yQFJdqG14OPKjjorac33BTgRuLmqqilf2ixOdgBwMPDNJLsneTlAkt2B44C7puFeJGnYTCYhTJJ9aCV8T6+q74z1BCZ9JQ2Tuf0OQJLUUlVbk5wN3ATMAS6vqnVJLgDWVNVK4DLgqiTraX2AXdpcuy7JdcAg4wkmAAAKIElEQVTdwFbgA1X1bJLXADe01pBkLnB1VX1u2m9Okma+bQlh4Hu0+t9TO+qMJoS/QVtCOMmewI3AuVX19WmMWZL6xmSDJA2QqloFrOooO7/t+CngpHGu/UPgDzvKNgBv6H2kkjS7TCYhDJwNHAScl+S8puy4ZnqbJA0lkw2SJElSF3Y2IVxVHwU+OuUBStIAcc0GSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPdVVsiHJ4iT3Jlmf5Jwxzu+a5Nrm/K1JFrSdO7cpvzfJO3oXuiQNn6nobydqU5LUHT8TS1L3Jkw2JJkDXAocDywETkmysKPaWcCjVXUQcDFwUXPtQmApcAiwGPjzpj1JUoep6G+7bFOSNAE/E0vSjulmZMNRwPqq2lBVzwDXAEs66iwBrmyOrweOSZKm/Jqqerqqvgusb9qTJL3QVPS33bQpSZqYn4klaQd0k2zYG3ig7fGmpmzMOlW1FXgMmNfltZKklqnob+2HJak3/EwsSTtgbhd1MkZZdVmnm2tJsgxY1jx8Ism9XcQ16F4NPDRVjeeiqWp56Ezp+wC+FztgGP4m9p/i9qeivx0rqfyCfhiGsi+e8r9/PjLWy64xTO174fvQrWH4m5jqfnh7/Ey8c6b2v/9n+vffpan/TOx70Y1heR+66ou7STZsAvZte7wPsHmcOpuSzAX2AB7p8lqqagWwopuAZ4oka6pqpN9xzHa+D4PD96IrU9XfTtgPw/D1xf7ODQ7fi8Hg+zBpfibeCf7eDQbfh8Ew296HbqZRrAYOTnJAkl1oLW6zsqPOSuCM5vhE4OaqqqZ8abMy7wHAwcA3exO6JA2dqehvu2lTkjQxPxNL0g6YcGRDVW1NcjZwEzAHuLyq1iW5AFhTVSuBy4Crkqynlb1d2ly7Lsl1wN3AVuADVfXsFN2LJM1oU9XfjtXmdN+bJM10fiaWpB2TVrJVvZZkWTMUTn3k+zA4fC803fydGxy+F4PB90H94O/dYPB9GAyz7X0w2SBJkiRJknqqmzUbJEmSJEmSumayAUjybJI7ktyV5K+SvLRH7b4myd8n+XaSu5Osasq/m+T1HXX/NMnvJlmU5LEktye5N8lXkryzF/EMGl/34ZDk/3Q8/u0kTyXZo61sUZK/H+Padzav+eh79d7piFmDx/6gP3zdh4P9sHrFPqE/fN2Hg33xC5lsaPlxVR1RVYcCzwDv6/bCtLY1Gs8FwBeq6g1VtRA4pym/hmbBoKaNF9FasfjapuirVfXGqno98EHgkiTHdH87M4av+xCoqn/bUXQKrRW7f3171yV5Ma3tvU6oqjcAbwRumYoYNSPYH/SHr/sQsB9WD9kn9Iev+xCwL34hkw0v9FXgoCQLktw1Wpjkw0mWN8e3JPmjJF8G/mOS+Un+Osnq5ufNzWWvpbWvMgBVdWdz+Bna/sCBtwEbq+q+zmCq6g5aHcXZPbzHQTRjXvckc5J8ssk+r03y25O68xkuyRNtx/8GeBnw+7Q62O15Oa0dcR4GqKqnq+reqYpTM8qM6Q+GzIx53e2Hn89+WFNkxvQJQ2bGvO72xc9nX/xCJhvaNJnB44G1XVTfs6p+uar+O/Ax4OKqehPwHuATTZ1LgcuSfCnJf07yOtj2h/5ckjc09ZbS+qMfz7eAn9vxO5oZZuDrfgSwd1UdWlWHAVd0EfdscQqt1/SrwOuT7DVexap6hNa+4/cl+UyS05rMumaxGdgfDIUZ+LrbD4/PfliTNgP7hKEwA193++Lx2RdjsmHUS5LcAawB7qe1R/JErm07/lVaw4vuoPWL8ookL6+qm4ADgb+k9Qd6e5L5zTWfAZY2ncoS4K+281zZobuZOWbq674BODDJ/0iyGPhhF3HPFkuBa6rqOeBvgJO2V7mq/h/gGOCbwIeBy6c8Qg2qmdofzHQz9XW3Hx6f/bAmY6b2CTPdTH3d7YvHZ19Ma7iGmnlS7QVJtvL8ZMxuHdc82Xb8IuCXqurHnQ03maqrgavTWgzkbcBf0/oD/zzwZeDOqnpwO/G9Ebiny3uZSWbE655kDnBbU7ayqs5vMsHvAD4A/Abwf2/3TmeBJIcDBwNfSAKwC63/CF26veuqai2wNslVwHeBM6c2Ug2oGdEfdHkvM8mMeN3th7tjP6wemBF9Qpf3MpPMiNfdvrg79sU/5ciG8f0LsFeSeUl2Bba3CuvnaZvHlOSI5t+3p1lNNsnLgX9DK1tJVX2H1rycC9nOsKXml/U8JvjlHCID97pX1bPNoj1HNJ3qq4EXVdVfN3V+Yedvd6icAiyvqgXNz+uAvZPsP1blJC9Lsqit6AjgBXMFNasNXH8wqbuZOQbudbcf7pr9sKbCwPUJk7qbmWPgXnf74q7ZFzcc2TCOqvpJkguAW2lllv5pO9U/CFya5E5ar+lXaK0ieyStIU2jmclPVNXqtus+A/xX4IaO9t6a5HbgpcCDwAer6os9uK2BN0Ne972BK9rmUp27I/c4xJbSmmfY7oam/FbgmCSb2s6dAvxuko8DP6aVoT9zGuLUDDFD+oOhM0Ned/vhsdkPq+dmSJ8wdGbI625fPDb74kaqqt8xSJIkSZKkIeI0CkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJEmSJPWUyQZJkiRJktRTJhs0qyTZ2OwJPKk6kqSdZ18sSf1lP6zpYLJBkiRJkiT1lMkGDbwkC5L8U5JPJLkryf9K8qtJvp7kn5McleRVSf53kjuT/GOSw5tr5yX5fJLbk3wcSFu7/z7JN5PckeTjSeb07SYlacDZF0tSf9kPa6Yx2aCZ4iDgY8DhwM8BpwJvAT4M/CfgI8DtVXV48/hTzXV/AHytqt4IrAT2A0jy88DJwJur6gjgWeC0absbSZqZ7Islqb/shzVjzO13AFKXvltVawGSrAO+WFWVZC2wANgfeA9AVd3cZG/3AN4G/Lum/MYkjzbtHQMcCaxOAvAS4MFpvB9JmonsiyWpv+yHNWOYbNBM8XTb8XNtj5+j9Xu8dYxrquPfdgGurKpzexahJA0/+2JJ6i/7Yc0YTqPQsPgKzZCvJIuAh6rqhx3lxwOvbOp/ETgxyV7NuVcl2X+6g5akIWNfLEn9ZT+sgeHIBg2L5cAVSe4EfgSc0ZR/BPhMkm8BXwbuB6iqu5P8PvD5JC8CfgJ8ALhvugOXpCGyHPtiSeqn5dgPa0CkaqzRNJIkSZIkSTvHaRSSJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6qn/H/5Eue+p2MGUAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X203WV95/33x0RAUUEx+MBTYKC24UEsR2jHh6ZSEFbV2BFKgClw39wrWmWc1d62hZlCI2M70DVTakfaMRUQcRAoLdO0RNGK+DQWEwQJgXI3xgAxWoJQFBQw8L3/2L8TN5tzcnZy9j57n33er7XOym9fv+t37e9v73MuNt99PaSqkCRJkiRJ6pXnDToASZIkSZI0Wkw2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2aCRkWRjkl8ZdBySJEmSNNeZbJAkSZIkST1lskFzTpL5g45Bkuayifph+2ZJmllJ5g06Bo02kw0aNUcmuTPJo0muTbJbksVJNiX5vSTfA64YdJCSNAqa6Wu/0/S7jye5LMkrknw6yQ+T/EOSlyZZmKSSnJ3kfuDmicoGfT+SNNOS7Jfkb5JsSfL9JB9J8rwkv5/kviQPJvlEkj2a+p9Jck5HG99M8u+a459N8rkkDye5N8mvt9X7eJK/SLIqyePALyf51SS3J/lBkgeSLO9o+4wmju8nOb992nIT57lJvtWcvy7Jy/r9mmn2MNmgUfPrwAnAgcARwFlN+SuBlwEHAMsGEpkkjaZ3AccBPwO8Hfg08J+Al9P6nPH+trq/BPwc8NYpyiRp5DUjC/4euA9YCOwDXEPr8+tZwC8DBwEvAj7SXHY1cGpbG4tofb69McnuwOeaOns39f48yaFtT3sa8IfAi4GvAI8DZwB7Ar8K/GaSd7a1/efA6cCrgD2aGMe9H3gnrX781cAjwKXTeEk0Ykw2aNT8WVVtrqqHgb8DjmzKnwH+oKqerKofDy48SRo5/6Oq/qWqvgN8Gbi1qm6vqieBG4DXtdVdXlWPd/TDE5VJ0lxwNK3/Sf+dph98oqq+Qut/7v+kqjZU1WPAecDSZrrZDbRG8h7QtHE68DdNn/s2YGNVXVFVW6vqG8BfAye1PeffVtVXq+qZ5vluqaq1zeM7gU/RSh7QXPd3VfWVqnoKuACotrbeDfznqtrUPP9y4CSnxWmcyQaNmu+1Hf+IViYYYEtVPTGAeCRp1P1L2/GPJ3j8orbHD0xw/URlkjQX7AfcV1VbO8pfTWu0w7j7gPnAK6rqh8CNwNLm3FLgfzXHBwDHJPnX8R9ayYhXtrX1rD43yTFJvtBM43gUeA+tkWnjcWyrX1U/Ar7fdvkBwA1tz3UP8DTwiq5fAY00kw2aK2rqKpKkPpuoL7Z/ljRXPQDsP8FIgM20/kd+3P7AVn6azP0UcGqSXwReAHyhrb0vVtWebT8vqqrfbGurs8+9GlgJ7FdVewD/E0hz7rvAvuMVk7wA2Ksj/hM7nm+3ZqSbZLJBkiRJkgbg67T+h/6iJLs3C5u/gVYy4beSHJjkRcAfAde2jYBYRSsZcWFT/kxT/vfAzyT5jSTPb35en+TnthPDi4GHq+qJJEfTWtNh3PXA25P82yS7AB/kp4kIaCUm/nB8SkeSBUmWTOP10Igx2SBJkiRJM6yqnqa1sO7BwP3AJuAU4HLgKuBLwLeBJ4D/0Hbdk8DfAL9Ca2TCePkPgeNpTa3YTGt68cXArtsJ473AhUl+SGtNhuva2lvXPO81tJIiPwQeBJ5sqnyY1qiIzzbX/yNwzA6/EBpZqXL0oiRJkiRpcs0oi38FDqmqbw86Hg0/RzZIkiRJkp4jyduTvLDZVvO/AWuBjYONSrOFyQZJkiRJ0kSW0JqSsRk4BFhaDo1Xl5xGIUmSJEmSesqRDZIkSZIkqadMNkiSJEmSpJ6aP+gAOr385S+vhQsXDjoMSXqO22677aGqWjDoOGaCfbGkYWQ/LEmD121fPHTJhoULF7JmzZpBhyFJz5HkvkHHMFPsiyUNI/thSRq8bvtip1FIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6imTDRpZixcvZvHixYMOQ5LmNPtiSRos+2ENiskGSRoiSU5Icm+S9UnOneD8rkmubc7fmmRhU350kjuan28m+bW2azYmWducWzNzdyNJkqS5av6gA5AktSSZB1wKHAdsAlYnWVlVd7dVOxt4pKoOTrIUuBg4BbgLGKuqrUleBXwzyd9V1dbmul+uqodm7m4kSZI0l5lskKThcTSwvqo2ACS5BlgCtCcblgDLm+PrgY8kSVX9qK3ObkD1P1xJkjRdh195eF/b3/C9DTPyPGvPXNvX9jX7OI1CkobHPsADbY83NWUT1mlGLTwK7AWQ5Jgk64C1wHvaRjUU8NkktyVZNtmTJ1mWZE2SNVu2bOnJDUmSJGlucmSDBmbhuTf2tf3vbfj+jDzPxot+ta/ta07JBGWdIxQmrVNVtwKHJvk54Mokn66qJ4A3VNXmJHsDn0vyT1X1pec0UrUCWAEwNjbmyAhJkiTtNEc2SNLw2ATs1/Z4X2DzZHWSzAf2AB5ur1BV9wCPA4c1jzc3/z4I3EBruoYkSZLUNyYbJGl4rAYOSXJgkl2ApcDKjjorgTOb45OAm6uqmmvmAyQ5AHgNsDHJ7kle3JTvDhxPazFJSZI0Bxx03kEcdN5Bgw5Dc5DTKCRpSDQ7SZwD3ATMAy6vqnVJLgTWVNVK4DLgqiTraY1oWNpc/kbg3CQ/AZ4B3ltVDyU5CLghCbT6/Kur6jMze2eSJEmaa0w2aGS98rSLBh2CtMOqahWwqqPsgrbjJ4CTJ7juKuCqCco3AK/tfaQaGcv36G/7Gx/v//Msf7R/bUttkpwAfJhWQvhjVXVRx/k3A38KHAEsrarrO86/BLgHuKGqzpmZqCVpMJxGIUmSJE0hyTzgUuBEYBFwapJFHdXuB84Crp6kmf8CfLFfMUrSMDHZIEmSJE3taGB9VW2oqqeAa4Al7RWqamNV3UlrOtuzJDkKeAXw2ZkIVpIGzWSDJEmSNLV9gAfaHm9qyqaU5HnAfwd+Z4p6y5KsSbJmy5YtOx2oJA0Dkw2SJEnS1DJBWXV57XuBVVX1wPYqVdWKqhqrqrEFCxbscICSNExcIFKSJEma2iZgv7bH+wKbu7z2F4E3JXkv8CJglySPVdW5PY5RkoZGVyMbkpyQ5N4k65M8p1NMsmuSa5vztyZZ2JQ/P8mVSdYmuSfJeb0NfzgtXryYxYsXDzoMSZIk9c5q4JAkBybZhdbWwyu7ubCqTq+q/atqIfAB4BMmGiSNuimTDV2uvHs28EhVHQxcAlzclJ8M7FpVhwNHAe8eT0RIkiRJs0VVbQXOAW6itX3ldVW1LsmFSd4BkOT1STbR+gz80STrBhexJA1WN9Motq28C5BkfOXdu9vqLAGWN8fXAx9JElrz2HZPMh94AfAU8IPehC5pthgf6XPLLbcMNA5JkqajqlYBqzrKLmg7Xk1resX22vg48PE+hCdJQ6WbZMNEK+8eM1mdqtqa5FFgL1qJhyXAd4EXAr9VVQ9PN2hJkjQ73HLW7oMOQZIkDUA3azZ0s/LuZHWOBp4GXg0cCPy/SQ56zhO4zY8kSZIkSSOjm2RDNyvvbqvTTJnYA3gYOA34TFX9pKoeBL4KjHU+gdv8SJIkSZI0OrqZRrFt5V3gO7RW3j2to85K4Ezga8BJwM1VVUnuB96S5JO0plH8AvCnvQp+Zy0898a+tv+9Dd/v+/NsvOhX+9a2JM12rhMiSZI0WFMmG5o1GMZX3p0HXD6+8i6wpqpWApcBVyVZT2tEw9Lm8kuBK4C7aE21uKKq7uzDfUiaBhNwkjTaTMBJkmZaNyMbull59wlaW/x0XvfYROWSJEmSJGl0dbNmgyRJkiRJUtdMNkiSJEmSpJ4y2SCp71552kW88rSLBh3GrJDkhCT3Jlmf5NwJzu+a5Nrm/K1JFjblRye5o/n5ZpJf67ZNSZIkqde6WrNBO8b/qZK0M5LMo7Ww7nG0thRenWRlVd3dVu1s4JGqOjjJUuBi4BRaC/GONYv6vgr4ZpK/A6qLNiVJkqSecmSDJA2Po4H1VbWhqp4CrgGWdNRZAlzZHF8PHJskVfWjqtralO9GK8nQbZuSJElSTzmyQZKGxz7AA22PNwHHTFanGcXwKLAX8FCSY4DLgQOA32jOd9MmAEmWAcsA9t9//+nfzfYs36O/7W98fGaeZ/mj/W1fkiRplnJkgyQNj0xQVt3Wqapbq+pQ4PXAeUl267JNmutXVNVYVY0tWLBgB8KWJEmSns2RDZI0PDYB+7U93hfYPEmdTUnmA3sAD7dXqKp7kjwOHNZlm5IGzdE+kqQR48gGSRoeq4FDkhyYZBdgKbCyo85K4Mzm+CTg5qqq5pr5AEkOAF4DbOyyTUmSJKmnTDZI0pBoFng8B7gJuAe4rqrWJbkwyTuaapcBeyVZD/w2ML6V5Rtp7UBxB3AD8N6qemiyNmfuriRpdHSxPfGbk3wjydYkJ7WVH5nka0nWJbkzySkzG7kkzTynUUjSEKmqVcCqjrIL2o6fAE6e4LqrgKu6bVOStGO63J74fuAs4AMdl/8IOKOq/jnJq4HbktxUVf86A6FL0kCYbJAkSZKmtm0rYYAk41sJb0s2VNXG5twz7RdW1f/Xdrw5yYPAAsBkg6SRZbJBkjRybjlr90GHIGn0dL2V8PYkORrYBfjWBOdmbgtiSeoz12yQJEmSptb1VsKTNpC8itaUt/+rqp7pPO8WxJJGickGSZIkaWrT2ko4yUuAG4Hfr6p/7HFskjR0nEYhSZI04pxa1BPbthIGvkNrK+HTurmw2Xr4BuATVfVX/QtRkoaHIxskSZKkKXSzPXGS1yfZRGvXoI8mGd9q+NeBNwNnJbmj+TlyALchSTPGkQ2SJElSF7rYnng1rekVndd9Evhk3wOUpCHiyAZJkiRJktRTJhskSZIkSVJPmWyQJEmSJEk9ZbJBkiRJkiT1lMkGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRpiCQ5Icm9SdYnOXeC87smubY5f2uShU35cUluS7K2+fctbdfc0rR5R/Oz98zdkSRJkuai+YMOQJLUkmQecClwHLAJWJ1kZVXd3VbtbOCRqjo4yVLgYuAU4CHg7VW1OclhwE3APm3XnV5Va2bkRiRJkjTnObJBkobH0cD6qtpQVU8B1wBLOuosAa5sjq8Hjk2Sqrq9qjY35euA3ZLsOiNRS5IkSR1MNkjS8NgHeKDt8SaePTrhWXWqaivwKLBXR513AbdX1ZNtZVc0UyjOT5Lehi1JkiQ9m8kGSRoeEyUBakfqJDmU1tSKd7edP72qDgfe1Pz8xoRPnixLsibJmi1btuxQ4JIkSVK7rpINO7tgWXPuiCRfS7KuWbhst96FL0kjZROwX9vjfYHNk9VJMh/YA3i4ebwvcANwRlV9a/yCqvpO8+8PgatpTdd4jqpaUVVjVTW2YMGCntyQJEmS5qYpkw1tC5adCCwCTk2yqKPatgXLgEtofas2/kH4k8B7qupQYDHwk55FL0mjZTVwSJIDk+wCLAVWdtRZCZzZHJ8E3FxVlWRP4EbgvKr66njlJPOTvLw5fj7wNuCuPt+HJI2kLr6Ae3OSbyTZmuSkjnNnJvnn5ufMzmsladR0M7JhpxcsA44H7qyqbwJU1fer6unehC5Jo6VZg+EcWjtJ3ANcV1XrklyY5B1NtcuAvZKsB34bGP+wew5wMHB+xxaXuwI3JbkTuAP4DvCXM3dXkjQauvwC7n7gLFqjyNqvfRnwB8AxtD5b/0GSl/Y7ZkkapG62vpxowbJjJqtTVVuTjC9Y9jNAJbkJWABcU1V/3PkESZYBywD233//Hb0HSRoZVbUKWNVRdkHb8RPAyRNc9yHgQ5M0e1QvY5SkOWrbF3AASca/gNu2PXFVbWzOPdNx7VuBz1XV+LS3zwEnAJ/qf9iSNBjdjGyYzoJl84E3Aqc3//5akmOfU9F5wpIkSRpu3ewYNK1rXahX0ijpJtkwnQXLNgFfrKqHqupHtL6t+/npBi1JkiTNsG6+gJvWtX4BJ2mUdJNs2OkFy2jNOz4iyQubJMQv0TbUTJIkSZoluvkCrh/XStKsNGWyYToLllXVI8Cf0EpY3AF8o6pu7P1tSJIkSX3VzRdwk7kJOD7JS5uFIY9vyiTNEYsXL2bx4sWDDmNGdbNA5E4vWNac+ySt7S8lSZKkWalZBH38C7h5wOXjX8ABa6pqZZLXAzcALwXenuSDVXVoVT2c5L/QSlgAXDi+WKQkjaqukg2SJEnSXNfFF3CraU2RmOjay4HL+xqgJA0Rkw2SJEmSpDnv8CsP71vbG763oe/PAbD2zLV9bX9HdLNApCRJkiRJUtdMNkiSJEmSpJ5yGoUkSZIkSX100HkHDTqEGefIBkmSJEmS1FMmGyRJkiRJUk+ZbJAkSZIkST1lskGSJElSzy1evJjFixcPOgxJA2KyQZIkSZIk9ZS7UUiSJElz0OFXHt7X9jd8b0Pfn2ftmWv71rak6XFkgyRJkiRJ6imTDZI0RJKckOTeJOuTnDvB+V2TXNucvzXJwqb8uCS3JVnb/PuWtmuOasrXJ/mzJJm5O5IkSdJc5DQKSRoSSeYBlwLHAZuA1UlWVtXdbdXOBh6pqoOTLAUuBk4BHgLeXlWbkxwG3ATs01zzF8Ay4B+BVcAJwKdn4p4kSXPXQecdNOgQJA2QIxskaXgcDayvqg1V9RRwDbCko84S4Mrm+Hrg2CSpqturanNTvg7YrRkF8SrgJVX1taoq4BPAO/t/K5IkSZrLTDZI0vDYB3ig7fEmfjo64Tl1qmor8CiwV0eddwG3V9WTTf1NU7QJQJJlSdYkWbNly5advglJkiTJZIMkDY+J1lKoHamT5FBaUyvevQNttgqrVlTVWFWNLViwoItwJWlumca6Os9PcmWzfs49Sc6b6dglaaaZbJCk4bEJ2K/t8b7A5snqJJkP7AE83DzeF7gBOKOqvtVWf98p2pQkTaFtXZ0TgUXAqUkWdVTbtq4OcAmt5C/AycCuVXU4cBTw7vFEhCSNKpMNkjQ8VgOHJDkwyS7AUmBlR52VwJnN8UnAzVVVSfYEbgTOq6qvjleuqu8CP0zyC80uFGcAf9vvG5GkEbTT6+rQGlG2e5MkfgHwFPCDmQlbkgbDZIMkDYlmDYZzaO0kcQ9wXVWtS3Jhknc01S4D9kqyHvhtYHwY7znAwcD5Se5ofvZuzv0m8DFgPfAt3IlCknbGdNbVuR54HPgucD/w36rq4c4ncO0cSaPErS8laYhU1Spa21O2l13QdvwEreG4ndd9CPjQJG2uAQ7rbaSSNOdMZ12do4GngVcDLwW+nOQfqmrDsypWrQBWAIyNjU24vo4kzRaObJAkSZKmNp11dU4DPlNVP6mqB4GvAmN9j1iSBshkgyRJkjS1nV5Xh9bUibekZXfgF4B/mqG4JWkgTDZIkiRJU5jmujqXAi8C7qKVtLiiqu6c0RuQpBnmmg2SJElSF6axrs5jE5VL0ihzZIMkSZIkSeopkw2SJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkiRJkiSpp7pKNiQ5Icm9SdYnOXeC87smubY5f2uShR3n90/yWJIP9CZsSZIkSZI0rKZMNiSZB1wKnAgsAk5Nsqij2tnAI1V1MHAJcHHH+UuAT08/XEmSJEmSNOy6GdlwNLC+qjZU1VPANcCSjjpLgCub4+uBY5MEIMk7gQ3Aut6ELEmSJEmShlk3yYZ9gAfaHm9qyiasU1VbgUeBvZLsDvwe8MHphypJkiRJkmaDbpINmaCsuqzzQeCSqnpsu0+QLEuyJsmaLVu2dBGSJEmSJEkaVvO7qLMJ2K/t8b7A5knqbEoyH9gDeBg4BjgpyR8DewLPJHmiqj7SfnFVrQBWAIyNjXUmMiRJkiRJ0izSzciG1cAhSQ5MsguwFFjZUWclcGZzfBJwc7W8qaoWVtVC4E+BP+pMNEiSfmpnd/9JsleSLzQ7/3yk45pbmjbvaH72npm7kSRJ0lw15ciGqtqa5BzgJmAecHlVrUtyIbCmqlYClwFXJVlPa0TD0n4GLUmjqG33n+NojRhbnWRlVd3dVm3b7j9JltLa/ecU4AngfOCw5qfT6VW1pq83IEmSJDW6mUZBVa0CVnWUXdB2/ARw8hRtLN+J+CRpLtm2+w9AkvHdf9qTDUuA5c3x9cBHkqSqHge+kuTgGYxXkiRJmlA30ygkSTNjp3f/6aLtK5opFOePb03cycV6JWn7dnaqW3PuiCRfS7Iuydoku81k7JI000w2SNLwmM7uP9tzelUdDryp+fmNiSpV1YqqGquqsQULFkwZrCTNJW1T3U4EFgGnJlnUUW3bVDfgElpT3WgWUP8k8J6qOhRYDPxkhkKXpIEw2SBJw2NHdv8Z//A6vvvPpKrqO82/PwSupjVdQ5K0Y7ZNdauqp4DxqW7tlgBXNsfXA8c2o8mOB+6sqm8CVNX3q+rpGYpbkgbCZIMkDY+d3v1nsgaTzE/y8ub4+cDbgLt6Hrkkjb7pTHX7GaCS3JTkG0l+d6IncDqbpFHS1QKRkqT+m+7uP0k2Ai8BdknyTlrfpN0H3NQkGuYB/wD85QzeliSNiulMdZsPvBF4PfAj4PNJbquqzz+rYtUKYAXA2NjYVFPkJGmomWyQpCEynd1/qmrhJM0e1av4JGkO25Gpbps6prptAr5YVQ8BJFkF/DzweSRpRDmNQpIkSZradKa63QQckeSFTRLil3j2tsaSNHIc2SBJkiRNYTpT3arqkSR/QithUcCqqrpxIDciSTPEZIMkSZLUhWlOdfskre0vJWlOcBqFJEmSJEnqKZMNkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkjREkpyQ5N4k65OcO8H5XZNc25y/NcnCpnyvJF9I8liSj3Rcc1SStc01f5YkM3M3kiRJmqtMNkjSkEgyD7gUOBFYBJyaZFFHtbOBR6rqYOAS4OKm/AngfOADEzT9F8Ay4JDm54TeRy9JkiT9lMkGSRoeRwPrq2pDVT0FXAMs6aizBLiyOb4eODZJqurxqvoKraTDNkleBbykqr5WVQV8AnhnX+9CkkbUzo4+azu/fzMCbaLEsCSNFJMNkjQ89gEeaHu8qSmbsE5VbQUeBfaaos1NU7QJQJJlSdYkWbNly5YdDF2SRts0R5+NuwT4dL9jlaRhYLJBkobHRGsp1E7U2an6VbWiqsaqamzBggXbaVKS5qSdHn0GkOSdwAZg3QzFK0kDZbJBkobHJmC/tsf7Apsnq5NkPrAH8PAUbe47RZuSpKnt9OizJLsDvwd8cAbilKShYLJBkobHauCQJAcm2QVYCqzsqLMSOLM5Pgm4uVmLYUJV9V3gh0l+ofl27Qzgb3sfuiSNvOmMPvsgcElVPbbdJ3A6m6QRMn/QAUiSWqpqa5JzgJuAecDlVbUuyYXAmqpaCVwGXJVkPa0RDUvHr0+yEXgJsEszXPf4qrob+E3g48ALaM0Vdr6wJO24HRl9tqlj9NkxwElJ/hjYE3gmyRNV9aytiqtqBbACYGxsbHtT5CRp6JlskKQhUlWrgFUdZRe0HT8BnDzJtQsnKV8DHNa7KCVpTto2+gz4Dq1k72kddcZHn32NZ48+e9N4hSTLgcc6Ew2SNGpMNkiSJElTmO7oM0maa7pKNiQ5AfgwrY71Y1V1Ucf5XWnt3X4U8H3glKramOQ44CJgF+Ap4Heq6uYexi9JkiTNiOmMPmurs7wvwUnSkJlygchp7in8EPD2qjqc1pCyq3oVuCRJkiRJGk7d7Eax03sKV9XtVTW+cM46YLdmFIQkSZIkSRpR3SQbdnpP4Y467wJur6onO5/AbX4kSZIkSRod3SQbprOncOtkciitqRXvnugJqmpFVY1V1diCBQu6CEmSJEmSJA2rbpINO7KnMB17CpNkX+AG4Iyq+tZ0A5YkSZIkScOtm2TDtj2Fk+xCawuflR11xvcUhrY9hZPsCdwInFdVX+1V0JIkSZIkaXhNmWxo1mAY31P4HuC68T2Fk7yjqXYZsFezp/BvA+c25ecABwPnJ7mj+dm753chSZIkSZKGxvxuKu3snsJV9SHgQ9OMUZIkSZIkzSLdTKOQJEmSJEnqmskGSZIkSZLUUyYbJEmSJElST5lskCRJkiRJPWWyQZIkSZIk9ZTJBkmSJEmS1FMmGyRpiCQ5Icm9SdYnOXeC87smubY5f2uShW3nzmvK703y1rbyjUnWJrkjyZqZuRNJkiTNZSYbJGlIJJkHXAqcCCwCTk2yqKPa2cAjVXUwcAlwcXPtImApcChwAvDnTXvjfrmqjqyqsT7fhiSNrJ1NCCc5LsltTeL3tiRvmenYJWmmmWyQpOFxNLC+qjZU1VPANcCSjjpLgCub4+uBY5OkKb+mqp6sqm8D65v2JEk9MJ2EMPAQ8PaqOhw4E7hqZqKWpMEx2SBJw2Mf4IG2x5uasgnrVNVW4FFgrymuLeCzzbdpy/oQtyTNBTudEK6q26tqc1O+Dtgtya4zErUkDYjJBkkaHpmgrLqss71r31BVP0/r27j3JXnzhE+eLEuyJsmaLVu2dBuzJM0V00kIt3sXcHtVPdmnOCVpKJhskKThsQnYr+3xvsDmyeokmQ/sATy8vWvHv02rqgeBG5hkekVVraiqsaoaW7BgwbRvRpJGzHQSwq2TyaG0pla8e8InMOkraYSYbJCk4bEaOCTJgUl2obXg48qOOitpzfcFOAm4uaqqKV/aLE52IHAI8PUkuyd5MUCS3YHjgbtm4F4kadRMJyFMkn1pJXzPqKpvTfQEJn0ljZL5gw5AktRiHqgJAAAKRElEQVRSVVuTnAPcBMwDLq+qdUkuBNZU1UrgMuCqJOtpfYBd2ly7Lsl1wN3AVuB9VfV0klcAN7TWkGQ+cHVVfWbGb06SZr9tCWHgO7T639M66ownhL9GW0I4yZ7AjcB5VfXVGYxZkgbGZIMkDZGqWgWs6ii7oO34CeDkSa79Q+APO8o2AK/tfaSSNLdMJyEMnAMcDJyf5Pym7PhmepskjSSTDZIkSVIXdjYhXFUfAj7U9wAlaYi4ZoMkSZIkSeopkw2SJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkSZIkSeopkw2SJEmSJKmnTDZIkiRJkqSe6irZkOSEJPcmWZ/k3AnO75rk2ub8rUkWtp07rym/N8lbexe6JI2efvS3U7UpSeqOn4klqXtTJhuSzAMuBU4EFgGnJlnUUe1s4JGqOhi4BLi4uXYRsBQ4FDgB+POmPUlSh370t122KUmagp+JJWnHdDOy4WhgfVVtqKqngGuAJR11lgBXNsfXA8cmSVN+TVU9WVXfBtY37UmSnqsf/W03bUqSpuZnYknaAd0kG/YBHmh7vKkpm7BOVW0FHgX26vJaSVJLP/pb+2FJ6g0/E0vSDpjfRZ1MUFZd1unmWpIsA5Y1Dx9Lcm8XcQ27lwMP9avxXNyvlkdOX98H8L3YAaPwN3FAn9vvR387UVL5Of0wjGRf3Pe/fz440cuuCfT3vfB96NYo/E30ux/eHj8T75z+/vf/LP/+u9T/z8S+F90Ylfehq764m2TDJmC/tsf7ApsnqbMpyXxgD+DhLq+lqlYAK7oJeLZIsqaqxgYdx1zn+zA8fC+60q/+dsp+GEavL/Z3bnj4XgwH34dp8zPxTvD3bjj4PgyHufY+dDONYjVwSJIDk+xCa3GblR11VgJnNscnATdXVTXlS5uVeQ8EDgG+3pvQJWnk9KO/7aZNSdLU/EwsSTtgypENVbU1yTnATcA84PKqWpfkQmBNVa0ELgOuSrKeVvZ2aXPtuiTXAXcDW4H3VdXTfboXSZrV+tXfTtTmTN+bJM12fiaWpB2TVrJVvZZkWTMUTgPk+zA8fC800/ydGx6+F8PB90GD4O/dcPB9GA5z7X0w2SBJkiRJknqqmzUbJEmSJEmSumayAUjydJI7ktyV5K+SvLBH7b4iyd8n+WaSu5Osasq/neQ1HXX/NMnvJlmc5NEktye5N8mXkrytF/EMG1/30ZDk/3Q8/q0kTyTZo61scZK/n+DatzWv+fh79e6ZiFnDx/5gMHzdR4P9sHrFPmEwfN1Hg33xc5lsaPlxVR1ZVYcBTwHv6fbCtLY1msyFwOeq6rVVtQg4tym/hmbBoKaN59FasfjapujLVfW6qnoN8H7gI0mO7f52Zg1f9xFQVf+2o+hUWit2/9r2rkvyfFrbe729ql4LvA64pR8xalawPxgMX/cRYD+sHrJPGAxf9xFgX/xcJhue68vAwUkWJrlrvDDJB5Isb45vSfJHSb4I/MckC5L8dZLVzc8bmsteRWtfZQCq6s7m8FO0/YEDbwY2VtV9ncFU1R20OopzeniPw2jWvO5J5iX5eJN9Xpvkt6Z157Ncksfajv8N8CLg92l1sNvzYlo74nwfoKqerKp7+xWnZpVZ0x+MmFnzutsPP5v9sPpk1vQJI2bWvO72xc9mX/xcJhvaNJnBE4G1XVTfs6p+qar+O/Bh4JKqej3wLuBjTZ1LgcuSfCHJf07yatj2h/5Mktc29ZbS+qOfzDeAn93xO5odZuHrfiSwT1UdVlWHA1d0EfdccSqt1/TLwGuS7D1Zxap6mNa+4/cl+VSS05vMuuawWdgfjIRZ+LrbD0/OfljTNgv7hJEwC193++LJ2RdjsmHcC5LcAawB7qe1R/JUrm07/hVaw4vuoPWL8pIkL66qm4CDgL+k9Qd6e5IFzTWfApY2ncoS4K+281zZobuZPWbr674BOCjJ/0hyAvCDLuKeK5YC11TVM8DfACdvr3JV/T/AscDXgQ8Al/c9Qg2r2dofzHaz9XW3H56c/bCmY7b2CbPdbH3d7YsnZ19Ma7iGmnlS7QVJtvLsZMxuHdc83nb8POAXq+rHnQ03maqrgavTWgzkzcBf0/oD/yzwReDOqnpwO/G9Driny3uZTWbF655kHnBbU7ayqi5oMsFvBd4H/Drwf2/3TueAJEcAhwCfSwKwC63/CF26veuqai2wNslVwLeBs/obqYbUrOgPuryX2WRWvO72w92xH1YPzIo+oct7mU1mxetuX9wd++KfcmTD5P4F2DvJXkl2Bba3CutnaZvHlOTI5t+3pFlNNsmLgX9DK1tJVX2L1ryci9jOsKXml/V8pvjlHCFD97pX1dPNoj1HNp3qy4HnVdVfN3V+fudvd6ScCiyvqoXNz6uBfZIcMFHlJC9Ksrit6EjgOXMFNacNXX8wrbuZPYbudbcf7pr9sPph6PqEad3N7DF0r7t9cdfsixuObJhEVf0kyYXArbQyS/+0nervBy5Nciet1/RLtFaRPYrWkKbxzOTHqmp123WfAv4rcENHe29KcjvwQuBB4P1V9fke3NbQmyWv+z7AFW1zqc7bkXscYUtpzTNsd0NTfitwbJJNbedOBX43yUeBH9PK0J81A3Fqlpgl/cHImSWvu/3wxOyH1XOzpE8YObPkdbcvnph9cSNVNegYJEmSJEnSCHEahSRJkiRJ6imTDZIkSZIkqadMNkiSJEmSpJ4y2SBJkiRJknrKZIMkSZIkSeopkw2aU5JsbPYEnlYdSdLOsy+WpMGyH9ZMMNkgSZIkSZJ6ymSDhl6ShUn+KcnHktyV5H8l+ZUkX03yz0mOTvKyJP87yZ1J/jHJEc21eyX5bJLbk3wUSFu7/z7J15PckeSjSeYN7CYlacjZF0vSYNkPa7Yx2aDZ4mDgw8ARwM8CpwFvBD4A/Cfgg8DtVXVE8/gTzXV/AHylql4HrAT2B0jyc8ApwBuq6kjgaeD0GbsbSZqd7IslabDshzVrzB90AFKXvl1VawGSrAM+X1WVZC2wEDgAeBdAVd3cZG/3AN4M/Lum/MYkjzTtHQscBaxOAvAC4MEZvB9Jmo3siyVpsOyHNWuYbNBs8WTb8TNtj5+h9Xu8dYJrquPfdgGurKrzehahJI0++2JJGiz7Yc0aTqPQqPgSzZCvJIuBh6rqBx3lJwIvbep/Hjgpyd7NuZclOWCmg5akEWNfLEmDZT+soeHIBo2K5cAVSe4EfgSc2ZR/EPhUkm8AXwTuB6iqu5P8PvDZJM8DfgK8D7hvpgOXpBGyHPtiSRqk5dgPa0ikaqLRNJIkSZIkSTvHaRSSJEmSJKmnTDZIkiRJkqSeMtkgSZIkSZJ6ymSDJEmSJEnqKZMNkiRJkiSpp0w2SJIkSZKknjLZIEmSJEmSespkgyRJkiRJ6qn/H8NKve/Ah4NQAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -1312,7 +1312,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -1321,7 +1321,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -1332,12 +1332,12 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 41, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHNhJREFUeJzt3XuYHFWd//H3BxJQIYCS4ZKQkBWzcdGFgHkiLsqOgiw3xXVRE1wgKxpvrOJlFV1XWRCN/lS8RMWAWeAnBBY1muUisCjLRUAHNoSAIDEGEwIkXAQiICR8949zBiqd7plOd89MJufzep5+prrqVJ1TVd2frjpd1aOIwMzMyrHFUDfAzMwGl4PfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn7rGEmvk3TXINX1TklXDFJd+0u6W9IaSW8ZjDo3Rm7XSwdguTMkXdfp5fZT586SrpH0uKSvSvq0pLMGsw0lcPAPAklHS+rJb9D7JF0m6bVD3a4qSd2SVmzkPCHpZb3PI+LaiJg0AG2bkOsaUanrvIg4uNN1NXAKMDsito2InwxSnXVJulrSu6vjcruWDlWbOmwm8CCwXUR8LCK+EBHv7m8m2zgO/gEm6aPA14EvADsD44HvAEcOZbtso+wO3N7KjNUPK2vK7sAd4TtLB1ZE+DFAD2B7YA3wtj7KbE36YFiZH18Hts7TuoEVwCeAVcB9wFuAw4DfAg8Dn64s62Tgh8CFwOPALcDelekBvKzy/Gzg88A2wJPAs7m9a4AxwFTgBuCPue7ZwFZ53mvy8v6Uy7+jt72V5f8VcHWe/3bgzTV1fxu4JLf1JmCPBtvoD7mu3ra9BpgBXFezbh8A7s7LOxXYI7f/MeA/e9ueyx8BLMxt+yWwV4O6f5e3y5O57q3ztlmQt/8S4D119sEPcr3vbvC6OBdYDdwDfAbYIk+bAVwPfAt4FLgTODBPOw1YBzyV2zK7dr/m7fod4LJc5npgF9Lr6pG8vH0qbTkpr+PjwB3A31emrbeN66zHa/O2+yOwHJjR5PpdB3wlt+f3wKGVtj8DPJ3bflDenj+o1HlsXuZDwL8By4CDmnw/fhK4N6/rXb3btcTHkDdgc34AhwBrgRF9lDkFuBHYCejKb6RT87TuPP9ngZHAe/Kb6XxgFPCKHAIvzeVPzm+co3L5j+c31sg8vW7wV+paUdO2VwH7ASOACcBvgBMr02uX99wycv1LgE8DWwFvyG+4SZW6HyZ9uIwAzgMuaLCNJuS6RlTGzWDD4F8AbJe3y5+Bq4CXkoLoDuC4XHZf0gfpq4EtgeNygGzdoP71wgX4H1K4vgCYnPfJgTX74C2kM+oX1lneucBP8z6cQPoQP76yXmuBj+Rt+A7SB8BL8vSrqfkwYcPgfzDvuxcAP8+vgWPzun4e+EVl3reRPsi2yHX9Cdi13jauqXN83p/Tczt3BCY3uX7PkF7LWwLvJx3wqPY1WdmeP8jDe5I+EF5Lek19JS+r3+AHJpE+nMZUXlN1DzRKeLirZ2DtCDwYEWv7KPNO4JSIWBURq4F/B46pTH8GOC0ingEuAEYD34iIxyPidtKR9F6V8jdHxA9z+a+R3vz7tdL4iLg5Im6MiLURsQz4HvC3Tc6+H7AtMCsino6InwMXk4Ki148j4ld5+5xHCtF2fCkiHsvbZTFwRUQsjYhHSUfA++Ry7wG+FxE3RcS6iDiH9EHR73aSNI4UPJ+MiKciYiFwFuvvsxsi4icR8WxEPFkz/5akgP1U3ofLgK/WzL8K+HpEPBMRF5KOTg/fiO0wP++7p4D5wFMRcW5ErCOdDfZuByLioohYmdt6IemMaWoTdbwT+O+ImJfb+VBELGxy/e6JiDNze84BdiV1g/bnKOC/IuK6iHiadEDUbJfQOtLZ2p6SRkbEsoj4XZPzbnYc/APrIWB0P/28Y0inrr3uyeOeW0Z+g0DqbgB4oDL9SVLA9lreOxARz5K6iqrLa5qkv5R0saT7JT1G+p5idJOzjwGW5zb0ugcYW3l+f2X4CdZfj1bUbpdG22l34GOS/tj7AMbR3HYaAzwcEY9XxtWu13IaG006Wq3d59X57418WFqZvjH7sNntgKRjJS2sbIdX0tw+HkfqIqrVzPo9t98j4ok82My+H8P6r+8nSO+xfkXEEuBE0hnEKkkXSGrpfbE5cPAPrBtIXTF9XQK4khREvcbnca0a1zsgaQtgt8ryngBeVCm7S2W43pHTd0l9whMjYjtSt42abMdKYFxuQ6/xpD7WjdXpL/qWk86idqg8XhQR85qYdyXwEkmjKuNq16uv9j5IOour3efV+cdKUs303n3YsW0haXfgTOAEYMeI2IF0ptTMPl5O+g6lVjPr16r7SK9nACS9kHRW3ZSIOD8iXpvbFsCXOtCmYcnBP4ByF8NngW9LeoukF0kaKelQSV/OxeYBn5HUJWl0Lv+DNqp9laS35rOME0ldGDfmaQuBoyVtKekQ1u+2eQDYUdL2lXGjSF9QrpH0clJ/LDXzNLp+/CZSf/En8jp3A28idVdtrNWkL1g7da36mcD7JL1ayTaSDq8J87oiYjnpe5gvSnqBpL2A40ldVf3KZ2//CZwmaVQO34+y/j7fCfhQ3m5vI31Jfmme1tc231jbkAJwNYCkfyId8TfjPOAgSW+XNELSjpImN7l+rfoh8CZJfyNpK1K36HMfUvmS5LofjJImSXqDpK1JB2NPkrp/iuTgH2AR8TXSC/8zpDfYctIRVu/14J8HeoBFwG2kK3E+30aVPyX1sT5C6ld9a+7vB/gwKXz/SOqjfe6a9Ii4k/QhtDSf9o8hfTl8NOlLvDNJ/cNVJwPn5PJvr1nvp4E3A4eSjgK/Axyb69ko+ZT+NOD6XFdL31lUltdD6uefTdpOS0hfOjZrOunLwZWkPvTPRcSVGzH/P5M+FJeSrnA5H5hbmX4TMJG03U4DjoqI3i6NbwBHSXpE0jc3os4NRMQdpP73G0gfKH9NugqomXn/QLq67GOkL+kXAnvnyf2tX6vtvT0v+wLS0f/jpO9D/pyLjMvrUs/WwCzSNr2f9OH66XbbNFxp/a5EG84knUy6uuMfh7ot1hpJM0hX7WxSN/htiiRtSzqImRgRv893+F4UEZcPcdM2eb65xMyGDUlvIl2mK9LlnLeRLrclfIdv09zVY2bDyZE8f7PjRGBauNtio7mrx8ysMD7iNzMrzCbZxz969OiYMGHCUDfDzGzYuPnmmx+MiK5mym6SwT9hwgR6enqGuhlmZsOGpHv6L5W4q8fMrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjMrQnd3N93d3UPdjE2Cg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDCb5P/cNbMyTDjpkkGr6/6lDw1qnctmHT4o9bTCR/xmZoVx8JuZFabfrh5Jc4EjgFUR8co87kJgUi6yA/DHiJhcZ95lwOPAOmBtREzpULvNzKxFzfTxnw3MBs7tHRER7+gdlvRV4NE+5n99RDzYagPNzKyz+g3+iLhG0oR60yQJeDvwhs42y8zMBkq7ffyvAx6IiLsbTA/gCkk3S5rZ14IkzZTUI6ln9erVbTbLzMwaaTf4pwPz+pi+f0TsCxwKfFDSAY0KRsSciJgSEVO6urrabJaZmTXScvBLGgG8FbiwUZmIWJn/rgLmA1Nbrc/MzDqjnSP+g4A7I2JFvYmStpE0qncYOBhY3EZ9ZmbWAf0Gv6R5wA3AJEkrJB2fJ02jpptH0hhJl+anOwPXSboV+BVwSUT8rHNNNzOzVjRzVc/0BuNn1Bm3EjgsDy8F9m6zfWZmHbHL0bOGugmbDN+5a2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhmvmfu3MlrZK0uDLuZEn3SlqYH4c1mPcQSXdJWiLppE423MzMWtPMEf/ZwCF1xp8eEZPz49LaiZK2BL4NHArsCUyXtGc7jTUzs/b1G/wRcQ3wcAvLngosiYilEfE0cAFwZAvLMTOzDmqnj/8ESYtyV9CL60wfCyyvPF+Rx9UlaaakHkk9q1evbqNZZmbWl1aD/7vAHsBk4D7gq3XKqM64aLTAiJgTEVMiYkpXV1eLzTIzs/60FPwR8UBErIuIZ4EzSd06tVYA4yrPdwNWtlKfmZl1TkvBL2nXytO/BxbXKfZrYKKkv5C0FTANWNBKfWZm1jkj+isgaR7QDYyWtAL4HNAtaTKp62YZ8N5cdgxwVkQcFhFrJZ0AXA5sCcyNiNsHZC3MzKxp/QZ/REyvM/r7DcquBA6rPL8U2OBSTzMzGzq+c9fMrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwvQb/JLmSlolaXFl3P+TdKekRZLmS9qhwbzLJN0maaGknk423MzMWtPMEf/ZwCE1464EXhkRewG/BT7Vx/yvj4jJETGltSaamVkn9Rv8EXEN8HDNuCsiYm1+eiOw2wC0zczMBkAn+vjfBVzWYFoAV0i6WdLMDtRlZmZtGtHOzJL+FVgLnNegyP4RsVLSTsCVku7MZxD1ljUTmAkwfvz4dpplZmZ9aPmIX9JxwBHAOyMi6pWJiJX57ypgPjC10fIiYk5ETImIKV1dXa02y8zM+tFS8Es6BPgk8OaIeKJBmW0kjeodBg4GFtcra2Zmg6eZyznnATcAkyStkHQ8MBsYReq+WSjpjFx2jKRL86w7A9dJuhX4FXBJRPxsQNbCzMya1m8ff0RMrzP6+w3KrgQOy8NLgb3bap2ZmXWc79w1MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjj4zcwK4+A3MyuMg9/MrDAOfjOzwjQV/JLmSlolaXFl3EskXSnp7vz3xQ3mPS6XuVvScZ1quJmZtabZI/6zgUNqxp0EXBURE4Gr8vP1SHoJ8Dng1cBU4HONPiDMzGxwNBX8EXEN8HDN6COBc/LwOcBb6sz6d8CVEfFwRDwCXMmGHyBmZjaI2unj3zki7gPIf3eqU2YssLzyfEUetwFJMyX1SOpZvXp1G80yM7O+DPSXu6ozLuoVjIg5ETElIqZ0dXUNcLPMzMrVTvA/IGlXgPx3VZ0yK4Bxlee7ASvbqNPMzNrUTvAvAHqv0jkO+GmdMpcDB0t6cf5S9+A8zszMhkizl3POA24AJklaIel4YBbwRkl3A2/Mz5E0RdJZABHxMHAq8Ov8OCWPMzOzITKimUIRMb3BpAPrlO0B3l15PheY21LrzMys43znrplZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv1mTuru76e7uHupmmLXNwW9mVhgHv5lZYZr6rR6zTdWEky4ZtLruX/rQoNe5bNbhg1aXlcNH/GZmhXHwm5kVxsFvZlYY9/GbNWmXo2cNdRPMOsJH/GZmhXHwm5kVpuXglzRJ0sLK4zFJJ9aU6Zb0aKXMZ9tvspmZtaPlPv6IuAuYDCBpS+BeYH6dotdGxBGt1mNmZp3Vqa6eA4HfRcQ9HVqemZkNkE4F/zRgXoNpr5F0q6TLJL2i0QIkzZTUI6ln9erVHWqWmZnVajv4JW0FvBm4qM7kW4DdI2Jv4FvATxotJyLmRMSUiJjS1dXVbrPMzKyBThzxHwrcEhEP1E6IiMciYk0evhQYKWl0B+o0M7MWdSL4p9Ogm0fSLpKUh6fm+h7qQJ1mZtaitu7clfQi4I3Aeyvj3gcQEWcARwHvl7QWeBKYFhHRTp1mZtaetoI/Ip4AdqwZd0ZleDYwu506zMyss3znrplZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFcfCbmRXGwW9mVhgHv5lZYRz8ZmaFaTv4JS2TdJukhZJ66kyXpG9KWiJpkaR9263TzMxa19b/3K14fUQ82GDaocDE/Hg18N3818zMhsBgdPUcCZwbyY3ADpJ2HYR6zcysjk4EfwBXSLpZ0sw608cCyyvPV+Rx65E0U1KPpJ7Vq1d3oFlmZlZPJ4J//4jYl9Sl80FJB9RMV515YoMREXMiYkpETOnq6upAs8zMrJ62gz8iVua/q4D5wNSaIiuAcZXnuwEr263XzMxa01bwS9pG0qjeYeBgYHFNsQXAsfnqnv2ARyPivnbqNTOz1rV7Vc/OwHxJvcs6PyJ+Jul9ABFxBnApcBiwBHgC+Kc26zQzsza0FfwRsRTYu874MyrDAXywnXrMzKxzfOeumVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBb2ZWGAe/mVlhHPxmZoVx8JuZFcbBP4i6u7vp7u4e6maYWeHa/Q9cw96Eky4ZtLruX/rQoNe5bNbhg1aXmQ0PPuI3MytMy0f8ksYB5wK7AM8CcyLiGzVluoGfAr/Po34cEae0Wudwt8vRs4a6CWZmbXX1rAU+FhG3SBoF3Czpyoi4o6bctRFxRBv1mJlZB7Xc1RMR90XELXn4ceA3wNhONczMzAZGR/r4JU0A9gFuqjP5NZJulXSZpFd0oj4zM2td21f1SNoW+BFwYkQ8VjP5FmD3iFgj6TDgJ8DEBsuZCcwEGD9+fLvNMjOzBto64pc0khT650XEj2unR8RjEbEmD18KjJQ0ut6yImJOREyJiCldXV3tNMvMzPrQcvBLEvB94DcR8bUGZXbJ5ZA0Ndf3UKt1mplZ+9rp6tkfOAa4TdLCPO7TwHiAiDgDOAp4v6S1wJPAtIiINuo0M7M2tRz8EXEdoH7KzAZmt1qHmZl1nu/cNTMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8I4+M3MCuPgNzMrjIPfzKwwDn4zs8K0FfySDpF0l6Qlkk6qM31rSRfm6TdJmtBOfWZm1r6Wg1/SlsC3gUOBPYHpkvasKXY88EhEvAw4HfhSq/WZmVlntHPEPxVYEhFLI+Jp4ALgyJoyRwLn5OEfAgdKUht1mplZm0a0Me9YYHnl+Qrg1Y3KRMRaSY8COwIP1i5M0kxgZn66RtJdbbRtUzaaOus/UORzrE7z/hveBm3/DcG+273Zgu0Ef70j92ihTBoZMQeY00Z7hgVJPRExZajbYa3x/hvevP+Sdrp6VgDjKs93A1Y2KiNpBLA98HAbdZqZWZvaCf5fAxMl/YWkrYBpwIKaMguA4/LwUcDPI6LuEb+ZmQ2Olrt6cp/9CcDlwJbA3Ii4XdIpQE9ELAC+D/x/SUtIR/rTOtHoYW6z787azHn/DW/ef4B8AG5mVhbfuWtmVhgHv5lZYTar4Je0TtJCSYslXSTpRR1a7s6SLpZ0q6Q7JF2ax/9e0qSasl+X9AlJ3ZIelfS/+WctrpF0RCfa06CNyySN7qfM1bktC/Njpzx+UH5ao+T90wxJa5ooM1fSKkmLa8afKmlR3r5XSBqTx0vSN/O+XSRp34Fqf7Mk/bLm+UckPSVp+8q4bkkX15n3iLzPevf1eweojc28n7aSNEfSbyXdKekf8vhN/6dqImKzeQBrKsPnAR/diHlH9DHte8CHK8/3yn+/CHyuMn4L0iWsuwPdwMWVaZOBZcCBrbajn/mWAaP7KXM1MKXO+A8AZ+ThacCF3j9trWer+3BNE2UOAPYFFteM364y/KHK/jwMuIx0T81+wE0DsW/b3F6/Aq4FZlTGrbd/8riRpEvGd8vPtwYmDdC+aOb99O/A5yuvrdF5eFDeT+08Nqsj/hrXAi+TNKF6dCTp45JOzsNXS/qCpP8BPiypS9KPJP06P/bPs+1KCgwAImJRHpzH+lcqHQAsi4h7ahsTEQuBU4ATaqdJOjkfOVwBnJvbfK2kW/Ljb3K57tzmH+YjjPOk9X8CQ9ILJf1M0ns2YlsNxU9rDKf9s6Wks/OZym2SPlKnzNmSvibpF8CXJE2V9Mt8dPrL3jMPSTMk/Tjvo7slfbnOskZLukHS4XXaeQ117oWJiMcqT7fh+RsljwTOjeRGYAdJu9bOP5iqZzaS9gC2BT4DTO9n1lGkKxEfAoiIP0fEBnf4D+L76V2kgwsi4tmI6L0jeJP/qZp27tzdZCndLHYo8LMmiu8QEX+b5zsfOD0irpM0nnSp6l+RfozuQqXLV/8b+I+IWBkRiyQ9K2nviLiVFDLz+qjrFuBfGkx7FfDaiHhSqQvkjRHxlKSJeZm9dxvuA7yCdORzPbA/cF2eti3pN5POjYhzG9TzH5LWAT8iHa0EG/HTGp0wDPfPZGBsRLwyt2OHBvP/JXBQRKyTtB1wQN6eBwFfAP6hsrx9gD8Dd0n6VkQsz8vemXT/y2ci4somts9zJJ0GHAs8Crw+j6730ypjgfs2ZtkDaDppn1wLTJK0U0SsqlcwIh6WtAC4R9JVwMXAvIh4tk7xAX0/VV4Dp0rqBn4HnBARDzDI76dWbG5H/C+UtBDoAf5Auo+gPxdWhg8CZudlLAC2kzQqIi4HXgqcCbwc+F9JXXmeecC0HGZHAhf1UVdfn/oLIuLJPDwSOFPSbXl51V89/VVErMgv9oXAhMq0n5JCr1HovzMi/hp4XX4c00e7BuI63+G6f5YCL5X0LUmHAI81KHdRRKzLw9sDF+WzmdNJ4dLrqoh4NCKeAu7g+d9YGQlcBXxiY0MfICL+NSLGkbrRes9cBmvftmoacEF+Pf8YeFtfhSPi3cCBpO6hjwNzGxQd6PfTCNKvFVwfEfsCNwBfydM29W2+2QX/kxExOT/+OdKvhq5l/fV8Qc08f6oMbwG8prKMsRHxOKSjjYg4PyKOId21fECeZx7wdlIoLWp0tJLtA/ymwbRqOz4CPADsTToy2aoy7c+V4XWsf9Z2PXBoo9PKiLg3/30cOJ/0C6sweD+tMSz2T+7a6f0C/JSIeIS0L64GPgic1WD+altPBX6RzxLeVLNejfbhWuBm4O/6aGMzzuf5s4tmflplSEjaC5gIXClpGelDoL/uHiLitog4HXgjz69nrYF+Pz0EPAHMz88vIn33AsPgp2o2t+Cv5wFgJ0k7Stoa6OvKjSuo9PFKmpz/viGfLiJpFLAH6YiViPgd6UUwiz66EfKL/N9I3RL92R64Lx+FHEO6M7oZn81t+U6d+kcoX6UgaSRpO/T2rQ/lT2tscvsnItZVPlw+m7fbFhHxo1ymmStjtgfuzcMzmigP6ajwXcDLVecfG/Uld2H0ejNwZx5eAByrZD/g0YjYlLp5To6ICfkxBhgrqe6vTEraNner9JoMbPB9TR0dfz/l98d/kb6EhnQWckce3uR/qmazD/6IeIb0pd1NpD7BO/so/iFgitJlb3cA78vjXwX0SFpEOqU7KyJ+XZlvHqmLYT7re13+cu8uUuB/KCKuaqLZ3wGOk3Qjqd/4T/2UrzoReEGdLw23Bi7P67CQFEpn5mnfB3ZU+mmNjwIbFTrtGCb7Zyxwde5iOhv4VBOr9mXgi5Kup/mgIXcVTQNeL+kDtdMlzSOt4yRJKyQdnyfNUvryeRFwMPDhPP5SUlfVEtL+3mCZQ2gaG+6T+Tz/hfyBeR1XSFpBOiP7hPIlyaSramY0Uc9AvJ8APgmcnLf5McDH8vghez81yz/ZYGZWmM3+iN/MzNbn4DczK4yD38ysMA5+M7PCOPjNzArj4DczK4yD38ysMP8HCFh3sSdTyLEAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGU9JREFUeJzt3Xu0XGV9//H3BxKCQgAlB0hi4BSkUdpC0CykBe1R0B+ICrZgAX9AfkVja6mgWKSUaspFo0vBVsSfRCiwhIDcSoqoUIRyEaKBpuFeMAYTEpIACkGuCd/+8TwHdibnZPaZmXNOzpPPa61Z2bNvz3f2nv2ZvZ/Zc6KIwMzMRr5NhrsAMzPrDAe6mVkhHOhmZoVwoJuZFcKBbmZWCAe6mVkhHOhWi6R3S3p4iNr6uKQbhqitfSQ9Iuk5SYcMRZsDkevaeRDWO03S7Z1eb5M2t5d0q6RVkr4h6RRJ3xvKGkrnQG+TpCMlzcsH3jJJP5K073DXVSWpR9KSAS4Tkt7a+zwibouIyYNQW3dua1SlrUsi4gOdbqsfpwHnRMSWEfFvQ9RmnyTdIukT1XG5roXDVVOHTQeeBLaKiBMj4ssR8YlmC1l9DvQ2SPoc8E3gy8D2wI7AucDBw1mXDchOwP2tLFj9ELJadgIeCP+acfBEhB8tPICtgeeAw9YzzxhS4C/Nj28CY/K0HmAJcBKwAlgGHAJ8EPgf4GnglMq6ZgBXApcDq4B7gD0q0wN4a+X5hcAZwBbAC8Crud7ngAnAXsCdwG9z2+cAm+Vlb83r+12e/y96662s/+3ALXn5+4GPNLT9beCHuda5wC79bKNf57Z6a/tjYBpwe8Nr+zTwSF7f6cAuuf5ngR/01p7n/xAwP9f2M2D3ftr+Zd4uL+S2x+RtMydv/0eBT/axD76f2/1EP++Li4GVwGPAqcAmedo04A7gW8AzwEPAfnnamcAa4MVcyzmN+zVv13OBH+V57gB2IL2vfpPXt2ellpPza1wFPAB8tDJtrW3cx+vYN2+73wKLgWk1X9/twNdzPb8CDqzU/grwcq59/7w9v19p8+i8zqeAfwQWAfvXPB6/ADyeX+vDvdt1Y3sMewEj9QEcAKwGRq1nntOAu4DtgK58gJyep/Xk5b8IjAY+mQ+SS4GxwB/kg3vnPP+MfEAcmuf/fD5gRufpfQZ6pa0lDbW9E9gbGAV0Aw8CJ1SmN67vtXXk9h8FTgE2A96XD6TJlbafJn1ojAIuAS7rZxt157ZGVcZNY91AnwNslbfLS8BNwM6kgHkAOCbP+w7SB+S7gE2BY3IwjOmn/bVCA/hPUmhuDkzJ+2S/hn1wCOnq9g19rO9i4Nq8D7tJH87HVl7XauCzeRv+BSnY35yn30LDhwTrBvqTed9tDvw0vweOzq/1DODmyrKHkT6gNslt/Q4Y39c2bmhzx7w/j8h1bgtMqfn6XiG9lzcF/pp0IqPG92Rle34/D+9GCvp9Se+pr+d1NQ10YDLpQ2dC5T3V5wlE6Q93ubRuW+DJiFi9nnk+DpwWESsiYiXwT8BRlemvAGdGxCvAZcA44J8jYlVE3E868929Mv/dEXFlnv8s0kG9dyvFR8TdEXFXRKyOiEXAd4E/rbn43sCWwMyIeDkifgpcRwqAXldHxM/z9rmEFI7t+GpEPJu3y33ADRGxMCKeIZ2x7pnn+yTw3YiYGxFrIuIi0gdA0+0kaRIpUL4QES9GxHzge6y9z+6MiH+LiFcj4oWG5TclBeff5324CPhGw/IrgG9GxCsRcTnpbPKgAWyHa/K+exG4BngxIi6OiDWkq7fe7UBEXBERS3Otl5OucPaq0cbHgf+IiNm5zqciYn7N1/dYRMzK9VwEjCd1RzZzKPDvEXF7RLxMOtGp2zWzhnR1tZuk0RGxKCJ+WXPZojjQW/cUMK5JP+oE0iVkr8fyuNfWkd/4kC77AZZXpr9ACs5ei3sHIuJVUpdNdX21Sfp9SddJekLSs6TvAcbVXHwCsDjX0OsxYGLl+ROV4edZ+3W0onG79LeddgJOlPTb3gcwiXrbaQLwdESsqoxrfF2L6d840tll4z6vLv945NPIyvSB7MO62wFJR0uaX9kOf0i9fTyJ1FXTqM7re22/R8TzebDOvp/A2u/v50nHWFMR8ShwAumMf4WkyyS1dFyMdA701t1J6hJZ361uS0kB02vHPK5Vk3oHJG0CvKWyvueBN1bm3aEy3NeZzndIfa67RsRWpO4T1axjKTAp19BrR1If5kB1+guyxaSrnm0qjzdGxOwayy4F3ixpbGVc4+taX71Pkq66Gvd5dfmJktQwvXcfdmxbSNoJmAUcB2wbEduQrmzq7OPFpO8oGtV5fa1aRno/AyDpDaSr4Foi4tKI2DfXFsBXO1DTiONAb1G+1P8i8G1Jh0h6o6TRkg6U9LU822zgVEldksbl+b/fRrPvlPRn+argBFJXwl152nzgSEmbSjqAtbtPlgPbStq6Mm4s6Yu95yS9jdTfScMy/d3/PJfUH3tSfs09wIdJ3UYDtZL0xWSn7rWeBfyVpHcp2ULSQQ0h3aeIWEz6nuMrkjaXtDtwLKnLqKl8tfUD4ExJY3Oofo619/l2wGfydjuM9OXy9Xna+rb5QG1BCraVAJL+H+kMvY5LgP0lfUzSKEnbSppS8/W16krgw5L+RNJmpO7J1z588q23fX7gSZos6X2SxpBOsl4gdcNsdBzobYiIs0hv6FNJB85i0hlR7/3MZwDzgAXAvaQ7U85oo8lrSX2YvyH1W/5Z7k8HOJ4Uqr8l9YG+dk91RDxE+nBZmC+/J5C+VD2S9OXXLFL/a9UM4KI8/8caXvfLwEeAA0lnbecCR+d2BiRfWp8J3JHbauk7gcr65pH60c8hbadHSV/W1XUE6Uu1paQ+6i9FxI0DWP5vSR92C0l3fFwKXFCZPhfYlbTdzgQOjYjeroV/Bg6V9BtJ/zKANtcREQ+Q+rfvJH1Q/BHprpg6y/6adLfViaQvt+cDe+TJzV5fq/Xen9d9GelsfRXp+4aX8iyT8mvpyxhgJmmbPkH60Dyl3ZpGIq3dnWcbKkkzSHc7/N/hrsVaI2ka6S6WDeqHZxsiSVuSTk52jYhf5V+UXhERPxnm0jZo/mGEmW0QJH2YdDuqSLct3ku6rZTwL0prcZeLmW0oDub1H+HtChwe7kIYEHe5mJkVwmfoZmaFGNI+9HHjxkV3d/dQNmlmNuLdfffdT0ZEV7P5hjTQu7u7mTdv3lA2aWY24kl6rPlc7nIxMyuGA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MRrSenh56enqGu4wNggPdzKwQDnQzs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40M3MCtE00CVNknSzpAcl3S/p+Dx+hqTHJc3Pjw8OfrlmZtafUTXmWQ2cGBH3SBoL3C3pxjzt7Ij4+uCVZ2YjUffJPxyytp5Y+NSQt7lo5kFD1tZANA30iFgGLMvDqyQ9CEwc7MLMzGxgBtSHLqkb2BOYm0cdJ2mBpAskvanDtZmZ2QDUDnRJWwJXASdExLPAd4BdgCmkM/hv9LPcdEnzJM1buXJlB0o2M7O+1Ap0SaNJYX5JRFwNEBHLI2JNRLwKzAL26mvZiDgvIqZGxNSurq5O1W1mZg3q3OUi4HzgwYg4qzJ+fGW2jwL3db48MzOrq85dLvsARwH3Spqfx50CHCFpChDAIuBTg1KhmZnVUucul9sB9THp+s6XY2ZmrfIvRc3MCuFANzMrhAPdzKwQDnQzs0I40M3MClHntkUzsw3WDkfOHO4SNhg+QzczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArRNNAlTZJ0s6QHJd0v6fg8/s2SbpT0SP73TYNfrpmZ9afOGfpq4MSIeDuwN/A3knYDTgZuiohdgZvyczMzGyZNAz0ilkXEPXl4FfAgMBE4GLgoz3YRcMhgFWlmZs0NqA9dUjewJzAX2D4ilkEKfWC7fpaZLmmepHkrV65sr1ozM+tX7UCXtCVwFXBCRDxbd7mIOC8ipkbE1K6urlZqNDOzGmoFuqTRpDC/JCKuzqOXSxqfp48HVgxOiWZmVkedu1wEnA88GBFnVSbNAY7Jw8cA13a+PDMzq2tUjXn2AY4C7pU0P487BZgJ/EDSscCvgcMGp0QzM6ujaaBHxO2A+pm8X2fLMTOzVvmXomZmhXCgm5kVwoFuZlYIB7qZWSEc6GZmhXCgm5kVwoFuZlYIB7qZWSEc6GZmhXCgm5kVwoFuZlYIB7qZWSEc6GZmhXCgm5kVwoFuZlYIB7qZWSEc6GZmhXCgm5kVwoFuZlYIB7qZWSEc6GZmhXCgm5kVwoFuZlYIB7qZWSEc6GZmhXCgm5kVwoFuZlYIB7qZWSEc6GZmhWga6JIukLRC0n2VcTMkPS5pfn58cHDLNDOzZuqcoV8IHNDH+LMjYkp+XN/ZsszMbKCaBnpE3Ao8PQS1mJlZG9rpQz9O0oLcJfOm/maSNF3SPEnzVq5c2UZzZma2Pq0G+neAXYApwDLgG/3NGBHnRcTUiJja1dXVYnNmZtZMS4EeEcsjYk1EvArMAvbqbFlmZjZQLQW6pPGVpx8F7utvXjMzGxqjms0gaTbQA4yTtAT4EtAjaQoQwCLgU4NYo5mZ1dA00CPiiD5Gnz8ItZiZWRv8S1Ezs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40M3MCuFANzMrhAPdzKwQDnQzs0I40G2j19PTQ09Pz3CXYdY2B7qZWSEc6GZmhXCgm5kVommgS7pA0gpJ91XGvVnSjZIeyf++aXDLNDOzZkbVmOdC4Bzg4sq4k4GbImKmpJPz8y90vjzbWHWf/MMha+uJhU8NeZuLZh40ZG3ZxqPpGXpE3Ao83TD6YOCiPHwRcEiH6zIzswFqtQ99+4hYBpD/3a6/GSVNlzRP0ryVK1e22JyZmTUz6F+KRsR5ETE1IqZ2dXUNdnNmZhutVgN9uaTxAPnfFZ0ryczMWlHnS9G+zAGOAWbmf6/tWEVmQ2yHI2cOdwlmHVHntsXZwJ3AZElLJB1LCvL3S3oEeH9+bmZmw6jpGXpEHNHPpP06XIuZmbXBvxQ1MyuEA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MrBAOdDOzQjjQzcwK4UA3MyuEA93MrBAOdDOzQjjQzcwK4UA3MyvEqHYWlrQIWAWsAVZHxNROFGVmZgPXVqBn742IJzuwHjMza4O7XMzMCtFuoAdwg6S7JU3vREFmZtaadrtc9omIpZK2A26U9FBE3FqdIQf9dIAdd9yxzebMzKw/bZ2hR8TS/O8K4Bpgrz7mOS8ipkbE1K6urnaaMzOz9Wg50CVtIWls7zDwAeC+ThVmZmYD006Xy/bANZJ613NpRPy4I1WZmdmAtRzoEbEQ2KODtZiZWRt826KZWSEc6GZmhXCgm5kVwoFuZlYIB7qZWSEc6B3S09NDT0/PcJdhZhuxTvy1xQ1W98k/HLK2nlj41JC3uWjmQUPWlplt+HyGbmZWiKLP0IfSDkfOHO4SzGwj5zN0M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK4QD3cysEA50M7NCtBXokg6Q9LCkRyWd3KmizMxs4FoOdEmbAt8GDgR2A46QtFunCjMzs4Fp5wx9L+DRiFgYES8DlwEHd6YsMzMbqFFtLDsRWFx5vgR4V+NMkqYD0/PT5yQ93EabG7pxwJND1Zi+OlQtbRS870a20vffTnVmaifQ1ce4WGdExHnAeW20M2JImhcRU4e7Dhs477uRzfsvaafLZQkwqfL8LcDS9soxM7NWtRPovwB2lfR7kjYDDgfmdKYsMzMbqJa7XCJitaTjgJ8AmwIXRMT9HatsZNooupYK5X03snn/AYpYp9vbzMxGIP9S1MysEA50M7NCjIhAl7RG0nxJ90m6QtIbO7Te7SVdJ+m/JT0g6fo8/leSJjfM+01JJ0nqkfSMpP/Kf/bgVkkf6kQ9/dS4SNK4JvPckmuZnx/b5fFjJF2e/zTDXEndg1TjRrt/6pD0XI15LpC0QtJ9DeNPl7Qgb98bJE3I4yXpX/K+XSDpHYNV/0BI+lnD889KelHS1pVxPZKu62PZD+X91ru/PzVINdY5pjaTdJ6k/5H0kKQ/z+OH5JhqWURs8A/gucrwJcDnBrDsqPVM+y5wfOX57vnfrwBfqozfhHSb5k5AD3BdZdoUYBGwX6t1NFluETCuyTy3AFP7GP9p4P/n4cOBy71/2nqdre7D52rM8x7gHcB9DeO3qgx/prI/Pwj8iPR7kL2BuYOxbzuwzX4O3AZMq4xbax/lcaNJtz2/JT8fA0wepP1R55j6J+CMyvtrXB4ekmOq1ceIOENvcBvwVknd1bMZSZ+XNCMP3yLpy5L+EzheUpekqyT9Ij/2yYuNJwUBABGxIA/OJu2sXu8BFkXEY43FRMR84DTguMZpkmbkT/kbgItzzbdJuic//iTP15NrvjKfDVwiSQ3reoOkH0v65AC21cHARXn4SmC/xvUOgpG0fzaVdGG+srhX0mf7mOdCSWdJuhn4qqS9JP0sn0n+rPdKQdI0SVfnffSIpK/1sa5xku6UdFAfdd4KPN3H+GcrT7fg9R/vHQxcHMldwDaSxjcuP9SqVyOSdgG2BE4Fjmiy6FjSXXdPAUTESxGxzq/Kh/CY+kvSiQMR8WpE9P4KdTiOqdra+aXokJM0ivTHwH5cY/ZtIuJP83KXAmdHxO2SdiTdavl20h8Xu1zp9sv/AP41IpZGxAJJr0raIyL+mxQes9fT1j3A3/Uz7Z3AvhHxglJXxPsj4kVJu+Z19v66bU/gD0hnKXcA+wC352lbkv5WzsURcXE/7fyrpDXAVaQzi6Dy5xki3Wb6DLAtg/QT6RG4f6YAEyPiD3Md2/Sz/O8D+0fEGklbAe/J23N/4MvAn1fWtyfwEvCwpG9FxOK87u1Jv9M4NSJurLF9XiPpTOBo4BngvXl0X396YyKwbCDrHmRHkPbLbcBkSdtFxIq+ZoyIpyXNAR6TdBNwHTA7Il7tY/ZBPaYq74PTJfUAvwSOi4jlDPExNVAj5Qz9DZLmA/OAXwPn11jm8srw/sA5eR1zgK0kjY2InwA7A7OAtwH/JakrLzMbODyH1MHAFetpa32f0HMi4oU8PBqYJenevL7qX6f8eUQsyW/g+UB3Zdq1pDDrL8w/HhF/BLw7P45aT12DcZ/qSN0/C4GdJX1L0gHAs/3Md0VErMnDWwNX5KuPs0mB0eumiHgmIl4EHuD1v78xGrgJOGmgYQ4QEf8QEZNI3Vm9VxpDtW/bcThwWX5PXw0ctr6ZI+ITwH6kbprPAxf0M+tgH1OjSL98vyMi3gHcCXw9T9ugt/tICfQXImJKfvxtpL/uuJq169+8YZnfVYY3Af64so6JEbEK0plBRFwaEUeRfv36nrzMbOBjpLBZ0N+ZRbYn8GA/06p1fBZYDuxBOovYrDLtpcrwGta+eroDOLC/S7uIeDz/uwq4lPSXMKHy5xly8G1NH5f1HTAi9k/uYun94vi0iPgNaV/cAvwN8L1+lq/Wejpwcz6r/3DD6+pvH64G7gb+z3pqrONSXr8a2KD/9Iak3YFdgRslLSKFe7NuFyLi3og4G3g/r7/WRoN9TD0FPA9ck59fQfp+A4bumGrJSAn0viwHtpO0raQxwPruZLiBSh+qpCn53/flSzYkjQV2IZ1hEhG/JO3Ymazncj6/cf+R1D3QzNbAsnzGcBTpF7Z1fDHXcm4f7Y9S/sZe0mjSdujtu54DHJOHDwV+mrtihsIGt38iYk3lQ+OLebttEhFX5Xnq3CmyNfB4Hp5WY35IZ3B/CbxNA/yPYHI3Qq+PAA/l4TnA0Ur2Bp6JiA2tu2VGRHTnxwRgoqQ+/2qgpC1z90avKcA634n0oePHVD5G/p305S2kq4YH8vBwHlNNjdhAj4hXSF92zSX1tz20ntk/A0xVur3rAeCv8vh3AvMkLSBdVn0vIn5RWW426VL/Gtb27vyl2MOkIP9MRNxUo+xzgWMk3UXql/1dk/mrTgA27+PLtjHAT/JrmE8Km1l52vnAtpIeBT4HDNn/KjVC9s9E4Jbc1XMh8Pc1XtrXgK9IuoP64UHusjkceK+kTzdOlzSb9BonS1oi6dg8aabSl7YLgA8Ax+fx15O6jB4l7e911jnMDmfd/XINr3+ZvV9+nUskLSFdRZ2kfPst6S6TaTXaGYxjCuALwIy83Y8CTszjh+2YqsM//TczK8SIPUM3M7O1OdDNzArhQDczK4QD3cysEA50M7NCONDNzArhQDczK8T/AoG8mRhZu9KQAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -1361,22 +1361,22 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 42, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 61, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF7RJREFUeJzt3X24XVVh5/HvjxcRJIKYC0ggpgpNBxgTbIwv1DbKi7yKjoxNxrGo+ESd8ogdHYpFAamttFNfWmBKI0TUkcAgDYMlCCkVgVGRAIGAQOUlSAiSQCSAvBn4zR97XTkczsm9Oeckl2T9Ps9zn7vP2mvvtfbZyW/vvfY+58o2ERFRj83GugMREbFhJfgjIiqT4I+IqEyCPyKiMgn+iIjKJPgjIiqT4I91JulkSf97rPuxqZJ0q6QZG6itxyW9bkO0FS8dCf5NlKSlkp4s/7F/KekcSduOUV/eKOmq0pcHJR3bMm+SpB9IekLS7ZL2H4s+Dkp530e9DWW/fLG1zPZetq9cD327UtJH29ra1vbdg24rXtoS/Ju2w21vC0wF9gE+u6E7IGk88H3gn4BXA7sDl7dUmQfcWOadAHxX0tCG7mdETRL8FbD9S+AymgMAAJIOlXSjpEcl3Sfp5JZ5kyRZ0lGSfiHpIUkndFq3pC0lzZN0oaSXdajy34HLbH/H9tO2H7N9W1n2d4E3AifZftL2hcAS4H1d2tpa0pcl3StptaRrJG1d5r27DJE8Us5s/0PLcksl/Q9JN0v6taSzJe0k6VJJj0n6V0mvatv22ZKWS3pA0qdb1vWCM3RJMyQtK9PfBiYC3ytXN8eV8gvKVdfqcuWzVymfDXwAOK7U/15Lf/cv01tJ+lrpy/IyvVVr25I+LWlF6euHu7x3fwW8HTi9tHV6Kbek3Vu27X+V9+VxSf9P0s6lzV+VK7J9Wta5S9nvKyXdI+mTndru0p8/l3R/ef/vkLTfaJeN/iX4KyBpV+Bg4M6W4l8DfwJsDxwKfELSe9oW/QNgMrAfcGJrmJb1bg1cBDwNvN/2Mx2afwuwStKPSjh9T9LEMm8v4G7bj7XUv6mUd/J3wO8DbwN2AI4DnisHkHnAp4AhYAFN+LYeiN4HHAD8LnA4cCnwF8B4mv8H7aH1DmAP4EDg+NEM39j+IPALypWW7b8tsy4t69oRuAH4Tqk/p0z/bal/eIfVnkDzHk4FpgDTgc+1zN8Z2A6YABwNnDF8EGvr2wnA1cAxpa1jumzG+8v6x9Ps1x+XPo8Hvgt8BUDSZsD3aPbXBJp/I5+S9K61vUdl2cnAMcCbbI8D3gUsHWm5GJwE/6btIkmPAfcBK4CThmfYvtL2EtvP2b6ZJjj/qG35L5Qz8Zto/oNPaZn3SpohnLuAD9t+tksfdgWOAo6lORu+p7QFsC2wuq3+amBc+0pK0HwEONb2/baftf0j208DfwxcYnuh7d/QHCC2pjlADDvN9oO276cJwGtt31iWn08zFNa+7b+2vQT4BjCry/aNyPbccqXzNHAyMEXSdqNc/APAKbZX2F4JfAH4YMv835T5v7G9AHic5mDdq/m2r7f9FM378pTtb5X9ez7Pv09vAoZsn2L7mXKf4OvAzFG08SywFbCnpC1tL7V9Vx99jnWU4N+0vaecUc0Afo/mrA0ASW9Wc1N1paTVwMdb5xe/bJl+giaoh70FeANwqtf+TX9P0oTJdSVMvgC8rQTf4zQHkFavBB7jxcYDL6c50LTbBbh3+IXt52gOdhNa6jzY1qf21+03vu9rmb63tLHOJG0u6VRJd0l6lOfPbNvf625esG0d+vKw7TUtr9v307oa7fv0WmCXMrT2iKRHaK6gdhqpAdt30lydnQyskHSepJ7e3+hNgr8Ctn8InENzJjzsXOBiYDfb2wFnAlqH1V4OfAm4QtLa/rPfDLQeGIanBdwKvE5S6xn+lFLe7iHgKeD1HeYtpwmiZsWSgN2A+0fYhrXZrWV6YmkDmiGybVrm7dy2XPtB8L8ARwD70wzJTBruZpf67V6wbW19WVeD/Cre+4B7bG/f8jPO9iGj6oh9ru0/oNk2A38zwL7FCBL89fgacICk4Ru844BVtp+SNJ0moNZJGcM+lyb8u53BfgN4r6SpkrYEPg9cY/sR2/8OLAZOkvRySe+luYq4sENbzwFzga+Um4qbS3prudH5f4BDJe1X2vg0zfj0j9Z1m1p8XtI25Ubsh2mGOSj9PUTSDpJ2pjlzbfUg0Ppc/LjSl4dpDhh/PUL9dvOAz0kaKu/xiUCvn6EYqa118VPg0XKTduuyP/aW9Cb47Y3njgcaSZMlvbPsu6doriS6DRXGepDgr0QZH/4WTfAC/DfglHIP4ESa8OxlvX9Jc4P3XyXt0GH+v9EMAVxCc59hd154kJkJTAN+BZwKHFn62slnaJ76uQ5YRXOWuJntO4D/CpxGc2VwOM0N1k43m0frhzQ3w68A/s728COo36a537GU5qrn/LblvkQT1I9I+gzNe34vzdXHz4CftNU/m2as+xFJF3XoxxeBRTRXTktobrR+sUO90fh74MjyhM4/9LgOAMqY/+E0N53voXnfz6K5qoHmiunHXRbfimZfP0QznLgjzb+R2ECUP8QS8TxJk2iCbMu2sfNYB5LOAi6wfdlY9yVeLMEf0SLBHzXIUE9ERGVyxh8RUZmc8UdEVGaLse5AJ+PHj/ekSZPGuhsRERuN66+//iHbo/qCw5dk8E+aNIlFixaNdTciIjYaku4duVYjQz0REZVJ8EdEVCbBHxFRmQR/RERlEvwREZUZ8akeSXOBw4AVtvcuZefz/B972B54xPbUDssupflu9WeBNbanDajfERHRo9E8znkOcDrNtwwCYPuPh6clfZkX/xWlVu+w/VCvHYyIiMEaMfhtX1W+uOpFyh+8eD/wzsF2KyIi1pd+x/jfDjxo++dd5hu4XNL1kmavbUWSZktaJGnRypXdvo49IiL61e8nd2fx/B/O7mRf28sl7QgslHS77as6VbQ9B5gDMG3atHxzXEQFJh1/yVh3Yb1ZeuqhY92Frno+45e0BfCfePFfIPot28vL7xXAfGB6r+1FRMRg9DPUsz9wu+1lnWZKesXwH9GW9ArgQOCWPtqLiIgBGDH4Jc2j+duZkyUtk3R0mTWTtmGe8kewF5SXOwHXSLqJ5g8zX2L7+4PrekRE9GI0T/XM6lL+oQ5ly4FDyvTdwJQ++xcREQOWT+5GRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFRmxOCXNFfSCkm3tJSdLOl+SYvLzyFdlj1I0h2S7pR0/CA7HhERvRnNGf85wEEdyr9qe2r5WdA+U9LmwBnAwcCewCxJe/bT2YiI6N+IwW/7KmBVD+ueDtxp+27bzwDnAUf0sJ6IiBigfsb4j5F0cxkKelWH+ROA+1peLytlHUmaLWmRpEUrV67so1sREbE2vQb/PwKvB6YCDwBf7lBHHcrcbYW259ieZnva0NBQj92KiIiR9BT8th+0/azt54Cv0wzrtFsG7NbyeldgeS/tRUTE4PQU/JJe0/LyvcAtHapdB+wh6XckvQyYCVzcS3sRETE4W4xUQdI8YAYwXtIy4CRghqSpNEM3S4GPlbq7AGfZPsT2GknHAJcBmwNzbd+6XrYiIiJGbcTgtz2rQ/HZXeouBw5peb0AeNGjnhERMXbyyd2IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyCf6IiMok+CMiKpPgj4ioTII/IqIyIwa/pLmSVki6paXsf0q6XdLNkuZL2r7LskslLZG0WNKiQXY8IiJ6M5oz/nOAg9rKFgJ7234D8O/AZ9ey/DtsT7U9rbcuRkTEII0Y/LavAla1lV1ue015+RNg1/XQt4iIWA8GMcb/EeDSLvMMXC7pekmzB9BWRET0aYt+FpZ0ArAG+E6XKvvaXi5pR2ChpNvLFUSndc0GZgNMnDixn25FRMRa9HzGL+ko4DDgA7bdqY7t5eX3CmA+ML3b+mzPsT3N9rShoaFeuxURESPoKfglHQT8OfBu2090qfMKSeOGp4EDgVs61Y2IiA1nNI9zzgN+DEyWtEzS0cDpwDia4ZvFks4sdXeRtKAsuhNwjaSbgJ8Cl9j+/nrZioiIGLURx/htz+pQfHaXusuBQ8r03cCUvnoXEREDl0/uRkRUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUJsEfEVGZBH9ERGUS/BERlUnwR0RUZlTBL2mupBWSbmkp20HSQkk/L79f1WXZo0qdn0s6alAdj4iI3oz2jP8c4KC2suOBK2zvAVxRXr+ApB2Ak4A3A9OBk7odICIiYsMYVfDbvgpY1VZ8BPDNMv1N4D0dFn0XsND2Ktu/Ahby4gNIRERsQP2M8e9k+wGA8nvHDnUmAPe1vF5Wyl5E0mxJiyQtWrlyZR/dioiItVnfN3fVocydKtqeY3ua7WlDQ0PruVsREfXqJ/gflPQagPJ7RYc6y4DdWl7vCizvo82IiOhTP8F/MTD8lM5RwP/tUOcy4EBJryo3dQ8sZRERMUZG+zjnPODHwGRJyyQdDZwKHCDp58AB5TWSpkk6C8D2KuAvgevKzymlLCIixsgWo6lke1aXWft1qLsI+GjL67nA3J56FxERA5dP7kZEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVKbn4Jc0WdLilp9HJX2qrc4MSatb6pzYf5cjIqIfW/S6oO07gKkAkjYH7gfmd6h6te3Dem0nIiIGa1BDPfsBd9m+d0Dri4iI9WRQwT8TmNdl3lsl3STpUkl7dVuBpNmSFklatHLlygF1KyIi2vUd/JJeBrwbuKDD7BuA19qeApwGXNRtPbbn2J5me9rQ0FC/3YqIiC4GccZ/MHCD7QfbZ9h+1PbjZXoBsKWk8QNoMyIiejSI4J9Fl2EeSTtLUpmeXtp7eABtRkREj3p+qgdA0jbAAcDHWso+DmD7TOBI4BOS1gBPAjNtu582IyKiP30Fv+0ngFe3lZ3ZMn06cHo/bURExGDlk7sREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERl+g5+SUslLZG0WNKiDvMl6R8k3SnpZklv7LfNiIjo3RYDWs87bD/UZd7BwB7l583AP5bfERExBjbEUM8RwLfc+AmwvaTXbIB2IyKig0EEv4HLJV0vaXaH+ROA+1peLytlERExBgYx1LOv7eWSdgQWSrrd9lUt89VhGbcXlIPGbICJEycOoFsREdFJ32f8tpeX3yuA+cD0tirLgN1aXu8KLO+wnjm2p9meNjQ01G+3IiKii76CX9IrJI0bngYOBG5pq3Yx8Cfl6Z63AKttP9BPuxER0bt+h3p2AuZLGl7Xuba/L+njALbPBBYAhwB3Ak8AH+6zzYiI6ENfwW/7bmBKh/IzW6YN/Gk/7URExODkk7sREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlBvU3dzdak46/ZKy7sF4tPfXQse5CRLzEVB/8sXHLgTti3WWoJyKiMgn+iIjKJPgjIiqT4I+IqEyCPyKiMgn+iIjKJPgjIirTc/BL2k3SDyTdJulWScd2qDND0mpJi8vPif11NyIi+tXPB7jWAJ+2fYOkccD1khba/llbvattH9ZHOxERMUA9n/HbfsD2DWX6MeA2YMKgOhYREevHQMb4JU0C9gGu7TD7rZJuknSppL3Wso7ZkhZJWrRy5cpBdCsiIjroO/glbQtcCHzK9qNts28AXmt7CnAacFG39dieY3ua7WlDQ0P9disiIrroK/glbUkT+t+x/c/t820/avvxMr0A2FLS+H7ajIiI/vTzVI+As4HbbH+lS52dSz0kTS/tPdxrmxER0b9+nurZF/ggsETS4lL2F8BEANtnAkcCn5C0BngSmGnbfbQZERF96jn4bV8DaIQ6pwOn99pGREQMXj65GxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmQR/RERlEvwREZVJ8EdEVCbBHxFRmb6CX9JBku6QdKek4zvM30rS+WX+tZIm9dNeRET0r+fgl7Q5cAZwMLAnMEvSnm3VjgZ+ZXt34KvA3/TaXkREDEY/Z/zTgTtt3237GeA84Ii2OkcA3yzT3wX2k6Q+2oyIiD5t0ceyE4D7Wl4vA97crY7tNZJWA68GHmpfmaTZwOzy8nFJd/TRt5ey8XTY/vVFucYatOy/jdsG239jsO9eO9qK/QR/pzN391CnKbTnAHP66M9GQdIi29PGuh/Rm+y/jVv2X6OfoZ5lwG4tr3cFlnerI2kLYDtgVR9tRkREn/oJ/uuAPST9jqSXATOBi9vqXAwcVaaPBP7Ndscz/oiI2DB6HuopY/bHAJcBmwNzbd8q6RRgke2LgbOBb0u6k+ZMf+YgOr2R2+SHszZx2X8bt+w/QDkBj4ioSz65GxFRmQR/RERlEvwjkPSspMWSbpF0gaRtBrTenST9i6SbJP1M0oJSfo+kyW11vybpOEkzJK2WdGP5qoyrJB02iP5sCrKv6iXpR22v/0zSU5K2aymbIelfOix7WNlPw/v3Yxuiz2MpwT+yJ21Ptb038Azw8dEuWB5h7eYUYKHtKbb3BIa/6+g8Wm6CS9qM5omo80vR1bb3sT0Z+CRwuqT9Rr85m7Tsq0rZfltb0SyaJw/fu7blJG1Jc8P3cNtTgH2AK9dHH19KEvzr5mpgd0mTJN0yXCjpM5JOLtNXSvprST8EjpU0JOlCSdeVn33LYq+h+ZwDALZvLpPzeOHTT38ILLV9b3tnbC+mCaVjBriNm4qNZl9J2lzSOeVKZYmkP+tryysk6fGW6dcD2wKfozkArM04mqcbHwaw/bTtTfVbA34rwT9K5YzwYGDJKKpvb/uPbH8Z+Hvgq7bfBLwPOKvUOQM4W9IPJJ0gaRf4bag8J2lKqTeTJmC6uQH4vXXfok3XRrivpgITbO9t+z8C3xhFv6O7WTT74WpgsqQdu1W0vYrm80b3Spon6QPlym2Ttslv4ABsLWkxsAj4Bc1nE0Zyfsv0/jSX+Itp/oG9UtI425cBrwO+ThMGN0oaKsvMA2aWADsCuGAtbeVL7563se6ru4HXSTpN0kHAo6Pod3Q3EzjP9nPAPwP/eW2VbX8U2A/4KfAZYO567+EY6+e7emrxpO2prQWS1vDCg+bL25b5dcv0ZsBbbT/ZvuJytnEucG656fSHwIU0YXI58EPgZtsr1tK/fYDbRrktm7qNYl+p+Urz60vZxbZPLFcN7wL+FHg/8JG1bml0JOkNwB7AQjVfBPwymgPrGWtbzvYSYImkbwP3AB9avz0dWznj782DwI6SXi1pK2BtT2tcTsu4rqSp5fc7h586kTQOeD3NWSq276IZczyVtQwdlH/kn2eEf9SVe8ntK9vPlpvQU0vojwc2s31hqfPG3je3erOAk21PKj+7ABMkdfzmSknbSprRUjQVeNE9mk1Nzvh7YPs3ar6a4lqas4Pb11L9k8AZkm6meb+vonna5PdphhWGz0jPsn1dy3LzgC8B89vW93ZJNwLbACuAT9q+YgCbtUnaSPbVBOAbLWPLn12XbYwXmElzf6fV/FJ+Lc3fBFnWMm8WcJykfwKepLkC/NAG6OeYylc2RERUJkM9ERGVSfBHRFQmwR8RUZkEf0REZRL8ERGVSfBHRFQmwR8RUZn/D4l8dOyzlPZ0AAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEv1JREFUeJzt3Xu4ZXVdx/H3h4uIMaI4h6vYJCCKFgOO5KWMQhMkQh8vQWZk9kwXecTSiMILkiX1pFlKFxTETEYlQkE0IBLRNGMQZCAkFQa5yQwiNwMU+PbHWkc3m3O/zOH8zvv1POc5a6/1W+v3XXud+ey1fmvvPakqJEmL32YLXYAkaW4Y6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQ9UNJjkvyzwtdR6uSXJlk/03U191Jnrwp+tIjh4G+yCRZn+Se/h/st5OcmmSbBapl3yQX9bXckuSogWUrknw2yf8l+VqSFyxEjXOlf96nvA/9cXnH4LyqenpVXTgPtV2Y5LeG+tqmqq6Z6770yGagL06HVNU2wEpgH+CPN3UBSZYD/wb8I/AEYHfgvIEma4BL+2XHAv+SZGRT1yktJQb6IlZV3wbOpQt2AJIcnOTSJHcmuT7JcQPLViSpJEck+VaSW5McO9a2k2yZZE2SM5I8aowmfwCcW1Ufqar7ququqrqqX/cpwL7A26rqnqo6A1gHvGycvrZO8q4k1yW5I8kXkmzdL/vlfqji9v5M9GkD661P8odJLk/yvSQnJ9khyWeS3JXk35M8fmjfVye5KcnNSd44sK2HnFEn2T/JDf30h4EnAWf3VyNH9/NP76+S7uivVJ7ez18NvAo4um9/9kC9L+int0rynr6Wm/rprQb7TvLGJBv6Wl8zznP3Z8DPAu/r+3pfP7+S7D6wb3/XPy93J/nPJDv2fX63v4LaZ2CbO/fHfWOSa5O8fqy+x6nnj5Lc2D//Vyc5YKrravYM9EUsyROBg4BvDMz+HvDrwOOAg4HfTfKSoVV/BtgTOAB462BI9tvdGvgEcB/wyqr6/hjdPxu4LckX+9A5O8mT+mVPB66pqrsG2n+1nz+WvwKeCTwX2A44Gniwf2FYA7wBGAE+TReqgy8wLwNeCDwFOAT4DPAnwHK6v+/hMPp5YA/gF4FjpjKMUlWvBr5Ff2VUVX/ZL/pMv63tga8AH+nbn9RP/2Xf/pAxNnss3XO4Etgb2A9488DyHYFtgV2A1wInjr44DdV2LPB54Mi+ryPH2Y1X9ttfTndcv9TXvBz4F+DdAEk2A86mO1670P2NvCHJiyZ6jvp19wSOBJ5VVcuAFwHrJ1tPc8dAX5w+keQu4HpgA/C20QVVdWFVrauqB6vqcrpA/Lmh9d/enzl/le4f7t4Dyx5LN5TyTeA1VfXAODU8ETgCOIru7PXavi+AbYA7htrfASwb3kgfIL8JHFVVN1bVA1X1xaq6D/gV4JyqOr+qfkAX/FvTBf+o91bVLVV1I12wfbmqLu3XP5NuSGp4379XVeuADwKHj7N/k6qqU/ork/uA44C9k2w7xdVfBRxfVRuqaiPwduDVA8t/0C//QVV9Grib7kV4ps6sqkuq6l665+Xeqvqn/vh+jB89T88CRqrq+Kr6fj8O/37gsCn08QCwFbBXki2ran1VfXMWNWuaDPTF6SX9GdD+wFPpzrIASPLT6W5GbkxyB/A7g8t73x6Y/j+6AB71bOCngBNq4m9uu4cuJC7uQ+LtwHP7QLub7oVh0GOBu3i45cCj6V5Ahu0MXDf6oKoepHsR22WgzS1DNQ0/Hr5hfP3A9HV9H9OWZPMkJyT5ZpI7+dGZ6PBzPZ6H7NsYtXynqu4feDx8nKZrqs/TjwM790Nctye5ne6KZ4fJOqiqb9BdTR0HbEjy0SQzen41Mwb6IlZVnwNOpTtzHXUacBawa1VtC/wDkGls9jzgncAFSSb6R3w5MBj4o9MBrgSenGTwjHzvfv6wW4F7gd3GWHYTXcB0G04C7ArcOMk+TGTXgekn9X1AN1T1mIFlOw6tN/zi9qvAocAL6IZGVoyWOU77YQ/Zt6FapmsuvzL1euDaqnrcwM+yqnrxlAqpOq2qfoZu3wr4izmsTZMw0Be/9wAvTDJ6Y3QZcFtV3ZtkP7rgmZZ+jPg0ulAf74zzg8BLk6xMsiXwFuALVXV7Vf0vcBnwtiSPTvJSurP+M8bo60HgFODd/c24zZM8p79B+HHg4CQH9H28kW7894vT3acBb0nymP4G5mvohhvo631xku2S7Eh3pjnoFmDwfd3L+lq+Q/dC8OeTtB+2BnhzkpH+OX4rMNPPAEzW13T8N3Bnf3Nz6/54PCPJs+CHN2zHfAFJsmeSX+iP3b10Z/7jDdlpHhjoi1w//vpPdIEK8HvA8f0Y+1vpQnEm2/1Tuhuj/55kuzGW/wfdpfg5dOP4u/PQF4/DgFXAd4ETgJf3tY7lTXTvgrkYuI3urG6zqroa+DXgvXRn8ofQ3Zgc6ybtVH2O7ibyBcBfVdXoWy0/THc/YT3dVcrHhtZ7J10A357kTXTP+XV0Vwv/A/zXUPuT6caSb0/yiTHqeAewlu5KZx3dDcp3jNFuKv4GeHn/jpW/neE2AOjH1A+hu1l7Ld3z/gG6qxDornC+NM7qW9Ed61vphvW2p/sb0SYS/4MLLQVJVtAF1JZDY9OahiQfAE6vqnMXuhY9nIGuJcFA11LgkIskNcIzdElqhGfoktSILTZlZ8uXL68VK1Zsyi4ladG75JJLbq2qSb/cbpMG+ooVK1i7du2m7FKSFr0k103eyiEXSWqGgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqxCb9pKikpWHFMecsdAnzav0JBy90CWPyDF2SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktSISQM9ya5JPpvkqiRXJjmqn79dkvOTfL3//fj5L1eSNJ6pnKHfD7yxqp4GPBt4XZK9gGOAC6pqD+CC/rEkaYFMGuhVdXNVfaWfvgu4CtgFOBT4UN/sQ8BL5qtISdLkpjWGnmQFsA/wZWCHqroZutAHth9nndVJ1iZZu3HjxtlVK0ka15QDPck2wBnAG6rqzqmuV1UnVdWqqlo1MjIykxolSVMwpUBPsiVdmH+kqv61n31Lkp365TsBG+anREnSVEzlXS4BTgauqqp3Dyw6Cziinz4C+OTclydJmqotptDmecCrgXVJLuvn/QlwAvDxJK8FvgW8Yn5KlCRNxaSBXlVfADLO4gPmthxJ0kz5SVFJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNWLSQE9ySpINSa4YmHdckhuTXNb/vHh+y5QkTWYqZ+inAgeOMf+vq2pl//PpuS1LkjRdkwZ6VV0E3LYJapEkzcJsxtCPTHJ5PyTz+PEaJVmdZG2StRs3bpxFd5Kkicw00P8e2A1YCdwMvGu8hlV1UlWtqqpVIyMjM+xOkjSZGQV6Vd1SVQ9U1YPA+4H95rYsSdJ0zSjQk+w08PClwBXjtZUkbRpbTNYgyRpgf2B5khuAtwH7J1kJFLAe+O15rFGSNAWTBnpVHT7G7JPnoRZJ0iz4SVFJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRkwZ6klOSbEhyxcC87ZKcn+Tr/e/Hz2+ZkqTJTOUM/VTgwKF5xwAXVNUewAX9Y0nSApo00KvqIuC2odmHAh/qpz8EvGSO65IkTdNMx9B3qKqbAfrf289dSZKkmZj3m6JJVidZm2Ttxo0b57s7SVqyZhrotyTZCaD/vWG8hlV1UlWtqqpVIyMjM+xOkjSZmQb6WcAR/fQRwCfnphxJ0kxN5W2La4AvAXsmuSHJa4ETgBcm+Trwwv6xJGkBbTFZg6o6fJxFB8xxLZKkWfCTopLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqxBazWTnJeuAu4AHg/qpaNRdFSZKmb1aB3vv5qrp1DrYjSZoFh1wkqRGzDfQCzktySZLVYzVIsjrJ2iRrN27cOMvuJEnjmW2gP6+q9gUOAl6X5PnDDarqpKpaVVWrRkZGZtmdJGk8swr0qrqp/70BOBPYby6KkiRN34wDPcmPJVk2Og38InDFXBUmSZqe2bzLZQfgzCSj2zmtqv5tTqqSJE3bjAO9qq4B9p7DWiRJs+DbFiWpEQa6JDViLj4p+oi14phzFrqEebX+hIMXugRJjyCeoUtSIwx0SWqEgS5JjTDQJakRTd8U1eLlDW1p+jxDl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiNmFehJDkxydZJvJDlmroqSJE3fjAM9yebAicBBwF7A4Un2mqvCJEnTM5sz9P2Ab1TVNVX1feCjwKFzU5Ykabq2mMW6uwDXDzy+Afjp4UZJVgOr+4d3J7l6Fn0+0i0Hbt1UneUvNlVPS4LHbnFr/fj9+FQazSbQM8a8etiMqpOAk2bRz6KRZG1VrVroOjR9HrvFzePXmc2Qyw3ArgOPnwjcNLtyJEkzNZtAvxjYI8lPJHkUcBhw1tyUJUmarhkPuVTV/UmOBM4FNgdOqaor56yyxWlJDC01ymO3uHn8gFQ9bNhbkrQI+UlRSWqEgS5JjViygZ7kgSSXJbkiyelJHjNH290hyaeSfDXJ/yT5dD//2iR7DrV9T5Kjk+yf5I4kl/ZfpXBRkl+ai3pa4LFaupJ8cejx7ye5N8m2A/P2T/KpMdb9pf44jR7f394UNS+kJRvowD1VtbKqngF8H/idqa6YZKKbyccD51fV3lW1FzD6HTcfpXsn0Og2NgNeDnysn/X5qtqnqvYEXg+8L8kBU9+dpnmslqiqeu7QrMPp3mH30onWS7Il3Y3SQ6pqb2Af4ML5qPGRZCkH+qDPA7snWZHkitGZSd6U5Lh++sIkf57kc8BRSUaSnJHk4v7nef1qO9G9Rx+Aqrq8n1zDQEgAzwfWV9V1w8VU1WV0YXPkHO5jKxbNsUqyeZJT+yuLdUl+f1Z7vgQluXtgejdgG+DNdME+kWV07+L7DkBV3VdVLX9KHTDQR8/gDgLWTaH546rq56rqXcDfAH9dVc8CXgZ8oG9zInByks8mOTbJzvDDsHgwyd59u8PogmM8XwGeOv09atciPFYrgV2q6hlV9ZPAB6dQt8Z3ON1x+DywZ5Ltx2tYVbfRfS7muiRrkryqv9JqWvM7OIGtk1wGrAW+BZw8hXU+NjD9ArpL7cvo/nAem2RZVZ0LPBl4P90/8kuTjPTrrAEO64PpUOD0Cfoa66sVlqrFeqyuAZ6c5L1JDgTunELdGt9hwEer6kHgX4FXTNS4qn4LOAD4b+BNwCnzXuECm813uSx291TVysEZSe7noS9yjx5a53sD05sBz6mqe4Y33J8dnAac1t+seT5wBl1InAd8Dri8qjZMUN8+wFVT3JfWLYpjle4rpS/p551VVW/tz/JfBLwOeCXwmxPuqcaU5KeAPYDzkwA8iu4F88SJ1quqdcC6JB8GrgV+Y34rXVhL+Qx9LLcA2yd5QpKtgInevXAeA+OmSVb2v39h9F0YSZYBu9GdVVJV36Qb0zuBCS7h+z/etzDJH+sS94g7VlX1QH/zdmUf5suBzarqjL7NvjPf3SXvcOC4qlrR/+wM7JJkzG8hTLJNkv0HZq0EHnYPpDVL+Qz9YarqB0mOB75M92r+tQmavx44McnldM/jRXTvvngm3eX96BnkB6rq4oH11gDvBM4c2t7PJrkUeAywAXh9VV0wB7vVpEVyrHYBPjgwdvvH09lHPcRhdPdPBp3Zz/8ycECSGwaWHQ4cneQfgXvorth+YxPUuaD86L8kNcIhF0lqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGvH/+paWIR7cDusAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -1399,7 +1399,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "By all means computing SVD on this dataset is much faster than ALS. This may, however, vary on other datasets due to a different sparsity structure. Nevertheless, you can still expect, that SVD-based models will be perfroming well due the usage of to highly optimized BLAS and LAPACK routines." + "By all means computing SVD on this dataset is much faster than ALS. This may, however, vary on other datasets due to a different sparsity structure. Nevertheless, you can still expect, that SVD-based models will be perfroming well due to the usage of highly optimized BLAS and LAPACK routines." ] }, { @@ -1418,7 +1418,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ @@ -1427,7 +1427,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -1436,7 +1436,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -1459,7 +1459,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 46, "metadata": {}, "outputs": [ { @@ -1469,11 +1469,11 @@ "\n", "100%\n", "60/60\n", - "[22:42<00:24, 22.70s/it]" + "[26:27<00:30, 26.45s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 60/60 [22:42<00:24, 22.70s/it]" + " [████████████████████████████████████████████████████████████] 60/60 [26:27<00:30, 26.45s/it]" ] }, "metadata": {}, @@ -1493,7 +1493,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 47, "metadata": {}, "outputs": [ { @@ -1503,11 +1503,11 @@ "\n", "100%\n", "15/15\n", - "[07:03<00:39, 28.18s/it]" + "[09:09<00:46, 36.58s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 15/15 [07:03<00:39, 28.18s/it]" + " [████████████████████████████████████████████████████████████] 15/15 [09:09<00:46, 36.58s/it]" ] }, "metadata": {}, @@ -1535,12 +1535,12 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 48, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XmcVNWd///Xp/bqfe8GGuim2ZcWsN3ABWPAJUTjkgF01BgdkxlNJs6q80uMMd9Ekzjf5OsXv2OcxCwmkSRmnBATYlAHSVwB2TeBBrsb6H3vrr3O7497u6luummgKZrl83w86lF3OffWuQVd7zrn3jpXjDEopZRSJ8sx0hVQSil1dtMgUUopNSwaJEoppYZFg0QppdSwaJAopZQaFg0SpZRSw6JBotRpICJrROS+k9x2nIh0iojzVNdrJIjIARH5uD39byLyg5GukxoeDRJ1StgfDgH7A69WRH4sImkjXa+zUeIHLYAxpsoYk2aMiY1kvZLBGPNNY8x9ACJSIiJGRFwjXS91YjRI1Kn0SWNMGjAbmAM8MsL1UUqdBhok6pQzxtQCr2IFCgAi4hWRp0SkSkTqRORZEfEnrL9JRDaJSLuI7BOR6+zlo0VkpYg0i8heEfmbhG0eE5Ffi8jPRKRDRLaKyGQReURE6kWkWkQWJZRfIyL/S0TetltOvxORXBH5uf2660SkJKH8VBFZbb/2bhH5q4R1PxaRZ0Tk9/ZrvyciZQnrF4rILhFpE5HlgCSsKxORN0SkSUQa7dfPste9AIwDfmfX8V/6f1M/jvfkVyLyU7te20WkImH9v4rIQXvdbhG5ZqB/Q/t9WWm/L++LyNdF5C/2uqNaDoldd8c6vgFe5zER+Zk9u9Z+brWP/Sr7GGcllC+wW775A+1PjQwNEnXKiUgxcD2wN2Hxt4DJWOEyERgDPGqXvxj4KfDPQBZwJXDA3u5FoAYYDdwGfLPfh98ngReAbGAjVoA57P0/Dny/X/WWAnfa68uAd4AfATnATuCrdp1SgdXAL4ACYBnw/0RkRsK+lgFfs197L/ANe9s84DfAl4E8YB8wP/EtAp6wj2kaMBZ4DMAYcydQhd26M8Z8m6MN9Z7cCKyw38uVwHK7XlOAB4GLjDHpwLUceZ/7ewYIAqOAz9qP4zXo8Q3hSvs5yz72N+3j+OuEMsuA14wxDSdQH5Vsxhh96GPYD6wPpE6gAzDA61gfCGB9sHQBZQnlLwP229PfB747wD7HAjEgPWHZE8CP7enHgNUJ6z5p18Fpz6fbdempxxrg/0so/+/Aqn7bb7KnlwB/7lef7wNftad/DPwgYd0NwC57+i7g3YR1gvXBf98g792ngI393suPJ8yX2MfhOs735LWEddOBgD09EagHPg64j/Fv6QQiwNSEZd8E/tK/Pgnr15zM8dn1/dkx9nsJUA047Pn1wF+N9P93ffR9aItEnUqfMtY33QXAVKxv4wD5QAqwQURaRaQV+KO9HKwPx30D7G800GyM6UhY9hFWa6JHXcJ0AGg0R05KB+zntGOU7z/fU3Y8cElPfe063wEUJZSvTZjuTth2NNaHHwDG+gTsnbe7Z1bYXUztwM848l4N5Xjek/718omIyxizF/gS1od3vV2H0QO8Rj5WaFUnLPvoOOs33OPrwxjzHtaXkKtEZCpWGK48mX2p5NEgUaecsbokfgw8ZS9qxPqQnmGMybIfmcY6MQ/WB1bZ0XviEJAjIukJy8YBB5NT8z6qgTcT6tvT3fK3x7HtYaxwBEBEJHEeqwVhgHJjTAZW140krD/WkNzDek+MMb8wxlyOFZQGq8uxvwYg2q/O4xKmu+znlIRliQE71PENWr1Blv/E3sedwEvGmOBx7EudRhokKlm+BywUkdnGmDjwn8B3RaQAQETGiMi1dtkfAveIyDUi4rDXTTXGVANvA0+IiE9EyoF7gZ+fhvq/AkwWkTtFxG0/LhKRacex7e+BGSJyi31C+ov0/aBNx+qCaxWRMVjnhhLVARMG2vFw3hMRmSIiHxMRL9b5jwBWN1n/14gB/wU8JiIpIjIduDthfQNWcP21iDhF5LP0/SIw1PENpgGIc/SxvwDcjBUmPz3OfanTSINEJYX9YfNT4Cv2on/FOiH9rt3d8RowxS77PnAP8F2gDXgT6xszWCdXS7C+ib+MdY5i9WmofwewCOvk/CGs7qJvAd7j2LYR+DTwJNAETALeSijyNWAu1rH+HutDO9ETwJftLrV/GuAlTvY98dp1arSPpwD4t0HKPojVVVeL1br8Ub/1f4MVEE3ADKxw6zHU8Q3IGNONdcHCW/axX2ovrwE+wGqx/Pl49qVOL7G6b5VSanAi8hmsk+mXj9DrPw8cMsZ8eSReXx2b/oJUKXVGE+u3Pbdg/chVnYG0a0spdcYSka8D24DvGGP2j3R91MC0a0sppdSwaItEKaXUsJwX50jy8vJMSUnJSFdDKaXOKhs2bGg0xgw5rtl5ESQlJSWsX79+pKuhlFJnFRE5rhENzosgUUqp08kYQ3NXGBHB73bidTlwOI7nx/1nJw0SpZQ6QfG4ob4jxMHWbmpaAgmPbg62BDjYGiAUjffZxuty4HM78bud+D1WuPg9Tnwua97vduJ1O6z1bqdVNqFc7zK7XIrHRW6qh/x0Lz73yN48U4NEKaX6icUNde1BaloCVlg020HRagXFodYg4VjfoMhN9VCc7WfaqAw+Pr2QUZk+HCIEIjEC4RjBaIxgOEYwEreWRWIE7UdDR7S3XChqPQciMeLHeVFths9FQYaP/DQvBRleCtK95Kd7KUj3UZBuLctP85Hhd2EN/XZqaZAopc55xhjCsTjhaJxQ9Mhzb1jYrYmesDjcGiTa71M8P93LmCw/M8dkct3MUYzJ9lOc7Wdstp/RWX5SPKf249QYQyRmCERihOzgscInbgdNlMaOMA2dIerbg9R3hGjoCLGxqpX6jiDBSPyofXpdDvJ7Q+ZI0OSn9wSQj/x0L7mpHlzO47+oN6lBItZd7v4P1v0NfmCMebLfei/WeEwXYo3Zs8QYc8C+0dFzPcWAx4wxL9vbHMC650UMiBpjKlBKnfVicUNjZ4jDbUFq2wIcbgvSGYwSjvX98A9FY4SjfUPBKjPA8mickB0gxyICBeleirNTmDsumzHlfoqzUyjO9jMm28+YLP9p7z4SETwuweNygN99QtsaY+gIRWnoCFHfHqK+I0iDHTQ9gbO/sYv39jfT2h05anuHQE7qkMPK9UpakIiIE+suawuxbuqzTkRWGmN2JBS7F2gxxkwUkaVYg+Itwfola4UxJioio4DNIvI7Y0zU3u5qe2A8pdRZIBqLU9/RExJBDrcFrOf2IIdbrem6jhCxAfpyHAIelwOP04HX7bSeXQ48riPPPreDTL8bj7Pvck9vOetcg7UPR2+5gnQfxdl+RmX58LpG9jzDqSQiZPjcZPjclOWnHbNsKBqjsTNMfXuwN2issAmy4ThfL5ktkouBvcaYSgARWQHcBCQGyU0cuQXnS8ByERF7FNAePo59fwal1AgK211Ete3BPq0JKzCs0GjoCB3V3+9zOxid6aco08dlZXmMyvRRlOnrfS7K8JHpd59QF4s6cV6XkzFZVqurvycHKD+QZAbJGPreYa0G67aZA5axWx9tQC7QKCKXAM9jDSd+Z0JrxAB/EhEDfN8Y8xxKqaQKRmIcaOqisqGL/Y1d7GvopLKhi4OtARo7Q/QfaSnV42RUlp9RmT4mFeTb4WB98x+V6WNUhj9pJ37V6ZfMIBnof0j/lsWgZexbbM6wbyT0ExFZZd8Zbb4x5pB9g6TVIrLLGLP2qBcXuR+4H2DcuHH9Vyul+onHDbXtQSobuqhs7LSfu6hs6ORga6BPWBRl+JiQn8rHphT0hkNRphUcozJ9pPtOrE9fnd2SGSQ19L1VZzHWjXgGKlNj30kuE2hOLGCM2SkiXcBMYL0x5pC9vF5EXsbqQjsqSOyWynMAFRUV2jWmlK0jGGF/o9W6qGzoZJ89faCxi0DkyA0TUz1OSvNTmTsum9suLGZCfhoT8lIpzUsl1asXfKojkvm/YR0wSURKsW7LuRS4vV+ZlVi38HwHuA14wxhj7G2q7e6u8Vh30jsgIqmAwxjTYU8vAh5P4jEodVbqCEaobQtS1dxtd0VZoVHZ2EVDR6i3nENgbE4KpXmpXDYhlwn5qUzIT6UsP42CdK92PanjkrQgsUPgQeBVrMt/nzfGbBeRx7FaFiux7tX9gojsxWqJLLU3vxx4WEQiWPdw/jtjTKOITABetv9zu4BfGGP+mKxjUOpM03OJbG2bdXK7zj7BXWfP17Zb013hvrdiz05xMyE/jQWT8ynNT2VCXhpl+amMy005p65WUiPjvLgfSUVFhdFBG9WZLhCOWWHQFqS2PUBtW8i6GiohNOoHuETW5RAKM3wUZngpyvRRmGFd8VSUaV3aOiEvjexUzwgdlTqbiciG4/mtnnZ0KnUaNXWG+LCukz31Heyp6+Sj5m5q7d9UtAejR5VP97ootC+FLbMvke2ZL8rwUZjpJS/Ve04PCKjOfBokSiVBS1eYD+s6+LC+kz11HXxYZwVHU1e4t0y610VJXirjc1O5dEJun5ZEof2cpie11VlA/5cqNQyt3WE+rOu0g6LDbm100th55IR2mtfFxII0rplWwOTCdCYVpjO5MI2iDJ+ezFbnBA0SpY5DW3eEPfUdR0LDnk68AirV42RiYTpXT8m3AyONyYXpjMrUwFDnNg0SpfoxxrCrtoPVO+p4f38zH9Z1UJ8QGCkeJ5MK0rhyUj6T7bCYVJjGmCy/BoY6L2mQKIU1qOD6j1r40/Y6Vu+spbo5gAhMK8rg8kl5TLa7oyYVpDMmy68nt5VKoEGizlvd4ShrP2zkTztqeWNXPa3dETwuB/PLcvm7BRO5ZloBBem+ka6mUmc8DRJ1XmnsDPH6zjr+tL2Ov+xtJBSNk+l387GpBSyaXsiVk/N1+A+lTpD+xahzXmVDJ6t31LF6Rx0bqlowBsZk+Vl28TgWzSjkopIc3DpUuVInTYNEnXPiccOmmtbe8Nhb3wnAjNEZ/P01k1g4vZDpozL0xLhSp4gGiTonhKIx3t7XxJ+21/HazjoaOkI4HcKlE3L460vG8fHphRRnp4x0NZU6J2mQqLNSNNBOw4fvU11Tza+aSli1N0RXOEaqx8mCKQUsnF7I1VMKyEzR+2IolWwaJOqMZYyhoSNEZUMHTfu3YmrWkd60ieKuHZTEPmKUGEYBc3Hwd+kXEJt8PeMuuxVvwcSRrrpS5xUNEjXi2oMRDjR29d43o7G2Gl/9RoratzIjvodyRyXpEgCgg1Sq/NN4L+djxEZfyKjCQspa32XCh6tg4zetR/5UmHI9TLkBxlwIDh0mXalk0mHk1WkRisaobu7uved3zx36DjY0U9j9IXMce5nj2Mtsx16KpRGAGE5aMyYTKpyLr/RisibOw5E3ERyDXGHVcgB2/xF2/wE+egviUUjNh8nXWqEy4Wrw6HkSpY7X8Q4jr0GiTrl43LDjcDtvftjAugPN7G/sorq5m7gxlEots2Uvl3krudBZSUm0EifWTZgi6cU4iitwjr0Iiitg1AXg9p9cJQItsPd1K1T2rIZQO7h8VphMuR4mXwfphafwqJU692iQJNAgSb6GjhB/3l3L5ne30bBxG/kNVUxqq6Es1IA3BdJSQqR5WvF6A7j9MVzpPlyl03BNvgjHhEthTEXyPtijYah6G3avgl1/gLYqa/mYiiNdYAXTQC8HVqoPDZIEGiSnXrC9g81rN7D/3U107dhJzuEDlLQfxheLWAUc4M2I4k6NEAs7iYZ9RLvARONH7cuRno6roABXQT6u/ALIyyaWm0kkO41gTgqBTD+dGW66JExXpIvuSDddUevZKU7K88uZUzCHXH/u0BU3Bup3WC2V3aswNRvAQDxtHKbkGszYK4nnzcREY8SDQRypqXgnTkQG605T6hymd0hUp4QxhmhdHcEdO6nduJX6jVth3x6yWurJwHABEPL6cBamkT3eRZqnHl92BBk3it2Tr2Jv7ng60/LoMlG6w12E21qhsQlHYxuulnY8zV34WwOktB0kveoAGTti5HSAy84bB5BqPzp84EqHWJrQnQbxDDdtvjhrI3HejRrynJmMcuVS4Mwi15GOP+bEhEKYUIh4KIQJBo9M9y4rhngciAKv2o++nHl5pM67jLT580mdNw9Xfv7pevuVOitoi0T1MuEwof37Ce7cSWjXbrp27KB75y6cHe29ZQ6l5lKXV0zu+Awmjuqg2LUdd/QjDrpdbB41ha05Y9gqUXZ1fEQkHumzf6/TS6o7lRRXCqnuVGvanTDds9yZQkZISG+PkNoawt8axNvSjaulHWdjG9LUQryhiWhjI8Ss8ytGIOp2EHQawi5D2AUxjwuPLxV/agZp6bmkpWbj9PkRrxfxeXF4fYjXi8PnRbw+xAWOtv1I03akfjOOWDvRkJuuznF0VYWJtXdbxzFlCqnz55M6fx4pF16Iw6cDO6pzk3ZtJdAgOZoxhtCePXS/+y7BHTsJ7t5NaM8eiFr3DY843ezPKGJfxmhqcorJmV7KlRO7mWs+IFb9BtsIssWXwtbsQrY54rTErMtz/S4/03OnU55Xzqz8WUzNmUqmN5MUVwoux6ltAJtYjHgggMPjAbcbESFu4uxv288H9R/wQd0HbKzfyMHOg711uyD/AuYWzmVuwVxm5c0ixT3IVVzxGNSsg52/g+3/jWmrIdSeQmd4Ol11XgK7azCRCOL1klJRYQfLfLyTJ+nQK+qccUYEiYhcB/wfwAn8wBjzZL/1XuCnwIVAE7DEGHNARC4GnuspBjxmjHn5ePY5EA0SS7y7m65336Nz7Zt0rl1L9NBhAKKZ2RzKG8tmbwHbUwqpzBxN1qQJXF8qfNy1gXDja2xr3sZWt5Mt/hQOuKzzBYIwIXMCs/JnUZ5fTnleOWVZZac8MIartquWTfWb2FC3gY31G/mw5UMMBqc4mZYzrTdYZhfMHvg8SzwOB9fD9pdh+39DxyHixku362K6Wgvo3FlLuHI/AK78fFLnzSP18stJnXcZrtzjOG+j1BlqxINERJzAh8BCoAZYBywzxuxIKPN3QLkx5vMishS42RizRERSgLAxJioio4DNwGjADLXPgZzPQRKurqZzjRUc3e+9hwmHkZQUYnMqeCNzEj+NjaLBn0VemofLy3K5PH8P7q43+LBxA1ujbezweAjZJ5pz3OmUF86lPP8CZuXPYkbuDNI96SN8hCeuI9zBpvpNbKzfyAf1H7C1YSvheBiAkowS5hbOZU7BHOYWzGVs+ti+LYx4HGreh23/BTt+C5214PITKVxAV6CMrsoOut55n1hbGwDe6dOscyvz5+OfO9dqPSl1ljgTguQyrJbEtfb8IwDGmCcSyrxql3lHRFxALZBvEiolIqXAu8AY4KKh9jmQ8ylITDhM94YNdL65ls433yS83/qm7Bk/ntSrrqJm8hyWN6WxZn8b6T64dk4HBe711LRtZGuwjkb7zn9eA9P8BcwqupjysVcyq6Cc0amjz8lum3AszI6mHXxQ/wEb66xwaQ9b54WyvdmUZZVRllXGhMwJvdO5vlzExKHqXaulsuO30FUP7hTMxEUE/ZfQVROn65336d64EaJRxOcj5eKLeoPFU1Z2Tr6f6txxJly1NQaoTpivAS4ZrIzd+mgDcoFGEbkEeB4YD9xprz+efQIgIvcD9wOMGzdu+EdzBovU19O1di2db66l6+23iXd1IW43KRdfTPaypaRecQV/CabwzP/s5YMNDZTmruNjMzawM76fV1uty6NKIlEu8+Yxq+hCyqfeyuTRl+J2nh8DHnqcHmYXzGZ2wWyYCXETp7K1kg/qP2B703YqWyv5Q+Uf6Ih09G6T6c2kLLOMCVkTKCudzYQLPsXEzmby97yB7Pwd/u7/xu9OJe+m64k99FW6mzPoencdXW+9Rd0TVm+sq7AQ/+zZuIuKcBUW4i4qxFVYiKuwCHdBPnKGtF5MPE6spYVoXR2R2jqiBw8QrTlA5PAhHG7BXZSHZ3Q+nqI83EV5OLwJ9T4qKPvND7XekwYp2eDPAW/G4KManEmMgUgAgm3gyzwvRlNIZpAM9FWrf/Nn0DLGmPeAGSIyDfiJiKw6zn1ib/8c9nmWioqKc+qKAhOLEdiyhc61VqsjtGMnAK6iIjI+8QnSFlxF6iWXYPwp/H7rYZ79700Q+D1pWVspmNxCowPCkRjXRIRrcmcxd+ptZE6+4bz4D388HOJgYvZEJmYfGfzRGENDoIF9rfuobKtkX+s+9rXuY/VHq3kp9FJvuXR3OhNmXEqZM5UJ7Q1MrH6Tsp0vU+hMIX36DfDpLxDxT6XzvfV0/eUtQrt20bl2LSYQOKoeztxc3IV2uBQV2tNFuAsLcBUV4SooxJmWOqxjjQeDRGsPE63aS6S6kujBj4jW1hJpaCDa2EK0pYtIewji/f6ExODyxonHhHik74e7yx/DkxbFnRbFkxbDkx7FnWYtc3qG8acoTvBnWaGSkpPwnG09EpYZbxaxkINIW4hoY4sVgLWHidTWEak9TLS2jmhdHbhcODMzcWZk4MzIwJGViTPDnk9PxeF34/Q7cHoFpzuG0x3F4QrjlG4k3A6BVmsUhWBr3+lY+Ei9fZmQPgrSiyB9tP08CjJGHVmeVghn8Re3ZAZJDTA2Yb4YODRImRq7aysTaE4sYIzZKSJdwMzj3Oc5KdbaSudf3qLzzTfp+vOfibW2gsOBf84c8v/hH0i76kq8kycjIoSiMV76859Zu+mHtHh2UJ8VIpQjZMdi3CCZLBo1n4tm3Ym7qFx/zX2cRISClAIKUgq4bPRlvcuNMTQFm6hsrWRf277egFnTtpv/CjZDjh9yxpCCk7KmvzBh9WrK4k7KiuZSdu/NjCv7MtF4nFhHG+HaWiL19UTq6onVNxCtbyTa0Ei4ah/xDeugvfOoepkUHyY3E5OXSTwnnVhuBrGcNKL2wx2Pk9bQQmpjK56GZmhsIdrUTrS1i2hHhFjw6A92ccVx++O4UuKk5HtwTUrBlZOBKz/PCrMxY3GNGo9kFIInlVh7J+HD9YQP1hE5VE/4UB3hww10Hqojtr+tz76dGWm4RxfgGVWAZ1S+NT06H8/oApyZ6f26+gyEOiHQbH1AdzdDoBnT2USsuYFI9QGizZuJtHQT7YgRCTiJdDuJdjuJBpyYeN//2+IEV7oLd5Yff34a7qllmFiMWHsnsa464nVVRPZHCAZjxIIGEzv234bDbXD6BIfPiTPVgzPVhzN1PM6MmTgyM3BmZuHN8+DLjuIIN0D7YWhca51Xi0f7v+vWuHDpRZCREDbpCWGTMdoKyjOwVZbMcyQurBPj1wAHsU6M326M2Z5Q5gFgVsLJ9luMMX9lnxeptruzxgPvAOVA61D7HMjZeI4k3tVFYNt2Ahs30rl2LYFNmyAex5mdTeoVl5N21VWkzZ+PMyvL2iASpGH3n/jV+z/hg+BuNvkh7BByY4aPpxSzqPQG5pbficufPbIHdh5pDjZT2VpJZVsle1v3Utm6l31Nu2hM6CI7Ee6IIacTcjogp8OQm/Cc3fPcCY5B/qTjQHsqtKdDV5qDUIaLaKYXslNx5mXjKSwkZfR4MgtKyMoaT1bGWLJ82aS4Uk76XE68q4twTQ3hqioiVVWEq6oJV31EpKqayOHD9o9BLZKSgmfsWDzjxuEeNxbP2HGYaHTAloSJ9P2NkrjduArycOdl4cpJx53px5Xpxp0muPxR3L4wTulAgi0JodQCLq/VmvFl2a2dI9PGnU4s5iMW9RCLuoiFHMRDhlgwTqw7TKy9g3h7G7G2dmJtbcTared4W1vf+ongKBmHe/pU/DNmkDZzFqklRThibdBRC+2HrOeOwwmPWuhqOPoNdbgTQsZuyaQVQlq+9ZxaAGn2w+U9qX+zPu/rSJ9stytxA/A9rEt1nzfGfENEHgfWG2NWiogPeAGYg9USWWqMqRSRO4GHgQjW///HjTH/Pdg+h6rHmR4kJholtGcPgS1bCWzZTHDLVkL79vX+kfmmTydtwVWkXXklvlmzEKc9LHpzJS07f8dre3/H64Eq3vN5iIqQE3MwL2Uyt82+nTmTb8Shw6ifUdpCbVQ27WbvvlW0tO7HIU6cIjjFgRMHDnHgFPvZnnfY6x3iwCXOI+sc/baJC86OEM7WAI6WABGHg47cTFry0mnM8tFKmJZQC62hVlqDrdZzqJW2UBtm4F5i3A432d5sMn2Z1rPXes7yZZHjyyHHl0OuL5dcfy65vlwyvBk4ZOhvzSYcJnzwIJHqasIfVRGuriLyURXh6moi1dW9H8biduMqKrJaQ6NGWeeSiorsc0tFuEcV4czOPvFhbIw5JS3ySDzC3pa9bGvaxrbGbWxr2Ep14z5Su2KMrzdMqDVMqIXSWuuLAFgfaofzHdSM9lI7NoWGcRm0jM3ClZqGz+XD7/Ljd3jwG4MvFsUfDeGLBPGHu/AFO0kJtuHvbqG4s5nRXS0Ddy35MvuFyyCBk5o/aLfaGREkZ4qkBUlnPbz2GESDR77R9D5nHr3Mk44RIXroEIEtW+zg2EJw+3ZMMAiAMzMT3wXl+GeV4y+fha+8HFe23YoId8OBv9D44e95o2YNq+linc9HTITMiJtxTOOWCz/LLbOvPq4/ZKV6xOIxOsIdA4ZMS6ilz3zP+rZwG3Fz9NhpLnGR7cvuDZYcX07vdK6/73y2L3vA3x2ZWIxofT3i8ZxcSCRJ3MSpaq86EhqN29jVvItQLARYF2HMzJvJzNyZTM+djsfpIRgNEogGCEQDROvrce+pxrf3IGmVdWTub8TfZv3txwWaCnzUjPHy0SgXlUWwJz9GmyNI1PTvCjvC5XAxPnU0Jb58St2ZlIiH0rhQEg6T0d0MnQ3QWWd9XoUHaQ2n5PYNl7RCSCtALv+SBkmPpARJaxX89FPQftDquwy0WldpmFifYrGwEGj2EGxyE2j2EGj2EgtY34LEKfhGpeArycVfNgr/lFLcY4sRf/aRAHI44cBb1O1ZxWtNW1jt9/CBz4sRIS3sJ9Y+kxlFn+Sfr/4Y00dnntpjVOoY4iaTd3BdAAAgAElEQVROW6iNpkATTcEmmgJNNAebe6ebgk00B47M9/xWp78sb1afFk2Ov28LJ8eX07vM5zq9w9HUddX1CY3tjdt7r97zu/xMy5nGzLyZzMqbxYy8GRSnFZ9wN2Ckrp7g9u3WY9s2Aju2E2uw7smDw4FnQine6dNwTJ2EmVpGpKyYkNv6PVR1RzUH2g+wv20/B9oPUN1e3Sd0cn25lGSWUJpZSklGCaWpoyl1pTE6Lji7Gq2A6UoIms76I9PRAPK1dg2SHqc8SBr3WCES7oDbfw3jrCuQTShEcOtGAhvXEdy6jcD23YQP1vVu5inMwD82A98oD/588GUGkXCbFUD2lR5BERqcDpqcThqdTqrcLt5ISWGzz+rvzDE5NDZdQLRzNjfPrODzV5VRkje8K3eUSjZjDJ2RTito+gfPAEHUFekacD8prpTeFk1vt1pPKyehxZPjyyHTm3lCLfO2UBvbG7f3CY6GgHWewiUuJmVPslob9mNC5oSkjeLQGy7bthHcvn3AcPFNnYb4vBCLQzyOiceJxyIEwt10BTvoDnfSHe4iGO4iEO4mFoviMNY5NKcR/A4vPocXv3jwOjx4ceMRJw4jmHgMYjEmvvqqBkmPUxokhzfDC7eACOaO39CxtY7udesIbN1CaMfO3n5dZ14e/vJy/OXleGfNJDR5LM2uII3djTQGG2nobqAx0EhToImGgDXdGGigc4A/oJKU8cQDF7JzbwleM4o7LhnHfVdMoChTBwtU56ZgNNjbomkONvdp6fRMNwebaQ400xJqGbCLzSlOq4utX6umJ2iyfdlUd1T3hkZVR1XvtiUZJX1CY0r2lNPeGurPCpdtBLfZrZc9H1oh4nBYXX89z04nOARxOMHptFpITicx4oRMhKAJE4iFCJgg3bEggXiQmFgDn8YFXG4PfncqKd40rnvhTxokPU5ZkHz0Dvzir8CbQey2Fzn8nR/QsXo1xuclPGksbWX51JVkcGCMmyp/Nw3BRhoDjTQHmgfs40xxpZDnz+t95Kfkk+HOJh7JIBBMoaXDy56DTtZVhsn0u7l7Xgn3zCshO/XM+KGaUmeCWDxGW7ittxutN3gSQydhXSDa9zc7BSkFzMqb1Rsa03Onk+HJGKGjOf0isQjVndUcaDvSRba/bT/72/bz9u1va5D0OCVBsuc1+OVfQ+YYwlf/P6of/l+E9u3jhY85+EMFxO2hRRziIMeXQ54/j1x/Lvn+/L5h4c8n15dLNJLG4RbDvobOI4/6Lmrbg70v6RAoyUtl6UVjuf2S8aR5z6zBEJU6G3VHumkONtMSbKEwtZCClIKRrtIZyRiDw+EY8SFSzh3bX4bf/A0UTKVz4iPU3PMFAvEg31kijFpwLcsnfqq3RZHlzertNw1GYuxv7LJC4mAX7zdagVHZUE93+MhJ+XSviwkFacybmEtZfhpl+amU5acxLjcFr0sv3VXqVEpxp5DiTqE4vXikq3JGO5GLBjRIhvLBC/C7L2LGXERz7Cbqv/gvHC5w8eTNwh0f/xfunHYnTV0R9jV08sHeTvbV11th0dhJTUuAxAbfmCw/ZQVpXFSSYweGFRr56V4dvE8pddbSIDmWd56BV/+N+LgFHN5aSvuq5bw/zcXPbsnkmwv/nbr6MVz6xBvUd4R6N/G7nUzIT2X22GxunVvcGxilean4Pdq6UEqdezRIBmIMrHkC3vwWkVHXUr0yTHD3q7x4lYPKT87imUu/xfLVjazcvJELxmbxtwvKrMAoSGNUhg+HQ1sXSqnzhwZJf/E4vPoIvPcsXRmfoPpHlQSDnfzv2xxMumEpn83+LHc+t5OmzjD/uHAyf7ugDJfzzPjVrVJKjQQNkkSxKKz8AmbTL2iJXkftDzdTl+3g3z/j5q7rvsLWXZO597ebmFSQxg/vvoiZY/SX5EoppUHSIxqClz5LfPsr1B68ira3t/DBJAe/WlLEZ+d8nf/7SpADTR9x3+Wl/NO1U/C59XyHUkqBBokl1Am/vIPItrVUb5tDqHIPv54vVH96Hhebz/FPP6tjVKafX9x3KZeV5Y50bZVS6oyiQRJogZ9/mu5NW6haN4FgsIGnb3Ew7vp7OLjxIlYfquO2C4v56ienk+47e+9gppRSyXJ+B0lHHfzsFlre/YjD6/NpyAjy9D2pTJryEL94NYt0b5jv33kh184oGumaKqXUGev8DZLWKsyPbqT2f9pp/TCdzRMM/3XHBGJd97HiTTcLp+fzxC2zyEsb/l3GlFLqXHZ+BknDh0S/fxNVq2OE6rz89lJh0ycXsHPrdWC8fOe26dx24YnfV0Appc5H51+QHN5M4Lu3cuB1F+Gwi/+4yUnN9KVsW3cBl5Tm8tSnL2BsTspI11Ippc4a51eQfPQ2bV+/g5p3fDSnCsvvzuTD+F107Snhy5+Ywmfnl+qv0pVS6gSdN0Fidr1K7cOfp3WXjx3jhB/eXMbug8uYXjCW7949m8mF6SNdRaWUOiudF0FiuprZ9/m/I1Lr4w8VwopLLqep+hM8uGAqX/jYJDwuHeJEKaVOVlI/QUXkOhHZLSJ7ReThAdZ7ReSX9vr3RKTEXr5QRDaIyFb7+WMJ26yx97nJfgx5V5pQ1SG6Gzz8xyfc/MeMpaSFbuelz1/JPy6aoiGilFLDlLQWiYg4gWeAhUANsE5EVhpjdiQUuxdoMcZMFJGlwLeAJUAj8EljzCERmQm8CoxJ2O4OY8xx3/IwivC127PZGP4Mfz19Hg9fP5UUz3nRGFNKqaRL5qfpxcBeY0wlgIisAG4CEoPkJuAxe/olYLmIiDFmY0KZ7YBPRLzGmBAnoSbbS6bzYX58x2VcOTn/ZHahlFJqEMns1xkDVCfM19C3VdGnjDEmCrQB/QezuhXY2C9EfmR3a31FBvmxh4jcLyLrRWS90+Typy9eryGilFJJkMwgGegD3pxIGRGZgdXd9bmE9XcYY2YBV9iPOwd6cWPMc8aYCmNMxaRR2WSm6DhZSimVDMkMkhpgbMJ8MXBosDIi4gIygWZ7vhh4GbjLGLOvZwNjzEH7uQP4BVYXmlJKqRGSzCBZB0wSkVIR8QBLgZX9yqwE7ranbwPeMMYYEckCfg88Yox5q6ewiLhEJM+edgOLgW1JPAallFJDSFqQ2Oc8HsS64mon8CtjzHYReVxEbrSL/RDIFZG9wD8APZcIPwhMBL7S7zJfL/CqiGwBNgEHgf9M1jEopZQamhjT/7TFuaeiosKsX3/cVwsrpZQCRGSDMaZiqHL6azyllFLDokGilFJqWDRIlFJKDYsGiVJKqWHRIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOiQaKUUmpYNEiUUkoNiwaJUkqpYdEgUUopNSwaJEoppYZFg0QppdSwaJAopZQaFg0SpZRSw6JBopRSalg0SJRSSg2LBolSSqlh0SBRSik1LEkNEhG5TkR2i8heEXl4gPVeEfmlvf49ESmxly8UkQ0istV+/ljCNhfay/eKyNMiIsk8BqWUUseWtCARESfwDHA9MB1YJiLT+xW7F2gxxkwEvgt8y17eCHzSGDMLuBt4IWGb/wDuBybZj+uSdQxKKaWGlswWycXAXmNMpTEmDKwAbupX5ibgJ/b0S8A1IiLGmI3GmEP28u2Az269jAIyjDHvGGMM8FPgU0k8BqWUUkNIZpCMAaoT5mvsZQOWMcZEgTYgt1+ZW4GNxpiQXb5miH0CICL3i8h6EVnf0NBw0gehlFLq2JIZJAOduzAnUkZEZmB1d33uBPZpLTTmOWNMhTGmIj8//ziqq5RS6mQMGSQi4hSRvIR5j/1tf+cQm9YAYxPmi4FDg5UREReQCTTb88XAy8Bdxph9CeWLh9inUkqp0+iYQSIiS7E+2LeIyJsicjVQiXUC/Y4h9r0OmCQipSLiAZYCK/uVWYl1Mh3gNuANY4wRkSzg98Ajxpi3egobYw4DHSJyqX211l3Ab4/nQJVSSiWHa4j1XwYuNMbsFZG5wDvAUmPMy0Pt2BgTFZEHgVcBJ/C8MWa7iDwOrDfGrAR+CLwgInuxAmupvfmDwETgKyLyFXvZImNMPfC3wI8BP7DKfiillBohYl38NMhKkQ+MMXMT5ncZY6aelpqdQhUVFWb9+vUjXQ2llDqriMgGY0zFUOWGapEUiMg/JMynJc4bY/73yVZQKaXUuWGoIPlPIP0Y80oppc5zxwwSY8zXBlsnIqmnvjpKKaXONsdz+e8YEamwr7xCRApE5JvAnqTXTiml1BlvqMt/vwRsAv4v8K6I3A3sxLpi6sLkV08ppdSZbqhzJPcDU4wxzSIyDtgLXGmMeTf5VVNKKXU2GKprK2iMaQYwxlQBH2qIKKWUSjRUi6RYRJ5OmC9InDfGfDE51VJKKXW2GCpI/rnf/IZkVUQppdTZaajLf39yrPVKKaXUMYNERPoPstiHMebGU1sdpZRSZ5uhurYuw7rx1IvAewx8PxCllFLnsaGCpAhYCCwDbsca2v1FY8z2ZFdMKaXU2eGYl/8aY2LGmD8aY+4GLsX6HckaEfnCaamdUkqpM95QLRJExAt8AqtVUgI8DfxXcqullFLqbDHUyfafADOxbh71NWPMttNSK6WUUmeNoVokdwJdwGTgi9bdbQHrpLsxxmQksW5KKaXOAkP9jmTI0YGVUkqd3zQolFJKDYsGiVJKqWFJapCIyHUisltE9orIwwOs94rIL+3174lIib08V0T+R0Q6RWR5v23W2PvcZD8KknkMSimljm3Iy39Plog4gWewftBYA6wTkZXGmB0Jxe4FWowxE0VkKfAtYAkQBL6CdcXYzAF2f4cxZn2y6q6UUur4JbNFcjGw1xhTaYwJAyuAm/qVuQnoGRjyJeAaERFjTJcx5i9YgaKUUuoMlswgGYM1TlePGnvZgGWMMVGgDcg9jn3/yO7W+ookXJOslFLq9EtmkAz0AW9Ookx/dxhjZgFX2I87B3xxkftFZL2IrG9oaBiyskoppU5OMoOkBhibMF8MHBqsjIi4gEyg+Vg7NcYctJ87gF9gdaENVO45Y0yFMaYiPz//pA5AKaXU0JIZJOuASSJSKiIeYCnQ//4mK4G77enbgDeMMYO2SETEJSJ59rQbWAzosC1KKTWCknbVljEmKiIPAq8CTuB5Y8x2EXkcWG+MWQn8EHhBRPZitUSW9mwvIgeADMAjIp8CFgEfAa/aIeIEXgP+M1nHoJRSamhyjAbAOaOiosKsX69XCyul1IkQkQ3GmIqhyukv25VSSg2LBolSSqlh0SBRSik1LBokSimlhkWDRCml1LBokCillBoWDRKllFLDokGilFJqWDRIlFJKDYsGiVJKqWHRIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOiQaKUUmpYknaHxDNdJBKhpqaGYDA40lU55/h8PoqLi3G73SNdFaXUaXDeBklNTQ3p6emUlJQgIiNdnXOGMYampiZqamooLS0d6eoopU6D87ZrKxgMkpubqyFyiokIubm52tJT6jxy3gYJoCGSJPq+KnV+Oa+DRCml1PAlNUhE5DoR2S0ie0Xk4QHWe0Xkl/b690SkxF6eKyL/IyKdIrK83zYXishWe5un5Sz++ut0Opk9ezYzZ87k05/+NN3d3adkv3V1dSxevJgLLriA6dOnc8MNNwBQWlrK7t27+5T90pe+xLe//W3WrFlDZmYmc+bMYcqUKVx55ZW88sorp6Q+SqlzW9KCREScwDPA9cB0YJmITO9X7F6gxRgzEfgu8C17eRD4CvBPA+z6P4D7gUn247pTX/vTw+/3s2nTJrZt24bH4+HZZ5897m2j0eig6x599FEWLlzI5s2b2bFjB08++SQAS5cuZcWKFb3l4vE4L730EkuWLAHgiiuuYOPGjezevZunn36aBx98kNdff/0kj04pdb5I5lVbFwN7jTGVACKyArgJ2JFQ5ibgMXv6JWC5iIgxpgv4i4hMTNyhiIwCMowx79jzPwU+BawaTkW/9rvt7DjUPpxdHGX66Ay++skZx13+iiuuYMuWLRw4cIDFixezbds2AJ566ik6Ozt57LHHWLBgAfPmzeOtt97ixhtv5K677uLzn/88VVVVAHzve99j/vz5HD58mEWLFvXuu7y8HIBly5axZMkSvvrVrwKwdu1aSkpKGD9+PPv37+9Tn9mzZ/Poo4+yfPlyrrnmmj7rYrEY9957L+vXr0dE+OxnP8tDDz104m+SUuqckMwgGQNUJ8zXAJcMVsYYExWRNiAXaDzGPmv67XPMQAVF5H6slgvjxo070bqfVtFolFWrVnHddUM3rlpbW3nzzTcBuP3223nooYe4/PLLqaqq4tprr2Xnzp088MADLFmyhOXLl/Pxj3+ce+65h9GjR1NeXo7D4WDz5s1ccMEFrFixgmXLlg36WnPnzuU73/nOUcs3bdrEwYMHe8OutbX1JI9cKXUuSGaQDHTuwpxEmZMqb4x5DngOoKKi4lj7PKGWw6kUCASYPXs2YLVI7r33Xg4dOnTMbXq6oQBee+01duw40sBrb2+no6ODa6+9lsrKSv74xz+yatUq5syZw7Zt28jPz2fZsmWsWLGCGTNm8Nvf/pbHH3980NcyZuC3bcKECVRWVvKFL3yBT3ziE31aP0qp808yg6QGGJswXwz0/5TsKVMjIi4gE2geYp/FQ+zzrNFzjiSRy+UiHo/3zvf/PUZqamrvdDwe55133sHv9x+175ycHG6//XZuv/12Fi9ezNq1a7n11ltZtmwZixYt4qqrrqK8vJyCgoJB67dx40amTZtGLBbjwgsvBODGG2/k8ccfZ/Pmzbz66qs888wz/OpXv+L5558/qfdAKXX2S+ZVW+uASSJSKiIeYCmwsl+ZlcDd9vRtwBtmsK/BgDHmMNAhIpfaV2vdBfz21Fd95BQWFlJfX09TUxOhUOiYV04tWrSI5cuPXNTWE0pvvPFG7xVgHR0d7Nu3r7d7r6ysjNzcXB5++OFjdmtt2bKFr3/96zzwwAM4nU42bdrEpk2bePzxx2lsbCQej3Prrbfy9a9/nQ8++OBUHLpS6iyVtBaJfc7jQeBVwAk8b4zZLiKPA+uNMSuBHwIviMherJbI0p7tReQAkAF4RORTwCJjzA7gb4EfA36sk+zDOtF+pnG73Tz66KNccskllJaWMnXq1EHLPv300zzwwAOUl5cTjUa58sorefbZZ9mwYQMPPvhgb+vmvvvu46KLLurdbtmyZTzyyCPcfPPNffb35z//mTlz5tDd3U1BQQFPP/30USfaAQ4ePMg999zT23J64oknTtHRK6XORnKMBsA5o6Kiwqxfv77Psp07dzJt2rQRqtG5T99fpc5+IrLBGFMxVDn9ZbtSSqlh0SBRSik1LBokSimlhkWDRCml1LBokCillBoWDRKllFLDokEygubNm9dn/rvf/S4+n4+2trbeZWvWrGHx4sVHbfvKK68wZ86c3qHiv//97ye9vkopNZDz9p7tZ4K33367z/yLL77IRRddxMsvv8xnPvOZQbeLRCLcf//9vP/++xQXFxMKhThw4EByK6uUUoPQIAFY9TDUbj21+yyaBdc/ecwiaWlpdHZ2ArBv3z46Ozv5zne+wze/+c1jBklHRwfRaJTc3FwAvF4vU6ZMOaqcDveulDodNEjOEC+++CLLli3jiiuuYPfu3dTX1w86oGJOTg433ngj48eP55prrmHx4sUsW7YMh6NvT6UO966UOh00SGDIlsPpsGLFCl5++WUcDge33HILv/71r3nggQcGLf+DH/yArVu38tprr/HUU0+xevVqfvzjH/cpo8O9K6VOBz3ZfgbYsmULe/bsYeHChZSUlLBixQpefPHFIbebNWsWDz30EKtXr+Y3v/kNsViM2bNn997dMDs7m82bN7NgwQKeeeYZ7rvvvtNwNEqp8422SM4AL774Io899hiPPPJI77LS0lI++uijAct3dnayfv16FixYAFhdWOPHj+8d7r1HY2MjHo+HW2+9lbKysmOed1FKqZOlQXIGWLFiBatW9R0N/+abb2bFihVccsklvP766xQXH7mf14svvsi3v/1tPve5z+H3+0lNTT2qWwt0uHel1Omhw8irpND3V6mznw4jr5RS6rTQIFFKKTUsGiRKKaWGRYNEKaXUsGiQKKWUGhYNEqWUUsOS1CARketEZLeI7BWRhwdY7xWRX9rr3xORkoR1j9jLd4vItQnLD4jIVhHZJCLr++/zbKLDyCulzgVJ+0GiiDiBZ4CFQA2wTkRWGmN2JBS7F2gxxkwUkaXAt4AlIjIdWArMAEYDr4nIZGNMzN7uamNMY7LqfrroMPJKqXNBMn/ZfjGw1xhTCSAiK4CbgMQguQl4zJ5+CVguImIvX2GMCQH7RWSvvb93klHRb73/LXY17zql+5yaM5V/vfhfj1km2cPIK6XU6ZDMrq0xQHXCfI29bMAyxpgo0AbkDrGtAf4kIhtE5P7BXlxE7heR9SKyvqGhYVgHcjoMNIz8YBKHkV+2bBk///nPe4dBUUqp0y2ZLRIZYFn/8VgGK3OsbecbYw6JSAGwWkR2GWPWHlXYmOeA58AaIuVYFR2q5XA6JGMYeaWUOh2SGSQ1wNiE+WLg0CBlakTEBWQCzcfa1hjT81wvIi9jdXkdFSRnk8Rh5AHC4TATJkw4ZpCANYz8rFmzuPPOOyktLdUgUUqNiGR2ba0DJolIqYh4sE6er+xXZiVwtz19G/CGsUaRXAksta/qKgUmAe+LSKqIpAOISCqwCNiWxGM4LXqGkT9w4AAHDhzg0KFDHDx48JjDyK9Zs6Z3vmcYeaWUGglJa5EYY6Ii8iDwKuAEnjfGbBeRx4H1xpiVwA+BF+yT6c1YYYNd7ldYJ+ajwAPGmJiIFAIvW+fjcQG/MMb8MVnHcLokaxh5pZQ6HXQYeZUU+v4qdfbTYeSVUkqdFhokSimlhkWDRCml1LBokCillBoWDRKllFLDokGilFJqWDRIRpDT6WT27NnMnDmTT3/603R3d5+S/dbV1bF48eLeIeZvuOEGAEpLS9m9e3efsl/60pf49re/zZo1a8jMzGTOnDlMmTKFK6+8kldeeeWU1EcpdW7TIBlBfr+fTZs2sW3bNjweD88+++xxbxuNRgdd9+ijj7Jw4UI2b97Mjh07ePLJJwFYunQpK1as6C0Xj8d56aWXWLJkCQBXXHEFGzduZPfu3Tz99NM8+OCDvP766yd5dEqp80Uyx9o6a9R+85uEdp7aYeS906ZS9G//dtzlr7jiCrZs2cKBAwdYvHgx27ZZI7889dRTdHZ28thjj7FgwQLmzZvHW2+9xY033shdd93F5z//eaqqqgD43ve+x/z58zl8+DCLFi3q3Xd5eTkAy5YtY8mSJXz1q18FYO3atZSUlDB+/Hj279/fpz6zZ8/m0UcfZfny5VxzzTXDei+UUuc2bZGcAaLRKKtWrWLWrFlDlm1tbeXNN9/kH//xH/n7v/97HnroIdatW8dvfvMb7rvvPgAeeOAB7r33Xq6++mq+8Y1vcOiQNVZmeXk5DoeDzZs3A9bQLMuWLRv0tebOncuuXac2YJVS5x5tkcAJtRxOpUAgwOzZswGrRXLvvff2fugPpqcbCuC1115jx44j9wlrb2+no6ODa6+9lsrKSv74xz+yatUq5syZw7Zt28jPz+f/b+/+g6wq6ziOvz8DK4ukKeAisQIrYhvYBo4I+ANQC39EKoNTLI5hyjhNEMroJDvMMIp/lNlkEgqhKUUmJqkgM2YGOJAiCS4ukJCQpOtYwIpEEuTKtz+eZ+Gy3Ls/uOyee3e/r5k7e87znHPu9z57733uec4531NeXs6iRYsYOHAgS5YsYdasWRmfqz2kz3HOZc87kgTVHSNJ1bFjx6NuUnXgwIGj6rt06XJ4+tChQ6xZs4bOnTsfs+2uXbsyYcIEJkyYwJgxY1i1ahXjxo2jvLyc0aNHM3LkSMrKyigqKsoYX2VlpefLcs41yoe2ckyPHj3YuXMnNTU1HDx4sMEzp0aPHs2cOXMOz9d1SitWrDh8Bti+ffvYvn07vXv3BqBfv35069aN6dOnNzisVVVVxX333dfoPVGcc873SHJMQUEBM2fOZOjQoZSUlFBaWppx2dmzZzN58mTKysqora1lxIgRzJs3j/Xr1zNlypTDezeTJk1iyJAhh9crLy+noqKCsWPHHrW91atXM3jwYPbv309RURGzZ8/2A+3OuUZ5GnnXIrx9nct/nkbeOedcq/COxDnnXFbadUfSHob1kuDt6lz70m47ksLCQmpqavxL7wQzM2pqaigsLEw6FOdcK2m3Z20VFxdTXV3Nrl27kg6lzSksLKS4uDjpMJxzraTddiQFBQWUlJQkHYZzzuW9Fh3aknSVpK2Stkmanqa+k6SnY/1aSX1T6ipi+VZJVzZ1m84551pXi3UkkjoADwNXAwOAckkD6i12K7DHzM4BHgTuj+sOAMYDA4GrgEckdWjiNp1zzrWiltwjuRDYZmZ/N7P/AYuA6+otcx3wqzi9GLhCkmL5IjM7aGbvAtvi9pqyTeecc62oJY+R9ALeT5mvBoZmWsbMaiXtBbrF8tfrrdsrTje2TQAk3QbcFmf/I2lruuVOkO7A7hbcfkvK59jB40+ax5+slo6/T1MWasmORGnK6p9rm2mZTOXp9qDSnr9rZvOB+Q0FeKJIWteUNAK5KJ9jB48/aR5/snIl/pYc2qoGzkqZLwbq32zj8DKSOgKfBz5qYN2mbNM551wrasmO5A2gv6QSSScRDp4vrbfMUmBinL4BWGHhCsGlwPh4VlcJ0B/4SxO36ZxzrhW12NBWPOYxBXgJ6AA8bmabJc0C1pnZUuCXwEJJ2wh7IuPjupsl/Q74K1ALTDazzwDSbbOlXkMztMoQWgvJ59jB40+ax5+snIi/XaSRd84513Laba4t55xzJ4Z3JIP7m+4AAAeLSURBVM4557LiHUkzSDpL0kpJb0vaLOn2WN5V0suS3ol/T0861obELAGVkpbF+ZKYouadmLLmpKRjzETSaZIWS9oS/w/D86X9JU2L75tNkp6SVJjrbS/pcUk7JW1KKUvb3gpmx/RFVZLOTy7yjLE/EN87VZKek3RaSl3atExJSRd/St1dkkxS9zifaNt7R9I8tcCdZvYlYBgwOaZomQ4sN7P+wPI4n8tuB95Omb8feDDGv4eQuiZXPQT8wcxKga8QXkfOt7+kXsBU4AIzO49wssh4cr/tFxDSFKXK1N5XE86w7E+4GHhuK8WYyQKOjf1l4DwzKwP+BlRA5rRMrRdqWgs4Nn4knQV8DXgvpTjZtjczfxznA1gS/6FbgZ6xrCewNenYGoi5mPDhvxxYRrj4czfQMdYPB15KOs4MsZ8KvEs8SSSlPOfbnyNZHLoSzpZcBlyZD20P9AU2NdbewC+A8nTL5Urs9erGAk/G6QqgIqXuJWB4rrV9LFtM+BG1A+ieC23veyTHKWYqHgysBXqY2YcA8W9RcpE16mfAD4BDcb4b8LGZ1cb51HQ0ueZsYBfwRByae0xSF/Kg/c3sA+AnhF+RHwJ7gfXkT9unytTe6dIi5fLruQV4MU7nReySrgU+MLO36lUlGr93JMdB0ueA3wN3mNm/k46nqSSNAXaa2frU4jSL5uo54R2B84G5ZjYY+IQcHMZKJx5HuA4oAb4AdCEMR9SXq23fFHnzXpI0gzBU/WRdUZrFcip2SScDM4CZ6arTlLVa/N6RNJOkAkIn8qSZPRuL/yWpZ6zvCexMKr5GXAxcK2kHIXPy5YQ9lNNiihrI7bQz1UC1ma2N84sJHUs+tP9XgXfNbJeZfQo8C1xE/rR9qkztnRcpjCRNBMYAN1ocByI/Yu9H+CHyVvwMFwNvSjqThOP3jqQZJIlwNf7bZvbTlKrUVC8TCcdOco6ZVZhZsZn1JRxYXGFmNwIrCSlqILfj/yfwvqQvxqIrCNkP8qH93wOGSTo5vo/qYs+Ltq8nU3svBb4dzyAaBuytGwLLFZKuAu4GrjWz/SlVmdIy5Qwz22hmRWbWN36Gq4Hz4+ci2bZP+mBSPj2ASwi7i1XAhvi4hnCcYTnwTvzbNelYm/BaRgHL4vTZhA/NNuAZoFPS8TUQ9yBgXfwfPA+cni/tD9wLbAE2AQuBTrne9sBThGM6nxK+uG7N1N6E4ZWHge3ARsIZarkW+zbCsYS6z++8lOVnxNi3AlfnYtvXq9/BkYPtiba9p0hxzjmXFR/acs45lxXvSJxzzmXFOxLnnHNZ8Y7EOedcVrwjcc45lxXvSFybJOmHkkZJul5Ss65+l3RGzMhbKenSenWvSLqgkfXviFchH5cY90UZ6jpJ+pOkDZK+1czt9pU04Xjjci4T70hcWzWUkAdtJLC6meteAWwxs8Fm1tx1Ae4AjrsjIVzjk7YjIeR3KzCzQWb2dDO32xdoVkeSctW9cxl5R+LalHi/iSpgCLAGmATMlXRMfiJJfSQtj/dvWC6pt6RBwI+Ba+Kv/s4NPNdcSesU7jFybyybSsiltVLSylg2WtIaSW9KeibmakPSDkn3xvKNkkpjMtDvAtPi81+a8nxFwG+AQbGun6SZkt5QuMfJ/HjVPJLOiXsub8Xt9wN+BFwa152mcD+UJ+JzV0q6LK57c4zzBeCPknpKWhXX21R/L825xK+c9Yc/TvQDuBD4OVAAvNrAci8AE+P0LcDzcfpmYE6GdV4hXjXMkSu6O8Tysji/gyNXHHcHVgFd4vzdwMyU5b4fp78HPBan7wHuyvD8o4gZCVJjiNMLgW/E6bXA2DhdSNhDqr/uncATcbqUkMalML7+6pTXdycwI+W1npL0/9gfufXwPRLXFg0mpL8oJeSzymQ48Ns4vZCQAqc5vinpTaCScEOkAWmWGRbLX5W0gZCbqk9KfV3iz/WEoafmuiwez9lISMI5UNIpQC8zew7AzA7Y0Xml6lxCeN2Y2RbgH8C5se5lM/soTr8BfEfSPcCXzWzfccTp2jAf/3RtRhyWWkDIfLqb8Ctc8Qt8uJn9t5FNNDlfUEzsdxcwxMz2SFpA+DV/zKKEL+XyDJs6GP9+RjM/j5IKgUcIe0jvxy/6QtKnFE+7iQbqPqmbMLNVkkYAXwcWSnrAzH7dnFhd2+Z7JK7NMLMNZjaIcAvVAcAK4EoLB6bTdSKvEbIgA9wI/LkZT3cq4ct2r6QeHH1vkX3AKXH6deBiSedAuKeEpHNpWOr6DanruHbH4y43AFi4R061pOvjc3aKZ5HV3+4qwusmxtSbkLDwKJL6EO5j8ygh+3Wi92J3ucc7EtemSDoD2GNmh4BSM2toaGsqYcimCriJcC/7JrFwh7pKYDPwOPBqSvV84EVJK81sF+GYw1PxeV4nDLk15AVgbP2D7Wli+Bh4lJDt9XnCEFSdm4Cp8TlfA84kZEyujQfgpxH2ZjrEYbGngZvN7CDHGgVskFQJjAMeaiR+18549l/nnHNZ8T0S55xzWfGOxDnnXFa8I3HOOZcV70icc85lxTsS55xzWfGOxDnnXFa8I3HOOZeV/wNfw82jPPH+zQAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XmcVNWd///Xp/bqfW+abqCbfRcUxQ2iUdDJEE2ihiWjjtExyWgmcVbNxGXMTGKM328SfzhjnMQxcaIkY8ZfGPNA4zKIoiIgi+z70kDvW1V37XW+f9zbTXXTTQNN0Syf5+NRj7rLuafOLZp617n31rlijEEppZQ6VY7BboBSSqlzmwaJUkqpAdEgUUopNSAaJEoppQZEg0QppdSAaJAopZQaEA0Spc4AEVkuIvec4rbDRSQoIs7T3a7BICL7ROR6e/o7IvLzwW6TGhgNEnVa2B8OIfsDr0ZEXhCRrMFu17ko9YMWwBhzwBiTZYxJDGa70sEY831jzD0AIlIpIkZEXIPdLnVyNEjU6fR5Y0wWMA2YDjw0yO1RSp0BGiTqtDPG1ABvYAUKACLiFZGnROSAiNSKyLMi4k9Zf7OIrBeRNhHZLSI32suHishSEWkSkV0i8hcp2zwmIv8lIv8pIgER+VRExorIQyJSJyIHRWRuSvnlIvLPIvKB3XP6HxEpFJFf26+7WkQqU8qPF5E37dfeLiJfTln3gog8IyJ/sF97lYiMSlk/R0S2iUiriCwGJGXdKBF5R0QaRaTBfv08e92LwHDgf+w2/n3Pb+on8J78VkR+Zbdrs4jMSFn/DyJyyF63XUSu6+3f0H5fltrvy8ci8j0Red9ed0zPIfXQ3fH2r5fXeUxE/tOeXWE/t9j7/hl7H6eklC+xe77FvdWnBocGiTrtRKQC+BNgV8riHwJjscJlNFAOPGKXvwz4FfB3QB4wG9hnb/cyUA0MBW4Fvt/jw+/zwItAPrAOK8Acdv2PAz/r0bwFwO32+lHAh8B/AAXAVuBRu02ZwJvAS0AJsBD4VxGZlFLXQuCf7NfeBfyLvW0R8Dvgu0ARsBu4KvUtAn5g79MEYBjwGIAx5nbgAHbvzhjzJMfq7z25CVhiv5dLgcV2u8YB9wOXGmOygRs4+j739AwQBsqAr9qPE9Xn/vVjtv2cZ+/7u/Z+/FlKmYXAW8aY+pNoj0o3Y4w+9DHgB9YHUhAIAAZ4G+sDAawPlnZgVEr5K4C99vTPgB/3UucwIAFkpyz7AfCCPf0Y8GbKus/bbXDa89l2WzrbsRz4x5Ty/wdY1mP79fb0fOC9Hu35GfCoPf0C8POUdZ8DttnTdwAfpawTrA/+e/p4774ArOvxXl6fMl9p74frBN+Tt1LWTQRC9vRooA64HnAf59/SCcSA8SnLvg+837M9KeuXn8r+2e39z+PUOxM4CDjs+TXAlwf7710f3R/aI1Gn0xeM9U33GmA81rdxgGIgA1grIi0i0gK8bi8H68Nxdy/1DQWajDGBlGX7sXoTnWpTpkNAgzl6UjpkP2cdp3zP+c6yI4CZne212/wVYEhK+ZqU6Y6UbYdiffgBYKxPwK55+/DMEvsQUxvwnxx9r/pzIu9Jz3b5RMRljNkFfBvrw7vObsPQXl6jGCu0DqYs23+C7Rvo/nVjjFmF9SXkMyIyHisMl55KXSp9NEjUaWesQxIvAE/ZixqwPqQnGWPy7EeusU7Mg/WBNerYmjgMFIhIdsqy4cCh9LS8m4PAuynt7Tzc8o0T2PYIVjgCICKSOo/VgzDAVGNMDtahG0lZf7whuQf0nhhjXjLGXI0VlAbrkGNP9UC8R5uHp0y3288ZKctSA7a//euzeX0s/6Vdx+3AK8aY8AnUpc4gDRKVLj8B5ojINGNMEvh34MciUgIgIuUicoNd9hfAXSJynYg47HXjjTEHgQ+AH4iIT0SmAncDvz4D7X8NGCsit4uI235cKiITTmDbPwCTRORL9gnpv6L7B2021iG4FhEpxzo3lKoWGNlbxQN5T0RknIh8VkS8WOc/QliHyXq+RgL4b+AxEckQkYnAnSnr67GC689ExCkiX6X7F4H+9q8v9UCSY/f9ReCLWGHyqxOsS51BGiQqLewPm18BD9uL/gHrhPRH9uGOt4BxdtmPgbuAHwOtwLtY35jBOrlaifVN/FWscxRvnoH2B4C5WCfnD2MdLvoh4D2BbRuA24AngEZgDLAypcg/ARdj7esfsD60U/0A+K59SO1ve3mJU31PvHabGuz9KQG+00fZ+7EO1dVg9S7/o8f6v8AKiEZgEla4depv/3pljOnAumBhpb3vl9vLq4FPsHos751IXerMEuvwrVJK9U1E/hzrZPrVg/T6zwOHjTHfHYzXV8envyBVSp3VxPptz5ewfuSqzkJ6aEspddYSke8Bm4AfGWP2DnZ7VO/00JZSSqkB0R6JUkqpAbkgzpEUFRWZysrKwW6GUkqdU9auXdtgjOl3XLMLIkgqKytZs2bNYDdDKaXOKSJyQiMaXBBBopRSZ5Ixhqb2KCKC3+3E63LgcJzIj/vPTRokSil1kpJJQ10gwqGWDqqbQymPDg41hzjUEiIST3bbxuty4HM78bud+D1WuPg9Tnwua97vduJ1O6z1bqdVNqVc1zK7XIbHRWGmh+JsLz734N48U4NEKaV6SCQNtW1hqptDVlg02UHRYgXF4ZYw0UT3oCjM9FCR72dCWQ7XTyylLNeHQ4RQLEEomiAcTxCOJgjHktayWIKw/agPxLvKReLWcyiWIHmCF9Xm+FyU5PgozvJSkuOlJNtLcbaXkmwfJdnWsuIsHzl+F9bQb6eXBolS6rxnjCGaSBKNJ4nEjz53hYXdm+gMiyMtYeI9PsWLs72U5/mZXJ7LjZPLKM/3U5HvZ1i+n6F5fjI8p/fj1BhDLGEIxRJE7OCxwidpB02chkCU+mCEurYwdYEI9YEI6w60UBcIE44lj6nT63JQ3BUyR4OmOLszgHwUZ3spzPTgcp74Rb1pDRKx7nL3U6z7G/zcGPNEj/VerPGYLsEas2e+MWaffaOj5zqLAY8ZY161t9mHdc+LBBA3xsxAKXXOSyQNDcEIR1rD1LSGONIaJhiOE010//CPxBNE491DwSrTy/J4kogdIMcjAiXZXiryM7h4eD7lU/1U5GdQke+nPN9PeZ7/jB8+EhE8LsHjcoDffVLbGmMIROLUByLUtUWoC4Spt4OmM3D2NrSzam8TLR2xY7Z3CBRk9jusXJe0BYmIOLHusjYH66Y+q0VkqTFmS0qxu4FmY8xoEVmANSjefKxfss4wxsRFpAzYICL/Y4yJ29tdaw+Mp5Q6B8QTSeoCnSER5khryHpuC3OkxZquDURI9HIsxyHgcTnwOB143U7r2eXA4zr67HM7yPW78Ti7L/d0lbPONVh1OLrKlWT7qMj3U5bnw+sa3PMMp5OIkONzk+NzM6o467hlI/EEDcEodW3hrqCxwibM2hN8vXT2SC4Ddhlj9gCIyBLgZiA1SG7m6C04XwEWi4jYo4B28nH8+zMopQZR1D5EVNMW7tabsALDCo36QOSY4/0+t4OhuX6G5Pq4YlQRZbk+huT6up6H5PjI9btP6hCLOnlel5PyPKvX1dMTvZTvTTqDpJzud1irxrptZq9l7N5HK1AINIjITOB5rOHEb0/pjRjgjyJigJ8ZY55DKZVW4ViCfY3t7KlvZ29DO7vrg+ypb+dQS4iGYISeIy1lepyU5fkpy/UxpqTYDgfrm39Zro+yHH/aTvyqMy+dQdLbX0jPnkWfZexbbE6ybyT0SxFZZt8Z7SpjzGH7Bklvisg2Y8yKY15c5F7gXoDhw4f3XK2U6iGZNNS0hdlT386ehqD93M6e+iCHWkLdwmJIjo+RxZl8dlxJVzgMybWCoyzXR7bv5I7pq3NbOoOkmu636qzAuhFPb2Wq7TvJ5QJNqQWMMVtFpB2YDKwxxhy2l9eJyKtYh9COCRK7p/IcwIwZM/TQmFK2QDjG3gard7GnPshue3pfQzuh2NEbJmZ6nFQVZ3Lx8HxuvaSCkcVZjCzKpKook0yvXvCpjkrnX8NqYIyIVGHdlnMBsKhHmaVYt/D8ELgVeMcYY+xtDtqHu0Zg3Ulvn4hkAg5jTMCengs8nsZ9UOqcFAjHqGkNc6Cpwz4UZYXGnoZ26gORrnIOgWEFGVQVZXLFyEJGFmcysjiTUcVZlGR79dCTOiFpCxI7BO4H3sC6/Pd5Y8xmEXkcq2exFOte3S+KyC6snsgCe/OrgQdFJIZ1D+e/NMY0iMhI4FX7j9sFvGSMeT1d+6DU2abzEtmaVuvkdq19grvWnq9ps6bbo91vxZ6f4WZkcRbXjC2mqjiTkUVZjCrOZHhhxnl1tZIaHBfE/UhmzJhhdNBGdbYLRRNWGLSGqWkLUdMasa6GSgmNul4ukXU5hNIcH6U5Xobk+ijNsa54GpJrXdo6siiL/EzPIO2VOpeJyNoT+a2eHuhU6gxqDEbYURtkZ12AnbVB9jd1UGP/pqItHD+mfLbXRal9Kewo+xLZzvkhOT5Kc70UZXrP6wEB1dlPg0SpNGhuj7KjNsCOuiA7awPsqLWCo7E92lUm2+uisiiTEYWZXD6ysFtPotR+ztKT2uocoH+lSg1AS0eUHbVBOygCdm8jSEPw6AntLK+L0SVZXDehhLGl2YwpzWZsaRZDcnx6MludFzRIlDoBrR0xdtYFjoaGPZ16BVSmx8no0myuHVdsB0YWY0uzKcvVwFDnNw0SpXowxrCtJsCbW2r5eG8TO2oD1KUERobHyZiSLGaPKWasHRZjSrMoz/NrYKgLkgaJUliDCq7Z38wfN9fy5tYaDjaFEIEJQ3K4ekwRY+3DUWNKsinP8+vJbaVSaJCoC1ZHNM6KHQ38cUsN72yro6Ujhsfl4KpRhfzlNaO5bkIJJdm+wW6mUmc9DRJ1QWkIRnh7ay1/3FzL+7saiMST5PrdfHZ8CXMnljJ7bLEO/6HUSdL/Meq8t6c+yJtbanlzSy1rDzRjDJTn+Vl42XDmTirl0soC3DpUuVKnTINEnXeSScP66pau8NhVFwRg0tAcvnXdGOZMLGViWY6eGFfqNNEgUeeFSDzBB7sb+ePmWt7aWkt9IILTIVw+soA/mzmc6yeWUpGfMdjNVOq8pEGizknxRJLq5hDrD7bwxy01vLu9nvZogkyPk2vGlTBnYinXjishN0Pvi6FUummQqLOWMYb6QMS+uVI7exuC1n00Gto50NhB3B68sDjby83Ty5kzsZQrRxXqaLZKnWEaJGrQtYVj7Gto77pvxt4GOzTq27sNh+5xOagqzGRsSTY3ThpCVVEm44ZkM3lorv6uQ6lBpEGizohIPMHBpo6ue3533aGvob3buFQiUJHvp6ooi8uHZTIhJ8LIjDDDvEEKaMXRsRM6GqC9EbbXw7YklF8Mw2ZCxQzw5Q7iXip1YdIgUaddMmnYcqSNd3fUs3pfE3sb2jnY1IHbRCmkjQJpo8ofYlZ2mDuKQpQP7aDY0UaeaSMj1oSjoxFqGuFAsPcXcHogowgyiyCZgN1vg0kCAqWTYNhlVrAMmwn5lVY6KaXSRoNE9c8YiHVAJAiRAEQD1nMkCNEgRNoItrVw8EgtjbuqiRyoJaMpyFUtYWaF44jP4PTHEH8MyUhi/EnISEIogfEnMR43rf48Wn25GH8+Jm8Sxp8LvjyMLxfjywN/LnhzrXm3H5OSDY5oCGnYgaNuK47aTTi2vYpj44uIAYe/AEfZVKTsIhxlF+EsmYS4vDjEgUMcCGI9i/XsFKc1jbXe4/Tgcuh/E6WOR/+HXMia98H6lyHUZIdCz4AIHF1ukt02TcaEcKuLSLObcIv1kBYXhQnrh30JBxwuhJpiISsE+c1uCg668fS4d5MAHV5ozoKm7Daas9poztpPc5bQlA3NWUJzFrRkQcx1gj2L4mwg++h8dDvs3w77f3tsWWNwJsETtx7u1OcYxLxO3KOqGF04lrH5YxmTN4ax+WMZkjnkgv8dStIkEeSCfx+UBsmFKdQC7/0fWPUsJOPgzQZvDniyrGlfDuQMBW8OxpNJPOQiXBOiaX8LzXvqiVc3kNHSSufHR8jrpX5oDvvHJ9iY18K+UiFaUcwVlbOZUToDr9NLXIQ6Y3AEw7ga23A2teBobMXZ2IazqZWixlZKGltx1rfi2NaKxBPHNDuZnYkpyiNZaD1MUT4mKwNiMYhEMdEoEoliIlGIRpGItVyiMSQcho52JBJCIjEklsARB0mAmON9ECboyN7L5pEH+WjYH3ihSmjNErLd2YzOH82YvDGMybfCZXT+aHI8OWn5JxtszeFmdjTvYGfzTnY072BH8w52t+wmw53BzCEzmVk2k8vKLmNY9rDBbqoaBHrP9gtJIgZrX4D//T6EmmHaIvjsd63QAEw0SmTvXsJbtxLZtp32LVvo2LoNZ6Ctq4rDmYXUFJfRPjKb2hER1uTuZY+vDYfDydSiqcyqmMXsitmMyx93yt9UTTJJorWVeF1dt0esro54Xf3RZQ0NkLADRwTx+XB4vYjXi/i8OLw+xOu1lvlSpr1exOPCEW9DQnU4QjVI8CCSDOJwGsTnR0rG4CibQNxRTPvm/bSv3USiuQWAUGUpB8cX8EllkuX5tTRx9FzOkMwh3cJlTP4YqnKqcDvPjd+zRBNR9rTu6RYYO5t3Uh+q7yqT781nbIHVO2sKN/Fxzcc0hBoAKM8qt0JlyGXMLJtJkb9osHZFnQYnes92DZILgTGw43X448PQuBMqZ2Hm/jORYAYdH31EeMtWwtu3E9m5E+LWsaeY083enCHszhlKdUE5rsmFRMZ1cMSzle0tm0iaJHnePK4uv5pZ5bO4cuiV5PnyzuxuJRIkQyEcHg+43QM7xGIMNO2Bg6vgwEdw8GOo39ptdaQtg2BTAe1H3IQORTEJg7idOMdX0DKlgl0Ti1hfGGVnYB97W/cST1rvpUtcVOZWdoVL5yGywTw8Zoyhpr2GnS12YDTtYGfLTva27iVhrHB2O9yMzhvdLRTH5o+l0FfYrd3GGPa07uGjIx/x8ZGPWV27mkA0AMCo3FHMLLN6LDOGzDhve2znq7MiSETkRuCngBP4uTHmiR7rvcCvgEuARmC+MWafiFwGPNdZDHjMGPPqidTZmws6SI5sgDf+Efa9RzJnFO3FiwjuChJ8bwXxw0cAiOfmc7hoGBu8JWzOKGVP7lByxgylsrKWpG8L29tWd30jnVg4kVnls5hVMYvJhZNxOs7jH/+FmqF+OwSOQKAG2g5bz4EjJJuO0LG7gfaDSYI1XqJtVo/D5UuQOTSJb1QmDROK2VWYww6XsNNE2Blr4Ui0tav6bHc25dnlZLgy8Lv8ZLit585H53K/y4/f3WO+l/J9XRQQjAbZ1bKrWw9jZ/NOArFAV5mhmUO7hcWY/DGMyBlxShcaJJIJtjVts4Kl5mM+qf2EcCKMQxxMLJjIZWVWb2V6yXT8Lv9J16/OnEEPEhFxAjuAOUA1sBpYaIzZklLmL4Gpxpivi8gC4IvGmPkikgFEjTFxESkDNgBDAdNfnb25IIOk7TC8889E3/8Nwfp8gh1j6Nh+2DqPkJFBYvoM3skdw68SZdT78yjMcnPJqBg5hbuoi2/g08Z1xJNxstxZXDH0CmZXzObq8qv1UEVPkQAEaojt2UL7Bx/QvnYz7ZsPkOiIAeAtgqySDjJLO/AXRWl3Cbs8bnZ4POzMyKbG4yPkchNyuOhwCCEMIZJ0JGNEkrGTaorH4cHv7h5CLZEWDgUPdZXJdGd2u2hgbMFYRueNJtuTfZyaByaaiLKhfgMf13zMqiOr+LT+U+Imjtvh5qLii7p6LJOLJuN2nBuHAC8UZ0OQXIHVk7jBnn8IwBjzg5Qyb9hlPhQRF1ADFJuURolIFfARUA5c2l+dvbmQgsQEm+l48RGCf3yN4CEX0TbrG6VnxAgyP/MZqsdOZ3FjFsv3tpLtj/PZaUFcWTvY3LKq6wNndN5oZlXMYlb5LKaVTNP/3CfJJBKEt2ylfeVK2leupGPdOojHEa+HjMmjyZo4lMyR2XiyOpBgLQTroL3Oeo6Hu+pJAGERQk43HZmFhDILCGXk0+HNIeTLpsOTScjjI+Ty0uFyE3I46DAJQvEQoXiIjngHWe6srsNpY/PHUpZZNuhXWXXEOlhbu7YrWLY1bcNg8Lv8XFJ6CZeXXc7MspmMzR+LQ3R4/8F0NgTJrcCNxph77PnbgZnGmPtTymyyy1Tb87vtMg0iMhN4HhgB3G6MefVE6kyp+17gXoDhw4dfsn///rTs59kgVldH+7vLCS59ifb120jGBHEKGTOmk3XdDWTOmsV7IT8/fXcVW5s/JTO3mvz8QzTF9nf9B545ZGZXeJRllQ32Lp1XEsF2OlZ/TPvKD2hfuZLo3r0AuEpL8U+bhnvIEFylpbhLS3EVZOLKcuH2xZFIY0rI2IHT+Wivs66468nphaxSyCqxHp6sk2ztsZ8HJmlIdMSIt0WJtUWJt0WJt0WItUVxeNy4SwvwlBXjqRiKe9gwHDmF1ggDnQ9vjvVw9B4KLeEWVteuZtWRVaw6sop9bfsAyPPmMTJ3JAW+AuvhLzg67Sug0FdIga+AHG/OWRM4iWSC9ng7gWiA9lg72e5sCv2FeJyewW7aKTnRIEnn5b+9fe3p+VfaZxljzCpgkohMAH4pIstOsE7s7Z/DPs8yY8aM8+qKApNIENq4keCKFQTffZfIFuuksMufIGdCNlm3/AXezy1iW/gA/7rhXd5561HaZTeOjDb8GeBzZTCmaCrTS/6E6SXTubj0YrxO7yDv1fnLmZVJ9rXXkn3ttQDEDh0i+MEHtL+/ksi2bQRXrMCEQsduV1hohUtpKa4hpbhLp+AqHYJ7SgmukhJcuV6cJtgjZFJ6OM37IdZO7/9tLMm4Id4O8XZDrL3ntLGngWSPDQVcfivLktFD3Va5/Ak8WXHcWXE8WQk82XHcWQk8hRk4c3K6B4wvlzxfLnPsByNuo8YBq0NH+Diwn0PRFva27GZtZC0tkRZML//dneIk35ffLWQKPPkMifkpDgj5bUmyW6L4m9pxNbRh6uqJ19QSr60Flwtnbi7OnBwcOTmQk0Uiy0c800c000PI7yTkd9DudxDwJmn1JmjxxmlyRWiLBwnGggSiga7n9lh7r+9zrjeXYn8xhf5Civ3FFPmLKPIXHZ3OsOaz3dmD3mM8FekMkmog9aLyCuBwH2Wq7UNbuUBTagFjzFYRaQcmn2Cd56VESwvB91cSfPdd2t97j0RLCzgc+Mt9FE9tQ8bls/WGO3g9I4tP6j5mw6s/J2asMawcznymFkznc2OvYEbpxYzJH6O/1h5E7vJy8m+7jfzbbgOsq56SgQCxmhritXXEa2uI1dYSr6klVldL7PBhQuvWWf/mPTiyso72ZroCZxKuCmsZIlZdtXXEa636OqfjtbUkWluPqVMyMnCXlOCqHEJGaQmuErvu0pKjr1NYiLisv6FESwvRfXuJ7tlBbO9uogf2E60+TPBQLYm9gW51OzMEd34YT24UT1YN7swIHl87Hm8bTm8CERgCfN5+dHH5iWcW0ZKRT6M/l5aEl0CHg1AgTrwthmkJ42yuwdeyj8yWKDltcdw9forU7oTGbGjOddJe6CMyOh+SCVzBAO6ORvy1cTL2GzLDkBUGX8z6WWtfZ49CPgfRDDfRTA/xLB8mKw+yK5CcbCuc8vJoK8umusxNnWmjIdRAfaieT2o/oSHUQDQZPaZOr9PbLWRSg6c442gAFfgKzqr/w+k8tOXCOjF+HXAI68T4ImPM5pQy9wFTUk62f8kY82X7vMhB+2T7COBDYCrQ0l+dvTkXz5Ek29sJbdpMaN06gitWEFq/HpJJnPn5ZF5+KdH8g2xxrOKTgkzW5Q9hV9T6tiY4IFpGJDiccv8E7r3sOr48bYqOjnseSIbD1u9pugVOHfGaGisgamqJ19dDsmf3wSaCs6gQd89gKOkeEo6srNP2rTjZ3k60uprogQPEDhwgeuAg0QP7iR04SOzIkW5tFb8fT3kZnqHFuEvz8RRlYUJtVqDWNxFrbCPe0kE8EMP0CAlxGFz+BO6MBK6MBO4sBybPT7gwi0BRDs1FOTTkZ9Ds8tDkgCaSNCUjuBwuslx+sl1+sp0+spw+sp0+sp1esoyL7JAhsyOJPxTH1x7DGYyQbO8gEWwnEeggGQyRaA+RCHaQCIZJtIdItkcwqT+oFcEzrBT/2FH4Jo7HN3kq3inTCPq9NIYbqQ/V0xBqsIKmo56GcAMNHQ1dwdMWbaMnQSj0F3JxycXMqpiVtgthBv0cid2IzwE/wbpU93ljzL+IyOPAGmPMUhHxAS8C07F6IguMMXvscx8PAjGsTvXjxpj/v686+2vH2R4kJh4nsnMnoY2fEtq4gfDGT4ns3t31n8w7cQKhyyayfYyPlYmPWBfcS719j/FMVwaTCqeSCFWybmcurc1DuaJqKH957SiuHl10TnaT1akz8Tjxxkar91FTAwbcpSVWcBQVIe6z58IJE40SPXSI2MGDRPcfIHrwALH9B4gePEjs4EFMzLpqTdxuXEOGWEFXVoZ7SOnR+cJc3DlunO4YEmq0R4ZugPZ66GjsMV3f7WKGtO2XAZMQEhEH4RYX4SYP4WY34SY38XDn5fIGT04CfzH4Sj34hmbhG5aPIzvPGlnCm9P1HPFk0OhwUO8wNJgEDckI9YkODocb+ah2DfXhRgAm5IxkVtFUZhVMYkpGOc5kDOIR65GIWvvebToKiYj1HA/3mI5CPILc+fvBD5KzxdkUJMYY4ocPE9q40Q6OjYQ3b8aErT9wZ24ujsnjaawqYEeZ4aO8BlaHtxGy/wMMjcWZ5ilk+sQvM6L0Gt751MlLHx2kPZrg+gklfOOa0VwyIn8wd1GpATOJBPG6OsTjwZmfj/Rxov7kKjUQbT96G4KOBuskjzhBHNbFAOIEhz3fbdpxksud1qjT0XZ7zLo2CLcRO1JNeNsuwrv2E95bQ2h/I4mAfRsFAU++E39REl9eBF9OAF9eFIer789oA2z3uHnP7+e9DB8bvF6SIuQmElwZCjOrI8RVoTAFffVSHW5w+cDlsS7UcNkPpwdcXuQv3tZyqfAeAAAgAElEQVQg6ZS2IEkmYNPvrLGruv4IXfYfk/VHlQhFCe06THjXIUI7DxLaeYBEi3XcWNwu3KOGEagqZF+Fi7XFIT5wH6Iuah0Ld4mLcZlDmdZaz7TGg0zPrqJ07g84mHcpP1uxm9+uqSaeSDJv6lC+cc0oJpTpr4aVOtfEausIb95sPTZtIrRlM4l6a8gZHA48I4bhH1uFb2Q5vqoSfENzcEjYGlDV4eoWAq0k+DCwj/dad/J+8xaaYgEEYXLeaGaVXsbVQ69kUvEUHC6/FRb9BPRZcWjrbJGWIIl2wO/uge1/6FpkEhBucRNqdBNu8hBq9BANHD0h5s6JES1OcGiIYeNQByvLPezyuTD24acRsRiTI1GmRCJMiUQZF43iNUDWELjuYXYOmce/rdjH7zccxinCLZdU8LXZI6ksyjy9+6aUGlRd4bJpE+HNm48Nl5FV+MZPQHxeSCQhmcQkk5BIYEwSEtZ8INxCU0cDzR1NtEcCiAGPuMh355DrziHbmYkTh719ontdySSjX1826Jf/nr86muCl+ZiDqwkU3UNHg5/Q5i1Etu/qOq7rLMiDScNpHJHN9tI4Kwua2BA7SNS+9r/Anc3U7BHckDWcKVnDmJQxlFyn1+rlmIT1nEyA08WnGVeweOVh3vjNSvxuJ3ddWck9s0YyJNc3mO+CUipN3KUluEtLyP7stV3LrHDZRHiT1Xvp+GSt9cHvcFiH/jqfnU5wCOJw4nM6KZcMyt3ZJNxJ2uJB2uJBWiKt1EWaMA7I9GSTl1FAQUYh2b5cxOG06nI64PVlJ9Re7ZGcrOb98J+3kKg/wJEDswl8tBnx+3FNGEvLqBL2lDv4OL+ZDxM7abPHMvK7/EwsnMiUoilMLprM1KKpfQ7Y1x6Js6e+nd31QXbXB1mzr5kP9zSS63dz55WV3HVlJfmZ5+aPm5RSZ4dEMsGmxk28f+h93qt+j82N1oWvRf4irhp6FbMqZnHF0CvI9ebqoa1Opy1IjmyEX99KtClC9drRRA7WsOaWibw0uZXqsDUAolOcjM4bbQVG8VQmF01mZO7Ibtd8G2M40hruFhi764Psrmunpu3oVSUOgcqiTBZcOoxFM0eQ5dUOpFLq9GsINfDB4Q94r/o9Vh5eSSAawClONty5QYOk02kJkt3/C7+5nWBDNodWZJJwCIu/6GXN8CizK2YzpWgKU4qmML5gPBnuDADCsQR7G9q7QmJPgxUYe+rb6Ygevc482+tiZEkWo4ozGVV89Hl4YQZe13k8uq5S6qwTT8b5tOFT3qt+j29d8i0Nkk4DDpKNv8W8+g2aDg6n7oMo4eHFPPi5FqR8CD+99qcUuCu79Sp21wfZ0xCkujlE6ttbnudnVLfAsKaLs736ew+l1FnnbBhr69xnDKz8KcnXH+PIljG0bQlSfelwHpp9iItHXMn1RX/N7f92kLrArq5N/G4nI4szmTYsn1surugKjKqiTPwe7V0opc4/GiR9SSbgje8Qe+ffObhmFJGadpb/aQX/OuUQC8ffyeE91/L37+zmomF5fOOaUVZglGRRluPT4UiUUhcUDZLexMLw6r20/+8yDn08jDiGZ7+Sz4eVLdw18mF+804hjcE6/mbOWL5xzShczrNjCGullBoMGiQ9hZoxLy+i+a311K4vJjI0l+/OCxCtyOJy+SZPLxXGlLj5xZ2XMrk8d7Bbq5RSg06DJFVrNckXbqHmjVpa9+RxZHo5D157hKrSSwjsuo3XGoR7rq7ib28Yh8+t5zuUUgo0SI6q3ULsZ7dQ/ccE4QY/K28o5+npR5iQeROrP7qcstwMXrrnIq4YVTjYLVVKqbOKBgnAvvfp+OmfUb3cR9z4+cWibJZXNVMQvItV28Zz6yUVPPr5iWT7zp4huJVS6myhQbL5VZqf/CY1q7OJFufx8M1hGko8BPfeCVTys9uncMOkIYPdSqWUOmtd0EFi3l9MzRNP0rIrm7rJQ/iH6+sxnnHUbf0y148bxQ++NIWiLL2XuVJKHc+FGSTJJPH//nuqf/rfhOozWfXZMv7vpXUkA7PhyDye/OIUbr2kQn9trpRSJ+DCC5J4lNAzt1P9wjpicT8v3JbH66Ob6Tg0n0sKr+epb13EsIKMwW6lUkqdMy6sIAm30frdmziyrIZoTgaPfFHYXeggeuAbPHjN9Xz1qir9VbpSSp2kCyZITFM1dfd9nqZ1YepH5fLgvHaazUhGdHydp++dxdjS7MFuolJKnZMuiCAxkQ4O3DaXjkOGVZfn8ZPZAcJtV3L3hG/xresm4HHpECdKKXWq0voJKiI3ish2EdklIg/2st4rIr+x168SkUp7+RwRWSsin9rPn03ZZrld53r7UdJfO6J79tBRk+T5eVk89ZkQ/vY/46Uv/ZC/u2GShohSSg1Q2nokIuIEngHmANXAahFZaozZklLsbqDZGDNaRBYAPwTmAw3A540xh0VkMvAGUJ6y3VeMMSd8g5Ekwne+4mVHiYdrc/6RJ78yjwzPBdEZU0qptEvnp+llwC5jzB4AEVkC3AykBsnNwGP29CvAYhERY8y6lDKbAZ+IeI0xkVNpyIFCcBVV8sTlT/L5yeNOpQqllFJ9SOdxnXLgYMp8Nd17Fd3KGGPiQCvQczCrW4B1PULkP+zDWg9LHz/2EJF7RWSNiKwxZPL2opc1RJRSKg3SGSS9fcD3vK/vccuIyCSsw11fS1n/FWPMFGCW/bi9txc3xjxnjJlhjJkxvqSSoiz9bYhSSqVDOoOkGhiWMl8BHO6rjIi4gFygyZ6vAF4F7jDG7O7cwBhzyH4OAC9hHUJTSik1SNIZJKuBMSJSJSIeYAGwtEeZpcCd9vStwDvGGCMiecAfgIeMMSs7C4uIS0SK7Gk3MA/YlMZ9UEop1Y+0BYl9zuN+rCuutgK/NcZsFpHHReQmu9gvgEIR2QX8NdB5ifD9wGjg4R6X+XqBN0RkI7AeOAT8e7r2QSmlVP/EmJ6nLc4/M2bMMGvWnPDVwkoppQARWWuMmdFfOf01nlJKqQHRIFFKKTUgGiRKKaUGRINEKaXUgGiQKKWUGhANEqWUUgOiQaKUUmpANEiUUkoNiAaJUkqpAdEgUUopNSAaJEoppQZEg0QppdSAaJAopZQaEA0SpZRSA6JBopRSakA0SJRSSg2IBolSSqkB0SBRSik1IBokSimlBkSDRCml1IBokCillBqQtAaJiNwoIttFZJeIPNjLeq+I/MZev0pEKu3lc0RkrYh8aj9/NmWbS+zlu0TkaRGRdO6DUkqp40tbkIiIE3gG+BNgIrBQRCb2KHY30GyMGQ38GPihvbwB+LwxZgpwJ/Biyjb/BtwLjLEfN6ZrH5RSSvUvnT2Sy4Bdxpg9xpgosAS4uUeZm4Ff2tOvANeJiBhj1hljDtvLNwM+u/dSBuQYYz40xhjgV8AX0rgPSiml+pHOICkHDqbMV9vLei1jjIkDrUBhjzK3AOuMMRG7fHU/dQIgIveKyBoRWVNfX3/KO6GUUur40hkkvZ27MCdTRkQmYR3u+tpJ1GktNOY5Y8wMY8yM4uLiE2iuUkqpU9FvkIiIU0SKUuY99rf9rf1sWg0MS5mvAA73VUZEXEAu0GTPVwCvAncYY3anlK/op06llFJn0HGDREQWYH2wbxSRd0XkWmAP1gn0r/RT92pgjIhUiYgHWAAs7VFmKdbJdIBbgXeMMUZE8oA/AA8ZY1Z2FjbGHAECInK5fbXWHcDvT2RHlVJKpYern/XfBS4xxuwSkYuBD4EFxphX+6vYGBMXkfuBNwAn8LwxZrOIPA6sMcYsBX4BvCgiu7ACa4G9+f3AaOBhEXnYXjbXGFMHfAN4AfADy+yHUkqpQSLWxU99rBT5xBhzccr8NmPM+DPSstNoxowZZs2aNYPdDKWUOqeIyFpjzIz+yvXXIykRkb9Omc9KnTfG/N9TbaBSSqnzQ39B8u9A9nHmlVJKXeCOGyTGmH/qa52IZJ7+5iillDrXnMjlv+UiMsO+8goRKRGR7wM70946pZRSZ73+Lv/9NrAe+P+Aj0TkTmAr1hVTl6S/eUoppc52/Z0juRcYZ4xpEpHhwC5gtjHmo/Q3TSml1Lmgv0NbYWNME4Ax5gCwQ0NEKaVUqv56JBUi8nTKfEnqvDHmr9LTLKWUUueK/oLk73rMr01XQ5RSSp2b+rv895fHW6+UUkodN0hEpOcgi90YY246vc1RSil1runv0NYVWDeeehlYRe/3A1FKKXUB6y9IhgBzgIXAIqyh3V82xmxOd8OUUkqdG457+a8xJmGMed0YcydwOdbvSJaLyDfPSOuUUkqd9frrkSAiXuBPsXollcDTwH+nt1lKKaXOFf2dbP8lMBnr5lH/ZIzZdEZapZRS6pzRX4/kdqAdGAv8lXV3W8A66W6MMTlpbJtSSqlzQH+/I+l3dGCllFIXNg0KpZRSA6JBopRSakDSGiQicqOIbBeRXSLyYC/rvSLyG3v9KhGptJcXisj/ikhQRBb32Ga5Xed6+1GSzn1QSil1fP1e/nuqRMQJPIP1g8ZqYLWILDXGbEkpdjfQbIwZLSILgB8C84Ew8DDWFWOTe6n+K8aYNelqu1JKqROXzh7JZcAuY8weY0wUWALc3KPMzUDnwJCvANeJiBhj2o0x72MFilJKqbNYOoOkHGucrk7V9rJeyxhj4kArUHgCdf+HfVjrYUm5JlkppdSZl84g6e0D3pxCmZ6+YoyZAsyyH7f3+uIi94rIGhFZU19f329jlVJKnZp0Bkk1MCxlvgI43FcZEXEBuUDT8So1xhyynwPAS1iH0Hor95wxZoYxZkZxcfEp7YBSSqn+pTNIVgNjRKRKRDzAAqDn/U2WAnfa07cC7xhj+uyRiIhLRIrsaTcwD9BhW5RSahCl7aotY0xcRO4H3gCcwPPGmM0i8jiwxhizFPgF8KKI7MLqiSzo3F5E9gE5gEdEvgDMBfYDb9gh4gTeAv49XfuglFKqf3KcDsB5Y8aMGWbNGr1aWCmlToaIrDXGzOivnP6yXSml1IBokCillBoQDRKllFIDokGilFJqQDRIlFJKDYgGiVJKqQHRIFFKKTUgGiRKKaUGRINEKaXUgGiQKKWUGhANEqWUUgOiQaKUUmpANEiUUkoNiAaJUkqpAdEgUUopNSAaJEoppQYkbXdIPNvFYjGqq6sJh8OD3ZTzjs/no6KiArfbPdhNUUqdARdskFRXV5OdnU1lZSUiMtjNOW8YY2hsbKS6upqqqqrBbo5S6gy4YA9thcNhCgsLNUROMxGhsLBQe3pKXUAu2CABNETSRN9XpS4sF3SQKKWUGri0BomI3Cgi20Vkl4g82Mt6r4j8xl6/SkQq7eWFIvK/IhIUkcU9trlERD61t3lazuGvv06nk2nTpjF58mRuu+02Ojo6Tku9tbW1zJs3j4suuoiJEyfyuc99DoCqqiq2b9/erey3v/1tnnzySZYvX05ubi7Tp09n3LhxzJ49m9dee+20tEcpdX5LW5CIiBN4BvgTYCKwUEQm9ih2N9BsjBkN/Bj4ob08DDwM/G0vVf8bcC8wxn7cePpbf2b4/X7Wr1/Ppk2b8Hg8PPvssye8bTwe73PdI488wpw5c9iwYQNbtmzhiSeeAGDBggUsWbKkq1wymeSVV15h/vz5AMyaNYt169axfft2nn76ae6//37efvvtU9w7pdSFIp1XbV0G7DLG7AEQkSXAzcCWlDI3A4/Z068Ai0VEjDHtwPsiMjq1QhEpA3KMMR/a878CvgAsG0hD/+l/NrPlcNtAqjjGxKE5PPr5SSdcftasWWzcuJF9+/Yxb948Nm3aBMBTTz1FMBjkscce45prruHKK69k5cqV3HTTTdxxxx18/etf58CBAwD85Cc/4aqrruLIkSPMnTu3q+6pU6cCsHDhQubPn8+jjz4KwIoVK6isrGTEiBHs3bu3W3umTZvGI488wuLFi7nuuuu6rUskEtx9992sWbMGEeGrX/0qDzzwwMm/SUqp80I6g6QcOJgyXw3M7KuMMSYuIq1AIdBwnDqre9RZ3ltBEbkXq+fC8OHDT7btZ1Q8HmfZsmXceGP/nauWlhbeffddABYtWsQDDzzA1VdfzYEDB7jhhhvYunUr9913H/Pnz2fx4sVcf/313HXXXQwdOpSpU6ficDjYsGEDF110EUuWLGHhwoV9vtbFF1/Mj370o2OWr1+/nkOHDnWFXUtLyynuuVLqfJDOIOnt3IU5hTKnVN4Y8xzwHMCMGTOOV+dJ9RxOp1AoxLRp0wCrR3L33Xdz+PDh427TeRgK4K233mLLlqMdvLa2NgKBADfccAN79uzh9ddfZ9myZUyfPp1NmzZRXFzMwoULWbJkCZMmTeL3v/89jz/+eJ+vZUzvb9vIkSPZs2cP3/zmN/nTP/3Tbr0fpdSFJ51BUg0MS5mvAHp+SnaWqRYRF5ALNPVTZ0U/dZ4zOs+RpHK5XCSTya75nr/HyMzM7JpOJpN8+OGH+P3+Y+ouKChg0aJFLFq0iHnz5rFixQpuueUWFi5cyNy5c/nMZz7D1KlTKSkp6bN969atY8KECSQSCS655BIAbrrpJh5//HE2bNjAG2+8wTPPPMNvf/tbnn/++VN6D5RS5750XrW1GhgjIlUi4gEWAEt7lFkK3GlP3wq8Y/r6GgwYY44AARG53L5a6w7g96e/6YOntLSUuro6GhsbiUQix71yau7cuSxefPSits5Qeuedd7quAAsEAuzevbvr8N6oUaMoLCzkwQcfPO5hrY0bN/K9732P++67D6fTyfr161m/fj2PP/44DQ0NJJNJbrnlFr73ve/xySefnI5dV0qdo9LWI7HPedwPvAE4geeNMZtF5HFgjTFmKfAL4EUR2YXVE1nQub2I7ANyAI+IfAGYa4zZAnwDeAHwY51kH9CJ9rON2+3mkUceYebMmVRVVTF+/Pg+yz799NPcd999TJ06lXg8zuzZs3n22WdZu3Yt999/f1fv5p577uHSSy/t2m7hwoU89NBDfPGLX+xW33vvvcf06dPp6OigpKSEp59++pgT7QCHDh3irrvu6uo5/eAHPzhNe6+UOhfJcToA540ZM2aYNWvWdFu2detWJkyYMEgtOv/p+6vUuU9E1hpjZvRXTn/ZrpRSakA0SJRSSg2IBolSSqkB0SBRSik1IBokSimlBkSDRCml1IBokAyiK6+8stv8j3/8Y3w+H62trV3Lli9fzrx5847Z9rXXXmP69OldQ8X/7Gc/S3t7lVKqNxfsPdvPBh988EG3+ZdffplLL72UV199lT//8z/vc7tYLMa9997Lxx9/TEVFBZFIhH379qW3sUop1QcNEoBlD0LNp6e3ziFT4E+eOG6RrKwsgsEgALt37yYYDPKjH/2I73//+8cNkkAgQDwep7CwEACv18u4ceOOKafDvSulzgQNkrPEyy+/zMKFC5k1axbbt2+nrq6uzwEVCwoKuOmmmxgxYgTXXXcd8+bNY+HChTgc3Y9U6nDvSqkzQYME+u05nAlLlizh1VdfxeFw8KUvfYn/+q//4r777uuz/M9//nM+/fRT3nrrLZ566inefPNNXnjhhW5ldLh3pdSZoCfbzwIbN25k586dzJkzh8rKSpYsWcLLL7/c73ZTpkzhgQce4M033+R3v/sdiUSCadOmdd3dMD8/nw0bNnDNNdfwzDPPcM8995yBvVFKXWi0R3IWePnll3nsscd46KGHupZVVVWxf//+XssHg0HWrFnDNddcA1iHsEaMGNE13HunhoYGPB4Pt9xyC6NGjTrueRellDpVGiRngSVLlrBsWffR8L/4xS+yZMkSZs6cydtvv01FxdH7eb388ss8+eSTfO1rX8Pv95OZmXnMYS3Q4d6VUmeGDiOv0kLfX6XOfTqMvFJKqTNCg0QppdSAaJAopZQaEA0SpZRSA6JBopRSakA0SJRSSg1IWoNERG4Uke0isktEHuxlvVdEfmOvXyUilSnrHrKXbxeRG1KW7xORT0VkvYis6VnnuUSHkVdKnQ/S9oNEEXECzwBzgGpgtYgsNcZsSSl2N9BsjBktIguAHwLzRWQisACYBAwF3hKRscaYhL3dtcaYhnS1/UzRYeSVUueDdP6y/TJglzFmD4CILAFuBlKD5GbgMXv6FWCxiIi9fIkxJgLsFZFddn0fpqOhP/z4h2xr2nZa6xxfMJ5/uOwfjlsm3cPIK6XUmZDOQ1vlwMGU+Wp7Wa9ljDFxoBUo7GdbA/xRRNaKyL19vbiI3Csia0RkTX19/YB25EzobRj5vqQOI79w4UJ+/etfdw2DopRSZ1o6eyTSy7Ke47H0VeZ4215ljDksIiXAmyKyzRiz4pjCxjwHPAfWECnHa2h/PYczIR3DyCul1JmQziCpBoalzFcAh/soUy0iLiAXaDretsaYzuc6EXkV65DXMUFyLkkdRh4gGo0ycuTI4wYJWMPIT5kyhdtvv52qqioNEqXUoEjnoa3VwBgRqRIRD9bJ86U9yiwF7rSnbwXeMdYokkuBBfZVXVXAGOBjEckUkWwAEckE5gKb0rgPZ0TnMPL79u1j3759HD58mEOHDh13GPnly5d3zXcOI6+UUoMhbT0SY0xcRO4H3gCcwPPGmM0i8jiwxhizFPgF8KJ9Mr0JK2ywy/0W68R8HLjPGJMQkVLgVet8PC7gJWPM6+nahzMlXcPIK6XUmaDDyKu00PdXqXOfDiOvlFLqjNAgUUopNSAaJEoppQZEg0QppdSAaJAopZQaEA0SpZRSA6JBMoicTifTpk1j8uTJ3HbbbXR0dJyWemtra5k3b17XEPOf+9znAKiqqmL79u3dyn7729/mySefZPny5eTm5jJ9+nTGjRvH7Nmzee21105Le5RS5zcNkkHk9/tZv349mzZtwuPx8Oyzz57wtvF4vM91jzzyCHPmzGHDhg1s2bKFJ554AoAFCxawZMmSrnLJZJJXXnmF+fPnAzBr1izWrVvH9u3befrpp7n//vt5++23T3HvlFIXinSOtXXOqPn+94lsPb3DyHsnjGfId75zwuVnzZrFxo0b2bdvH/PmzWPTJmvkl6eeeopgMMhjjz3GNddcw5VXXsnKlSu56aabuOOOO/j617/OgQMHAPjJT37CVVddxZEjR5g7d25X3VOnTgVg4cKFzJ8/n0cffRSAFStWUFlZyYgRI9i7d2+39kybNo1HHnmExYsXc9111w3ovVBKnd+0R3IWiMfjLFu2jClTpvRbtqWlhXfffZe/+Zu/4Vvf+hYPPPAAq1ev5ne/+x333HMPAPfddx9333031157Lf/yL//C4cPWWJlTp07F4XCwYcMGwBqaZeHChX2+1sUXX8y2bac3YJVS5x/tkcBJ9RxOp1AoxLRp0wCrR3L33Xd3fej3pfMwFMBbb73Fli1H7xPW1tZGIBDghhtuYM+ePbz++ussW7aM6dOns2nTJoqLi1m4cCFLlixh0qRJ/P73v+fxxx/v87UuhOFzlFIDp0EyiDrPkaRyuVzdblIVDoe7rc/MzOyaTiaTfPjhh/j9/mPqLigoYNGiRSxatIh58+axYsUKbrnlFhYuXMjcuXP5zGc+w9SpUykpKemzfevWrdPxspRS/dJDW2eZ0tJS6urqaGxsJBKJHPfKqblz57J48eKu+c5Qeuedd7quAAsEAuzevZvhw4cDMGrUKAoLC3nwwQePe1hr48aNfO97/6+98w+ysirj+Oc7uLJpmj8QJVZhVWxDI3BEwB+EWvkjU5mcghzDlHGaIJTRSXeYYbT+KLOp3FIITSkyNUkNmTEzxIH8QYIgPxKSknQdS0AkkzJXn/44z8rlcu/uXi5333t3n8/MO/e855z3vN/3uT+ee8553+d8u9M1UYIgCKJHUmXU1dUxc+ZMRo0aRWNjI01NTUXrtrS0MGXKFIYNG0ZbWxtjx45l9uzZrFixgqlTp37Qu5k8eTIjR4784LiJEyfS3NzM+PHjd2lv6dKljBgxgh07dtC/f39aWlpioj0Igk6JMPJBRQj7BkHtE2HkgyAIgm4hHEkQBEFQFr3akfSGYb0sCLsGQe+i1zqS+vp6tm7dGj96exkzY+vWrdTX12ctJQiCbqLX3rXV0NBAa2srmzdvzlpKj6O+vp6GhoasZQRB0E30WkdSV1dHY2Nj1jKCIAhqnooObUk6R9IGSRslXV+gvK+k+7x8maTBOWXNnr9B0tldbTMIgiDoXirmSCT1AW4FzgWGAhMlDc2rdgWwzcyOBX4I3OTHDgUmAMcD5wC3SerTxTaDIAiCbqSSPZKTgY1m9jcz+x9wL3BhXp0LgZ97ej5wliR5/r1m9o6ZvQRs9Pa60mYQBEHQjVRyjmQg8ErOfiswqlgdM2uTtB041POfyTt2oKc7axMASVcCV/ruvyVtKFRvL9EP2FLB9itJLWuH0J81oT9bKq1/UFcqVdKRqEBe/r22xeoUyy/Ugyp4/66ZzQHmdCRwbyFpeVfCCFQjtawdQn/WhP5sqRb9lRzaagWOzNlvAPIX2/igjqR9gI8Ab3RwbFfaDIIgCLqRSjqSZ4Ehkhol7UuaPF+QV2cBMMnTFwOPW3pCcAEwwe/qagSGAH/qYptBEARBN1KxoS2f85gKPAr0Ae40s3WSvgUsN7MFwM+AeZI2knoiE/zYdZJ+DfwZaAOmmNl7AIXarNQ1lEC3DKFViFrWDqE/a0J/tlSF/l4RRj4IgiCoHL021lYQBEGwdwhHEgRBEJRFOJISkHSkpMWSXpC0TtJVnn+IpMckveivB2ettSM8SsBKSQt9v9FD1LzoIWv2zVpjMSQdJGm+pPX+PoypFftLmu6fm7WS7pFUX+22l3SnpNclrc3JK2hvJVo8fNFqSSdmp7yo9pv9s7Na0oOSDsopKxiWKSsK6c8pu1aSSern+5naPhxJabQB15jZx4HRwBQP0XI9sMjMhgCLfL+auQp4IWf/JuCHrn8bKXRNtWQX2dIAAAbOSURBVHIL8DszawI+SbqOqre/pIHANOAkMzuBdLPIBKrf9nNJYYpyKWbvc0l3WA4hPQw8q5s0FmMuu2t/DDjBzIYBfwGaoXhYpu6TWpC57K4fSUcCnwFezsnO1vZmFtsebsBv/Q3dAAzwvAHAhqy1daC5gfTlPxNYSHr4cwuwj5ePAR7NWmcR7QcCL+E3ieTkV7392RnF4RDS3ZILgbNrwfbAYGBtZ/YGfgpMLFSvWrTnlY0H7vZ0M9CcU/YoMKbabO9580l/ojYB/arB9tEj2UM8UvEIYBlwuJm9BuCv/bNT1ik/Ar4JvO/7hwJvmlmb7+eGo6k2jgY2A3f50NwdkvanBuxvZq8C3yf9i3wN2A6soHZsn0sxexcKi1TN13M58Iina0K7pAuAV83s+byiTPWHI9kDJH0Y+A1wtZn9K2s9XUXS+cDrZrYiN7tA1Wq9J3wf4ERglpmNAN6mCoexCuHzCBcCjcBHgf1JwxH5VKvtu0LNfJYkzSANVd/dnlWgWlVpl7QfMAOYWai4QF636Q9HUiKS6khO5G4ze8Cz/ylpgJcPAF7PSl8nnApcIGkTKXLymaQeykEeogaqO+xMK9BqZst8fz7JsdSC/T8NvGRmm83sXeAB4BRqx/a5FLN3TYQwkjQJOB+4xHwciNrQfgzpj8jz/h1uAJ6TdAQZ6w9HUgKSRHoa/wUz+0FOUW6ol0mkuZOqw8yazazBzAaTJhYfN7NLgMWkEDVQ3fr/Abwi6WOedRYp+kEt2P9lYLSk/fxz1K69JmyfRzF7LwC+4ncQjQa2tw+BVQuSzgGuAy4wsx05RcXCMlUNZrbGzPqb2WD/DrcCJ/r3IlvbZz2ZVEsbcBqpu7gaWOXbeaR5hkXAi/56SNZau3At44CFnj6a9KXZCNwP9M1aXwe6hwPL/T14CDi4VuwP3AisB9YC84C+1W574B7SnM67pB+uK4rZmzS8civwV2AN6Q61atO+kTSX0P79nZ1Tf4Zr3wCcW422zyvfxM7J9kxtHyFSgiAIgrKIoa0gCIKgLMKRBEEQBGURjiQIgiAoi3AkQRAEQVmEIwmCIAjKIhxJ0COR9B1J4yRdJKmkp98lHeYReVdKOj2v7AlJJ3Vy/NX+FPIe4bpPKVLWV9IfJK2S9KUS2x0s6ct7qisIihGOJOipjCLFQfsUsLTEY88C1pvZCDMr9ViAq4E9diSkZ3wKOhJSfLc6MxtuZveV2O5goCRHkvPUfRAUJRxJ0KPw9SZWAyOBp4HJwCxJu8UnkjRI0iJfv2GRpKMkDQe+B5zn//o/1MG5ZklarrTGyI2eN40US2uxpMWe91lJT0t6TtL9HqsNSZsk3ej5ayQ1eTDQrwHT/fyn55yvP/BLYLiXHSNppqRnldY4meNPzSPpWO+5PO/tHwN8Fzjdj52utB7KXX7ulZLO8GMvc50PA7+XNEDSEj9ubX4vLQgyf3I2ttj29gacDPwYqAOe7KDew8AkT18OPOTpy4CfFDnmCfypYXY+0d3H84f5/iZ2PnHcD1gC7O/71wEzc+p9w9NfB+7w9A3AtUXOPw6PSJCrwdPzgM97ehkw3tP1pB5S/rHXAHd5uokUxqXer7815/quAWbkXOsBWb/HsVXXFj2SoCcyghT+ookUz6oYY4BfeXoeKQROKXxR0nPAStKCSEML1Bnt+U9KWkWKTTUop7w98OcK0tBTqZzh8zlrSEE4j5d0ADDQzB4EMLP/2q5xpdo5jXTdmNl64O/AcV72mJm94elnga9KugH4hJm9tQc6gx5MjH8GPQYflppLiny6hfQvXP4DPsbM/tNJE12OF+SB/a4FRprZNklzSf/md6tK+lGeWKSpd/z1PUr8PkqqB24j9ZBe8R/6egqHFC/YRAdlb7cnzGyJpLHA54B5km42s1+UojXo2USPJOgxmNkqMxtOWkJ1KPA4cLalielCTuQpUhRkgEuAP5ZwugNJP7bbJR3OrmuLvAUc4OlngFMlHQtpTQlJx9Exucd3RLvj2uLzLhcDWFojp1XSRX7Ovn4XWX67S0jXjWs6ihSwcBckDSKtY3M7Kfp1pmuxB9VHOJKgRyHpMGCbmb0PNJlZR0Nb00hDNquBS0lr2XcJSyvUrQTWAXcCT+YUzwEekbTYzDaT5hzu8fM8Qxpy64iHgfH5k+0FNLwJ3E6K9voQaQiqnUuBaX7Op4AjSBGT23wCfjqpN9PHh8XuAy4zs3fYnXHAKkkrgS8At3SiP+hlRPTfIAiCoCyiRxIEQRCURTiSIAiCoCzCkQRBEARlEY4kCIIgKItwJEEQBEFZhCMJgiAIyiIcSRAEQVAW/wcXsJfo2RDx4AAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -1579,20 +1579,20 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'alpha': 0.3,\n", - " 'epsilon': 0.1,\n", - " 'weight_func': ,\n", - " 'regularization': 1.0,\n", + "{'alpha': 0.1,\n", + " 'epsilon': 0.01,\n", + " 'weight_func': ,\n", + " 'regularization': 3.0,\n", " 'rank': 60}" ] }, - "execution_count": 68, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -1604,7 +1604,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ @@ -1613,7 +1613,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 51, "metadata": {}, "outputs": [ { @@ -1623,11 +1623,11 @@ "\n", "100%\n", "5/5\n", - "[03:00<00:36, 36.06s/it]" + "[03:36<00:44, 43.26s/it]" ], "text/plain": [ "\u001b[A\u001b[2K\r", - " [████████████████████████████████████████████████████████████] 5/5 [03:00<00:36, 36.06s/it]" + " [████████████████████████████████████████████████████████████] 5/5 [03:36<00:44, 43.26s/it]" ] }, "metadata": {}, @@ -1643,12 +1643,12 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 52, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+0nVV97/v3x0RAUVExWuVX4EBtg1BattBeq02lKIyqsUesQW6Fe7kjWuX2jHptC+cUGji2RzrOKbUH2msqIGIVLC23qUSjFfHXtZggSAyU2xijxGgJQlFQwMD3/rGejYuVneyV7Gevtfba79cYa+xnzWfOueYz2Xuy893zR6oKSZIkSZKktjxl2A2QJEmSJEnjxWCDJEmSJElqlcEGSZIkSZLUKoMNkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhls0NhIsiXJrw27HZIkSZI03xlskCRJkiRJrTLYoHknycJht0GS5rOpxmHHZkkarCQLht0GjTeDDRo3xyW5PckDSa5Nsl+SpUm2JvmDJN8Frhx2IyVpHDTL136vGXcfSnJ5khck+XiSHyT5pyTPSbI4SSU5O8m3gBunShv280jSoCU5JMnfJ9me5HtJLk3ylCR/mOSbSe5J8sEkBzT5P5HknJ46vprkPzbXP5PkU0nuS3JXkt/syveBJH+VZE2Sh4BfTfLrSW5N8v0kdydZ2VP3W5p2fC/J+d3Llpt2npvk6839jyZ57mz3meYOgw0aN78JnAIcDhwLnNWk/xTwXOAwYMVQWiZJ4+kNwMnATwOvBT4O/GfgeXR+z/idrry/Avws8Opp0iRp7DUzCz4GfBNYDBwEXEPn99ezgF8FjgCeAVzaFPswcHpXHUvo/H57Q5L9gU81eZ7f5PvLJEd3feybgT8Gngl8AXgIeAvwbODXgd9O8vquuv8SOAN4IXBA08ZJvwO8ns44/iLgfuCyGXSJxozBBo2bv6iqbVV1H/CPwHFN+uPAH1XVI1X1o+E1T5LGzv+sqn+rqm8Dnwdurqpbq+oR4Hrg57vyrqyqh3rG4anSJGk+OIHOP9J/rxkHH66qL9D5x/2fVdXmqnoQOA9Y3iw3u57OTN7DmjrOAP6+GXNfA2ypqiurakdVfQX4O+C0rs/8h6r6YlU93nzeTVW1oXl/O/AROsEDmnL/WFVfqKpHgQuA6qrrrcB/qaqtzeevBE5zWZwmGWzQuPlu1/UP6USCAbZX1cNDaI8kjbt/67r+0RTvn9H1/u4pyk+VJknzwSHAN6tqR0/6i+jMdpj0TWAh8IKq+gFwA7C8ubcc+Jvm+jDgxCT/PvmiE4z4qa66njTmJjkxyWeaZRwPAG+jMzNtsh1P5K+qHwLf6yp+GHB912fdCTwGvKDvHtBYM9ig+aKmzyJJmmVTjcWOz5Lmq7uBQ6eYCbCNzj/kJx0K7OAnwdyPAKcn+SXgacBnuur7bFU9u+v1jKr67a66esfcDwOrgUOq6gDg/wbS3PsOcPBkxiRPAw7saf+pPZ+3XzPTTTLYIEmSJElD8GU6/6B/T5L9m43NX0YnmPC7SQ5P8gzgT4Bru2ZArKETjLioSX+8Sf8Y8NNJfivJU5vXS5P87G7a8Ezgvqp6OMkJdPZ0mHQd8Nok/0uSfYAL+UkgAjqBiT+eXNKRZFGSZTPoD40Zgw2SJEmSNGBV9RidjXWPBL4FbAXeBFwBXA18DvgG8DDwf3aVewT4e+DX6MxMmEz/AfAqOksrttFZXnwxsO9umvF24KIkP6CzJ8NHu+rb2HzuNXSCIj8A7gEeabK8l86siE825f8ZOHGPO0JjK1XOXpQkSZIk7Vozy+LfgaOq6hvDbo9GnzMbJEmSJEk7SfLaJE9vjtX878AGYMtwW6W5wmCDJEmSJGkqy+gsydgGHAUsL6fGq08uo5AkSZIkSa1yZoMkSZIkSWqVwQZJkiRJktSqhcNuQK/nPe95tXjx4mE3Q5J2csstt9xbVYuG3Y5BcCyWNIochyVp+Podi0cu2LB48WLWr18/7GZI0k6SfHPYbRgUx2JJo8hxWJKGr9+x2GUUkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBrVi6dKlLF26dNjNkKR5zbFYkobLcVj6CYMNkjRCkpyS5K4km5KcO8X9fZNc29y/OcniJv2EJLc1r68m+Y2uMluSbGjurR/c00iSJGm+WjjsBkiSOpIsAC4DTga2AuuSrK6qO7qynQ3cX1VHJlkOXAy8CfgaMFFVO5K8EPhqkn+sqh1NuV+tqnsH9zSSJEmazww2SNLoOAHYVFWbAZJcAywDuoMNy4CVzfV1wKVJUlU/7MqzH1Cz31xJksbHMVcdM+M6Nn93c2t1AWw4c0Mr9UjD4DIKSRodBwF3d73f2qRNmaeZtfAAcCBAkhOTbAQ2AG/rmtVQwCeT3JJkxa4+PMmKJOuTrN++fXsrDyRJkqT5yZkNYvG5N8y4ju9u/l5rdQFsec+vt1KPNMdkirTeGQq7zFNVNwNHJ/lZ4KokH6+qh4GXVdW2JM8HPpXkX6rqcztVUrUKWAUwMTHhzAhJkiTtNWc2SNLo2Aoc0vX+YGDbrvIkWQgcANzXnaGq7gQeAl7SvN/WfL0HuJ7Ocg1JkiRp1hhskKTRsQ44KsnhSfYBlgOre/KsBs5srk8DbqyqasosBEhyGPBiYEuS/ZM8s0nfH3gVnc0kJUlSy4447wiOOO+IYTdDGgkuo5CkEdGcJHEOsBZYAFxRVRuTXASsr6rVwOXA1Uk20ZnRsLwp/svAuUl+DDwOvL2q7k1yBHB9EuiM+R+uqk8M9skkSZI03xhsUCt+6s3vGXYTpLFQVWuANT1pF3RdPwy8cYpyVwNXT5G+Gfi59luqVq08oJ16tjzUXn0rH5h5HdKYSXIK8F46AeH3V9V7eu6/Avhz4FhgeVVd13P/WcCdwPVVdc5gWi1Jw+EyCkmSJGkaSRYAlwGnAkuA05Ms6cn2LeAs4MO7qOa/Ap+drTZK0igx2CBJkiRN7wRgU1VtrqpHgWuAZd0ZqmpLVd1OZznbkyQ5HngB8MlBNFaShs1ggyRJkjS9g4C7u95vbdKmleQpwP8Afm+afCuSrE+yfvv27XvdUEkaBQYbJEmSpOllirTqs+zbgTVVdffuMlXVqqqaqKqJRYsW7XEDJWmUuEGkJEmSNL2twCFd7w8GtvVZ9peAlyd5O/AMYJ8kD1bVuS23UZJGRl8zG5KckuSuJJuS7DQoJtk3ybXN/ZuTLG7Sn5rkqiQbktyZ5Lx2mz/7li5dytKlS4fdDEmSJA3XOuCoJIcn2YfO0cOr+ylYVWdU1aFVtRh4F/BBAw2Sxt20wYY+d949G7i/qo4ELgEubtLfCOxbVccAxwNvnQxESJIkSXNFVe0AzgHW0jm+8qNVtTHJRUleB5DkpUm20vkd+H1JNg6vxZI0XP0so3hi512AJJM7797RlWcZsLK5vg64NEnorGPbP8lC4GnAo8D322m6pHE1OZvopptuGmo7JEnqVlVrgDU9aRd0Xa+js7xid3V8APjALDRPkkZKP8GGqXbePXFXeapqR5IHgAPpBB6WAd8Bng78blXdN9NGS3OV/4iWNJtuOmv/YTdBkiQJ6G/Phn523t1VnhOAx4AXAYcD/1eSI3b6AI/5kSRJkiRpbPQTbOhn590n8jRLJg4A7gPeDHyiqn5cVfcAXwQmej/AY34kSZIkSRof/SyjeGLnXeDbdHbefXNPntXAmcCXgNOAG6uqknwLeGWSD9FZRvGLwJ+31fjpLD73hhnX8d3N32utLoAt7/n1VuqRpPnEJUiSJElzy7TBhmYPhsmddxcAV0zuvAusr6rVwOXA1Uk20ZnRsLwpfhlwJfA1Okstrqyq22fhOSSNgLaCcm0G+QzwSZIkSYPXz8yGfnbefZjOET+95R6cKl2SJEmjy9lEkqSZ6mfPBkmSJEmSpL71NbNBmu9cHiBJ48G/2EuSNBjObJCkEZLklCR3JdmU5Nwp7u+b5Nrm/s1JFjfpJyS5rXl9Nclv9FunJEmS1DZnNkzjp978nmE3QdI8kWQBnY11T6ZzpPC6JKur6o6ubGcD91fVkUmWAxcDb6KzEe9Es6nvC4GvJvlHoPqoU5IkSWqVwQZJI2ceB/lOADZV1WaAJNcAy4DuwMAyYGVzfR1waZJU1Q+78uxHJ8jQb52SJElSqww2SNLoOAi4u+v9VuDEXeVpZjE8ABwI3JvkROAK4DDgt5r7/dQJQJIVwAqAQw89dOZPA7DygHbq2fJQe/WtfGDmdUiSJGm33LNBkkZHpkirfvNU1c1VdTTwUuC8JPv1WSdN+VVVNVFVE4sWLdqDZkuSJElP5swGaYDm8fIA9WcrcEjX+4OBbbvIszXJQuAA4L7uDFV1Z5KHgJf0WaekcdLGDKA2ZxOBM4okaR5yZoMkjY51wFFJDk+yD7AcWN2TZzVwZnN9GnBjVVVTZiFAksOAFwNb+qxTkiRJapXBBkkaEVW1AzgHWAvcCXy0qjYmuSjJ65pslwMHJtkEvBOYPMryl+mcQHEbcD3w9qq6d1d1Du6pJGl89HE88SuSfCXJjiSndaUfl+RLSTYmuT3JmwbbckkaPJdRSNIIqao1wJqetAu6rh8G3jhFuauBq/utU5qTXB6gIerzeOJvAWcB7+op/kPgLVX1r0leBNySZG1V/fsAmi5JQ2GwQZIkSZretEcJV9WW5t7j3QWr6v/rut6W5B5gEWCwQdLYMtggSRp5N521/7CbIEl9HyW8O0lOAPYBvj7FvfaPIJakIXHPBkmSJGl6fR8lvMsKkhfSWfL2v1XV4733PYJY0jgx2CBJkiRNb0ZHCSd5FnAD8IdV9c8tt02SRo7BBkmSJGl6e32UcJP/euCDVfW3s9hGSRoZ7tkgSZKkJ3GflJ1V1Y4kk0cJLwCumDyeGFhfVauTvJROUOE5wGuTXFhVRwO/CbyCztHFZzVVnlVVtw3+SSRpMAw2SJIkSX3o43jidXSWV/SW+xDwoVlvoCSNEJdRSJIkSZKkVjmzQZIkzRsuD5AkaTCc2SBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWyQpBGS5JQkdyXZlOTcKe7vm+Ta5v7NSRY36ScnuSXJhubrK7vK3NTUeVvzev7gnkiSJEnz0cJhN0CS1JFkAXAZcDKwFViXZHVV3dGV7Wzg/qo6Msly4GLgTcC9wGuraluSlwBrgYO6yp1RVesH8iCSJEma95zZIEmj4wRgU1VtrqpHgWuAZT15lgFXNdfXASclSVXdWlXbmvSNwH5J9h1IqyVJkqQeBhskaXQcBNzd9X4rT56d8KQ8VbUDeAA4sCfPG4Bbq+qRrrQrmyUU5ydJu82WJEmSnsxggySNjqmCALUneZIcTWdpxVu77p9RVccAL29evzXlhycrkqxPsn779u171HBJkiS1a+nSpSxdunTYzdhrfQUb9nbDsubesUm+lGRjs3HZfu01X5LGylbgkK73BwPbdpUnyULgAOC+5v3BwPXAW6rq65MFqurbzdcfAB+ms1xjJ1W1qqomqmpi0aJFrTyQJEmS5qdpgw1dG5adCiwBTk+ypCfbExuWAZfQ+ava5C/CHwLeVlVHA0uBH7fWekkaL+uAo5IcnmQfYDmwuifPauDM5vo04MaqqiTPBm4AzquqL05mTrIwyfOa66cCrwG+NsvPIUljqY8/wL0iyVeS7EhyWs+9M5P8a/M6s7esJI2bfk6jeGLDMoAkkxuWde+OvgxY2VxfB1zarAl+FXB7VX0VoKq+11K7JWnsVNWOJOfQOUliAXBFVW1MchGwvqpWA5cDVyfZRGdGw/Km+DnAkcD5Sc5v0l4FPASsbQINC4B/Av56YA8lSWOizxODvgWcBbyrp+xzgT8CJugsfbulKXv/INouabCOueqYVurZ/N3NrdW34cwNM65jT/UTbJhqw7ITd5Wn+WV5csOynwYqyVpgEXBNVf1p7wckWQGsADj00EP39BkkaWxU1RpgTU/aBV3XDwNvnKLcu4F376La49tsoyTNU9P+Aa6qtjT3Hu8p+2rgU1U1ueztU8ApwEdmv9mSNBz97Nkwkw3LFgK/DJzRfP2NJCftlNF1wpIkSRpt/ZwYNKOybtQraZz0M7NhTzYs29qzYdlW4LNVdS9AkjXALwCfnmG7JUmSpEHq5w9wMypbVauAVQATExP91i1pTB1x3hHDbsKM9DOzYa83LKOz7vjYJE9vghC/wpP3epAkSZLmgn7+ADcbZSVpTpo22FBVO+hsPLYWuBP46OSGZUle12S7HDiw2bDsncC5Tdn7gT+jE7C4DfhKVd3Q/mNIkiRJs6qfP8DtylrgVUmek+Q5dDbwXTtL7ZSkkdDPMoq93rCsufchOsdfSpIkSXNSPycGJXkpcD3wHOC1SS6sqqOr6r4k/5VOwALgosnNIiVpXPUVbJAkSZLmuz7+ALeOzhKJqcpeAVwxqw2UpBHSz54NkiRJkiS1YunSpSxdunTYzdAsM9ggSZIkSZJaZbBBkiRJkiS1yj0bJEmSJEnTOuaqY1qpZ/N3N7dW34YzN8y4Ds0OZzZIkiRJkqRWGWyQJEmSJEmtMtggSZIkSZJa5Z4NkiRJknZr8pjCm266aajt0Hg44rwjht0EDYAzGyRJkiRJUquc2SBJkiSNsTZ2/G/z9ADwBAFpPnBmgyRJkiRJapXBBkkaIUlOSXJXkk1Jzp3i/r5Jrm3u35xkcZN+cpJbkmxovr6yq8zxTfqmJH+RJIN7IkmSJM1HLqOQpBGRZAFwGXAysBVYl2R1Vd3Rle1s4P6qOjLJcuBi4E3AvcBrq2pbkpcAa4GDmjJ/BawA/hlYA5wCfHwQzyRJGg9u6CdpTzmzQZJGxwnApqraXFWPAtcAy3ryLAOuaq6vA05Kkqq6taq2Nekbgf2aWRAvBJ5VVV+qqgI+CLx+9h9FkiRJ85nBBkkaHQcBd3e938pPZifslKeqdgAPAAf25HkDcGtVPdLk3zpNnQAkWZFkfZL127dv3+uHkCRJkgw2SNLomGovhdqTPEmOprO04q17UGcnsWpVVU1U1cSiRYv6aK4kzS8z2FfnqUmuavbPuTPJeYNuuyQNmsEGSRodW4FDut4fDGzbVZ4kC4EDgPua9wcD1wNvqaqvd+U/eJo6JUnT6NpX51RgCXB6kiU92Z7YVwe4hE7wF+CNwL5VdQxwPPDWyUCEJI0rgw2SNDrWAUclOTzJPsByYHVPntXAmc31acCNVVVJng3cAJxXVV+czFxV3wF+kOQXm1Mo3gL8w2w/iCSNob3eV4fOjLL9myDx04BHge8PptmSNBwGGyRpRDR7MJxD5ySJO4GPVtXGJBcleV2T7XLgwCSbgHcCk9N4zwGOBM5Pclvzen5z77eB9wObgK/jSRSStDdmsq/OdcBDwHeAbwH/varu6/0A986RNE48+lKSRkhVraFzPGV32gVd1w/TmY7bW+7dwLt3Ued64CXttlSS5p2Z7KtzAvAY8CLgOcDnk/xTVW1+UsaqVcAqgImJiSn315GkucKZDZIkSdL0ZrKvzpuBT1TVj6vqHuCLwMSst1iShshggyRJkjS9vd5Xh87SiVemY3/gF4F/GVC7JWkoDDZIkiRJ05jhvjqXAc8AvkYnaHFlVd0+0AeQpAFzzwZJkiSpDzPYV+fBqdIlaZw5s0GSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRW9RVsSHJKkruSbEpy7hT3901ybXP/5iSLe+4fmuTBJO9qp9mSJEmSJGlUTRtsSLIAuAw4FVgCnJ5kSU+2s4H7q+pI4BLg4p77lwAfn3lzJUmSJEnSqOtnZsMJwKaq2lxVjwLXAMt68iwDrmqurwNOShKAJK8HNgMb22myJEmSJEkaZf0EGw4C7u56v7VJmzJPVe0AHgAOTLI/8AfAhTNvqiRJkiRJmgv6CTZkirTqM8+FwCVV9eBuPyBZkWR9kvXbt2/vo0mSJEmSJGlULewjz1bgkK73BwPbdpFna5KFwAHAfcCJwGlJ/hR4NvB4koer6tLuwlW1ClgFMDEx0RvIkCRJkiRJc0g/MxvWAUclOTzJPsByYHVPntXAmc31acCN1fHyqlpcVYuBPwf+pDfQIEn6ib09/SfJgUk+05z8c2lPmZuaOm9rXs8fzNNIkiRpvpp2ZkNV7UhyDrAWWABcUVUbk1wErK+q1cDlwNVJNtGZ0bB8NhstSeOo6/Sfk+nMGFuXZHVV3dGV7YnTf5Isp3P6z5uAh4HzgZc0r15nVNX6WX0ASZIkqdHPMgqqag2wpiftgq7rh4E3TlPHyr1onyTNJ0+c/gOQZPL0n+5gwzJgZXN9HXBpklTVQ8AXkhw5wPZKkiRJU+pnGYUkaTD2+vSfPuq+sllCcf7k0cS93KxXknZvb5e6NfeOTfKlJBuTbEiy3yDbLkmDZrBBkkbHTE7/2Z0zquoY4OXN67emylRVq6pqoqomFi1aNG1jJWk+6VrqdiqwBDg9yZKebE8sdQMuobPUjWYD9Q8Bb6uqo4GlwI8H1HRJGgqDDZI0Ovbk9J/JX14nT//Zpar6dvP1B8CH6SzXkCTtmSeWulXVo8DkUrduy4CrmuvrgJOa2WSvAm6vqq8CVNX3quqxAbVbkobCYIMkjY69Pv1nVxUmWZjkec31U4HXAF9rveWSNP5mstTtp4FKsjbJV5L8/lQf4HI2SeOkrw0iJUmzb6an/yTZAjwL2CfJ6+n8Je2bwNom0LAA+Cfgrwf4WJI0Lmay1G0h8MvAS4EfAp9OcktVffpJGatWAasAJiYmplsiJ0kjzWCDJI2QmZz+U1WLd1Ht8W21T5LmsT1Z6ra1Z6nbVuCzVXUvQJI1wC8An0aSxpTLKCRJkqTpzWSp21rg2CRPb4IQv8KTjzWWpLHjzAZJkiRpGjNZ6lZV9yf5MzoBiwLWVNUNQ3kQSRoQgw2SJElSH2a41O1DdI6/lKR5wWUUkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSRkiSU5LclWRTknOnuL9vkmub+zcnWdykH5jkM0keTHJpT5njk2xoyvxFkgzmaSRJkjRfGWyQpBGRZAFwGXAqsAQ4PcmSnmxnA/dX1ZHAJcDFTfrDwPnAu6ao+q+AFcBRzeuU9lsvSZIk/YTBBkkaHScAm6pqc1U9ClwDLOvJswy4qrm+DjgpSarqoar6Ap2gwxOSvBB4VlV9qaoK+CDw+ll9CkkaU3s7+6zr/qHNDLSpAsOSNFYMNkjS6DgIuLvr/dYmbco8VbUDeAA4cJo6t05TJwBJViRZn2T99u3b97DpkjTeZjj7bNIlwMdnu62SNAoMNkjS6JhqL4Xaizx7lb+qVlXVRFVNLFq0aDdVStK8tNezzwCSvB7YDGwcUHslaagMNkjS6NgKHNL1/mBg267yJFkIHADcN02dB09TpyRpens9+yzJ/sAfABcOoJ2SNBIMNkjS6FgHHJXk8CT7AMuB1T15VgNnNtenATc2ezFMqaq+A/wgyS82f117C/AP7TddksbeTGafXQhcUlUP7vYDXM4maYwsHHYDJEkdVbUjyTnAWmABcEVVbUxyEbC+qlYDlwNXJ9lEZ0bD8snySbYAzwL2aabrvqqq7gB+G/gA8DQ6a4VdLyxJe25PZp9t7Zl9diJwWpI/BZ4NPJ7k4ap60lHFVbUKWAUwMTGxuyVykjTyDDZI0gipqjXAmp60C7quHwbeuIuyi3eRvh54SXutlKR56YnZZ8C36QR739yTZ3L22Zd48uyzl09mSLISeLA30CBJ48ZggyRJkjSNmc4+k6T5pq9gQ5JTgPfSGVjfX1Xv6bm/L52z248Hvge8qaq2JDkZeA+wD/Ao8HtVdWOL7ZckSZIGYiazz7ryrJyVxknSiJl2g8gZnil8L/DaqjqGzpSyq9tquCRJkiRJGk39nEax12cKV9WtVTW5cc5GYL9mFoQkSZIkSRpT/QQb9vpM4Z48bwBurapHej/AY34kSZIkSRof/QQbZnKmcOdmcjSdpRVvneoDqmpVVU1U1cSiRYv6aJIkSZIkSRpV/QQb9uRMYXrOFCbJwcD1wFuq6uszbbAkSZIkSRpt/QQbnjhTOMk+dI7wWd2TZ/JMYeg6UzjJs4EbgPOq6ottNVqSJEmSJI2uaYMNzR4Mk2cK3wl8dPJM4SSva7JdDhzYnCn8TuDcJv0c4Ejg/CS3Na/nt/4UkiRJkiRpZCzsJ9PenilcVe8G3j3DNkqSJEmSpDmkn2UUkiRJkiRJfTPYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRphCQ5JcldSTYlOXeK+/smuba5f3OSxV33zmvS70ry6q70LUk2JLktyfrBPIkkSZLmM4MNkjQikiwALgNOBZYApydZ0pPtbOD+qjoSuAS4uCm7BFgOHA2cAvxlU9+kX62q46pqYpYfQ5LG1t4GhJOcnOSWJvB7S5JXDrrtkjRoBhskaXScAGyqqs1V9ShwDbCsJ88y4Krm+jrgpCRp0q+pqkeq6hvApqY+SVILZhIQBu4FXltVxwBnAlcPptWSNDwGGyRpdBwE3N31fmuTNmWeqtoBPAAcOE3ZAj7Z/DVtxSy0W5Lmg70OCFfVrVW1rUnfCOyXZN+BtFqShsRggySNjkyRVn3m2V3Zl1XVL9D5a9w7krxiyg9PViRZn2T99u3b+22zJM0XMwkId3sDcGtVPTJL7ZSkkWCwQZJGx1bgkK73BwPbdpUnyULgAOC+3ZWd/GtaVd0DXM8ulldU1aqqmqiqiUWLFs34YSRpzMwkINy5mRxNZ2nFW6dMJgyOAAALAklEQVT8AIO+ksaIwQZJGh3rgKOSHJ5kHzobPq7uybOaznpfgNOAG6uqmvTlzeZkhwNHAV9Osn+SZwIk2R94FfC1ATyLJI2bmQSESXIwnYDvW6rq61N9gEFfSeNk4bAbIEnqqKodSc4B1gILgCuqamOSi4D1VbUauBy4OskmOr/ALm/KbkzyUeAOYAfwjqp6LMkLgOs7e0iyEPhwVX1i4A8nSXPfEwFh4Nt0xt839+SZDAh/ia6AcJJnAzcA51XVFwfYZkkaGoMNkjRCqmoNsKYn7YKu64eBN+6i7B8Df9yTthn4ufZbKknzy0wCwsA5wJHA+UnOb9Je1Sxvk6SxZLBBkiRJ6sPeBoSr6t3Au2e9gZI0QtyzQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWyQJEmSJEmt6ivYkOSUJHcl2ZTk3Cnu75vk2ub+zUkWd907r0m/K8mr22u6JI2f2Rhvp6tTktQffyeWpP5NG2xIsgC4DDgVWAKcnmRJT7azgfur6kjgEuDipuwSYDlwNHAK8JdNfZKkHrMx3vZZpyRpGv5OLEl7pp+ZDScAm6pqc1U9ClwDLOvJswy4qrm+DjgpSZr0a6rqkar6BrCpqU+StLPZGG/7qVOSND1/J5akPdBPsOEg4O6u91ubtCnzVNUO4AHgwD7LSpI6ZmO8dRyWpHb4O7Ek7YGFfeTJFGnVZ55+ypJkBbCieftgkrv6aNcgPQ+4t42KcnEbtYws+6k/rfTTmPcRjGY/HdZqbTubjfF2qqDyTuMwzKOx+MKpumqs2E/Ta+3/V/ZTn9rrp9keh3fH34nb/F3vLH92+mE/TW/M+whGs5/6Gov7CTZsBQ7pen8wsG0XebYmWQgcANzXZ1mqahWwqp8GD0OS9VU1Mex2jDr7qT/2U3/maT/N1ng77TgMjsXjwn6ann3UH/tpJ/5O7PdEX+yn/thP/ZnL/dTPMop1wFFJDk+yD53NbVb35FkNnNlcnwbcWFXVpC9vduY9HDgK+HI7TZeksTMb420/dUqSpufvxJK0B6ad2VBVO5KcA6wFFgBXVNXGJBcB66tqNXA5cHWSTXSit8ubshuTfBS4A9gBvKOqHpulZ5GkOW22xtup6hz0s0nSXOfvxJK0Z9IJtmp3kqxoprVpN+yn/thP/bGf1Mvvif7YT9Ozj/pjP6mX3xP9sZ/6Yz/1Zy73k8EGSZIkSZLUqn72bJAkSZIkSerbWAYbkjyW5LYkX0vyt0me3lK9L0jysSRfTXJHkjVN+jeSvLgn758n+f0kS5M8kOTWJHcl+VyS17TRnj1su30yy5L8vz3vfzfJw0kO6EpbmuRjU5R9TdMfk/341kG0eRDsl/nJMWfKttsnA+CYszP7ZH5yzNll++2XWeaYM7V52S9VNXYv4MGu678B3rkHZRfu5t77gP/U9f7Y5ut/A/6oK/0pdI44OgxYCnys695xwBbgJPtkuH0ygD7/MvB54KyutCc9e5P2VDrHXx3cvN8XePGw22+/+Jrhf2fHHPtkJF6OOfbJfH055tgvo/JyzJm//TKWMxt6fB44MsniJF+bTEzyriQrm+ubkvxJks8C/ynJoiR/l2Rd83pZU+yFdAYHAKrq9ubyIzS7DTdeAWypqm/2NqaqbgMuAs5p8Rn31JzpkyQLknygiT5vSPK7M3ryWZTkwa7r/wA8A/hD4PRpij6Tzskw3wOoqkeq6q7ZauegzXa/zKXvkXlszow5AzRn+mSu/Yw5Fu/McVjMoTFnwOZMv8ylnzPH4anNx7F4rIMNSRYCpwIb+sj+7Kr6lar6H8B7gUuq6qXAG4D3N3kuAy5P8pkk/yXJi+CJweTxJD/X5FtOZ2DZla8AP7PnTzRzc7BPjgMOqqqXVNUxwJV9tHsUnE7neT8PvDjJ83eVsaruo3P+9jeTfCTJGUnG9WdzNvplrn6PzAtzcMyZdXOwT+byz5hj8c4ch+eZOTjmDMQc7Je5+nPmODy1eTEWj+t/vKcluQ1YD3yLzpnH07m26/rXgEubOlYDz0ryzKpaCxwB/DWdQeDWJIuaMh8BljcD1zLgb3fzWdmjp2nHXO2TzcARSf5nklOA7/fR7lGwHLimqh4H/h544+4yV9X/AZxEZzrVu4ArZr2FwzEb/TJXv0fG3Vwdc2bTXO2Tufwz5li8M8fh+WOujjmzba72y1z9OXMcntq8GIsXDrsBs+RHVXVcd0KSHTw5uLJfT5mHuq6fAvxSVf2ot+ImsvRh4MPpbN7xCuDv6AwinwQ+C9xeVffspn0/D9zZ57O0ZU70SZIFwC1N2uqquqCJBL8aeAfwm8D/vtsnHbIkxwJHAZ9KArAPnR/+y3ZXrqo2ABuSXA18Azhrdls6WG31S5KzmePfI/PEnBhz+nyWtsyJPhmHcRgci6fiODzvzIkxp89nadOc6JdxGIsdh6c2n8bicZ3ZMJV/A56f5MAk+wK72+n1k3StlUpyXPP1lWl2rE3yTOA/0ImIUlVfp7OO5j3sZmpU8811PtN8Mw3IyPVJVT1WVcc1rwuSPA94SlX9XZPnF/b+cQfmdGBlVS1uXi8CDkpy2FSZkzwjydKupOOAndbxjYFW+mVMvkfmq5Ebc2b0NO0YuT4Zo58xx+KdOQ5r5MacGT1Ne0auX8bk58xxeGrzZiwe15kNO6mqHye5CLiZToTsX3aT/XeAy5LcTqePPge8DTiezrSpyejn+6tqXVe5j9DZcfb6nvpenuRW4OnAPcDvVNWnW3isGZkjfXIQcGV+si7pvD15xiFZTmcNYLfrm/SbgZOSbO26dzrw+0neB/yITvT8rAG0c9Bmq1/m4vfIvDRHxpyBmiN9Mld/xhyLd+Y4PM/NkTFn4OZIv8zFnzPH4anNm7E41Tk+Q5IkSZIkqRXzaRmFJEmSJEkaAIMNkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYoHklyZbmDNoZ5ZEk7T3HYkkaLsdhDYLBBkmSJEmS1CqDDRp5SRYn+Zck70/ytSR/k+TXknwxyb8mOSHJc5P8P0luT/LPSY5tyh6Y5JNJbk3yPiBd9f6vSb6c5LYk70uyYGgPKUkjzrFYkobLcVhzjcEGzRVHAu8FjgV+Bngz8MvAu4D/DFwI3FpVxzbvP9iU+yPgC1X188Bq4FCAJD8LvAl4WVUdBzwGnDGwp5GkucmxWJKGy3FYc8bCYTdA6tM3qmoDQJKNwKerqpJsABYDhwFvAKiqG5vo7QHAK4D/2KTfkOT+pr6TgOOBdUkAngbcM8DnkaS5yLFYkobLcVhzhsEGzRWPdF0/3vX+cTrfxzumKFM9X7sFuKqqzmuthZI0/hyLJWm4HIc1Z7iMQuPiczRTvpIsBe6tqu/3pJ8KPKfJ/2ngtCTPb+49N8lhg260JI0Zx2JJGi7HYY0MZzZoXKwErkxyO/BD4Mwm/ULgI0m+AnwW+BZAVd2R5A+BTyZ5CvBj4B3ANwfdcEkaIytxLJakYVqJ47BGRKqmmk0jSZIkSZK0d1xGIUmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa36/wG5EhUVaMdJbgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAD7CAYAAAArSZxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+83VV95/vX20RAUUFjtMqvwEBtg1BajtBefzSVouFRNc6IJcBVuJd5xFaZzqOObWFGMDK2A33MFO1Ie00FRKyCpeU2lSha8fe1mCBIDJSZGKPEaAk/ioICBj73j/09cbNzkrOT8z377LPP6/l47Ee+e33XWnt9F+cszvmc9SNVhSRJkiRJUlueMtMNkCRJkiRJo8VggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbNDISLI5yW/OdDskSZIkaa4z2CBJkiRJklplsEFzTpL5M90GSZrLJhqHHZslabCSzJvpNmi0GWzQqDkuye1JHkxybZL9kixJsiXJHyX5AXDlTDdSkkZBs3ztD5px9+Eklyd5fpJPJvlRkn9M8uwki5JUknOSfBe4aaK0mX4eSRq0JIck+bsk25Lcl+T9SZ6S5J1JvpPkniQfTnJAk/9TSc7tqeMbSf5dc/0LST6T5P4kdyX57a58H0ryl0nWJHkY+I0kv5Xk1iQ/THJ3kpU9db+5acd9SS7oXrbctPO8JN9q7n88yXOmu880exhs0Kj5bWApcDhwLHB2k/5zwHOAw4AVM9IySRpNbwBOBn4eeC3wSeA/A8+l83PG73Xl/XXgF4FXT5ImSSOvmVnwCeA7wCLgIOAaOj+/ng38BnAE8Azg/U2xjwKnd9WxmM7Ptzck2R/4TJPneU2+v0hydNfHngH8MfBM4MvAw8CbgQOB3wJ+N8nru+r+C+BM4AXAAU0bx/0e8Ho64/gLgQeAy6bQJRoxBhs0av68qrZW1f3APwDHNelPAO+qqker6icz1zxJGjn/s6r+paq+B3wJuLmqbq2qR4HrgV/uyruyqh7uGYcnSpOkueAEOr+k/0EzDj5SVV+m88v9n1XVpqp6CDgfWN4sN7uezkzew5o6zgT+rhlzXwNsrqorq2p7VX0d+Fvg1K7P/Puq+kpVPdF83ueran3z/nbgY3SCBzTl/qGqvlxVjwEXAtVV11uA/1JVW5rPXwmc6rI4jTPYoFHzg67rH9OJBANsq6pHZqA9kjTq/qXr+icTvH9G1/u7Jyg/UZokzQWHAN+pqu096S+kM9th3HeA+cDzq+pHwA3A8ubecuCvm+vDgBOT/Ov4i04w4ue66nrSmJvkxCSfa5ZxPAj8Dp2ZaePt2JG/qn4M3NdV/DDg+q7PuhN4HHh+3z2gkWawQXNFTZ5FkjTNJhqLHZ8lzVV3A4dOMBNgK51f5McdCmznZ8HcjwGnJ/k14GnA57rq+0JVHdj1ekZV/W5XXb1j7keB1cAhVXUA8P8Aae59Hzh4PGOSpwELetp/Ss/n7dfMdJMMNkiSJEnSDPganV/oL06yf7Ox+UvpBBN+P8nhSZ4B/AlwbdcMiDV0ghEXNelPNOmfAH4+yZuSPLV5vSTJL+6mDc8E7q+qR5KcQGdPh3HXAa9N8n8k2Qd4Nz8LREAnMPHH40s6kixMsmwK/aERY7BBkiRJkgasqh6ns7HukcB3gS3AacAVwNXAF4FvA48A/6Gr3KPA3wG/SWdmwnj6j4BX0VlasZXO8uJLgH1304y3Ahcl+RGdPRk+3lXfhuZzr6ETFPkRcA/waJPlfXRmRXy6Kf9PwIl73BEaWaly9qIkSZIkadeaWRb/ChxVVd+e6fZo+DmzQZIkSZK0kySvTfL05ljN/w6sBzbPbKs0WxhskCRJkiRNZBmdJRlbgaOA5eXUePXJZRSSJEmSJKlVzmyQJEmSJEmtMtggSZIkSZJaNX+mG9Druc99bi1atGimmyFJO7nlllvuraqFM92OQXAsljSMHIclaeb1OxYPXbBh0aJFrFu3bqabIUk7SfKdmW7DoDgWSxpGjsOSNPP6HYtdRiFJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWxQK5YsWcKSJUtmuhmSNKc5FkvSzHIcln7GYIMkDZEkS5PclWRjkvMmuL9vkmub+zcnWdSkn5Dktub1jST/tqvM5iTrm3vrBvc0kiRJmqvmz3QDJEkdSeYBlwEnA1uAtUlWV9UdXdnOAR6oqiOTLAcuAU4DvgmMVdX2JC8AvpHkH6pqe1PuN6rq3sE9jSRJkuYygw2SNDxOADZW1SaAJNcAy4DuYMMyYGVzfR3w/iSpqh935dkPqOlvriRJo+OYq46Zch2bfrCptboA1p+1vpV6pJngMgpJGh4HAXd3vd/SpE2Yp5m18CCwACDJiUk2AOuB3+ma1VDAp5PckmTFrj48yYok65Ks27ZtWysPJEmSpLnJmQ1i0Xk3TLmOH2y6r7W6ADZf/Fut1CPNMpkgrXeGwi7zVNXNwNFJfhG4Ksknq+oR4KVVtTXJ84DPJPnnqvriTpVUrQJWAYyNjTkzQpIkSXvNmQ2SNDy2AId0vT8Y2LqrPEnmAwcA93dnqKo7gYeBFzfvtzb/3gNcT2e5hiRJkjRtDDZI0vBYCxyV5PAk+wDLgdU9eVYDZzXXpwI3VVU1ZeYDJDkMeBGwOcn+SZ7ZpO8PvIrOZpKSJKllR5x/BEecf8RMN0MaCi6jkKQh0ZwkcS5wIzAPuKKqNiS5CFhXVauBy4Grk2ykM6NheVP8ZcB5SX4KPAG8taruTXIEcH0S6Iz5H62qTw32ySRJkjTXGGxQK37ujItnugnSSKiqNcCanrQLu64fAd44QbmrgasnSN8E/FL7LVWrVh7QTj2bH26vvpUPTr0OacQkWQq8j05A+INVdXHP/VcA7wWOBZZX1XU9958F3AlcX1XnDqbVkjQzXEYhSZIkTSLJPOAy4BRgMXB6ksU92b4LnA18dBfV/FfgC9PVRkkaJgYbJEmSpMmdAGysqk1V9RhwDbCsO0NVba6q2+ksZ3uSJMcDzwc+PYjGStJMM9ggSZIkTe4g4O6u91uatEkleQrwP4A/mCTfiiTrkqzbtm3bXjdUkoaBwQZJkiRpcpkgrfos+1ZgTVXdvbtMVbWqqsaqamzhwoV73EBJGiZuEClp6CxZsgSAz3/+8zPaDkmSumwBDul6fzCwtc+yvwa8PMlbgWcA+yR5qKrOa7mNkjQ0+prZkGRpkruSbEyy06CYZN8k1zb3b06yqEl/apKrkqxPcmeS89tt/vRbsmTJjl98JEmSNGetBY5KcniSfegcPby6n4JVdWZVHVpVi4B3AB820CBp1E0abOhz591zgAeq6kjgUuCSJv2NwL5VdQxwPPCW8UCEJEmSNFtU1XbgXOBGOsdXfryqNiS5KMnrAJK8JMkWOj8DfyDJhplrsSTNrH6WUezYeRcgyfjOu3d05VkGrGyurwPenyR01rHtn2Q+8DTgMeCH7TRdmn1cHiBJ0uxVVWuANT1pF3Zdr6WzvGJ3dXwI+NA0NE+Shko/wYaJdt49cVd5qmp7kgeBBXQCD8uA7wNPB36/qu6faqMlSdLOPn/2/jPdBEmSJKC/PRv62Xl3V3lOAB4HXggcDvynJEfs9AEe8yNJkjQ03LNKkjRV/cxs6Gfn3fE8W5olEwcA9wNnAJ+qqp8C9yT5CjAGbOouXFWrgFUAY2Nj/R4hJGnILDrvhlbq+cGm+1qrb/PFvzXlOiRJkiTtmX6CDTt23gW+R2fn3TN68qwGzgK+CpwK3FRVleS7wCuTfITOMopfBd7bVuMn08YvKm3+0gP+4iNJe8P9TiRJkmaXSYMNzR4M4zvvzgOuGN95F1hXVauBy4Grk2ykM6NheVP8MuBK4Jt0llpcWVW3T8NzSNPKv9hL0mgwcCVJ0mD0M7Ohn513H6FzxE9vuYcmSpckSZIkSaOrnw0iJUmSJEmS+mawQZIkSZIktaqvZRSS2vFzZ1w8002YFeZyPyVZCryPzh45H6yqi3vu7wt8GDgeuA84rao2JzmB5lQfOnvkrKyq6/upU5IkSWqbwYZJzOVfeiQNVpJ5dDbWPZnOkcJrk6yuqju6sp0DPFBVRyZZDlwCnEZnI96xZlPfFwDfSPIPQPVRpyRJktQql1FI0vA4AdhYVZuq6jHgGmBZT55lwFXN9XXASUlSVT+uqu1N+n50ggz91ilJkiS1ypkNkjQ8DgLu7nq/BThxV3maWQwPAguAe5OcCFwBHAa8qbnfT50AJFkBrAA49NBDp/40ACsPaKeezQ+3V9/KB6dehyRJknbLmQ2SNDwyQVr1m6eqbq6qo4GXAOcn2a/POmnKr6qqsaoaW7hw4R40W5IkSXoyZzZI0vDYAhzS9f5gYOsu8mxJMh84ALi/O0NV3ZnkYeDFfdYpzQ5tzGxpc5YMDOdMGftJkjQEnNkgScNjLXBUksOT7AMsB1b35FkNnNVcnwrcVFXVlJkPkOQw4EXA5j7rlCRJklplsEGShkSzweO5wI3AncDHq2pDkouSvK7JdjmwIMlG4O3AeU36y+icQHEbcD3w1qq6d1d1Du6pJGl0JFma5K4kG5OcN8H9VyT5epLtSU7tSj8uyVeTbEhye5LTBttySRo8l1FI0hCpqjXAmp60C7uuHwHeOEG5q4Gr+61TkrRn+jye+LvA2cA7eor/GHhzVf3vJC8EbklyY1X96wCaLkkzwmCDJEmSNLkdRwkDJBk/SnhHsKGqNjf3nuguWFX/q+t6a5J7gIWAwQZJI8tggyRp6H3+7P1nugmS1PdRwruT5ARgH+BbE9xr/whiSZoh7tkgSZIkTa7vo4R3WUHyAjpL3v6vqnqi975HEEsaJQYbJEmSpMlN6SjhJM8CbgDeWVX/1HLbJGnouIxCkiTNGS7J0RTsOEoY+B6do4TP6Kdgc/Tw9cCHq+pvpq+JkjQ8nNkgSZIkTaKf44mTvCTJFjqnBn0gyfhRw78NvAI4O8ltzeu4GXgMSRoYZzZIkiRJfejjeOK1dJZX9Jb7CPCRaW+gJA0Rgw2SJEl6EpebSJKmymUUkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZI0RJIsTXJXko1Jzpvg/r5Jrm3u35xkUZN+cpJbkqxv/n1lV5nPN3Xe1ryeN7gnkiRJ0lw0f6YbIEnqSDIPuAw4GdgCrE2yuqru6Mp2DvBAVR2ZZDlwCXAacC/w2qramuTFwI3AQV3lzqyqdQN5EEmSJM15zmyQpOFxArCxqjZV1WPANcCynjzLgKua6+uAk5Kkqm6tqq1N+gZgvyT7DqTVkiRJUg+DDZI0PA4C7u56v4Unz054Up6q2g48CCzoyfMG4NaqerQr7cpmCcUFSdJusyVJktS2JUuWsGTJkpluxl4z2CBJw2OiIEDtSZ4kR9NZWvGWrvtnVtUxwMub15sm/PBkRZJ1SdZt27ZtjxouSZIkdesr2LC3G5Y1945N8tUkG5qNy/Zrr/mSNFK2AId0vT8Y2LqrPEnmAwcA9zfvDwauB95cVd8aL1BV32v+/RHwUTrLNXZSVauqaqyqxhYuXNjKA0mSJGlumnSDyKlsWNb8IPwR4E1V9Y0kC4Cftv4UkjQa1gJHJTkc+B6wHDijJ89q4Czgq8CpwE1VVUkOBG4Azq+qr4xnbsbhA6vq3iRPBV4D/OP0P4okjZ4kS4H3AfOAD1bVxT33XwG8FzgWWF5V13XdOwt4Z/P2PVV1FZJG0jFXHdNKPZt+sKm1+taftX7KdeypfmY27PWGZcCrgNur6hsAVXVfVT3eTtMlabQ0ezCcS+ckiTuBj1fVhiQXJXldk+1yYEGSjcDbgfHZZucCRwIX9BxxuS9wY5LbgdvoBDH+anBPJUmjoesPcKcAi4HTkyzuyfZd4Gw6s8i6yz4HeBdwIp2frd+V5NnT3WZJmkn9HH050YZlJ+4qT1VtTzK+YdnPA5XkRmAhcE1V/WnvByRZAawAOPTQQ/f0GSRpZFTVGmBNT9qFXdePAG+coNx7gPfsotrj22yjJM1RO/4AB5Bk/A9wO2b7VtXm5t4TPWVfDXymqsaXvX0GWAp8bPqbLUkzo5+ZDVPZsGw+8DLgzObff5vkpJ0yuk5YkiRJw62fE4OmVNaNeiWNkn5mNuzJhmVbejYs2wJ8oaruBUiyBvgV4LNTbLckSZI0SP38AW5KZatqFbAKYGxsrN+6JY2oI84/YqabMCX9zGzYsWFZkn3obFi2uifP+IZl0LVhGZ11x8cmeXoThPh1uqaaSZIkSbNEP3+Am46ykjQrTRpsmMqGZVX1APBndAIWtwFfr6ob2n8MSZIkaVr18we4XbkReFWSZzcbQ76qSZPmpCVLlrBkyZKZboamWT/LKPZ6w7Lm3kfoHH8pSZIkzUrNJujjf4CbB1wx/gc4YF1VrU7yEuB64NnAa5O8u6qOrqr7k/xXOgELgIvGN4uUpFHVV7BBkiRJmuv6+APcWjpLJCYqewVwxbQ2UJKGiMEGSZIkSdKkjrnqmFbq2fSDTa3Vt/6s9VOuQ9Ojnw0iJUmSJEmS+mawQZIkSZIktcplFJIkSZKkgTni/CNmugkaAGc2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSdJuLVmyhCVLlsx0MyTNIgYbJEmSJElSqzyNQpIkSRphx1x1zJTr2PSDTa3VBbD+rPWt1CNpeDmzQZIkSZIktcpggyQNkSRLk9yVZGOS8ya4v2+Sa5v7NydZ1KSfnOSWJOubf1/ZVeb4Jn1jkj9PksE9kSRJkuYil1FI0pBIMg+4DDgZ2AKsTbK6qu7oynYO8EBVHZlkOXAJcBpwL/Daqtqa5MXAjcBBTZm/BFYA/wSsAZYCnxzEM0mSRsMR5x8x002QNMs4s0GShscJwMaq2lRVjwHXAMt68iwDrmqurwNOSpKqurWqtjbpG4D9mlkQLwCeVVVfraoCPgy8fvofRZIkSXOZwQZJGh4HAXd3vd/Cz2Yn7JSnqrYDDwILevK8Abi1qh5t8m+ZpE4AkqxIsi7Jum3btu31Q0iSJEkGGyRpeEy0l0LtSZ4kR9NZWvGWPaizk1i1qqrGqmps4cKFfTRXkuaWKeyr89QkVzX759yZ5PxBt12SBs1ggyQNjy3AIV3vDwa27ipPkvnAAcD9zfuDgeuBN1fVt7ryHzxJnZKkSXTtq3MKsBg4Pcninmw79tUBLqUT/AV4I7BvVR0DHA+8ZTwQIUmjymCDJA2PtcBRSQ5Psg+wHFjdk2c1cFZzfSpwU1VVkgOBG4Dzq+or45mr6vvAj5L8anMKxZuBv5/uB5GkEbTX++rQmVG2fxMkfhrwGPDDwTRbkmaGwQZJGhLNHgzn0jlJ4k7g41W1IclFSV7XZLscWJBkI/B2YHwa77nAkcAFSW5rXs9r7v0u8EFgI/AtPIlCkvbGVPbVuQ54GPg+8F3gv1fV/b0f4N45kkaJR19K0hCpqjV0jqfsTruw6/oROtNxe8u9B3jPLupcB7y43ZZK0pwzlX11TgAeB14IPBv4UpJ/rKpNT8pYtQpYBTA2Njbh/jqSNFs4s0GSJEma3FT21TkD+FRV/bSq7gG+AoxNe4slaQYZbJAkSZImt9f76tBZOvHKdOwP/CrwzwNqtyTNCIMNkiRJ0iSmuK/OZcAzgG/SCVpcWVW3D/QBJGnA3LNBkiRJ6sMU9tV5aKJ0SRplzmyQJEmSJEmtMtggSZIkSZJaZbBBkiRJkiS1ymCDJEmSJElqlcEGSZIkSZLUKoMNkiRJkiSpVX0FG5IsTXJXko1Jzpvg/r5Jrm3u35xkUc/9Q5M8lOQd7TRbkiRJkiQNq0mDDUnmAZcBpwCLgdOTLO7Jdg7wQFUdCVwKXNJz/1Lgk1NvriRJkiRJGnb9zGw4AdhYVZuq6jHgGmBZT55lwFXN9XXASUkCkOT1wCZgQztNliRJkiRJw6yfYMNBwN1d77c0aRPmqartwIPAgiT7A38EvHvqTZUkSZIkSbNBP8GGTJBWfeZ5N3BpVT202w9IViRZl2Tdtm3b+miSJEmSJEkaVvP7yLMFOKTr/cHA1l3k2ZJkPnAAcD9wInBqkj8FDgSeSPJIVb2/u3BVrQJWAYyNjfUGMiRJkiRJ0izSz8yGtcBRSQ5Psg+wHFjdk2c1cFZzfSpwU3W8vKoWVdUi4L3An/QGGiRJP7O3p/8kWZDkc83JP+/vKfP5ps7bmtfzBvM0kiRJmqsmndlQVduTnAvcCMwDrqiqDUkuAtZV1WrgcuDqJBvpzGhYPp2NlqRR1HX6z8l0ZoytTbK6qu7oyrbj9J8ky+mc/nMa8AhwAfDi5tXrzKpaN60PIEmSJDX6WUZBVa0B1vSkXdh1/QjwxknqWLkX7ZOkuWTH6T8AScZP/+kONiwDVjbX1wHvT5Kqehj4cpIjB9heSZIkaUL9LKOQJA3GXp/+00fdVzZLKC4YP5q4l5v1StLu7e1St+besUm+mmRDkvVJ9htk2yVp0Aw2SNLwmMrpP7tzZlUdA7y8eb1pokxVtaqqxqpqbOHChZM2VpLmkq6lbqcAi4HTkyzuybZjqRtwKZ2lbjQbqH8E+J2qOhpYAvx0QE2XpBlhsEGShseenP4z/sPr+Ok/u1RV32v+/RHwUTrLNSRJe2bHUreqegwYX+rWbRlwVXN9HXBSM5vsVcDtVfUNgKq6r6oeH1C7JWlGGGyQpOGx16f/7KrCJPOTPLe5firwGuCbrbdckkbfVJa6/TxQSW5M8vUkfzjRB7icTdIo6WuDSEnS9Jvq6T9JNgPPAvZJ8no6f0n7DnBjE2iYB/wj8FcDfCxJGhVTWeo2H3gZ8BLgx8Bnk9xSVZ99UsaqVcAqgLGxscmWyEnSUDPYIElDZCqn/1TVol1Ue3xb7ZOkOWxPlrpt6VnqtgX4QlXdC5BkDfArwGeRpBHlMgpJkiRpclNZ6nYjcGySpzdBiF/nyccaS9LIcWaDJEmSNImpLHWrqgeS/BmdgEUBa6rqhhl5EEkaEIMNkiRJUh+muNTtI3SOv5SkOcFlFJIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRWGWyQJEmSJEmtMtggSZIkSZJaZbBBkoZIkqVJ7kqyMcl5E9zfN8m1zf2bkyxq0hck+VySh5K8v6fM8UnWN2X+PEkG8zSSJEmaqww2SNKQSDIPuAw4BVgMnJ5kcU+2c4AHqupI4FLgkib9EeAC4B0TVP2XwArgqOa1tP3WS5IkST9jsEGShscJwMaq2lRVjwHXAMt68iwDrmqurwNOSpKqeriqvkwn6LBDkhcAz6qqr1ZVAR8GXj+tTyFJI2pvZ5913T+0mYE2UWBYkkaKwQZJGh4HAXd3vd/SpE2Yp6q2Aw8CCyapc8skdQKQZEWSdUnWbdu2bQ+bLkmjbYqzz8ZdCnxyutsqScPAYIMkDY+J9lKovcizV/mralVVjVXV2MKFC3dTpSTNSXs9+wwgyeuBTcCGAbVXkmaUwQZJGh5bgEO63h8MbN1VniTzgQOA+yep8+BJ6pQkTW6vZ58l2R/4I+DdA2inJA0Fgw2SNDzWAkclOTzJPsByYHVPntXAWc31qcBNzV4ME6qq7wM/SvKrzV/X3gz8fftNl6SRN5XZZ+8GLq2qh3b7AS5nkzRC5s90AyRJHVW1Pcm5wI3APOCKqtqQ5CJgXVWtBi4Hrk6ykc6MhuXj5ZNsBp4F7NNM131VVd0B/C7wIeBpdNYKu15Ykvbcnsw+29Iz++xE4NQkfwocCDyR5JGqetJRxVW1ClgFMDY2trslcpI09Aw2SNIQqao1wJqetAu7rh8B3riLsot2kb4OeHF7rZSkOWnH7DPge3SCvWf05BmfffZVnjz77OXjGZKsBB7qDTRI0qgx2CBJkiRNYqqzzyRprukr2JBkKfA+OgPrB6vq4p77+9I5u/144D7gtKranORk4GJgH+Ax4A+q6qYW2y9JkiQNxFRmn3XlWTktjZOkITPpBpFTPFP4XuC1VXUMnSllV7fVcEmSJEmSNJz6OY1ir88Urqpbq2p845wNwH7NLAhJkiRJkjSi+gk27PWZwj153gDcWlWP9n6Ax/xIkiRJkjQ6+gk2TOVM4c7N5Gg6SyveMtEHVNWqqhqrqrGFCxf20SRJkiRJkjSs+gk27MmZwvScKUySg4HrgTdX1bem2mBJkiRJkjTc+gk27DhTOMk+dI7wWd2TZ/xMYeg6UzjJgcANwPlV9ZW2Gi1JkiRJkobXpMGGZg+G8TOF7wQ+Pn6mcJLXNdkuBxY0Zwq/HTivST8XOBK4IMltzet5rT+FJEmSJEkaGvP7ybS3ZwpX1XuA90yxjZIkSZIkaRbpZxmFJEmSJElS3ww2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkkaIkmWJrkrycYk501wf98k1zb3b06yqOve+U36XUle3ZW+Ocn6JLclWTeYJ5EkSdJcZrBBkoZEknnAZcApwGLg9CSLe7KdAzxQVUcClwKXNGUXA8uBo4GlwF809Y37jao6rqrGpvkxJGlk7W1AOMnJSW5pAr+3JHnloNsuSYNmsEGShscJwMaq2lRVjwHXAMt68iwDrmqurwNOSpIm/ZqqerSqvg1sbOqTJLVgKgFh4F7gtVV1DHAWcPVgWi1JM8dggyQNj4OAu7veb2nSJsxTVduBB4EFk5Qt4NPNX9NWTEO7JWku2OuAcFXdWlVbm/QNwH5J9h1IqyVphhhskKThkQnSqs88uyv70qr6FTp/jXtbkldM+OHJiiTrkqzbtm1bv22WpLliKgHhbm8Abq2qR6epnZI0FAw2SNLw2AIc0vX+YGDrrvIkmQ8cANy/u7Ljf03vGnfqAAALKUlEQVSrqnuA69nF8oqqWlVVY1U1tnDhwik/jCSNmKkEhDs3k6PpLK14y4QfYNBX0ggx2CBJw2MtcFSSw5PsQ2fDx9U9eVbTWe8LcCpwU1VVk7682ZzscOAo4GtJ9k/yTIAk+wOvAr45gGeRpFEzlYAwSQ6mE/B9c1V9a6IPMOgraZTMn+kGSJI6qmp7knOBG4F5wBVVtSHJRcC6qloNXA5cnWQjnR9glzdlNyT5OHAHsB14W1U9nuT5wPWdPSSZD3y0qj418IeTpNlvR0AY+B6d8feMnjzjAeGv0hUQTnIgcANwflV9ZYBtlqQZY7BBkoZIVa0B1vSkXdh1/Qjwxl2U/WPgj3vSNgG/1H5LJWlumUpAGDgXOBK4IMkFTdqrmuVtkjSSDDZIkiRJfdjbgHBVvQd4z7Q3UJKGiHs2SJIkSZKkVhlskCRJkiRJrTLYIEmSJEmSWmWwQZIkSZIktcpggyRJkiRJapXBBkmSJEmS1CqDDZIkSZIkqVUGGyRJkiRJUqsMNkiSJEmSpFYZbJAkSZIkSa0y2CBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVfQUbkixNcleSjUnOm+D+vkmube7fnGRR173zm/S7kry6vaZL0uiZjvF2sjolSf3xZ2JJ6t+kwYYk84DLgFOAxcDpSRb3ZDsHeKCqjgQuBS5pyi4GlgNHA0uBv2jqkyT1mI7xts86JUmT8GdiSdoz/cxsOAHYWFWbquox4BpgWU+eZcBVzfV1wElJ0qRfU1WPVtW3gY1NfZKknU3HeNtPnZKkyfkzsSTtgX6CDQcBd3e939KkTZinqrYDDwIL+iwrSeqYjvHWcViS2uHPxJK0B+b3kScTpFWfefopS5IVwIrm7UNJ7uqjXYP0XODeNirKJW3UMrTsp/600k8j3kcwnP10WKu17Ww6xtuJgso7jcMwh8bid0/UVSPFfppca/+/sp/61F4/Tfc4vDv+TNzmz3pn+73TD/tpciPeRzCc/dTXWNxPsGELcEjX+4OBrbvIsyXJfOAA4P4+y1JVq4BV/TR4JiRZV1VjM92OYWc/9cd+6s8c7afpGm8nHYfBsXhU2E+Ts4/6Yz/txJ+J/Zroi/3UH/upP7O5n/pZRrEWOCrJ4Un2obO5zeqePKuBs5rrU4Gbqqqa9OXNzryHA0cBX2un6ZI0cqZjvO2nTknS5PyZWJL2wKQzG6pqe5JzgRuBecAVVbUhyUXAuqpaDVwOXJ1kI53o7fKm7IYkHwfuALYDb6uqx6fpWSRpVpuu8XaiOgf9bJI02/kzsSTtmXSCrdqdJCuaaW3aDfupP/ZTf+wn9fJroj/20+Tso/7YT+rl10R/7Kf+2E/9mc39ZLBBkiRJkiS1qp89GyRJkiRJkvo2ksGGJI8nuS3JN5P8TZKnt1Tv85N8Isk3ktyRZE2T/u0kL+rJ+94kf5hkSZIHk9ya5K4kX0zymjbas4dtt0+mWZL/r+f97yd5JMkBXWlLknxigrKvafpjvB/fMog2D4L9Mjc55kzYdvtkABxzdmafzE2OObtsv/0yzRxzJjYn+6WqRu4FPNR1/dfA2/eg7Pzd3PsA8B+73h/b/PvfgHd1pT+FzhFHhwFLgE903TsO2AycZJ/MbJ8MoM+/BnwJOLsr7UnP3qQ9lc7xVwc37/cFXjTT7bdffE3xv7Njjn0yFC/HHPtkrr4cc+yXYXk55szdfhnJmQ09vgQcmWRRkm+OJyZ5R5KVzfXnk/xJki8A/zHJwiR/m2Rt83ppU+wFdAYHAKrq9ubyYzS7DTdeAWyuqu/0NqaqbgMuAs5t8Rn31KzpkyTzknyoiT6vT/L7U3ryaZTkoa7rfwM8A3gncPokRZ9J52SY+wCq6tGqumu62jlo090vs+lrZA6bNWPOAM2aPplt32OOxTtzHBazaMwZsFnTL7Pp+8xxeGJzcSwe6WBDkvnAKcD6PrIfWFW/XlX/A3gfcGlVvQR4A/DBJs9lwOVJPpfkvyR5IewYTJ5I8ktNvuV0BpZd+TrwC3v+RFM3C/vkOOCgqnpxVR0DXNlHu4fB6XSe90vAi5I8b1cZq+p+OudvfyfJx5KcmWRUvzeno19m69fInDALx5xpNwv7ZDZ/jzkW78xxeI6ZhWPOQMzCfpmt32eOwxObE2PxqP7He1qS24B1wHfpnHk8mWu7rn8TeH9Tx2rgWUmeWVU3AkcAf0VnELg1ycKmzMeA5c3AtQz4m918VvboadoxW/tkE3BEkv+ZZCnwwz7aPQyWA9dU1RPA3wFv3F3mqvr3wEl0plO9A7hi2ls4M6ajX2br18iom61jznSarX0ym7/HHIt35jg8d8zWMWe6zdZ+ma3fZ47DE5sTY/H8mW7ANPlJVR3XnZBkO08OruzXU+bhruunAL9WVT/prbiJLH0U+Gg6m3e8AvhbOoPIp4EvALdX1T27ad8vA3f2+SxtmRV9kmQecEuTtrqqLmwiwa8G3gb8NvB/7/ZJZ1iSY4GjgM8kAdiHzjf/ZbsrV1XrgfVJrga+DZw9vS0drLb6Jck5zPKvkTliVow5fT5LW2ZFn4zCOAyOxRNxHJ5zZsWY0+eztGlW9MsojMWOwxObS2PxqM5smMi/AM9LsiDJvsDudnr9NF1rpZIc1/z7yjQ71iZ5JvBv6EREqapv0VlHczG7mRrVfHFdwCRfTAMydH1SVY9X1XHN68IkzwWeUlV/2+T5lb1/3IE5HVhZVYua1wuBg5IcNlHmJM9IsqQr6Thgp3V8I6CVfhmRr5G5aujGnCk9TTuGrk9G6HvMsXhnjsMaujFnSk/TnqHrlxH5PnMcnticGYtHdWbDTqrqp0kuAm6mEyH7591k/z3gsiS30+mjLwK/AxxPZ9rUePTzg1W1tqvcx+jsOHt9T30vT3Ir8HTgHuD3quqzLTzWlMySPjkIuDI/W5d0/p484wxZTmcNYLfrm/SbgZOSbOm6dzrwh0k+APyETvT87AG0c9Cmq19m49fInDRLxpyBmiV9Mlu/xxyLd+Y4PMfNkjFn4GZJv8zG7zPH4YnNmbE41Tk+Q5IkSZIkqRVzaRmFJEmSJEkaAIMNkiRJkiSpVQYbJEmSJElSqww2SJIkSZKkVhlskCRJkiRJrTLYoDklyebmDNop5ZEk7T3HYkmaWY7DGgSDDZIkSZIkqVUGGzT0kixK8s9JPpjkm0n+OslvJvlKkv+d5IQkz0ny/ya5Pck/JTm2KbsgyaeT3JrkA0C66v0/k3wtyW1JPpBk3ow9pCQNOcdiSZpZjsOabQw2aLY4EngfcCzwC8AZwMuAdwD/GXg3cGtVHdu8/3BT7l3Al6vql4HVwKEASX4ROA14aVUdBzwOnDmwp5Gk2cmxWJJmluOwZo35M90AqU/frqr1AEk2AJ+tqkqyHlgEHAa8AaCqbmqitwcArwD+XZN+Q5IHmvpOAo4H1iYBeBpwzwCfR5JmI8diSZpZjsOaNQw2aLZ4tOv6ia73T9D5Ot4+QZnq+bdbgKuq6vzWWihJo8+xWJJmluOwZg2XUWhUfJFmyleSJcC9VfXDnvRTgGc3+T8LnJrkec295yQ5bNCNlqQR41gsSTPLcVhDw5kNGhUrgSuT3A78GDirSX838LEkXwe+AHwXoKruSPJO4NNJngL8FHgb8J1BN1ySRshKHIslaSatxHFYQyJVE82mkSRJkiRJ2jsuo5AkSZIkSa0y2CBJkiRJklplsEGSJEmSJLXKYIMkSZIkSWqVwQZJkiRJktQqgw2SJEmSJKlVBhskSZIkSVKrDDZIkiRJkqRW/f8uJyX4bIzTqQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -1666,7 +1666,7 @@ "metadata": {}, "source": [ "Surprisingly, the iALS model remains largely insensitive to the scaling trick. At least in the current settings and for the current dataset. \n", - "**Remark**: You may want to repeat all experiments in a different setting with `random_holdout` set to `True`. My own results indicate that in this case iALS becomes more responsive to scaling and gives the same result as the scaled version of SVD. However, SVD is still much easier to compute and tune." + "**Remark**: You may want to repeat all experiments in a different setting with `random_holdout` set to `True`. My own results indicate that in this case iALS performs slightly better than PureSVD and also becomes more responsive to scaling, giving the same result as the scaled version of SVD. However, SVD is still easier to compute and tune." ] }, { @@ -1680,9 +1680,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "With a proper tuning the quality of recommendations of one of the simplest SVD-based models can be substantially improved. Despite common beliefs, it turns out that PureSVD with simple scaling trick compares favorably to a much more popular iALS algorithm. It not only generates more relevant recommendations, but also makes them more diverse and potentially more interesting. In addition to that it has a number of unique advantages and merely requires to simply do `from scipy.sparse.linalg import svds` to start using it. Of course, the obtained results may not necessarily hold on all other datasets and require further verification. However, in the view of all its features, the scaled SVD-based model can be certainly considered as at least one of the default baseline candidates. The result obtained here resembles situation in the Natural Language Processing field, where simple SVD-based model with proper tuning [turns out to be competitive](http://www.aclweb.org/anthology/Q15-1016) on a variety of downstream tasks even in comparison with Neural Network-based models. In the end it's all about fair comparison.\n", + "With a proper tuning the quality of recommendations of one of the simplest SVD-based models can be substantially improved. Despite common beliefs, it turns out that PureSVD with simple scaling trick compares favorably to a much more popular iALS algorithm. The former not only generates more relevant recommendations, but also makes them more diverse and potentially more interesting. In addition to that it has a number of unique advantages and merely requires to do `from scipy.sparse.linalg import svds` to get started. Of course, the obtained results may not necessarily hold on all other datasets and require further verification. However, in the view of all its features, the **scaled SVD-based model certainly deserves to be included into the list of default baselines**. The result obtained here resembles situation in the Natural Language Processing field, where simple SVD-based model with proper tuning [turns out to be competitive](http://www.aclweb.org/anthology/Q15-1016) on a variety of downstream tasks even in comparison with Neural Network-based models. In the end it's all about adequate tuning and fair comparison.\n", "\n", - "As a final remark, Polara is designed to support opennes and reproducibility of research. It allows to perform thorough experiments with minimal efforts. It can be used to quickly test known ideas or implement something new by providing controlled environment and rich functionality with high level abstractions." + "As a final remark, this tutorial is a part of a series of tutorials demonstrating usage scenarios for the Polara framework. Polara is designed to support opennes and reproducibility of research. It provides controlled environment and rich functionality with high level abstractions, which allows conducting thorough experiments with minimal efforts and can be used to either quickly test known ideas or implement something new." ] }, { From 2350577e5e8421d0f4e435967efa5b5ef27f63d7 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sat, 4 May 2019 12:44:23 +0300 Subject: [PATCH 08/82] add cold start support for LCE --- polara/recommender/coldstart/models.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index 2c278f9..f2f3d15 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -2,6 +2,7 @@ from polara import SVDModel from polara.recommender.models import RecommenderModel, ScaledSVD +from polara.recommender.hybrid.models import LCEModel from polara.lib.similarity import stack_features from polara.lib.sparse import sparse_dot @@ -124,3 +125,23 @@ class ScaledSVDItemColdStart(ScaledSVD, SVDModelItemColdStart): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.method = 'PureSVDs(cs)' + + +class LCEModelItemColdStart(ItemColdStartEvaluationMixin, LCEModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.method = 'LCE(cs)' + + def get_recommendations(self): + Hu = self.factors[self.data.fields.userid].T + Hs = self.factors['item_features'].T + cold_info = self.item_features.reindex(self.data.index.itemid.cold_start.old.values, + fill_value=[]) + cold_item_features, _ = stack_features(cold_info, labels=self.feature_labels, normalize=False) + + cold_items_factors = cold_item_features.dot(Hs.T).dot(np.linalg.pinv(Hs @ Hs.T)) + cold_items_factors[cold_items_factors < 0] = 0 + + scores = cold_items_factors @ Hu + top_relevant_users = self.get_topk_elements(scores).astype(np.intp) + return top_relevant_users From 25119ff80bfaaf078cedc76501b1f32d80b5496b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sat, 4 May 2019 12:45:14 +0300 Subject: [PATCH 09/82] workaround graph construction with the help of scikit-learn --- polara/recommender/hybrid/models.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index e6dec48..f7a8a3f 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -1,9 +1,11 @@ +import math import scipy as sp import numpy as np from polara.recommender.models import RecommenderModel, ProbabilisticMF from polara.lib.optimize import kernelized_pmf_sgd, local_collective_embeddings from polara.lib.sparse import sparse_dot +from polara.lib.similarity import stack_features from polara.tools.timing import track_time @@ -149,19 +151,31 @@ def item_data(self): item_index = index_data self._item_data = self.item_features.reindex(item_index.old.values, # make correct sorting - fill_value=[]) + fill_value=[]) else: self._item_data = None return self._item_data + def build_item_graph(self, item_features, n_neighbors): + try: + from sklearn.neighbors import NearestNeighbors + except ImportError: + raise NotImplementedError('Install scikit-learn to construct graph for LCE model.') + else: + nbrs = NearestNeighbors(n_neighbors=1 + n_neighbors).fit(item_features) + if self.binary_features: + return nbrs.kneighbors_graph(item_features) + return nbrs.kneighbors_graph(item_features, mode='distance') + + def build(self): # prepare input matrix for learning the model Xs, lbls = stack_features(self.item_data, normalize=False) # item-features sparse matrix Xu = self.get_training_matrix().T # item-user sparse matrix n_nbrs = min(self.max_neighbours, int(math.sqrt(Xs.shape[0]))) - A = construct_A(Xs, n_nbrs, binary=self.binary_features) + A = self.build_item_graph(Xs, n_nbrs) with track_time(self.training_time, verbose=self.verbose, model=self.method): W, Hu, Hs = local_collective_embeddings(Xs, Xu, A, From d526447a26904ac173dd1ea89c70b35db072c842 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sat, 4 May 2019 12:47:35 +0300 Subject: [PATCH 10/82] simplify scaled svd definition in cold start --- polara/recommender/coldstart/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index f2f3d15..a47b42f 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -1,7 +1,7 @@ import numpy as np from polara import SVDModel -from polara.recommender.models import RecommenderModel, ScaledSVD +from polara.recommender.models import RecommenderModel, ScaledMatrixMixin from polara.recommender.hybrid.models import LCEModel from polara.lib.similarity import stack_features from polara.lib.sparse import sparse_dot @@ -121,7 +121,7 @@ def get_recommendations(self): return top_similar_users -class ScaledSVDItemColdStart(ScaledSVD, SVDModelItemColdStart): +class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.method = 'PureSVDs(cs)' From 6556804b3b1d1b876fb4edae5c94e1a0d7001a48 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Tue, 7 May 2019 06:31:08 +0300 Subject: [PATCH 11/82] remove old temporary test files --- tests/data_state_test.py | 59 -- tests/polara_new_evaluation.ipynb | 1569 ----------------------------- 2 files changed, 1628 deletions(-) delete mode 100644 tests/data_state_test.py delete mode 100644 tests/polara_new_evaluation.ipynb diff --git a/tests/data_state_test.py b/tests/data_state_test.py deleted file mode 100644 index b1366d3..0000000 --- a/tests/data_state_test.py +++ /dev/null @@ -1,59 +0,0 @@ -def define_state(usn, hsz, trt): - if usn: - if (hsz>0) and (trt>0): - state = 4 - else: - raise ValueError('Invalid parameters') - else: - if hsz == 0: - if trt == 0: - state = 1 - else: - state = 11 - else: #hsz > 0 - if trt == 0: - state = 2 - else: # trt > 0 - state = 3 - return state - -def assign_config(usn, hsz, trt, state): - data_model._test_ratio = trt - data_model._holdout_size = hsz - data_model._warm_start = usn - data_model._state = state - -fields = ['userid', 'itemid', 'rating'] -data = pd.DataFrame(columns=fields) -data_model = RecommenderData(data, *fields) - -for usn in [False, True]: - for hsz in [0, 0.25, 0.5, 1, 3]: - for trt in [0, 0.1, 0.2]: - try: - state = define_state(usn, hsz, trt) - except ValueError as e: - print '{}: usn {}, hsz {}, trt {}\n'.format(e, usn, hsz, trt) - continue - print 'current config: usn - {}, hsz - {}, trt - {}, state - {}'.format(usn, hsz, trt, state) - assign_config(usn, hsz, trt, state) - - for usn_new in [False, True]: - for hsz_new in [0, 0.25, 1]: - for trt_new in [0, 0.1]: - print 'usn: {:b}, hsz: {:4}, trt: {:4}'.format(usn_new, hsz_new, trt_new), - data_model.test_ratio = trt_new - data_model.holdout_size = hsz_new - data_model.warm_start = usn_new - try: - data_model._validate_config() - except ValueError as e: - print e - else: - new_stt, upd = data_model._check_state_transition() - print 'new state: {:3}, update rule: {} '.format(new_stt, upd), - print list(data_model._change_properties) - - assign_config(usn, hsz, trt, state) - data_model._change_properties.clear() - print \ No newline at end of file diff --git a/tests/polara_new_evaluation.ipynb b/tests/polara_new_evaluation.ipynb deleted file mode 100644 index 3e30b32..0000000 --- a/tests/polara_new_evaluation.ipynb +++ /dev/null @@ -1,1569 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import print_function\n", - "from collections import namedtuple\n", - "\n", - "import pandas as pd\n", - "import numpy as np\n", - "from scipy.sparse import csr_matrix" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from polara.recommender.evaluation import assemble_scoring_matrices, build_rank_matrix, matrix_from_observations, split_positive, generate_hits_data\n", - "from polara.recommender.evaluation import get_mrr_score, get_ndcr_discounts, get_ndcg_score, get_ndcl_score\n", - "from polara.recommender.evaluation import get_hits, get_relevance_scores, get_ranking_scores\n", - "from polara.datasets.movielens import get_movielens_data\n", - "from polara.recommender.data import RecommenderData\n", - "from polara.recommender.models import SVDModel" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Simple examples" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## from wiki\n", - "based on https://en.wikipedia.org/wiki/Discounted_cumulative_gain" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "swp = None\n", - "\n", - "data = pd.DataFrame({'userid': [0,0,0,0,0,0,0,0],\n", - " 'movieid': [0,1,2,3,4,5,6,7],\n", - " 'rating':[3, 2, 3, 0, 1, 2, 3, 2]})\n", - "recs = np.array([[0,1,2,3,4,5]])\n", - "hsz = data.shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "topk = recs.shape[1]\n", - "shp = (recs.shape[0], max(recs.max(), data['movieid'].max())+1)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "rankm, hrank, mrank, evalm, ehits, emiss = assemble_scoring_matrices(recs, data, 'userid', 'movieid', None, 'rating')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "discm, idisc = get_ndcr_discounts(rankm, evalm, topk)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.7561640298168335" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ndcg_score(ehits, discm, idisc, alternative=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "the result is slightly worse (expected value is 0.785), as normalization is based on the full holdout, not just topk elements \n", - "this is an intentional behavior in order to support NDCL score calculation when switch_positive is set" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## hand-crafted example" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "swp = 3\n", - "\n", - "data = pd.DataFrame({'userid': [0,0, 1,1, 2,2],\n", - " 'movieid': [0,1, 2,3, 4,5],\n", - " 'rating':[2,3, 1,3, 5,4]})\n", - "recs = np.array([[1,0], [2,3], [5,4]])\n", - "hsz = 2" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "topk = recs.shape[1]\n", - "shp = (recs.shape[0], max(recs.max(), data['movieid'].max())+1)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
rating
useridmovieid
002
13
121
33
245
54
\n", - "
" - ], - "text/plain": [ - " rating\n", - "userid movieid \n", - "0 0 2\n", - " 1 3\n", - "1 2 1\n", - " 3 3\n", - "2 4 5\n", - " 5 4" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "data.set_index(['userid', 'movieid']).sort_index()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "if swp is None:\n", - " is_positive = None\n", - "else:\n", - " is_positive = data.rating>=swp" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "rankm, hrank, mrank, evalm, ehits, emiss = assemble_scoring_matrices(recs, data, 'userid', 'movieid', is_positive, 'rating')" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "discm, idisc = get_ndcr_discounts(rankm, evalm, topk)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.8606251743711292" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ndcg_score(ehits, discm, idisc, alternative=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.861654166907052" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ndcl_score(emiss, discm, idisc, swp, alternative=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Movielens" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "ml_data = get_movielens_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
useridmovieidrating
0111935
116613
219143
3134084
4123555
\n", - "
" - ], - "text/plain": [ - " userid movieid rating\n", - "0 1 1193 5\n", - "1 1 661 3\n", - "2 1 914 3\n", - "3 1 3408 4\n", - "4 1 2355 5" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ml_data.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "dm = RecommenderData(ml_data, 'userid', 'movieid', 'rating', seed=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'holdout_size': 3,\n", - " 'negative_prediction': False,\n", - " 'permute_tops': False,\n", - " 'random_holdout': False,\n", - " 'shuffle_data': False,\n", - " 'test_fold': 5,\n", - " 'test_ratio': 0.2,\n", - " 'test_sample': None,\n", - " 'warm_start': True}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dm.get_configuration()" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Preparing data...\n", - "19 unique movieid's within 26 testset interactions were filtered. Reason: not in the training data.\n", - "1 unique movieid's within 1 holdout interactions were filtered. Reason: not in the training data.\n", - "1 of 1208 userid's were filtered out from holdout. Reason: not enough items.\n", - "1 userid's were filtered out from testset. Reason: inconsistent with holdout.\n", - "Done.\n" - ] - } - ], - "source": [ - "dm.random_holdout = True\n", - "dm.prepare()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "svd = SVDModel(dm)\n", - "svd.rank = 50" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "PureSVD training time: 0.4176868981995412s\n" - ] - } - ], - "source": [ - "svd.build()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "swp = 4\n", - "\n", - "svd.switch_positive = swp\n", - "data = dm.test.holdout\n", - "recs = svd.recommendations\n", - "hsz = dm.holdout_size" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "topk = recs.shape[1]\n", - "shp = (recs.shape[0], max(recs.max(), data['movieid'].max())+1)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "if swp is None:\n", - " is_positive = None\n", - "else:\n", - " is_positive = (data.rating>=swp).values" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "rankm, hrank, mrank, evalm, ehits, emiss = assemble_scoring_matrices(recs, data, 'userid', 'movieid', is_positive, 'rating')" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<1207x3687 sparse matrix of type ''\n", - "\twith 3621 stored elements in Compressed Sparse Row format>" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "evalm" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<1207x3687 sparse matrix of type ''\n", - "\twith 2346 stored elements in Compressed Sparse Row format>" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ehits" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<1207x3687 sparse matrix of type ''\n", - "\twith 1275 stored elements in Compressed Sparse Row format>" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emiss" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "discm, idisc = get_ndcr_discounts(rankm, evalm, topk)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<1207x3687 sparse matrix of type ''\n", - "\twith 12070 stored elements in Compressed Sparse Row format>" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "discm" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<1207x3687 sparse matrix of type ''\n", - "\twith 3621 stored elements in Compressed Sparse Row format>" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "idisc" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.1699440242225603" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ndcg_score(ehits, discm, idisc, alternative=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.06406889699069644" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ndcl_score(emiss, discm, idisc, swp, alternative=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Ranking(mrr=0.20079365079365077)" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_mrr_score(hrank)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "compare with previous implementation" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": { - "code_folding": [] - }, - "outputs": [], - "source": [ - "def get_matched_predictions(eval_data, holdout_size, recs):\n", - " userid, itemid = 'userid', 'movieid'\n", - " holdout_data = eval_data[itemid]\n", - " holdout_matrix = holdout_data.values.reshape(-1, holdout_size).astype(np.int64)\n", - "\n", - " matched_predictions = (recs[:, :, None] == holdout_matrix[:, None, :])\n", - " return matched_predictions\n", - "\n", - "def get_feedback_data(eval_data, holdout_size):\n", - " feedback = 'rating'\n", - " eval_data = eval_data[feedback].values\n", - " feedback_data = eval_data.reshape(-1, holdout_size)\n", - " return feedback_data\n", - "\n", - "def get_rnkng_scores(eval_data, holdout_size, recs, switch_positive=None, alternative=False):\n", - " matched_predictions = get_matched_predictions(eval_data, holdout_size, recs)\n", - " feedback_data = get_feedback_data(eval_data, holdout_size)\n", - " \n", - " users_num, topk, holdout = matched_predictions.shape\n", - " ideal_scores_idx = np.argsort(feedback_data, axis=1)[:, ::-1] #returns column index only\n", - " ideal_scores_idx = np.ravel_multi_index((np.arange(feedback_data.shape[0])[:, None],\n", - " ideal_scores_idx), dims=feedback_data.shape)\n", - " \n", - " where = np.where\n", - " is_positive = feedback_data >= switch_positive\n", - " positive_feedback = where(is_positive, feedback_data, 0)\n", - " negative_feedback = where(~is_positive, feedback_data-switch_positive, 0)\n", - " \n", - " relevance_scores_pos = (matched_predictions * positive_feedback[:, None, :]).sum(axis=2)\n", - " relevance_scores_neg = (matched_predictions * negative_feedback[:, None, :]).sum(axis=2)\n", - " ideal_scores_pos = positive_feedback.ravel()[ideal_scores_idx]\n", - " ideal_scores_neg = negative_feedback.ravel()[ideal_scores_idx]\n", - " \n", - " if alternative:\n", - " relevance_scores_pos = 2**relevance_scores_pos - 1\n", - " relevance_scores_neg = 2.0**relevance_scores_neg - 1\n", - " ideal_scores_pos = 2**ideal_scores_pos - 1\n", - " ideal_scores_neg = 2.0**ideal_scores_neg - 1\n", - "\n", - " disc_num = max(topk, holdout)\n", - " discount = np.log2(np.arange(2, disc_num+2)) \n", - " dcg = (relevance_scores_pos / discount[:topk]).sum(axis=1)\n", - " dcl = (relevance_scores_neg / -discount[:topk]).sum(axis=1)\n", - " idcg = (ideal_scores_pos / discount[:holdout]).sum(axis=1)\n", - " idcl = (ideal_scores_neg / -discount[:holdout]).sum(axis=1)\n", - " \n", - " with np.errstate(invalid='ignore'):\n", - " ndcg = np.nansum(dcg / idcg) / users_num\n", - " ndcl = np.nansum(dcl / idcl) / users_num\n", - "\n", - " ranking_score = namedtuple('Ranking', ['nDCG', 'nDCL'])._make([ndcg, ndcl])\n", - " return ranking_score" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Ranking(nDCG=0.1699440242225603, nDCL=0.06406889699069644)" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_rnkng_scores(data, hsz, recs, switch_positive=swp, alternative=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Ranking(nDCG=0.1699440242225603, nDCL=0.06406889699069644)" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ranking_scores(rankm, hrank, mrank, evalm, ehits, emiss, switch_positive=swp, topk=topk, alternative=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Hits(true_positive=602, false_positive=132, true_negative=1143, false_negative=1744)" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "svd.evaluate('hits', not_rated_penalty=None)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Relevance(precision=0.39215686274509803, recall=0.24247445457056063, fallout=0.06890361778514222, specifity=0.6096382214857774, miss_rate=0.6871030102181718)" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "svd.evaluate('relevance')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "from polara.recommender import defaults" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [], - "source": [ - "defaults.ndcg_alternative = False" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Ranking(nDCG=0.1699440242225603, nDCL=0.06406889699069644)" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "svd.evaluate('ranking')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Ranking(nDCG=0.07359347041824198, nDCL=0.022039537078199615)" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "svd.evaluate('ranking', topk=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Hand-picked test" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [], - "source": [ - "test_user = 98\n", - "test_data = svd.data.test.holdout.query('userid=={}'.format(test_user))\n", - "test_recs = svd.recommendations[test_user, :]" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [], - "source": [ - "topk = len(test_recs)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1045 2469 1126 1173 2489 846 2638 524 1130 2553]\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
useridmovieidrating
8201669811305
8201649811085
8201409810453
\n", - "
" - ], - "text/plain": [ - " userid movieid rating\n", - "820166 98 1130 5\n", - "820164 98 1108 5\n", - "820140 98 1045 3" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "print(test_recs)\n", - "test_data" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "820166 True\n", - "820164 False\n", - "820140 True\n", - "Name: movieid, dtype: bool" - ] - }, - "execution_count": 48, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "test_data.loc[:, 'movieid'].isin(test_recs)" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [], - "source": [ - "(rankm, hrank, mrank,\n", - " evalm, ehits, emiss) = assemble_scoring_matrices(test_recs, test_data,\n", - " svd._key, svd._target,\n", - " (test_data.rating>=swp).values, feedback='rating')" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([9], dtype=uint8)" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hrank.data" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1130], dtype=int64)" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hrank.indices" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([5, 5], dtype=int64)" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ehits.data" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1130, 1108])" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ehits.indices" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "discm, idisc = get_ndcr_discounts(rankm, evalm, topn=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1. , 0.63092975, 0.5 , 0.43067656, 0.38685281,\n", - " 0.35620719, 0.33333333, 0.31546488, 0.30103 , 0.28906483])" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "discm.data" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1045, 2469, 1126, 1173, 2489, 846, 2638, 524, 1130, 2553],\n", - " dtype=int32)" - ] - }, - "execution_count": 56, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "discm.indices" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1. , 0.63092975, 0.5 ])" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "idisc.data" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1108, 1130, 1045])" - ] - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "idisc.indices" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "NDCG" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.18457569677956817" - ] - }, - "execution_count": 59, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ndcg_score(ehits, discm, idisc, alternative=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "rec rank [1 9]\n", - "rec item [1045 1130]\n" - ] - } - ], - "source": [ - "print('rec rank', np.where(np.isin(test_recs, test_data.movieid))[0] + 1)\n", - "print('rec item', test_recs[np.isin(test_recs, test_data.movieid)])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "NDCL" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([3], dtype=int64)" - ] - }, - "execution_count": 61, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emiss.data" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1045])" - ] - }, - "execution_count": 62, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emiss.indices" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([3.])" - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emiss.multiply(discm).data" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<1x2639 sparse matrix of type ''\n", - "\twith 1 stored elements in Compressed Sparse Row format>" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emiss.multiply(idisc)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2.0" - ] - }, - "execution_count": 65, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_ndcl_score(emiss, discm, idisc, swp, alternative=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Why normalization in NDCG is changed" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "basically due to NDCL metric, which is \"the lower the better\" \n", - "this means that ideal score is 0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "regular case" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dcg 1.505149978319906\n", - "idcg 8.154648767857287\n", - "ndcg 0.1845756967795682\n" - ] - } - ], - "source": [ - "cg = lambda rel, pos: rel / np.log2(1+pos)\n", - "\n", - "print('dcg ', cg(5, 9))\n", - "print('idcg', cg(5, 1) + cg(5, 2))\n", - "print('ndcg', cg(5, 9) / (cg(5, 1) + cg(5, 2)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "singular, but still ok" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dcl 0\n", - "idcl 0\n", - "ndcl [nan]\n" - ] - } - ], - "source": [ - "cl = lambda rel, pos: (np.exp(rel-4)-1) / (-np.log2(1+pos))\n", - "\n", - "print('dcl ', 0)\n", - "print('idcl', 0)\n", - "with np.errstate(invalid='ignore'):\n", - " print('ndcl', np.array([0.]) / np.array([0.]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "broken case \n", - "when dcl is above zere and idcl is exactly 0 (due to only topk selected result, where negatove examples are not included at all)" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dcl 0.31606027941427883\n", - "idcl 0\n", - "ndcl [inf]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "C:\\Users\\evfro\\Anaconda3\\envs\\py3_polara\\lib\\site-packages\\ipykernel_launcher.py:6: RuntimeWarning: divide by zero encountered in true_divide\n", - " \n" - ] - } - ], - "source": [ - "cl = lambda rel, pos: (np.exp(rel-4)-1) / (-np.log2(1+pos))\n", - "\n", - "print('dcl ', cl(3, 3))\n", - "print('idcl', 0)\n", - "with np.errstate(invalid='ignore'): # will not catch an error\n", - " print('ndcl', cl(3, 3) / np.array([0.]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "therefore with standard normalization NDCL may generate inf doesn't make a lot of sense, especially when trying to average across many users" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.4" - }, - "toc": { - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 30b3b10126beafe060b084a602e854b67878168a Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 21 Aug 2019 14:08:52 +0300 Subject: [PATCH 12/82] update tutorial on tuning and evaluation --- ...uning and cross-validation tutorial.ipynb} | 177 ++++++------------ 1 file changed, 62 insertions(+), 115 deletions(-) rename examples/{Hyper_parameter_tuning_and_cross_validation_experiments.ipynb => Hyper-parameter tuning and cross-validation tutorial.ipynb} (95%) diff --git a/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb b/examples/Hyper-parameter tuning and cross-validation tutorial.ipynb similarity index 95% rename from examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb rename to examples/Hyper-parameter tuning and cross-validation tutorial.ipynb index fc7bde2..4dd2b5b 100644 --- a/examples/Hyper_parameter_tuning_and_cross_validation_experiments.ipynb +++ b/examples/Hyper-parameter tuning and cross-validation tutorial.ipynb @@ -11,20 +11,33 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In this tutorial we will go through a full cycle of model tuning and evaluation with Polara. This will include 2 phases: grid-search for finding (almost) optimal values of hyper-parameters and verification of results via 5-fold cross-validation.\n", - "\n", - "
We will focus on performing a fair comparison of popular ALS-based matrix factorization (MF) model called Weighted Regularized Matrix Factorization (WRMF a.k.a. iALS) [Hu, 2008] with less popular model called PureSVD [Cremonesi, 2010] based on standard SVD.
\n", + "
In this tutorial we will go through a full cycle of model tuning and evaluation to perform a fair comparison of recommendation algorithms with Polara.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This will include 2 phases: grid-search for finding (almost) optimal values of hyper-parameters and verification of results via 5-fold cross-validation. We will compare a popular ALS-based matrix factorization (MF) model called **Weighted Regularized Matrix Factorization (WRMF a.k.a. iALS)** [Hu, 2008] with much simpler SVD-based models.\n", "\n", - "We will use standard *Scipy*'s implementation for the latter and a great library called [*implicit*](https://github.com/benfred/implicit) for iALS. Both are wrapped by Polara and can be accessed via the corresponding classes. Due to its practicality the *implicit* library is often recommended to beginners and sometimes even serves as a default tool in production. On the other hand, there are some important yet often overlooked features, which make SVD-based models stand out. Ignoring them in my opinion leads to certain misconceptions and myths, not to say that it also overcomplicates things quite a bit.\n", + "We will use a standard sparse implementation of SVD from [*Scipy*](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.svds.html) for the latter and a great library called [*implicit*](https://github.com/benfred/implicit) for iALS. Both are wrapped by Polara and can be accessed via the corresponding recommender model classes. Due to its practicality the *implicit* library is often recommended to beginners and sometimes even serves as a default tool in production. On the other hand, there are some important yet often overlooked features, which make SVD-based models stand out. Ignoring them in my opinion leads to certain misconceptions and myths, not to say that it also overcomplicates things quite a bit.\n", "\n", "Note that by saying SVD I really mean *Singular Value Decomposition*, not just an arbitrary matrix factorization. In that sense, **methods like FunkSVD, SVD++, SVDFeature, etc., are not SVD-based at all**, even though historically they use SVD acronym in their names and are often referenced as if they are real substitutes for SVD. These methods utilize another optimization algorithm, typically based on stochastic gradient descent, and do not preserve the algebraic properties of SVD. This is really an important distinction, especially in the view of the following remarks:\n", "\n", - "1. **SVD-based approach has a number of unique and beneficial properties**. To name a few, it produces stable and determenistic output with global guarantees. It admits the same prediction formula for both known and previously unseen users (as long as at least one user rating is known). It can take a hybrid form to include side information via the generalized formulation (see Chapter 6 of [my thesis](https://www.skoltech.ru/en/2018/09/phd-thesis-defense-evgeny-frolov)). Even without hybridization it can be quite successfully applied in the cold start regime (paper on this is coming). It requires minimal tuning and allows to compute and store a single latent feature matrix - either for users or for items - instead of computing and storing both of them. This luxury is not available in the majority of other matrix factorization approaches, definitely not in popular ones. \n", - "2. Computational complexity of truncated SVD **scales linearly with the number of known observations** and quadratically with the rank of decomposition (thanks to Lanczos procedure). There are [open source implementations](https://github.com/criteo/rsvd), allowing to handle nearly *billion-scale problems* with its [efficient randomized version](https://medium.com/criteo-labs/sparkrsvd-open-sourced-by-criteo-for-large-scale-recommendation-engines-6695b649f519). \n", - "3. At least **in some cases PureSVD outperforms other more sophisticated matrix factorization methods** [Cremonesi, 2010].\n", - "4. Moreover, **PureSVD can be quite easily tuned to perform even better** [Nikolakopoulos, 2019].\n", + "1. **SVD-based approach has a number of unique and beneficial properties**. To name a few, it produces stable and determenistic output with global guarantees (can be critical for non-regression testing). It admits the same prediction formula for both known and previously unseen users as long as at least one user rating is known (this is especially handy for online regime). It requires minimal tuning and allows to compute and store a single latent feature matrix - either for users or for items - instead of computing and storing both of them. This luxury is not available in the majority of other matrix factorization approaches, definitely not in popular ones. \n", + "2. Thanks to the Lanczos procedure, computational complexity of truncated SVD **scales linearly with the number of known observations and with the number of users/items**. It scales quadratically only with the rank of decomposition. There are [open source implementations](https://github.com/criteo/rsvd), allowing to handle nearly *billion-scale problems* with one of its [efficient randomized versions](https://medium.com/criteo-labs/sparkrsvd-open-sourced-by-criteo-for-large-scale-recommendation-engines-6695b649f519). \n", + "3. At least **in some cases the simplest possible model called PureSVD outperforms other more sophisticated matrix factorization methods** [Cremonesi, 2010].\n", + "4. Moreover, **PureSVD can be quite easily tuned to perform much better** [Nikolakopoulos, 2019].\n", + "5. Finally, it can take a **hybrid form to include side information** via the generalized formulation (see Chapter 6 of [my thesis](https://www.skoltech.ru/en/2018/09/phd-thesis-defense-evgeny-frolov)). Even without hybridization it can be quite **successfully applied in the cold start regime** [Frolov, 2019]. \n", "\n", - "Despite that impresisve list, PureSVD technique (and especially its modifications) rarely gets into the list of baseline models to compare with. Hence, this tutorial also aims at performing a thorough assessment of the default choice of many practitioners to see whether it really provides the celebrated advantages over the simpler approach." + "Despite that impresisve list, SVD-based models **rarely get into the list of baselines** to compare or to start with." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Hence, this tutorial also aims at performing an assessment of the default choice of many practitioners to see whether it really stays advantageous over the simpler SVD-based approach after a thorough tuning of both models.
" ] }, { @@ -38,9 +51,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "* [Hu, 2008] Hu Y., Koren, Y. and Volinsky, C., 2008, December. *Collaborative Filtering for Implicit Feedback Datasets*. In ICDM (Vol. 8, pp. 263-272). [Link](http://yifanhu.net/PUB/cf.pdf). \n", - "* [Cremonesi, 2010] Cremonesi, P., Koren, Y. and Turrin, R., 2010, September. *Performance of recommender algorithms on top-n recommendation tasks*. In Proceedings of the fourth ACM conference on Recommender systems (pp. 39-46). ACM. [Link](https://www.researchgate.net/publication/221141030_Performance_of_recommender_algorithms_on_top-N_recommendation_tasks). \n", - "* [Nikolakopoulos, 2019] Nikolakopoulos, A.N., Kalantzis, V., Gallopoulos, E. and Garofalakis, J.D., 2019. *EigenRec: generalizing PureSVD for effective and efficient top-N recommendations*. Knowledge and Information Systems, 58(1), pp.59-81. [Link](https://arxiv.org/abs/1511.06033)." + "* [Hu, 2008] Hu Y., Koren, Y. and Volinsky, C., 2008. *Collaborative Filtering for Implicit Feedback Datasets*. In ICDM (Vol. 8, pp. 263-272). [Link](http://yifanhu.net/PUB/cf.pdf). \n", + "* [Cremonesi, 2010] Cremonesi, P., Koren, Y. and Turrin, R., 2010. *Performance of recommender algorithms on top-n recommendation tasks*. In Proceedings of the fourth ACM conference on Recommender systems (pp. 39-46). ACM. [Link](https://www.researchgate.net/publication/221141030_Performance_of_recommender_algorithms_on_top-N_recommendation_tasks). \n", + "* [Nikolakopoulos, 2019] Nikolakopoulos, A.N., Kalantzis, V., Gallopoulos, E. and Garofalakis, J.D., 2019. *EigenRec: generalizing PureSVD for effective and efficient top-N recommendations*. Knowledge and Information Systems, 58(1), pp.59-81. [Link](https://arxiv.org/abs/1511.06033).\n", + "* [Frolov, 2019] Frolov, E. and Oseledets, I., 2019. *HybridSVD: When Collaborative Information is Not Enough*. To appear in Proceedings of the Thirteenth ACM Conference on Recommender Systems. ACM. [Link](https://arxiv.org/abs/1802.06398)" ] }, { @@ -61,108 +75,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note that you'll need an internet connection in order to run the code below. It will automatically download data, store it in a temporary location, and convert into a `pandas` dataframe. \n", - "Alternatively, if you have already downloaded the dataset, you can use path to it as an input for the `get_movielens_data` function instead of `tmp_file`." + "
Note that you'll need an internet connection in order to run the cell below.
" ] }, { - "cell_type": "code", - "execution_count": 1, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "import urllib\n", - "from polara import (get_movielens_data, # returns data in the pandas dataframe format\n", - " RecommenderData) # provides common interface to access data " + "It will automatically download data, store it in a temporary location, and convert into a `pandas` dataframe. Alternatively, if you have already downloaded the dataset, you can use its local path as an input for the `get_movielens_data` function instead of `tmp_file`." ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
useridmovieidratingtimestamp
011225.0838985046
111855.0838983525
212315.0838983392
312925.0838983421
413165.0838983392
\n", - "
" - ], - "text/plain": [ - " userid movieid rating timestamp\n", - "0 1 122 5.0 838985046\n", - "1 1 185 5.0 838983525\n", - "2 1 231 5.0 838983392\n", - "3 1 292 5.0 838983421\n", - "4 1 316 5.0 838983392" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ + "import urllib\n", + "from polara import (get_movielens_data, # returns data in the pandas dataframe format\n", + " RecommenderData) # provides common interface to access data \n", + "\n", "url = 'http://files.grouplens.org/datasets/movielens/ml-10m.zip'\n", "tmp_file, _ = urllib.request.urlretrieve(url) # this may take some time depending on your internet connection\n", "\n", @@ -1293,7 +1225,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The difference between PureSVD and iALS is not significant. In contrast, the advantage of the scaled version of PureSVD denoted as `PureSVD-s` over the other models is much more pronounced making it a clear favorite. Interestingly, the difference is especially pronounced in terms of the `coverage` metric, which is defined as the ratio of unique recommendations generated for all test users to the total number of items in the training data. This indicates that generated recommendations are not only more relevant but also are significantly more diverse. " + "The difference between PureSVD and iALS is not significant.\n", + "
In contrast, the advantage of the scaled version of PureSVD denoted as `PureSVD-s` over the other models is much more pronounced making it a clear favorite.
\n", + "Interestingly, the difference is especially pronounced in terms of the `coverage` metric, which is defined as the ratio of unique recommendations generated for all test users to the total number of items in the training data. This indicates that generated recommendations are not only more relevant but also are significantly more diverse. " ] }, { @@ -1665,8 +1599,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Surprisingly, the iALS model remains largely insensitive to the scaling trick. At least in the current settings and for the current dataset. \n", - "**Remark**: You may want to repeat all experiments in a different setting with `random_holdout` set to `True`. My own results indicate that in this case iALS performs slightly better than PureSVD and also becomes more responsive to scaling, giving the same result as the scaled version of SVD. However, SVD is still easier to compute and tune." + "
Surprisingly, the iALS model remains largely insensitive to the scaling trick. At least in the current settings and for the current dataset.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Remark**: You may want to repeat all experiments in a different setting with `random_holdout` set to `True`. My own results indicate that in this case iALS performs slightly better than PureSVD and also becomes more responsive to scaling, *giving the same result as the scaled version of SVD*. However, the scaled version of SVD is still easier to compute and tune." ] }, { @@ -1680,17 +1620,24 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "With a proper tuning the quality of recommendations of one of the simplest SVD-based models can be substantially improved. Despite common beliefs, it turns out that PureSVD with simple scaling trick compares favorably to a much more popular iALS algorithm. The former not only generates more relevant recommendations, but also makes them more diverse and potentially more interesting. In addition to that it has a number of unique advantages and merely requires to do `from scipy.sparse.linalg import svds` to get started. Of course, the obtained results may not necessarily hold on all other datasets and require further verification. However, in the view of all its features, the **scaled SVD-based model certainly deserves to be included into the list of default baselines**. The result obtained here resembles situation in the Natural Language Processing field, where simple SVD-based model with proper tuning [turns out to be competitive](http://www.aclweb.org/anthology/Q15-1016) on a variety of downstream tasks even in comparison with Neural Network-based models. In the end it's all about adequate tuning and fair comparison.\n", - "\n", - "As a final remark, this tutorial is a part of a series of tutorials demonstrating usage scenarios for the Polara framework. Polara is designed to support opennes and reproducibility of research. It provides controlled environment and rich functionality with high level abstractions, which allows conducting thorough experiments with minimal efforts and can be used to either quickly test known ideas or implement something new." + "With a proper tuning the quality of recommendations of one of the simplest SVD-based models can be substantially improved. Despite common beliefs, it turns out that PureSVD with simple scaling trick compares favorably to a much more popular iALS algorithm. The former not only generates more relevant recommendations, but also makes them more diverse and potentially more interesting. In addition to that, it has a number of unique advantages and merely requires to do `from scipy.sparse.linalg import svds` to get started. Of course, the obtained results may not necessarily hold on all other datasets and require further verification." ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "
However, in the view of all its features and advantages, the scaled SVD-based model certainly deserves to be included into the list of the default baselines.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result obtained here resembles situation in the Natural Language Processing field, where simple SVD-based model with proper tuning [turns out to be competitive](http://www.aclweb.org/anthology/Q15-1016) on a variety of downstream tasks even in comparison with Neural Network-based models. In the end it's all about adequate tuning and fair comparison.\n", + "\n", + "As a final remark, this tutorial is a part of a series of tutorials demonstrating usage scenarios for the Polara framework. **Polara is designed to support openness and reproducibility of research**. It provides controlled environment and rich functionality with high level abstractions, which allows conducting thorough experiments with minimal efforts and can be used to either quickly test known ideas or implement something new." + ] } ], "metadata": { @@ -1709,7 +1656,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.4" + "version": "3.6.9" }, "toc": { "nav_menu": {}, @@ -1725,5 +1672,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From 2da78cbe7c0900964e6f6602f0428251ebf09410 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 21 Aug 2019 14:09:33 +0300 Subject: [PATCH 13/82] add method to read defaults in data config --- polara/recommender/data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index ca5c8ec..a672b84 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -173,6 +173,12 @@ def get_configuration(self): return config + @classmethod + def default_configuration(cls): + params = [prop[1:] for prop in cls._config] + return defaults.get_config(params) + + @property def test(self): self.update() From 53ba3688a2660b638352e3cd0d63f583a4d3ecb0 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 21 Aug 2019 14:10:20 +0300 Subject: [PATCH 14/82] fix recommendations not being updated after certain changes in holdout --- polara/recommender/models.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index d1b1763..0f4ffa1 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -414,18 +414,13 @@ def evaluate(self, metric_type='all', topk=None, not_rated_penalty=None, if not isinstance(metric_type, (list, tuple)): metric_type = [metric_type] - # support rolling back scenario for @k calculations if int(topk or 0) > self.topk: self.topk = topk # will also flush old recommendations - - # ORDER OF CALLS MATTERS!!! - # make sure to call holdout before getting recommendations - # this will ensure that model is renewed if data has changed - holdout = self.data.test.holdout # <-- call before getting recs + # support rolling back scenario for @k calculations recommendations = self.recommendations[:, :topk] # will recalculate if empty - switch_positive = switch_positive or self.switch_positive feedback = self.data.fields.feedback + holdout = self.data.test.holdout if (switch_positive is None) or (feedback is None): # all recommendations are considered positive predictions # this is a proper setting for binary data problems (implicit feedback) From f68b99950e046c8eac12fb4a64c52b0d348bba70 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 23 Aug 2019 14:04:33 +0300 Subject: [PATCH 15/82] implement simple one-hot-encoding-based similarity --- polara/lib/similarity.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/polara/lib/similarity.py b/polara/lib/similarity.py index 3b83d9f..3b6da9e 100644 --- a/polara/lib/similarity.py +++ b/polara/lib/similarity.py @@ -361,9 +361,24 @@ def _sim_func(func_type): raise NotImplementedError -def one_hot_similarity(meta_data): - raise NotImplementedError +def one_hot_similarity(meta_data, metric='common', assume_binary=True, fill_diagonal=True, ensure_csc=True): + features, labels = stack_features(meta_data, normalize=False) + similarity = None + + if metric == 'common': # common neighbours similarity + similarity = features.dot(features.T) + maxel = max(similarity.data.max(), abs(similarity.data.min())) + similarity = similarity / maxel + if fill_diagonal: + similarity.setdiag(1.0) + + if (metric == 'cosine') or (metric == 'salton'): + similarity = cosine_similarity(features, assume_binary=assume_binary, fill_diagonal=fill_diagonal) + + if ensure_csc and (similarity.format == 'csr'): + similarity = similarity.T # ensure CSC format (matrix is symmetric) + return similarity, labels def get_similarity_data(meta_data, similarity_type='jaccard'): features = meta_data.columns From 803d62323c85cd9e29176f32b1094c5daca3c2dd Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 23 Aug 2019 14:07:50 +0300 Subject: [PATCH 16/82] add wrapper for CHOLMOD cholesky factor object --- polara/lib/cholesky.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 polara/lib/cholesky.py diff --git a/polara/lib/cholesky.py b/polara/lib/cholesky.py new file mode 100644 index 0000000..1ca3db2 --- /dev/null +++ b/polara/lib/cholesky.py @@ -0,0 +1,35 @@ +class CholeskyFactor: + def __init__(self, factor): + self._factor = factor + self._L = None + self._transposed = False + + @property + def L(self): + if self._L is None: + self._L = self._factor.L() + return self._L + + @property + def T(self): + self._transposed = True + return self + + def dot(self, v): + if self._transposed: + self._transposed = False + return self.L.T.dot(self._factor.apply_P(v)) + else: + return self._factor.apply_Pt(self.L.dot(v)) + + def solve(self, y): + x = self._factor + if self._transposed: + self._transposed = False + return x.apply_Pt(x.solve_Lt(y, use_LDLt_decomposition=False)) + else: + raise NotImplementedError + + def update_inplace(self, A, beta): + self._factor.cholesky_inplace(A, beta=beta) + self._L = None From 636e86500b7aad351281348cafc903bf48b1444b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 23 Aug 2019 14:16:19 +0300 Subject: [PATCH 17/82] add HybridSVD model --- polara/recommender/coldstart/models.py | 41 +++++- polara/recommender/hybrid/models.py | 178 ++++++++++++++++++++++++- 2 files changed, 216 insertions(+), 3 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index a47b42f..97033ee 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -2,7 +2,7 @@ from polara import SVDModel from polara.recommender.models import RecommenderModel, ScaledMatrixMixin -from polara.recommender.hybrid.models import LCEModel +from polara.recommender.hybrid.models import LCEModel, HybridSVD from polara.lib.similarity import stack_features from polara.lib.sparse import sparse_dot @@ -145,3 +145,42 @@ def get_recommendations(self): scores = cold_items_factors @ Hu top_relevant_users = self.get_topk_elements(scores).astype(np.intp) return top_relevant_users + + +class HybridSVDItemColdStart(ItemColdStartEvaluationMixin, HybridSVD): + def __init__(self, *args, item_features=None, **kwargs): + super().__init__(*args, **kwargs) + self.method = 'HybridSVD(cs)' + self.item_features = item_features + self.use_raw_features = item_features is not None + + def build(self, *args, **kwargs): + super().build(*args, return_factors=True, **kwargs) + + def get_recommendations(self): + userid = self.data.fields.userid + + u = self.factors[userid] + v = self.factors['items_projector_right'] + s = self.factors['singular_values'] + + if self.use_raw_features: + item_info = self.item_features.reindex(self.data.index.itemid.training.old.values, + fill_value=[]) + item_features, feature_labels = stack_features(item_info, normalize=False) + w = item_features.T.dot(v).T + cold_info = self.item_features.reindex(self.data.index.itemid.cold_start.old.values, + fill_value=[]) + cold_item_features, _ = stack_features(cold_info, labels=feature_labels, normalize=False) + else: + w = self.data.item_relathions.T.dot(v).T + cold_item_features = self.data.cold_items_similarity + + wwt_inv = np.linalg.pinv(w @ w.T) + cold_items_factors = cold_item_features.dot(w.T) @ wwt_inv + scores = cold_items_factors @ (u * s[None, :]).T + top_similar_users = self.get_topk_elements(scores) + return top_similar_users + + +class ScaledHybridSVDItemColdStart(ScaledMatrixMixin, HybridSVDItemColdStart): pass diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index f7a8a3f..a3e0919 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -1,12 +1,25 @@ import math import scipy as sp import numpy as np - -from polara.recommender.models import RecommenderModel, ProbabilisticMF +from scipy.sparse.linalg import LinearOperator +from string import Template + +try: + from sksparse import __version__ as sk_sp_version +except ImportError: + SPARSE_MODE = False +else: + assert sk_sp_version >= '0.4.3' + SPARSE_MODE = True + from sksparse.cholmod import cholesky as cholesky_decomp_sparse + +from polara.recommender.models import RecommenderModel, ProbabilisticMF, ScaledMatrixMixin, SVDModel from polara.lib.optimize import kernelized_pmf_sgd, local_collective_embeddings from polara.lib.sparse import sparse_dot from polara.lib.similarity import stack_features from polara.tools.timing import track_time +from polara.lib.cholesky import CholeskyFactor + class SimilarityAggregation(RecommenderModel): @@ -210,3 +223,164 @@ def slice_recommendations(self, test_data, shape, start, stop, test_users=None): item_factors = self.factors[itemid] scores = user_factors.dot(item_factors.T) return scores, slice_data + + +class CholeskyFactorsMixin: + def __init__(self, *args, **kwargs): + self._sparse_mode = SPARSE_MODE + self.return_factors = True + + super().__init__(*args, **kwargs) + entities = [self.data.fields.userid, self.data.fields.itemid] + self._cholesky = dict.fromkeys(entities) + + self._features_weight = 0.999 + self.data.subscribe(self.data.on_change_event, self._clean_cholesky) + + def _clean_cholesky(self): + self._cholesky = {entity:None for entity in self._cholesky.keys()} + + def _update_cholesky(self): + for entity, cholesky in self._cholesky.items(): + if cholesky is not None: + self._update_cholesky_inplace(entity) + + @property + def features_weight(self): + return self._features_weight + + @features_weight.setter + def features_weight(self, new_val): + if new_val != self._features_weight: + self._features_weight = new_val + self._update_cholesky() + self._renew_model() + + @property + def item_cholesky_factor(self): + itemid = self.data.fields.itemid + return self.get_cholesky_factor(itemid) + + @property + def user_cholesky_factor(self): + userid = self.data.fields.userid + return self.get_cholesky_factor(userid) + + def get_cholesky_factor(self, entity): + cholesky = self._cholesky.get(entity, None) + if cholesky is None: + self._update_cholesky_factor(entity) + return self._cholesky[entity] + + def _update_cholesky_factor(self, entity): + entity_similarity = self.data.get_relations_matrix(entity) + if entity_similarity is None: + self._cholesky[entity] = None + else: + if self._sparse_mode: + cholesky_decomp = cholesky_decomp_sparse + mode = 'sparse' + else: + raise NotImplementedError + + weight = self.features_weight + beta = (1.0 - weight) / weight + if self.verbose: + print('Performing {} Cholesky decomposition for {} similarity'.format(mode, entity)) + + msg = Template('Cholesky decomposition computation time: $time') + with track_time(verbose=self.verbose, message=msg): + self._cholesky[entity] = CholeskyFactor(cholesky_decomp(entity_similarity, beta=beta)) + + def _update_cholesky_inplace(self, entity): + entity_similarity = self.data.get_relations_matrix(entity) + if self._sparse_mode: + weight = self.features_weight + beta = (1.0 - weight) / weight + if self.verbose: + print('Updating Cholesky decomposition inplace for {} similarity'.format(entity)) + + msg = Template(' Cholesky decomposition update time: $time') + with track_time(verbose=self.verbose, message=msg): + self._cholesky[entity].update_inplace(entity_similarity, beta) + else: + raise NotImplementedError + + def build_item_projector(self, v): + cholesky_items = self.item_cholesky_factor + if cholesky_items is not None: + if self.verbose: + print(f'Building {self.data.fields.itemid} projector for {self.method}') + msg = Template(' Solving triangular system: $time') + with track_time(verbose=self.verbose, message=msg): + self.factors['items_projector_left'] = cholesky_items.T.solve(v) + msg = Template(' Applying Cholesky factor: $time') + with track_time(verbose=self.verbose, message=msg): + self.factors['items_projector_right'] = cholesky_items.dot(v) + + def get_item_projector(self): + vl = self.factors.get('items_projector_left', None) + vr = self.factors.get('items_projector_right', None) + return vl, vr + + +class HybridSVD(CholeskyFactorsMixin, SVDModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.method = 'HybridSVD' + self.precompute_auxiliary_matrix = False + + def _check_reduced_rank(self, rank): + super()._check_reduced_rank(rank) + self.round_item_projector(rank) + + def round_item_projector(self, rank): + vl, vr = self.get_item_projector() + if (vl is not None) and (rank < vl.shape[1]): + self.factors['items_projector_left'] = vl[:, :rank] + self.factors['items_projector_right'] = vr[:, :rank] + + def build(self, *args, **kwargs): + if not self._sparse_mode: + raise NotImplementedError('Check the installation of scikit-sparse package.') + + # the order matters - trigger on_change events first + svd_matrix = self.get_training_matrix(dtype=np.float64) + cholesky_items = self.item_cholesky_factor + cholesky_users = self.user_cholesky_factor + + if self.precompute_auxiliary_matrix: + if cholesky_items is not None: + svd_matrix = cholesky_items.T.dot(svd_matrix.T).T + cholesky_items._L = None + if cholesky_users is not None: + svd_matrix = cholesky_users.T.dot(svd_matrix) + cholesky_users._L = None + operator = svd_matrix + else: + if cholesky_items is not None: + L_item = cholesky_items + else: + L_item = sp.sparse.eye(svd_matrix.shape[1]) + if cholesky_users is not None: + L_user = cholesky_users + else: + L_user = sp.sparse.eye(svd_matrix.shape[0]) + + def matvec(v): + return L_user.T.dot(svd_matrix.dot(L_item.dot(v))) + def rmatvec(v): + return L_item.T.dot(svd_matrix.T.dot(L_user.dot(v))) + operator = LinearOperator(svd_matrix.shape, matvec, rmatvec) + + super().build(*args, operator=operator, **kwargs) + self.build_item_projector(self.factors[self.data.fields.itemid]) + + def slice_recommendations(self, test_data, shape, start, stop, test_users=None): + test_matrix, slice_data = self.get_test_matrix(test_data, shape, (start, stop)) + vl, vr = self.get_item_projector() + scores = test_matrix.dot(vr).dot(vl.T) + return scores, slice_data + + +class ScaledHybridSVD(ScaledMatrixMixin, HybridSVD): pass From eadb3046b15fc19f64808c0866878bdb1475060f Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 23 Aug 2019 14:17:31 +0300 Subject: [PATCH 18/82] update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5d7d28e..87596ac 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ opts = dict(name="polara", description="Fast and flexible recommender system framework", keywords = "recommender system", - version = "0.6.4.dev", + version = "0.6.5.dev", license="MIT", author="Evgeny Frolov", platforms=["any"], From 521d92d97f55a9f7834456beebeaa385ee8c3d94 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sat, 24 Aug 2019 08:59:11 +0300 Subject: [PATCH 19/82] update package description --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 87596ac..6699306 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,8 @@ opts = dict(name="polara", - description="Fast and flexible recommender system framework", - keywords = "recommender system", + description="Fast and flexible recommender systems framework", + keywords = "recommender systems", version = "0.6.5.dev", license="MIT", author="Evgeny Frolov", From e42d71320d2985582861ad990e0eaeb439903dd2 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 25 Aug 2019 10:37:52 +0300 Subject: [PATCH 20/82] remove unnecessary attribute from Cholesky factors mixin --- polara/recommender/hybrid/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index a3e0919..323aef3 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -228,8 +228,6 @@ def slice_recommendations(self, test_data, shape, start, stop, test_users=None): class CholeskyFactorsMixin: def __init__(self, *args, **kwargs): self._sparse_mode = SPARSE_MODE - self.return_factors = True - super().__init__(*args, **kwargs) entities = [self.data.fields.userid, self.data.fields.itemid] self._cholesky = dict.fromkeys(entities) From 614a541bdb0eda11969da38cfbbb610ba181fe62 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 25 Aug 2019 10:38:29 +0300 Subject: [PATCH 21/82] change default features weight value for HybridSVD --- polara/recommender/hybrid/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index 323aef3..b4f657c 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -232,7 +232,7 @@ def __init__(self, *args, **kwargs): entities = [self.data.fields.userid, self.data.fields.itemid] self._cholesky = dict.fromkeys(entities) - self._features_weight = 0.999 + self._features_weight = 0.5 self.data.subscribe(self.data.on_change_event, self._clean_cholesky) def _clean_cholesky(self): From 021b6ff408486b6f380b223623fc09323a667a20 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 25 Aug 2019 11:10:18 +0300 Subject: [PATCH 22/82] improve feature_labels attribute naming for LCE model --- polara/recommender/hybrid/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index b4f657c..f85e2df 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -128,7 +128,7 @@ def __init__(self, *args, item_features=None, **kwargs): self.item_features = item_features self.binary_features = True self._item_data = None - self.feature_labels = None + self.item_features_labels = None self.seed = None self.show_error = False self.regularization = 1 @@ -139,7 +139,7 @@ def __init__(self, *args, item_features=None, **kwargs): def _clean_metadata(self): self._item_data = None - self.feature_labels = None + self.item_features_labels = None @property def rank(self): @@ -206,7 +206,7 @@ def build(self): self.factors[userid] = Hu.T self.factors[itemid] = W self.factors['item_features'] = Hs.T - self.feature_labels = lbls + self.item_features_labels = lbls def get_recommendations(self): if self.data.warm_start: From 8340d5f1404f46b11e27cb834adc12ef8c3b5c40 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 25 Aug 2019 11:48:15 +0300 Subject: [PATCH 23/82] respect memory limits and support parallel recommendations generation in cold start --- polara/recommender/coldstart/models.py | 176 +++++++++++++++++-------- 1 file changed, 119 insertions(+), 57 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index 97033ee..a42796a 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -15,6 +15,40 @@ def __init__(self, *args, **kwargs): self._prediction_target = self.data.fields.userid +class ItemColdStartRecommenderMixin: + def get_recommendations(self): + if self.verify_integrity: + self.verify_data_integrity() + + cold_item_meta = self.item_features.reindex( + self.data.index.itemid.cold_start.old.values, + fill_value=[] + ) + + n_test_items = cold_item_meta.shape[0] + try: + n_test_users = self.data.representative_users.shape[0] + except AttributeError: + n_test_users = self.data.index.userid.training.shape[0] + + test_shape = (n_test_items, n_test_users) + cold_slices_idx = self._get_slices_idx(test_shape) + cold_slices = zip(cold_slices_idx[:-1], cold_slices_idx[1:]) + + result = np.empty((test_shape[0], self.topk), dtype=np.int64) + if self.max_test_workers and len(cold_slices_idx) > 2: + self.run_parallel_recommender(result, cold_slices, cold_item_meta) + else: + self.run_sequential_recommender(result, cold_slices, cold_item_meta) + return result + + def _slice_recommender(self, cold_slice, cold_item_meta): + start, stop = cold_slice + scores = self.slice_recommendations(cold_item_meta, start, stop) + top_recs = self.get_topk_elements(scores) + return top_recs + + class RandomModelItemColdStart(ItemColdStartEvaluationMixin, RecommenderModel): def __init__(self, *args, **kwargs): self.seed = kwargs.pop('seed', None) @@ -82,43 +116,50 @@ def get_recommendations(self): return top_similar_users -class SVDModelItemColdStart(ItemColdStartEvaluationMixin, SVDModel): - def __init__(self, *args, item_features=None, **kwargs): +class SVDModelItemColdStart(ItemColdStartEvaluationMixin, ItemColdStartRecommenderMixin, SVDModel): + def __init__(self, *args, item_features, **kwargs): super().__init__(*args, **kwargs) self.method = 'PureSVD(cs)' self.item_features = item_features - self.use_raw_features = item_features is not None + self.item_features_labels = None + self.item_features_transform = None + + def _check_reduced_rank(self, rank): + super()._check_reduced_rank(rank) + try: + w = self.item_features_transform[0] + except TypeError: + return + if w.shape[0] < rank: + self.item_features_transform = None + else: + w = w[:rank, :] + self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) def build(self, *args, **kwargs): super().build(*args, return_factors=True, **kwargs) - def get_recommendations(self): - userid = self.data.fields.userid - itemid = self.data.fields.itemid - - u = self.factors[userid] - v = self.factors[itemid] - s = self.factors['singular_values'] + item_meta = self.item_features.reindex( + self.data.index.itemid.training.old.values, fill_value=[]) + item_one_hot, self.item_features_labels = stack_features( + item_meta, stacked_index=False, normalize=False) - if self.use_raw_features: - item_info = self.item_features.reindex(self.data.index.itemid.training.old.values, - fill_value=[]) - item_features, feature_labels = stack_features(item_info, normalize=False) - w = item_features.T.dot(v).T - wwt_inv = np.linalg.pinv(w @ w.T) + w = item_one_hot.T.dot(self.factors[self.data.fields.itemid]).T + self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) - cold_info = self.item_features.reindex(self.data.index.itemid.cold_start.old.values, - fill_value=[]) - cold_item_features, _ = stack_features(cold_info, labels=feature_labels, normalize=False) - else: - w = self.data.item_relations.T.dot(v).T - wwt_inv = np.linalg.pinv(w @ w.T) - cold_item_features = self.data.cold_items_similarity + def slice_recommendations(self, cold_item_meta, start, stop): + cold_slice_meta = cold_item_meta.iloc[start:stop] + cold_item_features, _ = stack_features( + cold_slice_meta, + labels=self.item_features_labels, + normalize=False) + u = self.factors[self.data.fields.userid] + s = self.factors['singular_values'] + w, wwt_inv = self.item_features_transform cold_items_factors = cold_item_features.dot(w.T) @ wwt_inv scores = cold_items_factors @ (u * s[None, :]).T - top_similar_users = self.get_topk_elements(scores).astype(np.intp) - return top_similar_users + return scores class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): @@ -127,60 +168,81 @@ def __init__(self, *args, **kwargs): self.method = 'PureSVDs(cs)' -class LCEModelItemColdStart(ItemColdStartEvaluationMixin, LCEModel): +class LCEModelItemColdStart(ItemColdStartEvaluationMixin, ItemColdStartRecommenderMixin, LCEModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.method = 'LCE(cs)' + self.item_features_invgram = None + + def build(self, *args, **kwargs): + super().build(*args, **kwargs) + Hs = self.factors['item_features'].T + self.item_features_invgram = np.linalg.pinv(Hs @ Hs.T) + + def slice_recommendations(self, cold_item_meta, start, stop): + cold_slice_meta = cold_item_meta.iloc[start:stop] + cold_item_features, _ = stack_features( + cold_slice_meta, + labels=self.item_features_labels, + normalize=False) - def get_recommendations(self): Hu = self.factors[self.data.fields.userid].T Hs = self.factors['item_features'].T - cold_info = self.item_features.reindex(self.data.index.itemid.cold_start.old.values, - fill_value=[]) - cold_item_features, _ = stack_features(cold_info, labels=self.feature_labels, normalize=False) - cold_items_factors = cold_item_features.dot(Hs.T).dot(np.linalg.pinv(Hs @ Hs.T)) + cold_items_factors = cold_item_features.dot(Hs.T).dot(self.item_features_invgram) cold_items_factors[cold_items_factors < 0] = 0 - scores = cold_items_factors @ Hu - top_relevant_users = self.get_topk_elements(scores).astype(np.intp) - return top_relevant_users + return scores -class HybridSVDItemColdStart(ItemColdStartEvaluationMixin, HybridSVD): - def __init__(self, *args, item_features=None, **kwargs): +class HybridSVDItemColdStart(ItemColdStartEvaluationMixin, ItemColdStartRecommenderMixin, HybridSVD): + def __init__(self, *args, item_features, **kwargs): super().__init__(*args, **kwargs) self.method = 'HybridSVD(cs)' self.item_features = item_features - self.use_raw_features = item_features is not None + self.item_features_labels = None + self.item_features_transform = None + self.data.subscribe(self.data.on_change_event, self._clean_metadata) + + def _clean_metadata(self): + self.item_features_labels = None + + def _check_reduced_rank(self, rank): + super()._check_reduced_rank(rank) + try: + w = self.item_features_transform[0] + except TypeError: + return + if w.shape[0] < rank: + self.item_features_transform = None + else: + w = w[:rank, :] + self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) def build(self, *args, **kwargs): super().build(*args, return_factors=True, **kwargs) - def get_recommendations(self): - userid = self.data.fields.userid + item_meta = self.item_features.reindex( + self.data.index.itemid.training.old.values, fill_value=[]) + item_one_hot, self.item_features_labels = stack_features( + item_meta, stacked_index=False, normalize=False) - u = self.factors[userid] - v = self.factors['items_projector_right'] - s = self.factors['singular_values'] + w = item_one_hot.T.dot(self.factors['items_projector_right']).T + self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) - if self.use_raw_features: - item_info = self.item_features.reindex(self.data.index.itemid.training.old.values, - fill_value=[]) - item_features, feature_labels = stack_features(item_info, normalize=False) - w = item_features.T.dot(v).T - cold_info = self.item_features.reindex(self.data.index.itemid.cold_start.old.values, - fill_value=[]) - cold_item_features, _ = stack_features(cold_info, labels=feature_labels, normalize=False) - else: - w = self.data.item_relathions.T.dot(v).T - cold_item_features = self.data.cold_items_similarity + def slice_recommendations(self, cold_item_meta, start, stop): + cold_slice_meta = cold_item_meta.iloc[start:stop] + cold_item_features, _ = stack_features( + cold_slice_meta, + labels=self.item_features_labels, + normalize=False) - wwt_inv = np.linalg.pinv(w @ w.T) - cold_items_factors = cold_item_features.dot(w.T) @ wwt_inv + u = self.factors[self.data.fields.userid] + s = self.factors['singular_values'] + w, w_invgram = self.item_features_transform + cold_items_factors = cold_item_features.dot(w.T) @ w_invgram scores = cold_items_factors @ (u * s[None, :]).T - top_similar_users = self.get_topk_elements(scores) - return top_similar_users + return scores class ScaledHybridSVDItemColdStart(ScaledMatrixMixin, HybridSVDItemColdStart): pass From e6d141b8ebfe37db43bae55ed87684a2e57f1c4a Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Mon, 26 Aug 2019 07:15:42 +0300 Subject: [PATCH 24/82] refactor item cold start code for SVD models --- polara/recommender/coldstart/models.py | 103 +++++++++++-------------- 1 file changed, 43 insertions(+), 60 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index a42796a..bc28868 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -116,58 +116,6 @@ def get_recommendations(self): return top_similar_users -class SVDModelItemColdStart(ItemColdStartEvaluationMixin, ItemColdStartRecommenderMixin, SVDModel): - def __init__(self, *args, item_features, **kwargs): - super().__init__(*args, **kwargs) - self.method = 'PureSVD(cs)' - self.item_features = item_features - self.item_features_labels = None - self.item_features_transform = None - - def _check_reduced_rank(self, rank): - super()._check_reduced_rank(rank) - try: - w = self.item_features_transform[0] - except TypeError: - return - if w.shape[0] < rank: - self.item_features_transform = None - else: - w = w[:rank, :] - self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) - - def build(self, *args, **kwargs): - super().build(*args, return_factors=True, **kwargs) - - item_meta = self.item_features.reindex( - self.data.index.itemid.training.old.values, fill_value=[]) - item_one_hot, self.item_features_labels = stack_features( - item_meta, stacked_index=False, normalize=False) - - w = item_one_hot.T.dot(self.factors[self.data.fields.itemid]).T - self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) - - def slice_recommendations(self, cold_item_meta, start, stop): - cold_slice_meta = cold_item_meta.iloc[start:stop] - cold_item_features, _ = stack_features( - cold_slice_meta, - labels=self.item_features_labels, - normalize=False) - - u = self.factors[self.data.fields.userid] - s = self.factors['singular_values'] - w, wwt_inv = self.item_features_transform - cold_items_factors = cold_item_features.dot(w.T) @ wwt_inv - scores = cold_items_factors @ (u * s[None, :]).T - return scores - - -class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.method = 'PureSVDs(cs)' - - class LCEModelItemColdStart(ItemColdStartEvaluationMixin, ItemColdStartRecommenderMixin, LCEModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -195,10 +143,9 @@ def slice_recommendations(self, cold_item_meta, start, stop): return scores -class HybridSVDItemColdStart(ItemColdStartEvaluationMixin, ItemColdStartRecommenderMixin, HybridSVD): +class ItemColdStartSVDModelMixin: def __init__(self, *args, item_features, **kwargs): super().__init__(*args, **kwargs) - self.method = 'HybridSVD(cs)' self.item_features = item_features self.item_features_labels = None self.item_features_transform = None @@ -207,8 +154,7 @@ def __init__(self, *args, item_features, **kwargs): def _clean_metadata(self): self.item_features_labels = None - def _check_reduced_rank(self, rank): - super()._check_reduced_rank(rank) + def round_item_features_transform(self, rank): try: w = self.item_features_transform[0] except TypeError: @@ -219,16 +165,26 @@ def _check_reduced_rank(self, rank): w = w[:rank, :] self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) - def build(self, *args, **kwargs): - super().build(*args, return_factors=True, **kwargs) + def _check_reduced_rank(self, rank): + super()._check_reduced_rank(rank) + self.round_item_features_transform(rank) + def encode_item_features(self): item_meta = self.item_features.reindex( self.data.index.itemid.training.old.values, fill_value=[]) item_one_hot, self.item_features_labels = stack_features( item_meta, stacked_index=False, normalize=False) + return item_one_hot - w = item_one_hot.T.dot(self.factors['items_projector_right']).T - self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) + def assemble_item_features_transform(self): + item_one_hot = self.encode_item_features() + mapping = self.compute_item_features_mapping(item_one_hot) # model dependent + mapping_invgram = np.linalg.pinv(mapping @ mapping.T) + self.item_features_transform = (mapping, mapping_invgram) + + def build(self, *args, **kwargs): + super().build(*args, return_factors=True, **kwargs) + self.assemble_item_features_transform() def slice_recommendations(self, cold_item_meta, start, stop): cold_slice_meta = cold_item_meta.iloc[start:stop] @@ -245,4 +201,31 @@ def slice_recommendations(self, cold_item_meta, start, stop): return scores +class SVDModelItemColdStart(ItemColdStartEvaluationMixin, + ItemColdStartRecommenderMixin, + ItemColdStartSVDModelMixin, + SVDModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.method = 'PureSVD(cs)' + + def compute_item_features_mapping(self, item_features): + return item_features.T.dot(self.factors[self.data.fields.itemid]).T + + +class HybridSVDItemColdStart(ItemColdStartEvaluationMixin, + ItemColdStartRecommenderMixin, + ItemColdStartSVDModelMixin, + HybridSVD): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.method = 'HybridSVD(cs)' + + def compute_item_features_mapping(self, item_features): + return item_features.T.dot(self.factors['items_projector_right']).T + + +class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): pass + + class ScaledHybridSVDItemColdStart(ScaledMatrixMixin, HybridSVDItemColdStart): pass From f0e7e3f67a0da922162e8a59b0a5bfb272e34c1d Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Mon, 26 Aug 2019 11:18:59 +0300 Subject: [PATCH 25/82] do not attempt to select non-existing fields --- polara/recommender/coldstart/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index 1ff2a3d..63ec010 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -74,7 +74,7 @@ def _sample_holdout(self, test_split, group_id=None): if self._holdout_size > 0: holdout = super(ItemColdStartData, self)._sample_holdout(test_split, group_id=itemid) else: - holdout = self._data.loc[test_split, list(self.fields)] + holdout = self._data.loc[test_split, [f for f in self.fields if f is not None]] itemid_cold = '{}_cold'.format(itemid) return holdout.rename(columns={itemid: itemid_cold}, copy=False) From 88c67dce968d882c9637ef3f32fb0c99f5ce85f3 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 30 Aug 2019 11:50:41 +0300 Subject: [PATCH 26/82] fix some comments --- polara/lib/similarity.py | 2 +- polara/recommender/data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polara/lib/similarity.py b/polara/lib/similarity.py index 3b6da9e..d449332 100644 --- a/polara/lib/similarity.py +++ b/polara/lib/similarity.py @@ -261,7 +261,7 @@ def feature2sparse(feature_data, ranking=None, deduplicate=True, labels=None): indices = [] indlens = [] for items in feature_data: - # wiil also remove unknown items to ensure index consistency + # will also remove unknown items to ensure index consistency inds = [feature_lbl[item] for item in items if item in feature_lbl] indices.extend(inds) indlens.append(len(inds)) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index a672b84..db47903 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -393,7 +393,7 @@ def _split_data(self): testset = holdout = None train_split = ~test_split else: # state 3 or state 4 - # NOTE holdout_size = None is also here; this can be used in + # NOTE holdout_size < 0 is also here; this can be used in # subclasses like ItemColdStartData to preprocess data properly # in that case _sample_holdout must be modified accordingly holdout = self._sample_holdout(test_split) From a8f01340a684b1ab449f49729a5cede5919fa5f1 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 30 Aug 2019 11:51:49 +0300 Subject: [PATCH 27/82] remove unnecessary settings from item cold start data --- polara/recommender/coldstart/data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index 63ec010..d13e183 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -14,7 +14,6 @@ def __init__(self, *args, **kwargs): self._test_ratio = 0.2 self._warm_start = False - self._holdout_size = -1 # needed for correct processing of test data # build unique items list to split them by folds itemid = self.fields.itemid From b30adf838a91ecd8339a442c7768e8253dc76e31 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 30 Aug 2019 11:52:33 +0300 Subject: [PATCH 28/82] clean representative users in item cold start via function --- polara/recommender/coldstart/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index d13e183..bec84f1 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -63,7 +63,7 @@ def _check_state_transition(self): # in standard state 3 scenario (as there's no testset) if '_test_sample' in self._change_properties: update_rule['test_update'] = True - self._repr_users = None + self._clean_representative_users() return new_state, update_rule From fa1dc1399bc02c31a55a267b70751bc2ce5be3c0 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 30 Aug 2019 11:55:20 +0300 Subject: [PATCH 29/82] improve attribute naming in item cold start data --- polara/recommender/coldstart/data.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index bec84f1..e2fe3d2 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -9,7 +9,7 @@ class ItemColdStartData(RecommenderData): def __init__(self, *args, **kwargs): - self.meta_data = kwargs.pop('meta_data', None) + self.item_meta = kwargs.pop('item_meta', None) super(ItemColdStartData, self).__init__(*args, **kwargs) self._test_ratio = 0.2 @@ -146,13 +146,13 @@ def _verify_cold_items_representatives(self): def _verify_cold_items_features(self): - if self.meta_data is None: + if self.item_meta is None: return - if self.meta_data.shape[1] > 1: - features_melted = self.meta_data.agg(lambda x: [f for l in x for f in l], axis=1) + if self.item_meta.shape[1] > 1: + features_melted = self.item_meta.agg(lambda x: [f for l in x for f in l], axis=1) else: - features_melted = self.meta_data.iloc[:, 0] + features_melted = self.item_meta.iloc[:, 0] feature_labels = defaultdict(lambda: len(feature_labels)) labels = features_melted.apply(lambda x: [feature_labels[i] for i in x]) From e6b7f4f0e3422c9838122659d11faf2020f6d914 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 30 Aug 2019 11:58:42 +0300 Subject: [PATCH 30/82] silence SparseEfficiency warning in one-hot similarity computation --- polara/lib/similarity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/lib/similarity.py b/polara/lib/similarity.py index d449332..a7de864 100644 --- a/polara/lib/similarity.py +++ b/polara/lib/similarity.py @@ -370,7 +370,7 @@ def one_hot_similarity(meta_data, metric='common', assume_binary=True, fill_diag maxel = max(similarity.data.max(), abs(similarity.data.min())) similarity = similarity / maxel if fill_diagonal: - similarity.setdiag(1.0) + set_diagonal_values(similarity, 1) if (metric == 'cosine') or (metric == 'salton'): similarity = cosine_similarity(features, assume_binary=assume_binary, fill_diagonal=fill_diagonal) From 85c6aea9043a1851b0c425f7afd2aaa5488ade19 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 1 Sep 2019 11:16:10 +0300 Subject: [PATCH 31/82] allow custom test data in cold start --- polara/recommender/coldstart/data.py | 29 +++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index e2fe3d2..37234bd 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -55,8 +55,6 @@ def _split_test_index(self): def _check_state_transition(self): assert not self._warm_start - assert self._holdout_size != 0 # needed for correct processing of test data - assert self._test_ratio > 0 new_state, update_rule = super(ItemColdStartData, self)._check_state_transition() # handle change of test_sample value which is not handled @@ -79,7 +77,7 @@ def _sample_holdout(self, test_split, group_id=None): return holdout.rename(columns={itemid: itemid_cold}, copy=False) - def _try_drop_unseen_test_items(self): + def _try_drop_unseen_test_items(self, *args, **kwargs): # there will be no such items except cold-start items pass @@ -90,13 +88,16 @@ def _filter_short_sessions(self, group_id=None): def _assign_test_items_index(self): - if self.build_index: + cold_items_are_initialized = self._test.holdout is not None + if self.build_index and cold_items_are_initialized: self._reindex_cold_items() def _reindex_cold_items(self): itemid_cold = '{}_cold'.format(self.fields.itemid) - cold_item_index = self.reindex(self._test.holdout, itemid_cold, inplace=True, sort=False) + holdout = self._test.holdout + cold_item_index = self.reindex( + holdout, itemid_cold, inplace=True, sort=False) try: # check if already modified item index to avoid nested assignemnt item_index = self.index.itemid.training @@ -115,10 +116,12 @@ def _try_sort_test_data(self): def _post_process_cold_items(self): self._clean_representative_users() - self._verify_cold_items_representatives() - self._verify_cold_items_features() - self._try_cleanup_cold_items() - self._sort_by_cold_items() + cold_items_are_initialized = self._test.holdout is not None + if cold_items_are_initialized: + self._verify_cold_items_representatives() + self._verify_cold_items_features() + self._try_cleanup_cold_items() + self._sort_by_cold_items() def _clean_representative_users(self): @@ -202,6 +205,14 @@ def _sort_by_cold_items(self): holdout = self._test.holdout holdout.sort_values(itemid_cold, inplace=True) + def set_test_data(self, *, holdout, **kwargs): + itemid = self.fields.itemid + itemid_cold = '{}_cold'.format(itemid) + if itemid_cold not in holdout.columns: + holdout = holdout.rename(columns={itemid: itemid_cold}, copy=kwargs.pop('copy', True)) + super().set_test_data(holdout=holdout, copy=False, **kwargs) + self._post_process_cold_items() + class ColdSimilarityMixin(object): @property From 468578b0ff935bb22d94aaad249b649d9a4330c6 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 1 Sep 2019 11:18:22 +0300 Subject: [PATCH 32/82] improve LightFM model handling --- polara/recommender/coldstart/models.py | 7 ++++ .../external/lightfm/lightfmwrapper.py | 41 +++++++++++-------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index bc28868..a057113 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -3,6 +3,7 @@ from polara import SVDModel from polara.recommender.models import RecommenderModel, ScaledMatrixMixin from polara.recommender.hybrid.models import LCEModel, HybridSVD +from polara.recommender.external.lightfm.lightfmwrapper import LightFMWrapper, ItemColdStartLightFMMixin from polara.lib.similarity import stack_features from polara.lib.sparse import sparse_dot @@ -229,3 +230,9 @@ class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): pass class ScaledHybridSVDItemColdStart(ScaledMatrixMixin, HybridSVDItemColdStart): pass + + +class LightFMItemColdStart(ItemColdStartEvaluationMixin, + ItemColdStartRecommenderMixin, + ItemColdStartLightFMMixin, + LightFMWrapper): pass diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index 2c16a3f..3d78f8f 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -1,6 +1,3 @@ -# python 2/3 interoperability -from __future__ import print_function - import numpy as np from numpy.lib.stride_tricks import as_strided from lightfm import LightFM @@ -15,15 +12,16 @@ def __init__(self, *args, item_features=None, user_features=None, **kwargs): self.method='LightFM' self.rank = 10 self.fit_method = 'fit' + self.fit_params = {} self.item_features = item_features - self.item_feature_labels = None + self.item_features_labels = None self.item_alpha = 0.0 self.item_identity = True self._item_features_csr = None self.user_features = user_features - self.user_feature_labels = None + self.user_features_labels = None self.user_alpha = 0.0 self.user_identity = True self._user_features_csr = None @@ -50,21 +48,32 @@ def build(self): matrix = self.get_training_matrix() + try: + item_index = self.data.index.itemid.training + except AttributeError: + item_index = self.data.index.itemid + if self.item_features is not None: - item_features = self.item_features.reindex(self.data.index.itemid.old.values, fill_value=[]) - self._item_features_csr, self.item_feature_labels = stack_features(item_features, - add_identity=self.item_identity, - normalize=True, - dtype='f4') + item_features = self.item_features.reindex( + item_index.old.values, + fill_value=[]) + self._item_features_csr, self.item_features_labels = stack_features( + item_features, + add_identity=self.item_identity, + normalize=True, + dtype='f4') if self.user_features is not None: - user_features = self.user_features.reindex(self.data.index.userid.training.old.values, fill_value=[]) - self._user_features_csr, self.user_feature_labels = stack_features(user_features, - add_identity=self.user_identity, - normalize=True, - dtype='f4') + user_features = self.user_features.reindex( + self.data.index.userid.training.old.values, + fill_value=[]) + self._user_features_csr, self.user_features_labels = stack_features( + user_features, + add_identity=self.user_identity, + normalize=True, + dtype='f4') with track_time(self.training_time, verbose=self.verbose, model=self.method): - fit(matrix, item_features=self._item_features_csr, user_features=self._user_features_csr) + fit(matrix, item_features=self._item_features_csr, user_features=self._user_features_csr, **self.fit_params) def slice_recommendations(self, test_data, shape, start, stop, test_users=None): From aea095f63502463ceb1346d611b1198639c8b9cb Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 1 Sep 2019 11:20:02 +0300 Subject: [PATCH 33/82] add proper cold start support in LightFM --- .../external/lightfm/lightfmwrapper.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index 3d78f8f..72eb53c 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -96,3 +96,25 @@ def slice_recommendations(self, test_data, shape, start, stop, test_users=None): item_features=self._item_features_csr ).reshape(n_users, n_items) return scores, slice_data + + +class ItemColdStartLightFMMixin: + def slice_recommendations(self, cold_item_meta, start, stop): + cold_slice_meta = cold_item_meta.iloc[start:stop] + cold_item_features, _ = stack_features( + cold_slice_meta, + labels=self.item_features_labels, + add_identity=False, + normalize=True) + + user_embeddings = self._model.user_embeddings + repr_users = self.data.representative_users + if repr_users is not None: + user_embeddings = user_embeddings[repr_users.new.values, :] + + # proper handling of cold-start (instead of built-in predict) + n_items = self.data.index.itemid.training.shape[0] + item_features_embeddings = self._model.item_embeddings[n_items:, :] + cold_items_embeddings = cold_item_features.dot(item_features_embeddings) + scores = cold_items_embeddings @ user_embeddings.T + return scores From 785bfda6e2ddda0b3ae4b720107bd1825510ab42 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Tue, 3 Sep 2019 08:16:04 +0300 Subject: [PATCH 34/82] move LightFM cold start mixin to a proper location --- polara/recommender/coldstart/models.py | 24 ++++++++++++++++++- .../external/lightfm/lightfmwrapper.py | 22 ----------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index a057113..a3e87b2 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -3,7 +3,7 @@ from polara import SVDModel from polara.recommender.models import RecommenderModel, ScaledMatrixMixin from polara.recommender.hybrid.models import LCEModel, HybridSVD -from polara.recommender.external.lightfm.lightfmwrapper import LightFMWrapper, ItemColdStartLightFMMixin +from polara.recommender.external.lightfm.lightfmwrapper import LightFMWrapper from polara.lib.similarity import stack_features from polara.lib.sparse import sparse_dot @@ -232,6 +232,28 @@ class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): pass class ScaledHybridSVDItemColdStart(ScaledMatrixMixin, HybridSVDItemColdStart): pass +class ItemColdStartLightFMMixin: + def slice_recommendations(self, cold_item_meta, start, stop): + cold_slice_meta = cold_item_meta.iloc[start:stop] + cold_item_features, _ = stack_features( + cold_slice_meta, + labels=self.item_features_labels, + add_identity=False, + normalize=True) + + user_embeddings = self._model.user_embeddings + repr_users = self.data.representative_users + if repr_users is not None: + user_embeddings = user_embeddings[repr_users.new.values, :] + + # proper handling of cold-start (instead of built-in predict) + n_items = self.data.index.itemid.training.shape[0] + item_features_embeddings = self._model.item_embeddings[n_items:, :] + cold_items_embeddings = cold_item_features.dot(item_features_embeddings) + scores = cold_items_embeddings @ user_embeddings.T + return scores + + class LightFMItemColdStart(ItemColdStartEvaluationMixin, ItemColdStartRecommenderMixin, ItemColdStartLightFMMixin, diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index 72eb53c..3d78f8f 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -96,25 +96,3 @@ def slice_recommendations(self, test_data, shape, start, stop, test_users=None): item_features=self._item_features_csr ).reshape(n_users, n_items) return scores, slice_data - - -class ItemColdStartLightFMMixin: - def slice_recommendations(self, cold_item_meta, start, stop): - cold_slice_meta = cold_item_meta.iloc[start:stop] - cold_item_features, _ = stack_features( - cold_slice_meta, - labels=self.item_features_labels, - add_identity=False, - normalize=True) - - user_embeddings = self._model.user_embeddings - repr_users = self.data.representative_users - if repr_users is not None: - user_embeddings = user_embeddings[repr_users.new.values, :] - - # proper handling of cold-start (instead of built-in predict) - n_items = self.data.index.itemid.training.shape[0] - item_features_embeddings = self._model.item_embeddings[n_items:, :] - cold_items_embeddings = cold_item_features.dot(item_features_embeddings) - scores = cold_items_embeddings @ user_embeddings.T - return scores From 3a3f01f56a7ead41c5197dbf3aca898d2df552ca Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 4 Sep 2019 05:45:12 +0300 Subject: [PATCH 35/82] report holdout size in custom test data scenario --- polara/recommender/data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index db47903..98379dd 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -912,6 +912,11 @@ def set_test_data(self, testset=None, holdout=None, warm_start=False, test_users self._try_reindex_test_data() # either assign known index, or reindex (if warm_start) self._try_sort_test_data() + if self.verbose: + if holdout is not None: + num_events = self.test.holdout.shape[0] + print(f'Done. There are {num_events} events in the holdout.') + class LongTailMixin(object): def __init__(self, *args, **kwargs): From c1688ecfa0007c465bc2b093c0d4a75b0ac8562d Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 4 Sep 2019 05:49:40 +0300 Subject: [PATCH 36/82] improve handling os psarse cholesky factors in hybridsvd --- polara/recommender/hybrid/models.py | 36 ++++++++++++++++++----------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index f85e2df..0e21a98 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -238,6 +238,14 @@ def __init__(self, *args, **kwargs): def _clean_cholesky(self): self._cholesky = {entity:None for entity in self._cholesky.keys()} + def _clear_cholesky_cache(self): + cholesky_items = self.item_cholesky_factor + cholesky_users = self.user_cholesky_factor + if cholesky_items is not None: + cholesky_items._L = None + if cholesky_users is not None: + cholesky_users._L = None + def _update_cholesky(self): for entity, cholesky in self._cholesky.items(): if cholesky is not None: @@ -350,29 +358,31 @@ def build(self, *args, **kwargs): if self.precompute_auxiliary_matrix: if cholesky_items is not None: svd_matrix = cholesky_items.T.dot(svd_matrix.T).T - cholesky_items._L = None if cholesky_users is not None: svd_matrix = cholesky_users.T.dot(svd_matrix) - cholesky_users._L = None operator = svd_matrix else: - if cholesky_items is not None: - L_item = cholesky_items - else: - L_item = sp.sparse.eye(svd_matrix.shape[1]) - if cholesky_users is not None: - L_user = cholesky_users - else: - L_user = sp.sparse.eye(svd_matrix.shape[0]) - def matvec(v): - return L_user.T.dot(svd_matrix.dot(L_item.dot(v))) + if cholesky_items is not None: + v = cholesky_items.dot(v) + mat_vec = svd_matrix @ v + if cholesky_users is not None: + mat_vec = cholesky_users.T.dot(mat_vec) + return mat_vec + def rmatvec(v): - return L_item.T.dot(svd_matrix.T.dot(L_user.dot(v))) + if cholesky_users is not None: + v = cholesky_users.dot(v) + r_mat_vec = svd_matrix.T @ v + if cholesky_items is not None: + r_mat_vec = cholesky_items.T.dot(r_mat_vec) + return r_mat_vec + operator = LinearOperator(svd_matrix.shape, matvec, rmatvec) super().build(*args, operator=operator, **kwargs) self.build_item_projector(self.factors[self.data.fields.itemid]) + self._clear_cholesky_cache() def slice_recommendations(self, test_data, shape, start, stop, test_users=None): test_matrix, slice_data = self.get_test_matrix(test_data, shape, (start, stop)) From dc079ba634c5c6d7abd1064946b7fe7b79f51f4f Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 6 Sep 2019 17:02:51 +0300 Subject: [PATCH 37/82] add human-friendly __str__ for data models --- polara/recommender/data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 98379dd..8f31f38 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -149,6 +149,11 @@ def __init__(self, data, userid, itemid, feedback=None, custom_order=None, seed= # on_update indicates whether only test data has been changed -> renew recommendations self.verbose = True + def __str__(self): + name = self.__class__.__name__ + fields = self.fields + return f'{name} with {fields}' + def subscribe(self, event, model_callback): self._notify.subscribe(event, model_callback) From 771bec5b77e5cb4254161d5ddda044943c71767f Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 6 Sep 2019 17:05:14 +0300 Subject: [PATCH 38/82] improve meta data attribute naming and handling for cold start data --- polara/recommender/coldstart/data.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index 37234bd..199436e 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -8,10 +8,9 @@ class ItemColdStartData(RecommenderData): - def __init__(self, *args, **kwargs): - self.item_meta = kwargs.pop('item_meta', None) + def __init__(self, *args, item_features=None, **kwargs): super(ItemColdStartData, self).__init__(*args, **kwargs) - + self.item_features = item_features self._test_ratio = 0.2 self._warm_start = False @@ -149,13 +148,13 @@ def _verify_cold_items_representatives(self): def _verify_cold_items_features(self): - if self.item_meta is None: + if self.item_features is None: return - if self.item_meta.shape[1] > 1: - features_melted = self.item_meta.agg(lambda x: [f for l in x for f in l], axis=1) + if self.item_features.shape[1] > 1: + features_melted = self.item_features.agg(lambda x: [f for l in x for f in l], axis=1) else: - features_melted = self.item_meta.iloc[:, 0] + features_melted = self.item_features.iloc[:, 0] feature_labels = defaultdict(lambda: len(feature_labels)) labels = features_melted.apply(lambda x: [feature_labels[i] for i in x]) From d63ac0e69fec149d5f5d8ad41d839dca3627d23a Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 6 Sep 2019 17:15:50 +0300 Subject: [PATCH 39/82] refactor handling of learned factor matrices --- polara/recommender/coldstart/models.py | 62 +++++++++++++++++--------- polara/recommender/hybrid/models.py | 15 ++++--- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index a3e87b2..36b9544 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -149,43 +149,57 @@ def __init__(self, *args, item_features, **kwargs): super().__init__(*args, **kwargs) self.item_features = item_features self.item_features_labels = None - self.item_features_transform = None + self._item_features_transform_helper = None self.data.subscribe(self.data.on_change_event, self._clean_metadata) def _clean_metadata(self): self.item_features_labels = None - def round_item_features_transform(self, rank): + @property + def item_features_embeddings(self): + itemid = self.data.fields.itemid + item_features_key = f'{itemid}_features' + return self.factors.get(item_features_key, None) + + def _round_item_features_transform(self): try: - w = self.item_features_transform[0] - except TypeError: - return - if w.shape[0] < rank: - self.item_features_transform = None + rank = self.item_features_embeddings.shape[1] + except AttributeError: # embeddings are None (not computed yet) + self._item_features_transform_helper = None else: - w = w[:rank, :] - self.item_features_transform = (w, np.linalg.pinv(w @ w.T)) + transform_rank = self._item_features_transform_helper.shape[0] + if transform_rank > rank: # round transform + self.update_item_features_transform() + else: + raise ValueError(f'Unable to round: the rank of factors is not lower than the rank of transform!') def _check_reduced_rank(self, rank): super()._check_reduced_rank(rank) - self.round_item_features_transform(rank) + self._round_item_features_transform() def encode_item_features(self): - item_meta = self.item_features.reindex( - self.data.index.itemid.training.old.values, fill_value=[]) + training_items = self.data.index.itemid.training.old.values + item_features = self.item_features.reindex(training_items, fill_value=[]) item_one_hot, self.item_features_labels = stack_features( - item_meta, stacked_index=False, normalize=False) + item_features, stacked_index=False, normalize=False) return item_one_hot - def assemble_item_features_transform(self): + def update_item_features_transform(self): + mapping = self.item_features_embeddings + mapping_invgram = np.linalg.pinv(mapping.T @ mapping) + self._item_features_transform_helper = mapping_invgram + + def prepare_item_features_transformation(self): item_one_hot = self.encode_item_features() mapping = self.compute_item_features_mapping(item_one_hot) # model dependent - mapping_invgram = np.linalg.pinv(mapping @ mapping.T) - self.item_features_transform = (mapping, mapping_invgram) + item_features_key = f'{self.data.fields.itemid}_features' + # this will take care of truncating the matrix when the rank is reduced: + self.factors[item_features_key] = mapping + self.update_item_features_transform() def build(self, *args, **kwargs): super().build(*args, return_factors=True, **kwargs) - self.assemble_item_features_transform() + self.prepare_item_features_transformation() def slice_recommendations(self, cold_item_meta, start, stop): cold_slice_meta = cold_item_meta.iloc[start:stop] @@ -196,8 +210,9 @@ def slice_recommendations(self, cold_item_meta, start, stop): u = self.factors[self.data.fields.userid] s = self.factors['singular_values'] - w, w_invgram = self.item_features_transform - cold_items_factors = cold_item_features.dot(w.T) @ w_invgram + w = self.item_features_embeddings + w_invgram = self._item_features_transform_helper + cold_items_factors = (cold_item_features @ w) @ w_invgram scores = cold_items_factors @ (u * s[None, :]).T return scores @@ -211,7 +226,9 @@ def __init__(self, *args, **kwargs): self.method = 'PureSVD(cs)' def compute_item_features_mapping(self, item_features): - return item_features.T.dot(self.factors[self.data.fields.itemid]).T + itemid = self.data.fields.itemid + item_factors = self.factors[itemid] + return item_features.T.dot(item_factors) class HybridSVDItemColdStart(ItemColdStartEvaluationMixin, @@ -223,7 +240,10 @@ def __init__(self, *args, **kwargs): self.method = 'HybridSVD(cs)' def compute_item_features_mapping(self, item_features): - return item_features.T.dot(self.factors['items_projector_right']).T + itemid = self.data.fields.itemid + right_projector_key = f'{itemid}_projector_right' + item_factors = self.factors[right_projector_key] + return item_features.T.dot(item_factors) class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): pass diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index 0e21a98..551ca0e 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -318,15 +318,17 @@ def build_item_projector(self, v): if self.verbose: print(f'Building {self.data.fields.itemid} projector for {self.method}') msg = Template(' Solving triangular system: $time') + itemid = self.data.fields.itemid with track_time(verbose=self.verbose, message=msg): - self.factors['items_projector_left'] = cholesky_items.T.solve(v) + self.factors[f'{itemid}_projector_left'] = cholesky_items.T.solve(v) msg = Template(' Applying Cholesky factor: $time') with track_time(verbose=self.verbose, message=msg): - self.factors['items_projector_right'] = cholesky_items.dot(v) + self.factors[f'{itemid}_projector_right'] = cholesky_items.dot(v) def get_item_projector(self): - vl = self.factors.get('items_projector_left', None) - vr = self.factors.get('items_projector_right', None) + itemid = self.data.fields.itemid + vl = self.factors.get(f'{itemid}_projector_left', None) + vr = self.factors.get(f'{itemid}_projector_right', None) return vl, vr @@ -343,8 +345,9 @@ def _check_reduced_rank(self, rank): def round_item_projector(self, rank): vl, vr = self.get_item_projector() if (vl is not None) and (rank < vl.shape[1]): - self.factors['items_projector_left'] = vl[:, :rank] - self.factors['items_projector_right'] = vr[:, :rank] + itemid = self.data.fields.itemid + self.factors[f'{itemid}_projector_left'] = vl[:, :rank] + self.factors[f'{itemid}_projector_right'] = vr[:, :rank] def build(self, *args, **kwargs): if not self._sparse_mode: From f4e69c6b5572a972ac268896c60b7668165f7803 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 6 Sep 2019 17:48:07 +0300 Subject: [PATCH 40/82] improve imput argument handling for cold start models --- polara/recommender/coldstart/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index 36b9544..901a166 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -145,8 +145,11 @@ def slice_recommendations(self, cold_item_meta, start, stop): class ItemColdStartSVDModelMixin: - def __init__(self, *args, item_features, **kwargs): + def __init__(self, *args, item_features=None, **kwargs): super().__init__(*args, **kwargs) + if item_features is None: # assume features are provided via data model + item_features = self.data.item_features + assert item_features is not None self.item_features = item_features self.item_features_labels = None self._item_features_transform_helper = None From 978942d9e25364b8816e56fa0f2960b1e52f7529 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 6 Sep 2019 17:48:28 +0300 Subject: [PATCH 41/82] improve rank value handling for lightfm --- .../recommender/external/lightfm/lightfmwrapper.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index 3d78f8f..d061e2e 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -10,7 +10,7 @@ class LightFMWrapper(RecommenderModel): def __init__(self, *args, item_features=None, user_features=None, **kwargs): super(LightFMWrapper, self).__init__(*args, **kwargs) self.method='LightFM' - self.rank = 10 + self._rank = 10 self.fit_method = 'fit' self.fit_params = {} @@ -35,6 +35,18 @@ def __init__(self, *args, item_features=None, user_features=None, **kwargs): self._model = None + @property + def rank(self): + return self._rank + + @rank.setter + def rank(self, new_value): + if new_value != self._rank: + self._rank = new_value + self._is_ready = False + self._recommendations = None + + def build(self): self._model = LightFM(no_components=self.rank, item_alpha=self.item_alpha, From 80e80b4cbc0472bc600ebe6499f10a1bf1de026c Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 6 Sep 2019 17:49:57 +0300 Subject: [PATCH 42/82] align LCE model with naming convention for item fetures embeddings --- polara/recommender/hybrid/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/hybrid/models.py b/polara/recommender/hybrid/models.py index 551ca0e..a09f911 100644 --- a/polara/recommender/hybrid/models.py +++ b/polara/recommender/hybrid/models.py @@ -205,7 +205,7 @@ def build(self): itemid = self.data.fields.itemid self.factors[userid] = Hu.T self.factors[itemid] = W - self.factors['item_features'] = Hs.T + self.factors[f'{itemid}_features'] = Hs.T self.item_features_labels = lbls def get_recommendations(self): From 22b6f6ff6ac74b45bf3959d687c5a8d82f5d58d8 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 6 Sep 2019 17:50:54 +0300 Subject: [PATCH 43/82] improve input argument handling for similarity (side relations) data models --- polara/recommender/hybrid/data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polara/recommender/hybrid/data.py b/polara/recommender/hybrid/data.py index 960a868..9e14fb9 100644 --- a/polara/recommender/hybrid/data.py +++ b/polara/recommender/hybrid/data.py @@ -6,15 +6,15 @@ class SideRelationsMixin: - def __init__(self, rel_mat, rel_idx, *args, **kwargs): + def __init__(self, *args, relations_matrices, relations_indices, **kwargs): super().__init__(*args, **kwargs) entities = [self.fields.userid, self.fields.itemid] self._rel_idx = {entity: pd.Series(index=idx, data=np.arange(len(idx)), copy=False) if idx is not None else None - for entity, idx in rel_idx.items() + for entity, idx in relations_indices.items() if entity in entities} - self._rel_mat = {entity: mat for entity, mat in rel_mat.items() if entity in entities} + self._rel_mat = {entity: mat for entity, mat in relations_matrices.items() if entity in entities} self._relations = dict.fromkeys(entities) self.subscribe(self.on_change_event, self._clean_relations) From daffe8e096496e49442eb50ebe431ccf6f4a7e6e Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 8 Sep 2019 14:47:39 +0300 Subject: [PATCH 44/82] avoid csr to coo conversion in lightfm --- polara/recommender/external/lightfm/lightfmwrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index d061e2e..20f0df8 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -58,7 +58,7 @@ def build(self): random_state=self.seed) fit = getattr(self._model, self.fit_method) - matrix = self.get_training_matrix() + matrix = self.get_training_matrix(sparse_format='coo') # as reqired by LightFM try: item_index = self.data.index.itemid.training From 0fa94ee2602745b76079e2360b957711f59b9024 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 8 Sep 2019 15:06:51 +0300 Subject: [PATCH 45/82] minor code refactoring in lightfm --- .../external/lightfm/lightfmwrapper.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index 20f0df8..5be8c5a 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -96,15 +96,17 @@ def slice_recommendations(self, test_data, shape, start, stop, test_users=None): all_items = self.data.index.itemid.new.values n_users = stop - start n_items = len(all_items) + test_shape = (n_users, n_items) + test_users_index = test_users[start:stop].astype('i4', copy=False) + test_items_index = all_items.astype('i4', copy=False) # use stride tricks to avoid unnecessary copies of repeated indices # have to conform with LightFM's dtype to avoid additional copies itemsize = np.dtype('i4').itemsize - useridx = as_strided(test_users[start:stop].astype('i4', copy=False), - (n_users, n_items), (itemsize, 0)) - itemidx = as_strided(all_items.astype('i4', copy=False), - (n_users, n_items), (0, itemsize)) - scores = self._model.predict(useridx.ravel(), itemidx.ravel(), - user_features=self._user_features_csr, - item_features=self._item_features_csr - ).reshape(n_users, n_items) + scores = self._model.predict( + as_strided(test_users_index, test_shape, (itemsize, 0)).ravel(), + as_strided(test_items_index, test_shape, (0, itemsize)).ravel(), + user_features=self._user_features_csr, + item_features=self._item_features_csr, + num_threads=self.fit_params.get('num_threads', 1) + ).reshape(test_shape) return scores, slice_data From 37b7380bd82e6f932fd1d27ebfe7a41427cf580b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 8 Sep 2019 15:09:55 +0300 Subject: [PATCH 46/82] reuse the lightfm's built-in predict method in item cold start + refactor code --- polara/recommender/coldstart/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index 901a166..4fc74f6 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -1,4 +1,6 @@ import numpy as np +from numpy.lib.stride_tricks import as_strided +import scipy as sp from polara import SVDModel from polara.recommender.models import RecommenderModel, ScaledMatrixMixin From 59aeda9be224ae137449a5dc435f053cc1c55268 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 8 Sep 2019 15:11:09 +0300 Subject: [PATCH 47/82] reuse lightfm built-in predict method in item cold start + refactor code --- polara/recommender/coldstart/models.py | 44 +++++++++++++++----------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index 4fc74f6..4c6fc66 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -257,29 +257,35 @@ class ScaledSVDItemColdStart(ScaledMatrixMixin, SVDModelItemColdStart): pass class ScaledHybridSVDItemColdStart(ScaledMatrixMixin, HybridSVDItemColdStart): pass -class ItemColdStartLightFMMixin: +class LightFMItemColdStart(ItemColdStartEvaluationMixin, + ItemColdStartRecommenderMixin, + LightFMWrapper): def slice_recommendations(self, cold_item_meta, start, stop): cold_slice_meta = cold_item_meta.iloc[start:stop] cold_item_features, _ = stack_features( cold_slice_meta, labels=self.item_features_labels, - add_identity=False, - normalize=True) - - user_embeddings = self._model.user_embeddings - repr_users = self.data.representative_users - if repr_users is not None: - user_embeddings = user_embeddings[repr_users.new.values, :] + add_identity=self.item_identity, + normalize=True + ) - # proper handling of cold-start (instead of built-in predict) - n_items = self.data.index.itemid.training.shape[0] - item_features_embeddings = self._model.item_embeddings[n_items:, :] - cold_items_embeddings = cold_item_features.dot(item_features_embeddings) - scores = cold_items_embeddings @ user_embeddings.T + n_cold_items = stop - start + cold_items_index = np.arange(n_cold_items, dtype='i4') + + test_users = self.data.representative_users + if test_users is None: + test_users = self.data.index.userid.training + + n_test_users = test_users.shape[0] + test_users_index = test_users.new.values.astype('i4', copy=False) + + test_shape = (n_cold_items, n_test_users) + itemsize = np.dtype('i4').itemsize + scores = self._model.predict( + as_strided(test_users_index, test_shape, (0, itemsize)).ravel(), + as_strided(cold_items_index, test_shape, (itemsize, 0)).ravel(), + user_features=self._user_features_csr, + item_features=cold_item_features, + num_threads=self.fit_params.get('num_threads', 1) + ).reshape(test_shape) return scores - - -class LightFMItemColdStart(ItemColdStartEvaluationMixin, - ItemColdStartRecommenderMixin, - ItemColdStartLightFMMixin, - LightFMWrapper): pass From 75ece1dbe7d101948bf8810bc4050764be2aca47 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 8 Sep 2019 15:43:13 +0300 Subject: [PATCH 48/82] add explanation to warm_start error for lightfm --- polara/recommender/external/lightfm/lightfmwrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index 5be8c5a..27a1679 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -90,7 +90,7 @@ def build(self): def slice_recommendations(self, test_data, shape, start, stop, test_users=None): if self.data.warm_start: - raise NotImplementedError + raise NotImplementedError('Not supported by LightFM.') slice_data = self._slice_test_data(test_data, start, stop) all_items = self.data.index.itemid.new.values From 73cb16c76f4250bd1a3ea9d21957b6a98627a9b4 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sun, 8 Sep 2019 15:45:13 +0300 Subject: [PATCH 49/82] fix items representation inconsistency in lightfm for item cold start --- polara/recommender/coldstart/models.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/polara/recommender/coldstart/models.py b/polara/recommender/coldstart/models.py index 4c6fc66..899806b 100644 --- a/polara/recommender/coldstart/models.py +++ b/polara/recommender/coldstart/models.py @@ -265,13 +265,20 @@ def slice_recommendations(self, cold_item_meta, start, stop): cold_item_features, _ = stack_features( cold_slice_meta, labels=self.item_features_labels, - add_identity=self.item_identity, + add_identity=False, normalize=True ) n_cold_items = stop - start cold_items_index = np.arange(n_cold_items, dtype='i4') + if self.item_identity: + n_items = self._item_features_csr.shape[0] + no_identity = sp.sparse.csr_matrix((n_cold_items, n_items)) + cold_item_features = sp.sparse.hstack( + [no_identity, cold_item_features], format='csr' + ) + test_users = self.data.representative_users if test_users is None: test_users = self.data.index.userid.training From d6e3544f62b7b87b4b355a0b7166184db4ea529f Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 11 Sep 2019 14:50:58 +0300 Subject: [PATCH 50/82] add convenience method for gatherin evaluation metrics into a single dataframe --- polara/evaluation/evaluation_engine.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/polara/evaluation/evaluation_engine.py b/polara/evaluation/evaluation_engine.py index 190db10..009d808 100644 --- a/polara/evaluation/evaluation_engine.py +++ b/polara/evaluation/evaluation_engine.py @@ -48,6 +48,14 @@ def average_results(scores): return averaged, errors +def consolidate_metrics(scores, label='scores', include_metric_types=True): + metric_types = None + if include_metric_types: + metric_types = [s.__class__.__name__.lower() for s in scores] + + labeled_scores = [pd.DataFrame([s], index=[label]) for s in scores] + metrics_df = pd.concat(labeled_scores, keys=metric_types, axis=1) + return metrics_df def evaluate_models(models, metrics, **kwargs): scores = [] @@ -55,11 +63,8 @@ def evaluate_models(models, metrics, **kwargs): model_scores = model.evaluate(metric_type=metrics, **kwargs) # ensure correct format model_scores = model_scores if isinstance(model_scores, list) else [model_scores] - # concatenate all scores - name = [model.method] - metric_types = [s.__class__.__name__.lower() for s in model_scores] - scores_df = pd.concat([pd.DataFrame([s], index=name) for s in model_scores], - keys=metric_types, axis=1) + # gather all scores + scores_df = consolidate_metrics(model_scores, label=model.method) scores.append(scores_df) res = pd.concat(scores, axis=0) res.columns.names = ['type', 'metric'] From 11393543a376716d9a789897f7c08c3dfbb47e3a Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 11 Sep 2019 14:51:50 +0300 Subject: [PATCH 51/82] accept tuples and lists in print_dataframes function --- polara/tools/display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/tools/display.py b/polara/tools/display.py index 006cf25..d346dab 100644 --- a/polara/tools/display.py +++ b/polara/tools/display.py @@ -4,7 +4,7 @@ def print_frames(dataframes): - if not isinstance(dataframes, tuple): + if not isinstance(dataframes, (tuple, list)): return dataframes border_style = u'\"border: none\"' From 1424b579bd681f578a7fe4aa6cf9f109fd9a380d Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 11 Sep 2019 14:53:18 +0300 Subject: [PATCH 52/82] disable currently unsupported holdout size attribute modification in cold start data --- polara/recommender/coldstart/data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index 199436e..56a5890 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -13,6 +13,7 @@ def __init__(self, *args, item_features=None, **kwargs): self.item_features = item_features self._test_ratio = 0.2 self._warm_start = False + self._holdout_size = -1 # build unique items list to split them by folds itemid = self.fields.itemid @@ -22,6 +23,17 @@ def __init__(self, *args, item_features=None, **kwargs): self._test_sample = None # fraction of representative users from train self._repr_users = None + @property + def holdout_size(self): + return -1 + + @holdout_size.setter + def holdout_size(self, new_value): + if new_value == 0: # enable setting test data + self._holdout_size = 0 + else: + raise NotImplementedError('Setting holdout size is currently not supported in item cold start.') + @property def representative_users(self): if self._repr_users is None: From 45cea3dde6400eb732efb220c71498dead6ea345 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 11 Sep 2019 14:54:01 +0300 Subject: [PATCH 53/82] align item cold start data model class name with naming convention --- polara/recommender/coldstart/data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polara/recommender/coldstart/data.py b/polara/recommender/coldstart/data.py index 56a5890..be384fe 100644 --- a/polara/recommender/coldstart/data.py +++ b/polara/recommender/coldstart/data.py @@ -253,7 +253,7 @@ def get_cold_similarity(self, entity): return sim_mat[:, seen_idx][cold_idx, :] -class ColdStartSimilarityDataModel(ColdSimilarityMixin, - IdentityDiagonalMixin, - SideRelationsMixin, - ItemColdStartData): pass +class ItemColdStartSimilarityData(ColdSimilarityMixin, + IdentityDiagonalMixin, + SideRelationsMixin, + ItemColdStartData): pass From c93999003bc41c8ac1ea425c1ba58b4bc09a2ee7 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 11 Sep 2019 14:54:45 +0300 Subject: [PATCH 54/82] refactor some print statements --- polara/recommender/data.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 8f31f38..9a1f5fc 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -208,7 +208,7 @@ def _update_data_property(self, data_property, new_value): def _verified_data_property(self, data_property): if data_property in self._change_properties: - print('The value of {} might be not effective yet.'.format(data_property[1:])) + print(f'The value of {data_property[1:]} might be not effective yet.') return getattr(self, data_property) @@ -239,8 +239,8 @@ def prepare(self): if self.verbose: num_train_events = self.training.shape[0] if self.training is not None else 0 num_holdout_events = self.test.holdout.shape[0] if self.test.holdout is not None else 0 - stats_msg = 'Done.\nThere are {} events in the training and {} events in the holdout.' - print(stats_msg.format(num_train_events, num_holdout_events)) + print(f'Done.\nThere are {num_train_events} events in the training ' + f'and {num_holdout_events} events in the holdout.') def prepare_training_only(self): self.holdout_size = 0 # do not form holdout @@ -565,8 +565,10 @@ def _filter_short_sessions(self, group_id=None): invalid_session_index = invalid_sessions.index[invalid_sessions] holdout.query('{} not in @invalid_session_index'.format(group_id), inplace=True) if self.verbose: - msg = '{} of {} {}\'s were filtered out from holdout. Reason: incompatible number of items.' - print(msg.format(n_invalid_sessions, len(invalid_sessions), group_id)) + n_sessions = len(invalid_sessions) + incompatible = 'incompatible number of items' + print(f'{n_invalid_sessions} of {n_sessions} {group_id} entities ' + f'were filtered out from holdout. Reason: {incompatible}.') def _align_test_users(self): if (self._test.testset is None) or (self._test.holdout is None): @@ -584,18 +586,18 @@ def _align_test_users(self): n_unique_users = invalid_holdout_users.nunique() holdout.drop(invalid_holdout_users.index, inplace=True) if self.verbose: - REASON = 'Reason: inconsistent with testset' - msg = '{} {}\'s were filtered out from holdout. {}.' - print(msg.format(n_unique_users, userid, REASON)) + inconsistent = 'inconsistent with testset' + print(f'{n_unique_users} {userid} entities were filtered out ' + f'from holdout. Reason: {inconsistent}.') if not testset_in_holdout.all(): invalid_testset_users = testset.loc[~testset_in_holdout, userid] n_unique_users = invalid_testset_users.nunique() testset.drop(invalid_testset_users.index, inplace=True) if self.verbose: - REASON = 'Reason: inconsistent with holdout' - msg = '{} {}\'s were filtered out from testset. {}.' - print(msg.format(n_unique_users, userid, REASON)) + inconsistent = 'inconsistent with holdout' + print(f'{n_unique_users} {userid} entities were filtered out ' + f'from testset. Reason: {inconsistent}.') def _reindex_train_users(self): userid = self.fields.userid @@ -662,9 +664,10 @@ def _filter_unseen_entity(self, entity, dataset, label, mapping): # unseen_index = dataset.index[unseen_entities] # dataset.drop(unseen_index, inplace=True) if self.verbose: - UNSEEN = 'not in the training data' - msg = '{} unique {}\'s within {} {} interactions were filtered. Reason: {}.' - print(msg.format(n_unseen_entities, entity, (~seen_data).sum(), label, UNSEEN)) + unseen = 'not in the training data' + print(f'{n_unseen_entities} unique {entity} entities within ' + f'{(~seen_data).sum()} {label} interactions were filtered. ' + f'Reason: {unseen}.') def _reindex_testset_users(self): userid = self.fields.userid From 2c4399db654572e8bc8dde169ede4b8f19dbb24b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 11 Sep 2019 14:55:56 +0300 Subject: [PATCH 55/82] handle implicit feedback without requiring to specify random_holdout attribute --- polara/recommender/data.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 9a1f5fc..875215b 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -709,30 +709,31 @@ def reindex(data, col, sort=True, inplace=True): def _sample_holdout(self, test_split, group_id=None): # TODO order_field may also change - need to check it as well order_field = self._custom_order or self.fields.feedback or [] + sample_at_random = self._random_holdout or (order_field == []) selector = self._data.loc[test_split, order_field] # data may have many items with the same top ratings # randomizing the data helps to avoid biases in that case - if self._permute_tops and not self._random_holdout: + if self._permute_tops and not sample_at_random: random_state = np.random.RandomState(self.seed) selector = selector.sample(frac=1, random_state=random_state) group_id = group_id or self.fields.userid grouper = selector.groupby(self._data[group_id], sort=False, group_keys=False) - if self._random_holdout: # randomly sample data for evaluation + if sample_at_random: # randomly sample data for evaluation random_state = np.random.RandomState(self.seed) - if self._holdout_size >= 1: # pick at most _holdout_size elements + if self._holdout_size >= 1: # pick at most _holdout_size elements holdout = grouper.apply(random_choice, self._holdout_size, random_state) else: holdout = grouper.apply(random_sample, self._holdout_size, random_state) - elif self._negative_prediction: # try to holdout negative only examples - if self._holdout_size >= 1: # pick at most _holdout_size elements + elif self._negative_prediction: # try to holdout negative only examples + if self._holdout_size >= 1: # pick at most _holdout_size elements holdout = grouper.nsmallest(self._holdout_size, keep='last') else: raise NotImplementedError - else: # standard top-score prediction mode - if self._holdout_size >= 1: # pick at most _holdout_size elements + else: # standard top-score prediction mode + if self._holdout_size >= 1: # pick at most _holdout_size elements holdout = grouper.nlargest(self._holdout_size, keep='last') else: frac = self._holdout_size From 7054038bb3c2b369af28eab6d51571912b2b4cbd Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 11 Sep 2019 14:56:37 +0300 Subject: [PATCH 56/82] add tutorial on comparing LightFM with HybridSVD --- .../Comparing LightFM with HybridSVD.ipynb | 2625 +++++++++++++++++ 1 file changed, 2625 insertions(+) create mode 100644 examples/Comparing LightFM with HybridSVD.ipynb diff --git a/examples/Comparing LightFM with HybridSVD.ipynb b/examples/Comparing LightFM with HybridSVD.ipynb new file mode 100644 index 0000000..4be090d --- /dev/null +++ b/examples/Comparing LightFM with HybridSVD.ipynb @@ -0,0 +1,2625 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "LightFM is a very popular tool, which finds its applications both in small projects and in production systems. Aside from the technical aspects, the general idea of representing users and items in terms of feature combinations, used in the LightFM model, is not new. It was implemented in a number of preceeding works, and I'm not even saying about the obvious link to [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf). Another notable member of this family is [SVDFeature](https://arxiv.org/pdf/1109.2271.pdf) , which came around in 2011 and in some sense is even closer to LightFM than FM (the naming, though, is misleading, as it is not an SVD-based model). In fact, the feature combination approach can be traced back to the Microsoft's [MatchBox system](https://www.microsoft.com/en-us/research/wp-content/uploads/2009/01/www09.pdf) from 2009, where they use the term *trait vector* for a combined representation. Still, even though the idea itself was not new, the LightFM model was made convenient and extreemly easy to use and had a fairly good computational performance, which, I believe, has made this framework so popular among practitioners.\n", + "\n", + "On the other hand, I have also seen some complains from data scientists about actual prediction quality of LightFM. Moreover, I could not find more or less rigorous comparison of LightFM with strong baselines, where all compared models would undergo appropriate tuning. The [examples section](https://lyst.github.io/lightfm/docs/examples.html) of the LightFM's documentation merely provides quick-start demos, not real performance tests. The [paper from the RecSys workshop](http://ceur-ws.org/Vol-1448/paper4.pdf) also does not provide too much evidence on the model's performance. Numerous online tutorials only repeat basic configuration steps, leaving aside the whole tuning aspect. Considering how often practitioners are advised to start with LightFM (according to data science chats and forums I'm aware of), I have decided **to put the LightFM capabilities to the real test**." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
In this tutorial we will conduct a thorough experiment with proper tuning and testing of LightFM against some strong baselines in the cold strat setting.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The baseline models in this tutorial will be based on my favorite SVD (the real one, not just some matrix factorization). I have summarized the reasons why I promote SVD-based recommendation algorithms as the default choice in my [previous post](//slug:to-svd-or-not-to-svd). However, I have not yet described how to use SVD in the cold start scenario. The theoretical foundation of this can be found in our [recent paper](https://arxiv.org/abs/1802.06398), where we demonstrate how to adopt both PureSVD and HybridSVD models for the cold start scenario. Here, I will provide only the minimum necessary material for understanding what is going on, focusing more on technical aspects instead. For all experiments we will be using [Polara](https://github.com/evfro/polara), as it provides easy-to-use high-level API for both LightFM and SVD-based models, as well as a set of convenient tools for building the entire experimentation pipeline.\n", + "\n", + "Unlike many online materials, we will employ an *advanced optimization procedure for LightFM* to ensure that the obtained model is close to its optimal configuration.\n", + "Conversely, as we will see, *tuning of the SVD-based models is very straightforward*. They depend on fewer hyper-parameters and also have deterministic output. The question remains, whether it is possible to outperform LightFM with these simpler models. Let's find out." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Teleport me to:\n", + "[Getting data](#getting-data) \n", + "[Recommender models](#recommender-models) \n", + ">[LightFM](#LightFM) \n", + ">[PureSVD](#PureSVD) \n", + ">[HybridSVD](#HybridSVD) \n", + "\n", + "[Evaluation of models](#evaluation-of-models) \n", + "[Conclusion](#conclusion) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "# Getting data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use the **StackExchange** dataset provided in [one of the examples](https://lyst.github.io/lightfm/docs/examples/hybrid_crossvalidated.html) from the LightFM documentation. It consist of users who have answered certain questions. In addition to the user-answer interaction matrix, the dataset also provides additional information in the form of tags assigned to each answered question. The data-reading code below is simply copied from the documentation." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Downloading and preprocessing data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from lightfm.datasets import fetch_stackexchange\n", + "from polara.recommender.coldstart.data import ItemColdStartData\n", + "from polara.tools.display import print_frames # to print df's side-by-side" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "data = fetch_stackexchange('crossvalidated',\n", + " test_set_fraction=0.1,\n", + " indicator_features=False,\n", + " tag_features=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `data` variable contains both training and test datasets, as well as tag assignments and their labels:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['train', 'test', 'item_features', 'item_feature_labels'])\n" + ] + } + ], + "source": [ + "print(data.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we need to convert it into the Polara-compatible format of `pandas DataFrames`. Mind the field names for users and items in the result:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "entities = ['users', 'items']\n", + "training_data = pd.DataFrame(dict(zip(entities, data['train'].nonzero())))\n", + "test_data = pd.DataFrame(dict(zip(entities, data['test'].nonzero())))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "seed = 321 # to be used in data model, LightFM, and optimization routines\n", + "training_data = training_data.sample(frac=1, random_state=seed) # shuffle data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also need to convert item features (tags) into the dataframe with a special structure. This will enable many Polara's built-in functions for data manipulation." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# convert sparse matrix into `item - tag list` Series format\n", + "# make use of sparse CSR format\n", + "item_tags = (\n", + " pd.Series(\n", + " np.array_split(\n", + " # convert back fron indices to tag labels\n", + " np.take(data['item_feature_labels'],\n", + " data['item_features'].indices),\n", + " # split tags into groups by items\n", + " data['item_features'].indptr[1:-1]))\n", + " .rename_axis('items')\n", + " .to_frame('tags')\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
usersitems
35417108841856
1238920538838
48719208120186
2510463549982
46833187830914
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
usersitems
01670628
11866569
21866609
31868964
41869780
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tags
items
0[bayesian, prior, elicitation]
1[distributions, normality]
2[software, open-source]
3[distributions, statistical-significance]
4[machine-learning]
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print_frames([training_data.head(),\n", + " test_data.head(),\n", + " item_tags.head()])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preparing data model for training and validation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to wrap our data into an instance of Polara's recommender data model, which allows to share data among many recommender models and propagate consistent state across all of them. More details on its usage can be found in other [Polara examples](https://github.com/evfro/polara/tree/master/examples). In order to fine-tune models we will additionally split training data into the traininig and validation sets using the Polara's built-in functionality." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Once optimal-hyperparameters are found, the model will be retrained on the full training data and verified against the initially provided test set.
" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ItemColdStartData with Fields(userid='users', itemid='items', feedback=None)\n" + ] + } + ], + "source": [ + "data_model = ItemColdStartData(\n", + " training_data,\n", + " *training_data.columns, # userid, itemid\n", + " item_features=item_tags,\n", + " seed=seed)\n", + "print(data_model)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing data...\n", + "1 unique users entities within 2 holdout interactions were filtered. Reason: not in the training data.\n", + "Done.\n", + "There are 54975 events in the training and 2853 events in the holdout.\n" + ] + } + ], + "source": [ + "data_model.test_ratio = 0.05 # take 5% of items as cold (at random)\n", + "data_model.prepare()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let us look at the holdout data:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
usersitems_cold
12389390
4871711
25266752
2790112422
30278323
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
usersitems_cold
8223291920
363486771921
13834531922
385583431923
419792251924
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print_frames([data_model.test.holdout.head(), data_model.test.holdout.tail()])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**The procedure for both validation and final evaluation will be the following**:\n", + "1. For every unique cold item, we generate a list of candidate users who, according to our model, are most likely to be intersted in this item.\n", + "2. We then verify the list of predicted users against the actual users from the holdout.\n", + "\n", + "We will use standard evaluation metrics to assess the quality of predictions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "# Recommender models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LightFM" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As stated in the introduction, LightFM is an efficient and convenient instrument for building hybrid recommender systems. There are, however, several downsides in its implementation (at least at the moment of writing this post).\n", + "\n", + "First of all, **there is no off-the-shelf support for the warm start scenario** (e.g., for new users with several known interactions), which would allow estimating strong generalization of the algorithm rather then weak generalization that we test in the standard setup (with only known users).\n", + "Basically, there's no folding-in implementation, even though it seems to be in a high demand (see github issues [here](https://github.com/lyst/lightfm/issues/300), [here](https://github.com/lyst/lightfm/issues/322#issuecomment-401951114) or [here](https://github.com/lyst/lightfm/issues/194#issuecomment-310775451)) and it is not that difficult to implement. Likewise, there is no [session-based recommendations support](https://github.com/lyst/lightfm/issues/362) as well. The [sometimes recommended](https://github.com/lyst/lightfm/issues/347#issuecomment-407383263) `fit_partial` method is not really an answer to these problems. It is going to update both user and item embeddings and sometimes it is not what you may want. For example, fitting newly introduced users would affect latent representation of some know items as well, potentially changing recommendations for already present users without any of their actions. Such an unintended change in recommendations can be confusing for users and lead to a lower satisfaction with a recommendation service.\n", + "\n", + "Another aspect, which is critical in certain situations is **the lack of determinism**. LightFM's output is deterministic only in strict conditions: single-threaded training with the fixed input that never changes (even the order of elements). In multithredaded execution the LightFM's optimization process causes racing conditions that are resolved at the hardware level, which you have no control of. This is not unique to LightFM and is shared across all matrix factorization algorithms based on naive SGD optimization. Moreover, even tiny reorderings (e.g., when two items corresponding to a single user switch their places) may lead to [noticably different results](https://github.com/lyst/lightfm/issues/225). Even in this notebook, if you set the number of LightFM threads to 1 (see below), once you restart the notebook kernel, the results may change. The problem seems to be outside of LightFM itself, however, it demonstrates its non-deterministic nature. If your business has specific requirements on, e.g., non-regression testing, this model is not for you. Conversely, **SVD-based models are free of such issues.** This is also among the reasons why matrix factorization models like SVDFeature or FunkSVD should not be mixed with real SVD-based models." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The model" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.coldstart.models import LightFMItemColdStart" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "num_threads = 4 # number of parallel threads used by LightFM\n", + "max_rank = 200 # the max value or latent features used in tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting the number of threads greater than 1 is not critical in our setup, as the result will fluctuate by a relatively small margin, which should not drammatically change the whole picture of the algorithm's performance. Nevertheless, you can try running in a fully deterministic setting and verify that. However, you'll have to wait a bit longer during the LightFM tuning phase due to a sequential execution." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def create_lightfm_model(data_model, item_features, num_threads, seed=None):\n", + " 'Prepare LightFM model and fix initial configuration.'\n", + " model = LightFMItemColdStart(data_model, item_features=item_features)\n", + " model.loss = 'warp'\n", + " model.learning_schedule = 'adagrad'\n", + " model.seed = seed\n", + " model.fit_params['num_threads'] = num_threads\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "lfm = create_lightfm_model(data_model, item_tags, num_threads, seed)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As the LightFM algorithm depends on many hyper-parameters and has a stochastic nature, it presents a hard problem for optimization. We will have to give a fair bit of tuning to LightFM. There are many great tools for doing this. Two popular options are [HyperOpt](https://hyperopt.github.io/) and [Optuna](https://optuna.readthedocs.io/); both of them are great. I personally prefer the latter as it typically allows me to write a more concise code. We will be using it in this tutorial as well. If you don't have `optuna` installed you can simply run\n", + "```\n", + "pip install optuna\n", + "```\n", + "in your python environment.\n", + "\n", + "Instead of performing *random search* for hyper-parameter optimization we will employ a more advanced and flexible techniqe based on *Tree-structured Parzen Estimator* (TPE). It will help to iteratively narrow-down the hyper-parameter search subspace and preemptively disregard unmpromising search regions. A bit more details along with further reading can be found in the [optuna documentation](https://optuna.readthedocs.io/en/latest/reference/samplers.html). There is also a nice comparison of different techniques in this [blog post](http://neupy.com/2016/12/17/hyperparameter_optimization_for_neural_networks.html#tree-structured-parzen-estimators-tpe)." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "import optuna\n", + "try: # import lightweight progressbar\n", + " from ipypb import track\n", + "except ImportError: # fallback to default\n", + " from tqdm.auto import tqdm as track" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that tuning for all possible types of hyper-parameters exponentially increases the complexity of the task. The authors of `optuna` even recommend to refrain from tuning unimportant variables. We will therefore **concentrate on 3 main hyper-parameters**:\n", + "1. dimensionality of the latent space (rank of the factor matrices),\n", + "2. importance of item features, controlled by the `item_alpha` regularization parameter,\n", + "3. number of epochs.\n", + "\n", + "We will leave other hyper-parameters with their default values. My preliminary experiments showed that there was no big difference in the final result after changing them, as long as their values remained within a reasonable range. As always, you are free verify that on your own by adding more hyper-parameters into the search space. You will need to modify the `objective` function defined below.\n", + "\n", + "To aid the learning process, we will sample `item_alpha` from a log-uniform distribution and `rank` values from a range of positive integer numbers up to `max_rank`. Note that we also use `set_user_attr` routine to store additional information related to each tuning trial." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_lightfm(model):\n", + " 'Convenience function for evaluating lightfm.'\n", + " # disabling user bias terms improves cold start prediction quality\n", + " model._model.user_biases *= 0.0\n", + " return model.evaluate()\n", + "\n", + "def find_target_metric(metrics, target_metric):\n", + " 'Convenience function to quickly extract the required metric.'\n", + " for metric in metrics:\n", + " if hasattr(metric, target_metric):\n", + " return getattr(metric, target_metric)\n", + "\n", + "def lightfm_objective(model, target_metric):\n", + " 'Objective function factory for optuna trials.'\n", + " def objective(trial):\n", + " # sample hyper-parameter values\n", + " model.rank = trial.suggest_int('rank', 1, max_rank)\n", + " model.item_alpha = trial.suggest_loguniform('item_alpha', 1e-10, 1e-0)\n", + " # train model silently and evaluate\n", + " model.verbose = False\n", + " model.build()\n", + " metrics = evaluate_lightfm(model)\n", + " target = find_target_metric(metrics, target_metric)\n", + " # store trial-specific information for later use\n", + " trial.set_user_attr('epochs', model.fit_params['epochs'])\n", + " trial.set_user_attr('metrics', metrics)\n", + " return target\n", + " return objective" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One of the limitations of TPE is that it does not take into account possible dependencies between hyper-parameters. In addition to that, parameters like `rank` or `epochs` significantly influence the training time. In order to avoid spending too much time on tuning yet being able to achieve a reasonably good model, we will use a specific procedure. We take the number of `epochs` out of the main loop.\n", + "Based on that, we want to early exclude bad hyper-parameter values that lead to high enough scores solely due to high number of epochs. Hence, we start from a smaller number of epochs and increase it gradually, leaving out unpromising configurations along the way due to TPE optimization procedure. Hopefully, it won't discard potentially good search directions that will later produce good results with the higher number of epochs. Likewise, we also gradually decrease the trials budget for each subsequent number of epochs, assuming that narrower search space requires less exploration." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "n_trials = {\n", + "# epochs: # trials\n", + " 15: 30,\n", + " 25: 25,\n", + " 50: 20,\n", + " 75: 15,\n", + " 100: 10,\n", + " 150: 5\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will target specifically the `Precision@10` metric and will anylize other metrics during the final evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "6/6\n", + "[06:10<00:36, 61.66s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 6/6 [06:10<00:36, 61.66s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "target_metric = 'precision'\n", + "objective = lightfm_objective(lfm, target_metric)\n", + "\n", + "study = optuna.create_study(\n", + " direction = 'maximize',\n", + " sampler = optuna.samplers.TPESampler(seed=seed)\n", + ")\n", + "\n", + "optuna.logging.disable_default_handler() # do not report progress\n", + "for num_epochs, num_trials in track(n_trials.items()):\n", + " lfm.fit_params['epochs'] = num_epochs\n", + " study.optimize(objective, n_trials=num_trials, n_jobs=1, catch=None)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our optimal configuration can be retrieved as follows (note that it can be slightly different on your machine):" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The best value of precision=0.0344 was achieved with rank=9 and item_alpha=7.26e-07 within 100 epochs.\n" + ] + } + ], + "source": [ + "print(f'The best value of {target_metric}={study.best_value:0.4f} was achieved with '\n", + " f'rank={study.best_params[\"rank\"]} and item_alpha={study.best_params[\"item_alpha\"]:.02e} '\n", + " f'within {study.best_trial.user_attrs[\"epochs\"]} epochs.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualizing tuning results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot the result to see whether there are any trends and whether we need to continue our parameter search in some new region of the hyper-parameter subspace." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "%config InlineBackend.figure_format = 'retina'\n", + "plt.style.use(['seaborn-notebook']) # see plt.style.available" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
precisionitem_alpharankepochsmetrics
920.0339742.986615e-076100[(0.03397402597402597, 0.25204923647780786, No...
990.0340781.901221e-0612100[(0.034077922077922075, 0.25326630512344794, N...
530.0340785.043306e-062525[(0.034077922077922075, 0.2526167166167166, No...
600.0342343.091518e-09950[(0.034233766233766234, 0.2541265401265401, No...
910.0343907.262623e-079100[(0.034389610389610394, 0.2558374958374958, No...
\n", + "
" + ], + "text/plain": [ + " precision item_alpha rank epochs \\\n", + "92 0.033974 2.986615e-07 6 100 \n", + "99 0.034078 1.901221e-06 12 100 \n", + "53 0.034078 5.043306e-06 25 25 \n", + "60 0.034234 3.091518e-09 9 50 \n", + "91 0.034390 7.262623e-07 9 100 \n", + "\n", + " metrics \n", + "92 [(0.03397402597402597, 0.25204923647780786, No... \n", + "99 [(0.034077922077922075, 0.25326630512344794, N... \n", + "53 [(0.034077922077922075, 0.2526167166167166, No... \n", + "60 [(0.034233766233766234, 0.2541265401265401, No... \n", + "91 [(0.034389610389610394, 0.2558374958374958, No... " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trials_df = (study.trials_dataframe() # all trials history\n", + " .loc[:, ['value', 'params', 'user_attrs']]\n", + " .rename(columns={'': target_metric}))\n", + "trials_df.columns = trials_df.columns.get_level_values(1)\n", + "trials_df.sort_values('precision').tail()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will treat low-score points (likely obtained at the initial tuning stage) as outliers and only take into account the top-20% results." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAx4AAAIvCAYAAAD6aw2CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdebhdVZ3n//c3ogiZgAgihWVEGdJIayvFqEBIi2jKghLstruLkqhQNFAgyK8NUcJUQCgHFHACNQJWNVrY6lMyiIYhFpGiCrD5gZEheBUqjAlFSJiEfPuPvQ8cTu5w7r1n3XOH9+t5zrNz9l7ru9e5m0PuJ2sPkZlIkiRJUkmTuj0ASZIkSeOfwUOSJElScQYPSZIkScUZPCRJkiQVZ/CQJEmSVJzBQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSNK5ExHciIiPitA7UOq2u9Z3hj0ySJjaDhyRJkqTiDB6SpPHmIeBu4PEO1Hq8rvVQB2pJ0oQWmdntMUiSJEka55zxkCRJklScwUOSJriI6KkvoN4vIv44Ir4ZEQ9ExLMR8duI+HxETO+l30sXcUfExhHxmYi4IyKeqtdv1tJ+ZkRcEBF3R8TTdbtbI+LTETF5gDHuHhGX1mN9NiIej4jbIuKciNixr3H1UmeriPhcRNwZEevqWg9ExLKIOCMi3tTSvt+LyyNiUkR8PCJujIjVTT+ziyLirX302a+u2VO/3zsiflJ/pmci4v9GxLEREf39TCRprNmo2wOQJI0abwW+D2wJrAUSmAl8CjgoIvbJzN6udXgtsBTYDfgD8HRrg4j4EPB3dVuAZ4DXAO+sX/8jIt6bmY+09AtgEfC/mlavAaYC/6l+vQE4fKAPV4eKX9btAV6sa/0RsC2wJ7AS+PpAtep6mwI/BA6oVzU++0zgCOCwiPhIZv64nxqHA9+k+ofANVQ/n/8IXEB1PD7ZzlgkaSxwxkOS1PB54EngPZk5FZgMHEx1gfVbgUv66HcMsAPwEWBKZm5G9cv3OoCI+BPgcuDVwLnAm+ramwJ7AP8M7AJc2kvtk3g5dHwNmJmZ04FN6n0cBdzb5uc7lSp03AfsA7wmM7eoa+0C/A3wcJu1AL5IFTqeq8cxtf7sOwI3UIWIv4+IHfrovyXwjfpzvaHuuzlV6AA4LiJ2HsR4JGlUc8ZDktSwMfD+zLwPIDPXAz+OiDXAdcB7I+LdmflPLf2mAO/LzGsbKzLzd03bz6MKHSdm5nlN618E/jki3g/cCRwQEbtm5r8CRMQM4LS67TmZuaCp/nrgd1S/uLdrj3r52cz8RVOt5+r939luoXr25Ij67fGZ+dI4MvOeiJgL3AG8Bfgs8Je9lNkU+GZm/nVT33+nChz7UYWhQ4C72h2XJI1mznhIkhq+3wgdzTLzemBZ/fbQXvrd0Rw6mkXEW4C9qU6t6vUUpsx8Ari6fvvepk0fpvrl/AngzHY+wADW1Ms39NuqPR+i+jv0YapTpV4hM58G/rbRNiJe1Uedc/pY3zg9623DGaQkjSbOeEiSGm7oZ9uNwF5U12O0+mU//faql68BftvP9dJT6uUbm9Y1Ziiuz8xn+tlHu64CdgfOjYjtgSuAm4dYu/Fz+EVmvthHm+vq5WSq069+3bJ9dWbe30fff6uXmw9hbJI0Khk8JEkN/9bGti172fZYP/0aswuvAl7fxhg2bfpzo/3v2+jXjnOBdwF/Bhxdv16IiH+hukj84vpUp3Y0fg79/cwe7KV9s6f66ftsvXx1m+ORpFHPU60kSe3o79auff2LP7z898ztmRltvA5vc5+DlpnPZeZBVHev+lvgZqo7dzXe3xMRbx9k2Y372+WQBipJ45TBQ5LUsE0/2xozF/3NbvSmcXvc7SNisLPsjTtMvanfVoOUmTdn5qczc0+qU5n+G9Wsypb0cr1GHxo/h/7G1nza2GB/bpI07hg8JEkN+7ax7bZB1mxc/zGFl5930a6b6+V+EbHJIPu2JTPXZeblwJH1qncN9DDDWuPnsHv9PI/e7F8v1wF3D2OYkjQuGDwkSQ3/NSK2a10ZEftQ3ZkK4B8GUzAzf8PLAeLc/n6pj4hNIqL51KV/oLob1ubAwsHst4/6r+lnc+MC86C6EH4g/wdYD8zg5dDSvK9Ngf+v0bafC9AlacIweEiSGp4Hro6IvQAiYlJEfJDq7k8AP8vMm4ZQ96+pHrL3NuAXEfGfG6dd1fvYOSI+C6yg6Va3mbkKOL1+Oz8iLoyIP27q98cRcWJEtBtK7oyIsyPiTxohJCq78fJD+/6lvr1vv+rnlFxUv10UEUc2QlP9wMArqR66+DTVgwklacLzrlaSpIaTgLOBmyJiLdWdqBqnON0HfHQoRTPzXyPiz4H/Dfwn4GfA8xHxFDCNV965qfWC7L8FtgY+SfWE9GMi4sl6XI2Zib6eqN5qK+Dk+vViXWdq0/4fBz4xiI/2KaoHBL6X6kGGF0bEOmCzevtzwH/PzHsGUVOSxi1nPCRJDfcBuwLfBp6kCh49wBeAXTPzoaEWzsyrgR2o/vX/NqrbxW5G9VC/ZVSnUs1qeeI5WTkB2Af4HtXtazepx3cbVVA6q81hHET1wL6bgJVU1508T/WE8UXAzpl5xyA+09PA+6nCyi+oZjc2pXqi+jeBXTLzx31XkKSJJTK9258kTWQR0UN1d6bZmXlDd0cjSRqvnPGQJEmSVJzBQ5IkSVJxBg9JkiRJxRk8JEmSJBXnxeWSJEmSinPGQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEbdXsAE01E/BaYBvR0eSiSJEnSYM0E1mTmmwfb0eAx8qZtsskmW8yaNWuLbg9EkiRJGozly5fzzDPPDKmvwWPk9cyaNWuLW2+9tdvjkCRJkgblXe96F7fddlvPUPp6jYckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqbiNuj0ASZKksS4iurbvzOzavqXBcMZDkiRJUnHOeEiSJA3TUGcdZs6/EoCeRXM7ORxpVHLGQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScDxCUpDEmIrqy36E+IE2SJHDGQ5IkSdIIcMZDksaYoc48zJx/JQA9i+Z2cjiSJLXFGQ9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSJElScT5AUJoAIqJr+x7qw+4kSdL40rEZj4jYNiK+HRErI+K5iOiJiC9FxOaDrLNF3a+nrrOyrrttH+3PjYglEfFARDwTEasj4vaIODUiZrS5z29FRNavtw5mvJIkSZIG1pEZj4h4C7AM2Ar4MfAbYDfgeODAiNg7M1e1UWdGXWcH4DrgcmAnYB4wNyL2zMz7W7qdANwG/Ax4FJgM7AGcBhwZEXtk5gP97PODwMeAtcCUdj+zNJYMZ9Zh5vwrAehZNLdTw5EkSRNQp061+ipV6DguMy9orIyIL1IFg7OAo9qoczZV6DgvM09sqnMc8OV6Pwe29JmWmc+2FoqIs4AFwMnA0b3tLCK2BC4GvgdsDezbxhglSZIkDdKwT7WKiO2AA4Ae4Cstm08F1gGHRcTkAepMBg6r25/asvnCuv776v29pLfQUft+vdy+n91eVC+P6W9skiRJkoanE9d47F8vr83M9c0bMvMp4CZgU6rTn/qzJ7AJcFPdr7nOeuDa+u3sNsf1wXp5R28bI+Jw4GDgqHZOA5MkSZI0dJ041WrHenlPH9vvpZoR2QFYMsw61HU2EBEnUV2jMR3YFXg3VehY1EvbN1GduvXdzPxRP2MasojwVj6SJElSrRPBY3q9fLKP7Y31mxWucxLw+qb31wCHZ+ZjzY0iYhJwCdXF5McNMCZJkiRJHTASz/FoPEBguDMA/dbJzK0BIuL1wF5UMx23R8SfZuZtTU1PoLqIfG5mPjHMMfUpM3t9cEJE3Aq8s9R+JUnS0M1bfAvX3/3YwA07rHEHwZEye8ctWTxvtxHdp9SJazwaMxHT+9g+raVd0TqZ+Uhm/pDq9K4ZwKWNbRGxPdUdthZn5lUDjEeSJE0w3Qgd3TBRPqdGl07MeNxdL3u99oKX7yrV17Ubna4DQGb+LiJ+DbwjIl6XmY8DOwMbA/MiYl4fXe+tn/L856Wu/5AkSaPbeH520UjPrkgNnQge19fLAyJiUvOdrSJiKrA38Axw8wB1bq7b7R0RU5vvbFVfl3FAy/7asU29fLFe9gDf6qPtXKpnefwDsKZuK0mSJKkDhh08MnNFRFxLFQyOAS5o2nw61ZPEv5GZ6xorI2Knuu9vmuqsjYjLgCOpnjr+qaY6xwIzgZ82P7m8rvPvmflw85jqoHIm1UMNlzWu5cjMXwGf6O1zRMQNVMFjQWbe1/5PQJIkSdJAOnVx+dHAMuD8iJgDLAd2p3rmxj3AZ1raL6+XrRdgLwD2A06MiHcAtwCzgIOAR9nwQX8HAp+LiKXACmAV1Z2t9gW2Ax4GjhjmZ5MkSZI0TB0JHvWsx67AGVRh4APAQ8D5wOmZubrNOqsiYk+qJ5cfDLyHKkwsBhZm5oMtXX5O9fTxvYG3U91qdx1V2LkMOL/dfUuSJEkqp2O3083MB4C+LthubdvrrWbrbauB4+vXQHXuZMNZkCHJzP06UUeSJEnShjpxO11JkiRJ6pfBQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSJElScRt1ewCSJEnSaBERXdt3ZnZt3yPBGQ9JkiRJxTnjIUmSJNWGM+swc/6VAPQsmtup4YwrznhIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzrtaSZIkadyZt/gWrr/7sa7su3F3q5Ewe8ctWTxvtxHb33A44yFJkqRxp1uhY6SNpc/pjIckSZLGrfH8TI2RnFnpBGc8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSJElScQYPSZIkScUZPCRJkiQVZ/CQJEmSVJzBQ5IkSVJxBg9JkiRJxW3U7QFIkiRJnTZ11nwAdrlkfpdHUs7UWY0/ze3mMNrmjIckSZKk4pzxkCRJ0rjz1PJFAPQsGhuzAUMxc/6V3R7CoDjjIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkorbqNsDkCRpooqIru07M7u2b0kTkzMekiRJkopzxkOSpC4Z6qzDzPlXAtCzaG4nhyNJRTnjIUmSJKk4ZzwkqQvmLb6F6+9+rCv7bvxr+UiZveOWLJ6324juUxqqqbPmA7DLJfO7PJJyps5q/MkZM40sZzwkqQu6FTq6YSJ9VklS3zo24xER2wJnAAcCM4CHgB8Bp2fmE4OoswWwEDgYeAOwCrgGWJiZD/bS/lxgV2AH4HXAM8Dv6n1fmJmrWtq/ETgZeBfwJmDzeh8rgG8D383MP7T9wSVpGMb7OfojPbsiDddTyxcB4/u76fdS3dKRGY+IeAtwKzAPuAU4D7gfOB74ZUTMaLPODOCXdb8VdZ1b6rq3RsR2vXQ7AZgM/Az4MvB3wAvAacAdddBo9hbgfwBPUoWTLwD/SBVCvg1cGxGegiZJkiR1UKd+wf4qsBVwXGZe0FgZEV+kCgZnAUe1UedsqpmL8zLzxKY6x1GFiq9Szag0m5aZz7YWioizgAVUsxtHN21aBmyemetb2r8auBbYD/gQ8P02xitJkiSpDcOe8ahnIQ4AeoCvtGw+FVgHHBYRkweoMxk4rG5/asvmC+v672ud9egtdNQawWH7lvbPt4aOev0fqGZANugjSZIkaXg6carV/vXy2tZf6DPzKeAmYFNgjwHq7AlsAtxU92uus55qNgJgdpvj+mC9vKOdxhHxKuADg+kzQL3s7QW8c7i1JUmSpLGmE6da7Vgv7+lj+71UMyI7AEuGWYe6zgYi4iRgCjCd6mLzd1MFiEV9tH8dcCwQwJbAe4G3An8P/KSfcU4YEdG1fQ/1oVqSJEkanToRPKbXyyf72N5Yv1nhOicBr296fw1weGb2dR/H1/HKU7oS+DywIDvwW29m9vpbe0TcirMekiRJmmBG4jkejV/Ah/vLfL91MnPr+pf9rakuDt8OuD0iev0lPzN/U7ffiOqOVicARwJL61v6TniZOeTXmz79E9706Z8Mub8kSZLGl04Ej8ZMxPQ+tk9raVe0TmY+kpk/pDq9awZw6QDtX8zM32fml4G/oroW5YwBxipJkiRpEDoRPO6ul71ee8HLd4jq69qNTtcBIDN/B/wa2Lm+nqMdV9fL/dpsL0mSJKkNnQge19fLAyLiFfUiYiqwN9XTxG8eoM7Ndbu9637NdSZRzWA0768d29TLF9ts/0f18oVB7EOSJEnSAIYdPDJzBdWtbmcCx7RsPp3qqeKXZua6xsqI2Ckidmqpsxa4rG5/WkudY+v6P83M+1vqbN06poiYVD9AcCtgWWY+0bRt94jYtJc+U6geUghwZT8fWZIkSdIgderJ5UdTPRH8/IiYAywHdqd65sY9wGda2i+vl613flpAdZrTiRHxDuAWYBZwEPAoGwabA4HPRcRSYAWwiurOVvtSXVz+MHBES5+Tgf0i4kbg98DTwBuB91PdMWsZcE77H12SJEnSQDoSPDJzRUTsSnVR9oFUD+J7CDgfOD0zV7dZZ1VE7El1m9uDgfdQhYnFwMLMfLCly8+Bi6hO53o7VXBYRxV2LgPO72XfF9dt/oQq5GwKPAHcSvW0829npqdaSZIkSR3UqRkPMvMBYF6bbft8Ml0dFI6vXwPVuZMNZ0EG6nMlnkolSZIkjaiReI6HJEmSpAnO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSituo2wOQ1L55i2/h+rsf68q+Z86/ckT3N3vHLVk8b7cR3ackSSrHGQ9pDOlW6OiGifRZJUmaCJzxkMagnkVzuz2EokZ6dkWSJJXnjIckSZKk4pzxkKQumDprPgC7XDK/yyMpa+qsxp/G7yyd115JUnuc8ZAkaRgm0vVIE+mzSuo8ZzwkqQueWr4I8Hqd8cRjKUn9c8ZDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSJElScQYPSZIkScUZPCRJkiQVZ/CQJEmSVJzBQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJU3EbdHoAkSWPZ1FnzAdjlkvldHklZU2c1/jS3m8OQBm3m/Cu7PQTVnPGQJEmSxqjZO27Z7SG0zRkPSZKG4anliwDoWTS+ZwL8V2ONNd34Tja+J+P9/wdD5YyHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKm4jbo9AJU3b/EtXH/3Y13Z98z5V47YvmbvuCWL5+02Yvvrhqmz5gOwyyXzuzySsqbOavxpbjeHIUmagCJi+DXOHVq/zBz2vkczZzwmgG6FjpE2UT6nJEnSWOSMxwTSs2j8/uvxSM6sdNNTyxcB4/tYwsQ5npKk0We8zzp0kzMekiRJkoozeEiSJEkqzuAhSZIkqbiOBY+I2DYivh0RKyPiuYjoiYgvRcTmg6yzRd2vp66zsq67bR/tz42IJRHxQEQ8ExGrI+L2iDg1Imb00n77iPh0RFxX93k+Ih6JiB9HxOyhfn5JkiRJfetI8IiItwC3AvOAW4DzgPuB44Ff9hYA+qgzA/hl3W9FXeeWuu6tEbFdL91OACYDPwO+DPwd8AJwGnBHRLyxpf2ZwCLg9cBVwBeAm6ju23ldRBzX1oeWJEmS1LZO3dXqq8BWwHGZeUFjZUR8kSoYnAUc1Uads4EdgPMy88SmOsdRhYqvAge29JmWmc+2FoqIs4AFwMnA0U2brgHOzczbW9rvSxVePhcR/5CZD7UxXkmSJEltGPaMRz0LcQDQA3ylZfOpwDrgsIiYPECdycBhdftTWzZfWNd/X+usR2+ho/b9erl9S/vvtIaOev2NwA3Aa4C9+hurJEmSpMHpxKlW+9fLazNzffOGzHyK6jSmTYE9BqizJ7AJcFPdr7nOeuDa+m2712F8sF7e0WZ7gD/UyxcG0UeSJEnSADpxqtWO9fKePrbfSzUjsgOwZJh1qOtsICJOAqYA04FdgXdThY5F/eyzuf+bgDnA08DSdvoMUM+nz0iSJEm1TgSP6fXyyT62N9ZvVrjOSVQXjDdcAxyemY8NsF8iYmOqi9I3Bv5XZj4xUB9JkiRJ7RuJ53hEvRzuDEC/dTJz68wMYGvgQ8B2wO0R8c5+i0a8CrgM2Bv4HvD5YY6zMZ7o7QXc1on6kiRJ0ljSieDRmImY3sf2aS3titbJzEcy84dUp3fNAC7tq20dOr4LfJjqYvS/yExPkZIkSZI6rBPB4+562eu1F7x8V6m+rt3odB0AMvN3wK+BnSPida3bI2Ij4H8DHwH+HvjvmelF5ZIkSVIBnQge19fLAyLiFfUiYirVKUzPADcPUOfmut3edb/mOpOoZjCa99eOberliy31XgNcQTXTcSlwWGa+iCRJkqQihh08MnMF1a1uZwLHtGw+neqp4pdm5rrGyojYKSJ2aqmzlupai8lUTx1vdmxd/6eZeX9Lna1bxxQRk+oHCG4FLGu+WLy+kPyHwEHAt4B5rbcBliRJktRZnXpy+dHAMuD8iJgDLAd2p3rmxj3AZ1raL6+X0bJ+AbAfcGJEvAO4BZhFFRIeZcNgcyDVk8aXAiuAVVR3ttqX6uLyh4EjWvp8HfgA8Djwb8DCiNZhcENm3jDAZ5YkSZLUpo4Ej8xcERG7AmdQhYEPAA8B5wOnZ+bqNuusiog9qZ5cfjDwHqowsRhYmJkPtnT5OXAR1elcb6e61e46qrBzGXB+L/t+c718HbCwn+Hc0M6YJUmSJA2sUzMeZOYDwLw2224wxdC0bTVwfP0aqM6dbDgLMlCf/QbTXpIkSdLwjcRzPCRJkiRNcAYPSZIkScUZPCRJkiQVZ/CQJEmSVJzBQ5IkSVJxBg9JkiRJxXXsdrqSpMGbOf/Kbg9BkqQR4YyHJKmo2Ttu2e0hSJJGAWc8JoCps+YDsMsl87s8knKmzmr8aW43hyG1rWfRyP+32phd6ca+JUlyxkOSJElScc54TABPLV8EjO9/5fQ8eUmSpNHNGQ9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSJElScT5AUJKkDvBBppLUP2c8JElSW2bvuGW3hyBpDHPGQ5KkYehZNHfE99mYXenGviVpqJzxkCRJklScwUOSJElScQYPSZIkScUZPCRJkiQVZ/CQJEmSVJzBQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFbdRtwcgSZI02sycf2W3hyCNO854SJIkTTCzd9yy20PQBOSMhyRJUq1n0dwR3V9jZmWk9yt1gzMekiRJkoozeEiSJEkqzlOtpDHIix4lSdJY44yHpFHJCx8lSRpfnPGQxpBuXHzohY+SJKkTnPGQJEmSVJzBQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSJElScQYPSZIkScUZPCRJkiQVZ/CQJEmSVJzBQ5IkSVJxHQseEbFtRHw7IlZGxHMR0RMRX4qIzQdZZ4u6X09dZ2Vdd9s+2p8bEUsi4oGIeCYiVkfE7RFxakTM6KX9qyPi+IhYHBG/iojnIyIj4hND/eySJEmS+rdRJ4pExFuAZcBWwI+B3wC7AccDB0bE3pm5qo06M+o6OwDXAZcDOwHzgLkRsWdm3t/S7QTgNuBnwKPAZGAP4DTgyIjYIzMfaGo/GfhS/edHgIeBNw72M0uSJElqX0eCB/BVqtBxXGZe0FgZEV+kCgZnAUe1UedsqtBxXmae2FTnOODL9X4ObOkzLTOfbS0UEWcBC4CTgaObNj0NfAD4VWY+FBGnAae2MTZJkiRJQzTsU60iYjvgAKAH+ErL5lOBdcBhETF5gDqTgcPq9q1B4MK6/vvq/b2kt9BR+3693L6l/fOZeXVmPtTfeCRJkiR1Tieu8di/Xl6bmeubN2TmU8BNwKZUpz/1Z09gE+Cmul9znfXAtfXb2W2O64P18o4220uSJEkqpBOnWu1YL+/pY/u9VDMiOwBLhlmHus4GIuIkYAowHdgVeDdV6FjUzz6LiYjsxn4lSZKk0agTwWN6vXyyj+2N9ZsVrnMS8Pqm99cAh2fmYwPsV5IkSVJhI/Ecj6iXw50B6LdOZm6dmQFsDXwI2A64PSLeOcz9DklmRm8vqjtwSZIkSRNKJ4JHYyZieh/bp7W0K1onMx/JzB9Snd41A7h0gP1KkiRJKqwTwePuetnrtRe8fFepvq7d6HQdADLzd8CvgZ0j4nXt9JEkSZJURieCx/X18oCIeEW9iJgK7A08A9w8QJ2b63Z71/2a60yimsFo3l87tqmXLw6ijyRJkqQOG3bwyMwVVLe6nQkc07L5dKonhV+amesaKyNip4jYqaXOWuCyuv1pLXWOrev/tPnJ5XWdrVvHFBGT6gcIbvXmuwAAAB4CSURBVAUsy8wnhvThJEmSJHVEp55cfjSwDDg/IuYAy4HdqZ65cQ/wmZb2y+tltKxfAOwHnBgR7wBuAWYBBwGPsmGwORD4XEQsBVYAq6jubLUv1cXlDwNHtA42IuYDjeDzjno5LyLeXf/5nzLzmwN+akmSJElt6UjwyMwVEbErcAZVGPgA8BBwPnB6Zq5us86qiNiT6snlBwPvoQoTi4GFmflgS5efAxdRnc71dqpb7a6jCjuXAef3se8DqcJJs73qV4PBQ5IkSeqQTs14kJkPAPPabNs609G8bTVwfP0aqM6dbDgL0s7+9xtsH0mSJElDNxLP8ZAkSZI0wRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFWfwkCRJklScwUOSJElScQYPSZIkScUZPCRJkiQVZ/CQJEmSVJzBQ5IkSVJxBg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFGTwkSZIkFbdRtwegkTNz/pXdHoIkSZImKIOHxo3ZO27Z7SFIIyIihtf/3KH1y8xh7VeSNLEZPCaAnkVzR3yfjdmVbuxbkiRJo4/BQ5LGGGceJEljkReXS5IkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqbiNuj0ASZImqogYXv9zh943M4e1b0karI7NeETEthHx7YhYGRHPRURPRHwpIjYfZJ0t6n49dZ2Vdd1t+2h/bkQsiYgHIuKZiFgdEbdHxKkRMaOf/ewVEVfV7Z+OiDsi4pMR8arBfnZJkiRJ/evIjEdEvAVYBmwF/Bj4DbAbcDxwYETsnZmr2qgzo66zA3AdcDmwEzAPmBsRe2bm/S3dTgBuA34GPApMBvYATgOOjIg9MvOBlv0cBPwAeBb4HrAa+CBwHrA38OFB/ggkSRo0Zx0kTSSdOtXqq1Sh47jMvKCxMiK+SBUMzgKOaqPO2VSh47zMPLGpznHAl+v9HNjSZ1pmPttaKCLOAhYAJwNHN62fBlwMvAjsl5n/Wq8/hSrsHBoRH8nMy9sYrzQmDPd0Dhj6KR3+YiVJkqADp1pFxHbAAUAP8JWWzacC64DDImLyAHUmA4fV7U9t2XxhXf999f5e0lvoqH2/Xm7fsv5QYEvg8kboaKrz2frt/+xvrJIkSZIGpxPXeOxfL6/NzPXNGzLzKeAmYFOq05/6syewCXBT3a+5znrg2vrt7DbH9cF6eUcf472mlz5LgaeBvSJi4zb306uIyN5ewDuHU1caiszs2kuSJAk6c6rVjvXynj6230s1I7IDsGSYdajrbCAiTgKmANOBXYF3U4WORe3uJzNfiIjfAjsD2wHL+xmvJEmSpDZ1InhMr5dP9rG9sX6zwnVOAl7f9P4a4PDMfKzD+2lLZvZ6Un1E3IqzHpIkSZpgRuIBgo1fwId7zkW/dTJz6/qX/a2BD1HNWNweEYP9Jb9T45UkSZJU60TwaMwQTO9j+7SWdkXrZOYjmflDqtO7ZgCXltiPJEmSpPZ1InjcXS97vfaCl+8q1de1G52uA0Bm/g74NbBzRLyunf1ExEbAm4EXgNbnhUiSJEkaok4Ej+vr5QER8Yp6ETGV6oF8zwA3D1Dn5rrd3nW/5jqTqGYwmvfXjm3q5YtN666rl63PAwHYh+oOXMsy87lB7EeSJElSP4YdPDJzBdWtbmcCx7RsPp3qSeKXZua6xsqI2Ckidmqpsxa4rG5/WkudY+v6P21+cnldZ+vWMUXEpPoBgltRhYgnmjZfATwOfCQidm3q81rgb+q3X+v/U0uSJEkajE49ufxoYBlwfkTMoboN7e5Uz9y4B/hMS/vGbWpb7/y0ANgPODEi3gHcAswCDgIeZcNgcyDwuYhYCqwAVlHd2WpfqovLHwaOaO6QmWsi4giqAHJDRFwOrAb+jOpWu1cA3xvcx5ckSZLUn44Ej8xcUc8enEEVBj4APAScD5yemavbrLMqIvakenL5wcB7qMLEYmBhZj7Y0uXnwEVUp3O9neoWuOuows5lwPm97TszfxQR+1IFokOA1wL3ASfWfbyjlSRJktRBnZrxIDMfAOa12bbXZ1zU21YDx9evgercyYazIG3JzJuoApIkSZKkwkbiOR6SJEmSJjiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpuI26PQBJo9ddd93FkiVLWLNmDdOmTWPOnDnsvPPO3R6WJEkagwwekjawZMkSzjjjDJYuXbrBtn322YeFCxcyZ86cLoxMkiSNVQYPSa/wrW99iyOPPJL169czZcoUDj30ULbZZhtWrlzJFVdcwdKlSznggAO4+OKL+djHPtbt4UqSpDHCazwkvWTJkiUvhY6TTz6ZlStXsnjxYs466ywWL17MypUrOfnkk1m/fj1HHHEES5Ys6faQJUnSGGHwkPSSM84446XQcfbZZzN16tRXbJ86dSpnn332S+HjzDPP7NJIJUnSWGPwkARUF5IvXbqUKVOmcPLJJ/fbdv78+UyZMoUbb7yRu+66a4RGKEmSxjKDhySAl06bOvTQQzeY6Wg1bdo0DjnkkFf0kyRJ6o/BQxIAa9asAWCbbbZpq32jXaOfJElSfwwekoBqFgNg5cqVbbVvtGv0kyRJ6o/BQxLAS8/luOKKK3jqqaf6bbtmzRp+8IMfvKKfJElSfwwekgDYeeed2WeffVi7di3nnHNOv20XLVrE2rVr2XfffX2SuSRJaovBQ9JLFi5cyKRJkzjnnHNYsGDBBtdvrFmzhgULFnDOOecwadIkTjnllC6NVJIkjTU+uVzSS+bMmcNFF13EkUceyTnnnMMFF1zAIYcc8tKTy3/wgx+wdu1aJk2axMUXX+xpVpIkqW0GD0mv8PGPf5yZM2dy5plncuONN3LJJZe8Yvu+++7LKaecYuiQJEmDYvCQtIE5c+YwZ84c7rrrLpYsWcKaNWuYNm0ac+bM8ZoOSZI0JB0LHhGxLXAGcCAwA3gI+BFwemY+MYg6WwALgYOBNwCrgGuAhZn5YEvbGcCfA3OBXYA/Ap4H/n9gMbA4M9f3so8pwKeBQ4E3A88CtwJfyMyr2v/U0vi28847GzQkSVJHdCR4RMRbgGXAVsCPgd8AuwHHAwdGxN6ZuaqNOjPqOjsA1wGXAzsB84C5EbFnZt7f1OXDwNeoQs71wO+B1wMfAr4JvD8iPpyZ2bSPzYBfAG8D7gK+AUwG/gy4MiKOz8zzh/qzkCRJkrShTs14fJUqdByXmRc0VkbEF4ETgLOAo9qoczZV6DgvM09sqnMc8OV6Pwc2tb+HOjA0z2xExALgFuAQqhDyg6Y+p1GFjv8D/NfMfKHus2Xd5/MRcXVm3tvWJ5ckSZI0oGHfTjcitgMOAHqAr7RsPhVYBxwWEZMHqDMZOKxuf2rL5gvr+u+r9wdAZl6Xmf/YejpVZj4MfL1+u19LrQ/Vy4WN0FH3eQz4AvBq2gtJkiRJktrUied47F8vr+0lADwF3ARsCuwxQJ09gU2Am+p+zXXWA9fWb2e3Oa4/1MsXWtZvXS/vZ0ONdd6uR5IkSeqgTgSPHevlPX1sb5yytMMI1SEiNgL+sn57Tcvmx+vlm3vp2phN2WmgfbQxhuztBbxzuLUlSZKksaYTwWN6vXyyj+2N9ZuNUB2ARVTXcVyVmT9t2faTenlaRLyqsbK+sL1xXcnGEbFJG/uRJEmS1IaReI5H1Mvst1WH6tQXon+K6s5ah/XSZCHVNSkfBmZFxBKqU8EOAp4Cnq7fvzicwWZm9LY+Im7FWQ9JkiRNMJ2Y8WjMREzvY/u0lnbF6kTEMVR3v/o1MDszV7e2qS88/xPgfKrb6B5NFTp+AvxnqutMnszM5wcYryRJkqQ2dWLG4+562de1F9vXy76u3ehInYj4JHAecCcwJzMf7WtH9R2sjq9fzTVmU82s/MsAY5UkSXpJRK8nOrTf/9yh9216XJk0qnVixuP6enlARLyiXkRMBfYGngFuHqDOzXW7vet+zXUmUZ0e1by/5u2fpgodv6Ka6egzdAzgiHr5d0PsL0mSJKkXww4embmC6la3M4FjWjafTnU606WZua6xMiJ2iohX3DkqM9cCl9XtT2upc2xd/6ctTy4nIk6hupj8VqqZjsfpR0RMiogpvaz/BPDfqMKLwUOSJLUtM7v2ksaKTl1cfjSwDDg/IuYAy4HdqZ65cQ/wmZb2y+tl67zkAqoH/p0YEe+gepL4LKprMB6lJdhExEeBM6guBP8FcFwvU509mfmdpvebAo9ExM+A++p17wF2A1YAf56Zf0CSJElSx3QkeGTmiojYlSoEHAh8AHiI6gLu03u7yLuPOqsiYk+qJ5cfTBUIVgGLqZ40/mBLl8azOF4FfLKPsjcC32l6/xxwOfBu4L31uhX1Pr9Yz7yI4Z+vCkM/Z9V/wZEkSRpfOnY73cx8AJjXZts+f6OtQ8oGF3730fY0Njwta6A+fwA+Ppg+kiRJkoZnJJ7joTHKWQdJkiR1SifuaiVJkiRJ/TJ4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHpIkSZKKi8zs9hgmlIhYtckmm2wxa9asbg9FkiRJGpTly5fzzDPPrM7MGYPta/AYYRHxW2Aa0NPloZT2znp5W1dHoU7wWI4fHsvxw2M5fngsx5eJcDxnAmsy882D7WjwUBERkQCZGd0ei4bHYzl+eCzHD4/l+OGxHF88nv3zGg9JkiRJxRk8JEmSJBVn8JAkSZJUnMFDkiRJUnEGD0mSJEnFeVcrSZIkScU54yFJkiSpOIOHJEmSpOIMHpIkSZKKM3hIkiRJKs7gIUmSJKk4g4ckSZKk4gwekiRJkoozeEiSJEkqzuAhSZIkqTiDhyRJkqTiDB6SJEmSijN4SJIkSSrO4CFJkiSpOIOHJEmSpOIMHhqUiDg0Ii6IiF9ExJqIyIj4bh9tZ9bb+3pdPtLj18siYkZEfCIifhgR90XEMxHxZET8U0R8PCImtbT3eI5iEdHTz7F5uI8+e0XEVRGxOiKejog7IuKTEfGqkR6/KhFx+ADfs4yIF5va+73sssH8vdjUZ9DfvYj404i4of7/9NqI+OeI+GjnP9HENlK/50TERyPilvpYPlkf2z8t98lGh426PQCNOZ8F3g6sBR4Edmqjz/8FftTL+js7OC4N3oeBrwEPAdcDvwdeD3wI+Cbw/oj4cGZmSz+P5+j1JPClXtavbV0REQcBPwCeBb4HrAY+CJwH7E3134dG3q+A0/vY9h5gf+DqXrb5veyeQf29OJTvXkQcC1wArAK+CzwPHAp8JyJ2ycyTOvVhVP73nIj4PPCpuv7FwGuAjwD/GBF/nZkXDmHcY0Nm+vLV9guYDWwPBLAfkMB3+2g7s97+nW6P21evx2d/qr/sJrWs35oqhCRwiMdzbLyAHqCnzbbTgEeB54Bdm9a/FlhWH+ePdPsz+drguP2yPjZ/1rTO72X3j8tg/l4c9HevPsbPUoWOmU3rNwfuq/vs2e2fw3h5lf49B9ir7nMfsHlLrVX1sZ45nM8wml+eaqVByczrM/PerL8lGrsy87rM/MfMXN+y/mHg6/Xb/UZ8YBoJhwJbApdn5r82Vmbms1T/2gfwP7sxMPUuIt4G7AH8G3Bll4ejJoP8e3Eo372PARsDF2ZmT1OfJ4Cz67dHDXH4ajECv+c0jtVZ9TFs7LcH+ArVsZ5XaN9d56lWGgnbRMRfATOo0vwvM/OOLo9J/ftDvXyhl20ez9Fr44j4C+CPgXXAHcDSzHyxpd3+9fKaXmosBZ4G9oqIjTPzuWKj1WD8Vb38Vi/HE/xejhVD+e711+fqljbqjsF8/wY6nqfUbU7t+ChHAYOHRsJ769dLIuIG4KOZ+fuujEh9ioiNgL+s3/b2P0aP5+i1NXBZy7rfRsS8zLyxad2O9fKe1gKZ+UJE/BbYGdgOWF5kpGpbRGwC/AWwnur6q974vRwbhvLd66/PQxGxDtg2IjbNzKcLjFkDa+v7FxGTgT8C1mbmQ73Uubde7lBonF3nqVYq6WngTOBdVOeibg7sS3Uh837AkvpLqNFlEfA24KrM/GnTeo/n6LYYmEMVPiYDuwDfoDpv+OqIeHtT2+n18sk+ajXWb9b5YWoI/gvVsbg6Mx9o2eb3cmwZynev3T7T+9iucgb7/Zvw/+81eKiYzHw0Mxdm5m2Z+e/1aylwAPDPwFuBT3R3lGoWEcdR3WnjN8Bhzds8nqNbZp5eX7fzSGY+nZl3ZuZRwBeBTYDTBlEuGmU7PU4NyZH18hutG/xejjtD+e75fe2Sgt+/cXssDR4acZn5Ai+fLrBPN8eil0XEMcCXgV8DszNzdTv9PJ6jXuNGAc3HZqB/IZ3W0k5dEhH/geouOA8CV7Xbz+/lqDWU7167fdYMY1zqoH6+fwMdy4FmRMY8g4e65bF66SkAo0BEfBK4kOqe47PrO1sNhsdz9Hq0XjYfm7vr5QbnEdfX+LyZ6sYC95cdmtow0EXl/fF7OfoM5bvXX583UB3fB72+Y9TZ4PuXmeuo7kw3pT52rbavlxtczzNeGDzULXvUS3+x6bKI+DTVg6t+RRU6Hh2gS288nqPXnvWy+dhcVy8P7KX9PsCmwDLvaNVdEfFaqlMe1wPfGkIJv5ejz1C+e/31eX9LG40efX3/JvTxNHiomIjYPSJe08v6/YET6rffHdlRqVlEnEJ1MfmtwJzMfLyfth7PUSoido6ILXpZ/yaqmSx45bG5Angc+EhE7NrU/rXA39Rvv1ZouGrfh6kuVr2ql4vKAb+XY9BQvnuLqR44eGxEzGzqszmwoH77dTTihvj9axyrz9THsNFnJnAM1bFe3PHBjhLhc+A0GBFxMHBw/XZr4H1Uaf4X9brHM/Okuu0NVLcFvIHq/GSA/8jL97A+JTMb/6PVCIuIjwLfAV6E/9fevcbYVdVhGH9eKzcBCYJR7qWoINEoRFBBQ5VgCCgpIBigEpWgojZGIWiChBqMRBEvUPDyAcGCESkRAkEgIRTRQAQRDBgTaSlQhKICBYUCsX8/rD1wcphpOzPstkyfXzJZc/ZZa+995uTk7Hf2unAeo/cpXVJVF3X1F+L7uV5KMhf4Om0mlfuBp4HdgENpKyJfCxxeVc8PtJlFuwhaAfwKeBw4jDZ15wLgaBcKXbeS3AJ8gLZS+dVj1FmIn8t1ajzfiwP1x/XZSzIHOJe2RsRlwPO0xQh3BM4Z3L8mZ21c5yQ5B/hq12YBsDHwCdo6IHOqat5wm6nC4KFx6S5wVrWozQNVNb2rewJwOG1q1m2BjYBlwK20FVhvGWsn6t8avJcAN1fVzK6+7+d6KskBtNVw9+Kl6XSfpHWfmw/MHy1EJNkfOI3WHWtT4D7gQuDcCYwn0CsoydtpEz0sBaaP9X74uVz3xvO9ONBm3J+9JB8DTgH2pvVY+SvtPb54ki9BA9bWdU73z78vAXvSulPeCZxdVddM/lWsvwwekiRJknrnGA9JkiRJvTN4SJIkSeqdwUOSJElS7wwekiRJknpn8JAkSZLUO4OHJEmSpN4ZPCRJkiT1zuAhSZIkqXcGD0mSJEm9M3hIkiRJ6p3BQ5IkSVLvDB6SJA1IsiRJJZm+rs9FkqYSg4ckSZKk3hk8JEmSJPXO4CFJkiSpdwYPSdKkJdk8yalJbk/yVJJnk9ybZG6SLYbqzu3GUMxNsmuSS5IsS7Kia3NykteOcZwk+WSShUme6NosSnJ+kp1Wc36nJLk1yZPd+S1OcnmSQ1bR7qAkNyZZnuSZJLclOWyMutsnmZfkvu68nknyYJLrknx2Tf+WkjRVparW9TlIkl7FkuwIXA/sCfwT+DOwAtgH2A74CzCzqp7o6s8FzgB+AXy0q/t74PXAh4BNgCuBI6tq5cBxAlwCHAu8ACwEHgf2BXbtfj+4qm4fOr9duvPbHfhPd6zlwE7Au4A7qmrmQP0lwC7At4DTgNuBxV37vYACjq6qBQNttgPuBN4MPADcBTwH7AC8A3i0qvYYx59VkqacUf+jJEnSmujCwK9poWMe8LWqeqZ7bjPgZ8Bs4AfAp4aaHw9cAcyuqhVdm7cCNwGzgM8DFwzUP4kWOpYBB1bVvV2bad3+5wCXJ9m9qp7rnnsN8BtaaLgK+PRIAOqe35IWXEZzKnBIVV03UP8bwJnAWcCCgbon0kLHT4GTauC/ekk2Ad47xjEkaYNhVytJ0mQcDLwfuA348kjoAKiqZ2nh4THguCRbD7V9FvjCSOjo2vwdOL17+JWh+id35ekjoaNr8z/gFOAh2p2Kjw+0OYx2l2IJcMxg6OjaPl1VN47x2s4bDB2d79Lulrwlyc4D29/UldfVUFeCqnquqn43xjEkaYNh8JAkTcbI+IgrBrtFjaiq/wJ30O6w7zP09A1V9dgo+7wUWEm7uN8BXuzONaPbPn+U4zzftQOYOfDUwSP77ILQeFwzxnEWdw+3H3jqj135nSSzkmw+zmNJ0pRn8JAkTcaMrjy7GzD+sh9eCidvHGp7/2g77C7uH+ke7tiVO3TlI4N3SIYsGqoL7Q4IwN9W90JG8eAY25/qyk0Hts0Hfgm8jda1a3mSu5Kcm2S/CRxbkqYcx3hIkiZjWlfeTOvOtCoPTGD/I92WMvR4NFnFcxPxsjs4Y+nu9hyX5CzagPn9u585wJwkF1bVCa/w+UnSq4rBQ5I0GQ915eVVdf44204fbWOSjWmzYQH8oyuXduX2STYZGTw+ZNeufHhg20jY2X2c5zYhVXUPcA+8OLD9ENqdkM8kuayqblgb5yFJ6yO7WkmSJuO3XXnUBNp+JMlw9yuAY2jfT4uqailAVy7uts8ebpBkI9qMV9Cm2R1xfVfOTrIpa1FVrayqa2izaUGbuleSNlgGD0nSZFwJ/Ak4IMlPkrxhuEKSGUm+OErb1wHzuulmR+ruRpuuFuBHQ/W/35VnJtljoM002mxTu9DucAxOc3sVbU2N6cClSbYaOrctkxy42le5GkmOT7L3KNu3oc36BRPraiZJU4ZdrSRJE1ZVK5PMAq4FPgccm+RuWteobYGdaQOulwHDXbHmA4cCi5L8AdgC+DBt0PbVo9S/gDZu4hjg7iQ3AU/Q1uGY0f1+1GA3rO78jgBuAI4ADkoyuIDgu2mzbo01pe6aOgK4OMnDtKDzJLAN8EFgc+AW2qBzSdpgGTwkSZNSVUuT7AucABwNvJO2YN6/aeMtvsfoF92LaVPsfpsWOLbqtl0I/HB4et6qqiTH0bp3nQi8D9iMNg7kx8BZVfUQQ6rq/u5uxBzgSFoYmAY8Spsy9+eTef2dc2iD6/cD3gNsDfyLtpr5RbTpfF94BY4jSa9aGVrnSJKkXiWZC5wBfLOq5q7bs5EkrS2O8ZAkSZLUO4OHJEmSpN4ZPCRJkiT1zjEekiRJknrnHQ9JkiRJvTN4SJIkSeqdwUOSJElS7wwekiRJknpn8JAkSZLUO4OHJEmSpN4ZPCRJkiT1zuAhSZIkqXcGD0mSJEm9M3hIkiRJ6p3BQ5IkSVLvDB6SJEmSemfwkCRJktS7/wNuDDm337CoKAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "image/png": { + "height": 279, + "width": 399 + }, + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# take only top-20% of the points for analysis\n", + "(trials_df.query(f'{target_metric}>{0.8*study.best_value}')\n", + " .boxplot(column=target_metric, by='epochs', grid=False))\n", + "plt.suptitle(''); # hide auto-generated box-plot title" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Even though this plot is not suitable for making conclusions about the *actual perfromance* of the LightFM model, it roughly indicates that, *with the current selection of optimization mechanism*, **increasing the number of epochs further is unlikely to significantly improve the quality of predictions**. In other words, our tuning procedure reached a nearly stable state and, most likely, it is somewhere close to a local optimum. We will, therefore, proceed with the best found configuration without any further adjustments. Below is a characterizing plot of all trials." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABLEAAAKfCAYAAACRyVooAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdebxcdX3/8ddn5s5y1+wkhJBAwhYCEvYAgoqiWKVY97pCfy31oVbr2lYtilZbq3VBrVVrXWvd6lJbQRZl3wlbIAuE7Htubm7uvsx8fn98z5BhMnP3mXsueT8fj3kMd8453+/3zAbz5vP9HnN3RERERERERERE4iwx2QMQEREREREREREZjkIsERERERERERGJPYVYIiIiIiIiIiISewqxREREREREREQk9hRiiYiIiIiIiIhI7CnEEhERERERERGR2FOIJSIiIiIiIiIisacQS0REREREREREYk8hloiIiIiIiIiIxJ5CLBERERERERERiT2FWCIiIiIiIiIiEnsKsUREREREREREJPYUYomIiIiIiIiISOwpxBIROYyY2RVm5mZ2ywS3+92o3U9MZLtTkZndEj0XV0z2WEQkXszsuOj7YXCyxyIiIjIVKcQSEZmiioKjWyZ7LONhZsvN7BPDhT5F4dBwt1eVHLexaNujIxjPl0raO2ZcJygSQ2Y2M/rcXT3ZY5loZvb+6NwWTvZYREREZGLVTfYARESkptqBtcDmyR5IkeXAx4Fbge+OYP9ewnkMtb2SU81subs/XG6jmdUBfzqCMYhMdTMJn7sc8MlJHstEez9wFHAT8fquA+gnfAerEktERGQMFGKJiBxG3P2XwC8nexzj9BN3v2IMx20GFgJvA8qGWMClwBHAJmDRmEYnIlKBu28GTprscYiIiExVmk4oIiKHi/8EHHiTmSUr7PO26P6HtRmSiIiIiIiMlEIsEZHDyHALu5tZ0sz+2sweNbMeM9tjZv9rZhdE24ddJ6qojUfMrNvM9kVtnFVmXwe+E/35gjLrW71wvOdcZDNhyuJc4KVlxjINuAzoBv57Ijo0s/pobZ610fO528x+bGbHl+xnZvZUdM7vHqbNW6P9PlP02LNeVzN7u5ndY2YHzKzdzG42s0tHMN7LzOzXZrbTzPqj8f7GzF5WYf/Sft8cja+13Ppkw/S9NTrm+Wa2yMy+HT3Wa2ZPm9k/m1lLhWMz0di/Hb3vWqPjNpnZD8zs9BH2e7SZfd3MNphZn5k9ULTfUWb2LjO7LnqtuqPnd6WZfTx6/5Rr/yVR+09Ff788ej3aotsNZnZu0f7TzewfzezJ6Bw2R39nh3n+LjKzn0Tn0xc9Bzea2RvK7HsH8GT0Z7LM5+5jZY451sy+ambris79ATP7sJk1lNm/rqi9BWa2zMy+H41vwMx+XrTvXDP7vJk9bmZd0XlvMbO7zOwaG+HaVmb2Dxa+U46KHrq95LxuKtr3h5XOtfh5ivZ5S8njf17cnpldbmHNvv1m1mlmd5vZ6yu0WXFh9+IxWfgefb+F7+LC9+j/mNkZwzwHF5rZb6P9O83sYTN7r5klRnLOIiIicafphCIiAoCZpYBfAy+PHhok/HviFcDLzOyNI2imDvhfwrS8AaAPmBG18WIzu9jd7y7afxdQD7RE++8raa9/bGdT0Q+AFxIqrq4r2fYGIAv8COiYgL5agDuB0wnPQx6YE/VziZmd4+7rAdzdzew/gE8DVwJfLdegmS0BLoz+/E6Ffb4I/HXU3wFgGnAxcLGZfcjdP1/mmFTU3puLHj4QjfeVwCvN7HPu/uFKJ2tm1wJ/FfXbHt2PxQnAr4BZQCeheu5Y4EPAH5vZC9x9V8kxL+fZ02S7o+MWAm8B3mhmb3f3Hw3R71LC+39mdHxpyPA14PKiv/cTXuPTo9ubzeyF7r69Ugdm9h7gS9HYOqPjLwGeb2YXA08DfwBOBroI/7PxaOBvo/EdEgqamQGfAz5Q9HAH4XP3EuAlZnYZ8DZ3L7wmrcBeYHb0d+nz2VnSx+sIn51M9FBP9M9nRrc3mdkl7r6nwqm/EPgm4bN+gLAOV6HtY4G7gHnRQ7lo/Aui23nAFuDfK7RdrCM6lzmE524f4XuloPT7ZdzM7BrgasL7vQNoBFYAPzGzI9y97Gd5GCngesLr1x/dZhBC9pdE77P7yozlSuDbgEUP7QeWEd5zFzDx36ciIiI1p0osEREp+BghDMgRQpAWd58BHEP4QTWSH5HvAs4hBDVN7t4MnAasIgREXy7e2d3nAe+N/rzL3eeV3O4a/2k9y88IP8Avt0OregpTCb8/QX1dQ/jheSnhh20TcBGwlRCU/GPJ/t8hPPdnmNnzKrR5JeEH6u3u/mSZ7csJr91ngZnR63cUYSolwD+b2fPLHPfPhABrI/AmoNndpwHNwF8SgocPmVmlRe/PBN5NWCh8lrvPjM59LK/fFwghywXR+6cJeDUhgDiR8uFdB+HH+8VR/43uXk94736FEK5+y8yOKnNscb9bgBXR8Y1AcXC7FvgoIUyqj57bbNTng8DxwNeHaH8e8HngHwivzTRgMXAvIdz5IvAtwuv7fHdvis79Lwnvi8vN7JAKQsIi5h8AdgPvAGa4ewvhPfdGYCfhtX0m5HL3ywnhEECuzOfuS4V9zWwFIdhNEt6zC929AWgAzgceIHzGvzvEuX8duAc4JTrveuBvom3XRM/NOkJAm4qe23rgeYRgtzRkK8vdPxt9p+yIHrq85LzKVkeNw5mE98RHCa/pdOBIDgaqnzWz6WNo9z2Ez/LrCO+BlujvJwjPy5dKDzCzZcA3CO+f3wDHRM/jNOB9hM/QK8YwFhERkXhxd91000033abgjfCj0YFbRnHMFeWOIfxQKlS9fKTMcSnCYuge3Y6pMBYn/AAvPf7Mou2LRjKmMm3cEu3XQ/hhXu72yTLHbYyOe0f094+iv/+saJ/F0WPbCT/Wj6t0riN4jgvj7AaOK7P9NdH2XiBdsu3X0bYvljkuQZgS6cAVFZ5DB75V5lgDfh9tv6lk2/GEkKQNWFzhnF4fHbtqiH4/M87389ai5+3YMttfUtTXilG2/b3ouI8O0W8rMGeMY58dHZ8Hjh5i3OVem8VF2/sqnHth/N8seXwmoWKrDzizwtguiI7dC9QVPV54jw8Oc253R/v9VYXtswihkQPLix6vKzqvdUC2wvHron1eM573T4XX9JDvoqJ9fhjt87Eh9rkj2uctJY//edG5/U2Z4xqi59uBN5Vsq/i8F40pX+49Dpxb1O9RJdsKa/49TAgCS4/9SNGxFc9ZN91000033eJ+UyWWiIgAvIxQudELXFu60d0HCJUqw7nd3e8oc/yDhB+WEKa3jEeWsK5VuVvZNZNKFCqt3lb0WOGf/9Pdc0yMn7v7U2Ue/x/CD8kM4QdtsUK121uiKX7FLiFMLesgVJRV8pnSB9zdOVj5dbGZzSza/DZCQPYrd3+6Qpu/IAQly8zsyDLbc4zs/TES/+XuG0ofdPebgMIUqteOss3fRPcXDLHPd73ydLghufteQthjHKxwKqe0+o7oOS887z8ud+7AzdH9KSWPv44QmNwWfcbKje1OwtU2ZxGmPY6YmZ1ImBrXRZgOWK79VuB30Z+XVGjqK+7eW2Hbgei+3Psq7rop/33ZDdwY/Vn6mo3ELe5+T5l27yWE9VD0PWrhQhWFqa5fjL6vS11L+B8AIiIiU5rWxBIRETj44/Zhd++ssM/tI2jn/iG2bSOscTNjNAMr43vufsU4jr+RUDlykZktcvdNhHWTYOKmEkKF58LdB8xsNyF0K30ufkuoBptPWP/mF0Xb/iy6/4m7d1Xoc3OFEARCVUmOUGm2nFCZBWFKGMBrzezl5Q6MFEK1ozk4XavgqSjImQi3DLHtVsJ01UMWtzazWYQpjZcSph22EM612Pwh2r57iG2FPlYQpvedT5im2Vhmt0p9dA0REu4mVGStqrC9MJ2u9P1SeO3ON7OdVFYILY9m6M9oqUL7GWBTWH6rrKai9ssZ6rn9LaFS8/NRaPZz4N4hQq84WeXulYKhbdH9WL7vhvsenVfS7vEcfC8e8j8RANy908we4uBrKiIiMiUpxBIRETi4wHNpOFGs4oLVRYZaEL3wo7S0wqim3D1nZj8irBH0FjO7FVhCCPAem8CuRv1cRGP7LmHqz5VEIVZUOVWotPiPIdrdVmmDu/eYWRvhtZ5TtKlQAVNYg2k4h1yJDqhYwWRmf0NYk6fUBncvV7VU8RyKthWPHzM7FbgJOKLo4Q5C5YkDacKP/nKhU8GQVVhm9reEKrdCkjNImIJZWCx7GqFKsFIfQ4VMheq/Sp+/wvbSz07htSusUTWckexTrv06Qug61vaHem4/QwglX0EIId8NDJrZfYS1pb7l7u0jG27NVev7brTtzi765/F+h4uIiMSaphOKiAgc/GE+FK/6KGqnUHH1ViZ+Qffx+jbhuX65mRWu2PYmQjXMGn/21R1Hq9zrXPhvgfe6u43gdkuZNoaagtlM+amfc4Y4ZjTjh7DY+xGERcZfSrioQIu7z/Ww0PefDnM8DHEOZnYaBwOsLxOuHph195keLRpOuKLicH1MtMJr97kRvnY/HGP794+w/T+v0E7F59bde939lYQKoc8RFrr3or/XmtlYpuQdTmr5nhMREZlUCrFERAQOVkoMtS7NUFOxphR3fxR4lDDt7ArCj+wfTeaYCqIpZ78nTIV7a/RwYSrhUFVYMMRrZGZZoHCltOLKmMJUtZNHN9KRcfePVQg8StcDKxjqfVZ4fz4zfjNbTJiONghc5u43lpluOZIqoqG8hhAU/J+7/7W7ry6zdtp4+xiLqr52Re2fGK27VDXufre7f9jdVxCmP76ZsI7eXMJVGyfaYHSfHWKfaVXotxqKP89DfYdPxXXHREREnkUhloiIADwU3S83s0pTyi6sUt/56L7W1QSFyqsUcIO77xpq5xorLPB+ZVQFdDrhR/dw1WKLzOyYCtueTwjGClcwKyhUdl1WZjH5yfCCEWxbWfTYguh+p7tXmrL3knGOqdDHQ+U2mlkzYa2uWiu8di8ys9GuvTSSz12h/RbgxaNsf8zcvdPdfwS8I3roHDOrH0UTIzm3/dH9gnIbo9f0xFH0OZmeJCy+D+Fzfggza2SUC/uLiIjEkUIsEREBuIHwIygLvKt0o5nVUX5do4lQuDrZ9CH3mng/AP4lun26xn0P55dAK7AU+Fr02P+NMGj7u9IHLKzI/bfRnze7+76izd8j/OifX+7YknbGuyj/SLzJzBaV6ftFwLnRn8VXZyyslzQ/Wty99LjlwBvGOaZCH6dW2P73DL3eVrX8hHCFvAbgs0PtWOa1K3zuElFgcwh3X0WYognwz2ZWcU0tM2sws/SIRv3s44Y6prBoeoLRreM6ku+Uwvp3l5pZpsz2DzDJ6/eNVFQV+D/Rn38dfV+XejejXxNNREQkdhRiiYhMfSkzmz3MbcgfY+7eAXwx+vMfzOyvCpUPZraQcMWwY6s0/sej+5PN7Nwh95xA7r7b3T8Y3e6sVb8j4e59hJAN4ILofriphBB+vF9lZp8xs2kA0bpa3yNU0jhwTUlfq4EvRX9eY2Zfi6boER3fZGaXmNkPeHZ4VC0DwPXRlQAxs4SZXV7U93Xufm/R/qsIi1kngJ8Uxm5mKTN7LfA7hl4oeyRujO4vN7O/KfpsHGFm/wJ8iBA61pS77wE+Fv35F2b2X2a2rLDdzLJmdqGZfR24reTYvYSrIkK4iEAl7yYsXn8acJuZXVyYWhi9NqeY2dXAep69sP5IrTazT5vZWYXvKQtWANdG+9wdfUeNVOE75U3RNNpy/gfoI0xX/K6ZzYn6nh6dz8c4GF5OBZ8hfHaWAz+Lvrcxs3ozew/wDxysPhMREZmyFGKJiEx95xPWRBnqdkHFow/6FKEiq47w47HdzPYBm4A/4uC6TBB+/E0Id3+S8AO7DrjHzFrNbGN0WzFR/UxB/170z7uA347gmIcIgdTfAa3R67edg2trfdjd7yhz3IeBr0f//E5gvZkdiK5meIDwvngLYTpitX2AsOj73WZ2AOgkLJo+C1hHSeASVaG8h1BN9uLC2KPjfkaoVHr/eAbk7r/lYKXLPwFd0XO7M2r7m8D14+ljHGP7IvAJQkD5RmCVmRXG10X4bL2D8ms/Fd5jXzazjqLP3buL2r+XsCZYB2HtsZuBbjPbS7hS3mOEYHQeY7v4w1zC1TjvB3rMrJXw/XI3sIwQtP3FKNv8dnT/RsL32JbovJ5Z2D4KAD9StN/u6P3eGp3P1YSAdEqIqubeRXgNXgVsKvr8fpnwWSh8h0zY97eIiEitKcQSEREA3L2fcJn7DxB+vOUJC57/BrgI+EPR7hP9f/RfDfwrsAFoAhZFt6EWXX5Oc/fHCaENwPfdfXCo/YuOex9hsfoHCcFgJ+G1e7m7f77CMTl3fydhPZ0fEoLLNFAPbCZMb3w74cdxta0lhCXfIQQnScL74nPA2eWmVLr7zwnrXt0cHZMCNkbHnE4I8sbrtYTQYw2h4gXgDuCt7v6XE9D+mLn7NYTz/HfgKcJaUI2ECrXrCCHW+WUO/Tgh8HyM8DwXPnfPmobn7v8LHE+o9nmIEF5NJwQkdxKmU57k7tvGMPzLCMHgXdF4mwiVX49E/S2LPgsj5u43EF6v2whTEo+KzmteyX5fIFy58l5C2JkgvKaXu/tnxnAuk8rdvwW8iFB92E74HKwC/oqwUH5hoXpVZImIyJRl7s+lK6aLiEi1mNmLgZuATe5+zCQP5znPzI4mBDEJYKm7rxli3ysIoc+t7v7CWoxvopnZVkLYcGGFajERGSMzSwBbCGvf6TMmIiJTliqxRERkpD4U3d845F4yUa4i/Hv69qECLBGREXgzIcDaz8HF+kVERKYchVgiIgKAmSXN7OdmdmlhUfDo8WVm9nPgZYRpVNdWbEQmhJmdDrw3+vNLQ+0rIgJgZn9vZu8yswXRFUkxs5lm9j7Cum0AX3X33skbpYiIyPiM5nLFIiLy3GaEBZxfAxAtjl3Hwcuy54F3u/tj5Q+X8TKzO4DFhLV7jLCmzy8ndVAiMlUsA94AfBXoN7MuwtplFm3/HeEqhSIiIlOWQiwRESnIEa5M9zLgVOAIwmLPmwhhypfcfeXkDe+wsAA4knA1wv8F/sa1eKWIjMxXgTbC1WiPJARY+wiL5P+QcIGI3OQNT0REZPy0sLuIiIiIiIiIiMSe1sQSEREREREREZHYU4glIiIiIiIiIiKxpxBLRERERERERERiTyGWiIiIiIiIiIjEnkIsERERERERERGJvbrJHoBUl5ltAFqAjZM8FBEREREREZl4xwAH3P3YyR5INZjZfwIn1bDLNe7+5hr2J6OgEOu5r6W+vn7m0qVLZ072QERERERERGRirV69mp6enskeRjWdVF+fOmPpiXOr3tHqtbvo6Rmoej8ydgqxnvs2Ll26dOaDDz442eMQERERERGRCXbmmWeycuXKjZM9jmpaeuJc7r/tQ1Xv5+yLPsfKh7dWvR8ZO4VYIiIiIiIiIhJf+S68/96a9CPxphBLREREREREROIr0Qjpc2rQz21AW/X7kTFTiCUiIiIiIiIiseX5Lrz/npr0I/GmEEtERERERERE4ivRCOlza9DPHagSK94UYomIiIiIiIhIfOW7yPVVvxJLa2LFn0IsEREREREREYmvRCOJzIoa9HMnqsSKN4VYIiIiIiIiIhJf+S5yfXfXpB+JN4VYIiIiIiIiIhJfNavEugtVYsWbQiwRERERERERiS3PdzJYg0osz3dWvQ8ZH4VYIiIiIiIiIhJbZk0k0tWvxDK7G9hf9X5k7BRiiYiIiIiIiEhsudeoEstViRV3CrFEREREREREJL6siWTmvBr0cw+qxIo3hVgiIiIiIiIiElvunQz03VWTfiTeFGKJiIiIiIiISGyZNVJXg0osUyVW7CnEEhEREREREZHYcu+ivyZrYnVVvQ8ZH4VYIiIiIiIiIhJbZo2kVIklQGKyBzBWZrbAzP7DzLabWZ+ZbTSzL5nZjFG2MzM6bmPUzvao3QUV9v+smd1sZlvMrMfM9pnZQ2b2cTObNUQ/ZmZvN7NbomN6zGyDmf3UzE4Y7fmLiIiIiIiIiBxOpmQllpktAe4CjgB+DawBzgHeC1xqZhe4e+sI2pkVtXMC8Hvgx8BJwJXAK8zsPHd/uuSw9wErgRuB3UAjsAL4BHCVma1w9y0l/WSBnwGvBNYCPwI6gPnAhVH/60b3LIiIiIiIiIg897l30VeThd01nTDupmSIBfwrIcB6j7t/pfCgmX2BEDJ9GnjHCNr5DCFA+qK7v7+onfcAX476ubTkmBZ37y1tyMw+DXwE+DvgnSWb/4UQYP0j8DF3z5ccmxrBWEVEREREREQOO7WbTngvmk4Yb1MuxDKzxcBLgY3A10o2fxy4CnirmX3Ah4hRzawReCvQFR1X7KuEMOxlZra4uBqrXIAV+SkhxDq+pJ8lhEDtfuCj7u6lB7r7QKVxioiIiIiIiBzO8jWqxMqrEiv2plyIBVwc3d9QWtHk7h1mdich5FoB3DxEO+cB9VE7HSXt5M3sBkIg9iKgdEphOZdF94+WPP6nhLXHvge0mNllwNFAK/B7d39qBG2LiIiIiIiIHJbMGklnzq9BP/ehSqx4m4oh1onRfaU1pJ4khFgnMHSINZJ2iNo5hJl9EGgCpgFnAc8nBFj/VLLr2dH9NGA9ULz4u5vZ1wnTInNDjHVYZnZIhZeIiIiIiIjIVKc1saRgKoZY06L79grbC49Pr3I7HwTmFv19PXCFu+8p2e+I6P6TwE3RcRsJC9F/g7B+1h7CwvAiIiIiIiIiUkSVWFIwFUOs4Vh0P97KpCHbcfd5AGY2FzifUIH1kJm90t1XFu2ajO53AH/i7j3R3783s9cSrnT4fjP7jLv3j3Ww7m7lHjezB4EzxtquiIiIiIiIyGTKexc9vVoTS6ZmiFWokJpWYXtLyX5VbcfddwG/NLOVhKmJ3wdOKdqlLbq/vijAKhz7iJltAJYAS4FHhhmziIiIiIiIyGElYY1kalCJlVAlVuxNxRBrbXRfdq0qDl4dsNJaVxPdDgDuvsnMngCWm9lsd99b1M9LqfxJKIRc9SPpR0RERERERORwkvcuenR1QmFqhlh/iO5famaJ4isUmlkzcAHQA9wzTDv3RPtdYGbNxVcoNLMEIXgq7m8k5kf3xYu03wz8Fc+uzir0k+FgWLZxFP2IiIiIiIiIHBYS1khWlVjCFAyx3H29md1ACJneBXylaPM1QCPwDS+6rICZnRQdu6aonU4z+wFwFWFR9Q8UtfNu4Bjgd+7+dEk7+919Z/GYotDrU4RF3O9y97aizdcBTwMvM7NL3P3Gom1/T5jOeGtpmyIiIiIiIiKiSiw5aMqFWJF3AncB15rZi4HVwLnAiwjT/z5asv/q6L508fOPAC8kLKy+HLiPsDbV5cBuQkhW7FLgc2Z2G7AeaCVcofAFwGJgJ/AXxQe4e7+ZvR24AbjOzH4JbALOBi4iXJnwqtGdvoiIiIiIiMjhwWpUiaWrE8bflAyxomqss4BPEoKlPyJc/e9a4Bp33zfCdlrN7Dzg48CrgAsJwdR3gKvdfWvJITcB3yRMWTwNmA50EYKzHwDXluvb3e+IxvtxQtA2HdgVtfWpMv2IiIiIiIiICKFCqluVWMIUDbEA3H0LcOUI9y2twCretg94b3Qbrp1VHFqdNSLu/gTwhrEcKyIiIiIiInK4Slgj9VoTS5jCIZaIiIiIiIiIPPflvYuu3rtr0o/Em0IsEREREREREYmthDVSn61FJda9qBIr3hKTPQAREREREREREZHhqBJLRERERERERGIrTCfUwu6iEEtEREREREREYixhjTRoOqGgEEtEREREREREYiznXXTWYGH3nCqxYk8hloiIiIiIiIjEVsKaaMyoEksUYomIiIiIiIhIjOXznXTUoBIrn++seh8yPgqxRERERERERCS2EokmGrPn1aCfe1AlVrwpxBIRERERERGR2MrlO+novacm/Ui8KcQSERERERERkdhKJppoqkElVjJxN6rEijeFWCIiIiIiIiISW7l8FwdqUomlqxPGnUIsEREREREREYmtZKKR5ppUYt0FtFW9Hxk7hVgiIiIiIiIiElu5fBftqsQSFGKJiIiIiIiISIwlEo00Z1fUoJ87USVWvCnEEhEREREREZHYyuW7aO+5tyb9SLwpxBIRERERERGR2EomGmmpQSVWMnEHqsSKN4VYIiIiIiIiIhJbuXwX+3tViSUKsUREREREREQkxpKJRqbVpBLrdlSJFW+JyR6AiIiIiIiIiIjIcFSJJSIiIiIiIiKxNZjvpq33vpr0I/GmEEtEREREREREYiuZaGBa9twa9HMrsK/q/cjYKcQSERERERERkdjK1agSK6dKrNhTiCUiIiIiIiIisZVMNDJdlViCQiwRERERERERibFcvot9PbWoxOqqeh8yPgqxRERERERERCS2kolGZtSfU4N+bkGVWPGmEEtEREREREREYmsw30Vrz/016UfiTSGWiIiIiIiIiMRWXY0qseoSf0CVWPGmEEtEREREREREYkuVWFKgEEtEREREREREYiuZaGRmthZrYv0eVWLFm0IsEREREREREYmtwXw3e3seqEk/Em8KsUREREREREQktuoSDcyqP7sG/dwEtFa9Hxk7hVgiIiIiIiIiEluD+W72qBJLUIglIiIiIiIiIjFWu0qsG1ElVrwpxBIRERERERGR2FIllhQoxBIRERERERGR2KpLNDJblVgCJCZ7ACIiIiIiIiIiIsNRJZaIiIiIiIiIxNZAvpvd3Q/WpB+JN4VYIiIiIiIiIhJbddbAnPqzatDP76reh4yPQiwRERERERERia1B72ZXT/UrsQZdlVhxpxBLRERERERERGKrdpVY11e9DxkfhVgiIiIiIiIiElsDNarEGlAlVuwpxBIRERERERGR2EpZA0fUn1mDfq6reh8yPgqxRERERERERCS2BvLd7OxeWZN+JN4UYomIiIiIiIhIbKUSDcxtqEElVuK3Ve9DxkchloiIiIiIiIjE1kC+h3l78iAAACAASURBVB3dD9WkH4m3KRtimdkC4JPApcAsYAfwK+Aad28bRTszgauBVwFHAq3A9cDV7r61zP6fBc4CTgBmAz3Apqjvr7p7a8n+xwAbhhjCT9z9jSMdr4iIiIiIiMjhpC5Rz9z6M2rQz/9WvQ8ZnykZYpnZEuAu4Ajg18Aa4BzgvcClZnZBaZhUoZ1ZUTsnAL8HfgycBFwJvMLMznP3p0sOex+wErgR2A00AiuATwBXmdkKd99SprtHCEFXqVXDjVNERERERETkcKVKLCmYkiEW8K+EAOs97v6VwoNm9gVCyPRp4B0jaOczhADri+7+/qJ23gN8Oern0pJjWty9t7QhM/s08BHg74B3lunrYXf/xAjGJCIiIiIiIiKRVKKBeQ3Vr8RKqRIr9qZciGVmi4GXAhuBr5Vs/jhwFfBWM/uAu3cN0U4j8FagKzqu2FcJYdjLzGxxcTVWuQAr8lNCiHX8yM9GRERERERERIYykO9me00qsXR1wribciEWcHF0f4O754s3uHuHmd1JCLlWADcP0c55QH3UTkdJO3kzu4EQiL0IKJ1SWM5l0f2jFbbPN7O/JKzf1Qrc7e6V9hURERERERERQiXWkTWpxPpN1fuQ8ZmKIdaJ0f26CtufJIRYJzB0iDWSdojaOYSZfRBoAqYRFnp/PiHA+qcK7V0S3YrbuAV4u7tvHmKcI2JmPt42REREREREROKmP9/Ntu6Ha9KPxNtUDLGmRfftFbYXHp9e5XY+CMwt+vt64Ap331OyXzfwKcKi7oWKrucRFoJ/EXCzmS0fauqjiIiIiIiIyOEqVGKdXoN+fl31PmR8EpM9gCqw6H68lUlDtuPu89zdgHnAq4HFwENmdkbJfrvd/Wp3X+nu+6PbbYRqsXuB44A/H+dYcXcrdyNcSVFEREREREREZEqbipVYhQqpaRW2t5TsV9V23H0X8EszW0mYmvh94JRh+sbdB83s34FzgYsIV0MUERERERERkSID+R62dVV/OuFAvqfqfcj4TMUQa210X3atKg5eHbDSWlcT3Q4A7r7JzJ4AlpvZbHffO4LDClMPG0fSh4iIiIiIiMjhJpWoZ37D8hr086uq9yHjMxVDrD9E9y81s0TxFQrNrBm4AOgB7hmmnXui/S4ws+biKxSaWYIw3a+4v5GYH93nRrj/iuh+JFc/FBERERERETnsDOR72Nr9SE36kXibciGWu683sxsIIdO7gK8Ubb6GUNX0jeKF0s3spOjYNUXtdJrZD4CrCIusf6ConXcDxwC/c/enS9rZ7+47i8cUhV6fAo4A7nL3tqJt5wIPuXt/yTEXA++L/vzhKJ4CERERERERkcNGKlHPUTWpxPpl1fuQ8ZlyIVbkncBdwLVm9mJgNWFtqRcRpv99tGT/1dG9lTz+EeCFwPvNbDlwH7AUuBzYTQjJil0KfM7MbgPWA62EKxS+gLCw+07gL0qO+SywzMxuAbZGjz0PuDj6579397tGctIiIiIiIiIih5v+fA9balCJ1a9KrNibkiFWVI11FvBJQrD0R8AO4FrgGnffN8J2Ws3sPODjwKuACwnB1HeAq919a8khNwHfJExZPA2YDnQRgrMfANeW6fsHwJ8AZwMvB1LALuCnwFfd/fZRnLqIiIiIiIjIYSWdqGdBDSqx0olfVL2PsTKz44FXAy8jrOE9F2gjLJX0JXcfzVJIU9aUDLEA3H0LcOUI9y2twCretg94b3Qbrp1VHFqdNdwx3wa+PZpjRERERERERCToz/ewuevRmvQTY58C3gA8AfwW2AecCPwx8Mdm9l53v3YSx1cTUzbEEhEREREREZHnvlSigQWNp9Wgn/+ueh/jcD3wWXd/qPhBM3sBcCNh6aOfufuOSRldjSjEEhEREREREZHYUiUWuPt3Kzx+a7QG9yXA+UCsk7jxUoglIiIiIiIiIrGVTtRzdEP1K7HSiZ9XvY8qGYjuByd1FDWgEEtEREREREREYqvGlVhnmJmX2z7UetuTxcwWAS8GuoHbJnk4VacQS0RERERERERiK52o5+jG59Wkn6nEzDLAfwIZ4MPu3jbJQ6o6hVgiMuHcna6+fvoHc+TyecyMukSCxmyaVDI52cMTEREREZEppD/fw6aux2rSD7DS3c+sRvtmthFYNIpD/tPd31KhrSTwA+AC4CfA58c9wClAIZaIjFv/4CDrd+9j/a5Wdh/o4kBPL/0DOfI47mAABklL0FyfZnpDPYtmz+D4ebOY3dw42cMXEREREZEYSyfqWfjcqMRaD/SOYv/t5R6MAqwfAq8Dfgq8xd3LToF8rlGIJSJj1trZzW1rNvD0rlb6BnM0ZFKkkslnbuXk8s6eji62tO7n1tXrmdXUyIrjF7J0/hHUJRM1PgMREREREYm7vlwPGzurX4nVl6vu1Qnd/cXjbcPM6oAfEQKsHwFvc/fceNudKhRiiciobWtr57pH1rH3QBfZVB3ZdIpsOjXi4xNmNGTSAPQNDvLbh9dw06qnOG3hPF6wdDHJhMIsEREREREJ0skaVWIl470mlpmlCZVXlwPfB6509/zkjqq2FGKJyIgN5vLctOpJHt60g8ZMmub6zLjbNDOasqGdBzZsY/X2Pbz27FOZO71p3G2LiIiIiIg8F0SLuP8C+CPg28BVh1uABQqxRGSEtrW184v7H6e3f3BCwqtyGjNpcvk837n9Ac46dgEvOllVWSIiIiIih7v+XC8bu1bVpJ8Y+zdCgLUX2AZcbWal+9zi7rfUeFw1pRBLRIb1xLbd/PrBx2nKZGjIjHza4FgkEwmasxke3LCNHW0HeOP5p+mKhiIiIiIih7F0MsuixlNr0k+MHRvdzwauHmK/W6o/lMmjEEtEhvTwpu1c98g6mrMZyiT9VdOUTbPrQCc/uP0h3vL800nXKcgSERERETkc9ed62XCYV2K5+wsnewxxoBBLRCpas3031z2ylpb6yfk/EvXpFG1d3fz47kd40/nLdfVCEREREZHDUDpZzzE1qcSK98LuohBLRCrY19nNrx58guZsdda/GqlsOsWu9g6uf2Qtrzxj6aSORUREREREaq8v18PTndWvxOrL9VS9DxkfhVgicoh83vnv+1dRn0rVdAphJQ2ZNI9t3cWpC49k0ezpkz0cERERERGpoUyinmMbT6lJPxJvCrFE5BB3PbmJts4eGrPpyR7KM5qyaX794OO848UrtD6WiIiIiMhhpC/fw/rOx2vSj8SbQiwReZa2zm7uWLdx0qcRlkqY0dM/wA2PreOVp2taoYiIiIjI4SKdqOfYpupXYqVViRV7CrFE5Fl+v/ppsql4fjXUZ9I8sW03Fy87joZ0arKHIyIiIiIiNaBKLCmI5y9VEZkUfQODbNi9j/oYB0QG3L9+Cy9YuniyhyIiIiIiIjWQSdSzuAaVWFoTK/4UYonIM1Zu3E7OfbKHMaSGTJpHNu/gwhOPJZGY/EXnRURERESkuvryvTWqxOqteh8yPgqxRAQAd+fBjdtonKAqLHcnl8vjecfMSCSNRCIxIW139Q3w9J5Wjps7e0LaExERERGR+MoksjWqxMpWvQ8ZH4VYIgJAZ28/Xb19NI1xQXd3p6uzjz272unu6qO/b5B8Lo8TpgBikErXkcmmmDGziVlzmkkmxxZqNWZSPLZ5p0IsEREREZHDQF++l6c6VIklCrFEJLJjfwe5/OinEubzzt7dB9i5rY2+vgHq6pIkEhbdks/a1/NOT1c/B/bvZuumvUyb3sCCRbPJZEdX/ZVMJNjT0T3qsYqIiIiIyNSTSWRZ0rSsJv1IvCnEEhEA1u9qHfWC7j3dfTy5Zgf9vQOk0nWk08N/pZjxzH4H2nt4bOUmjjx6BvMXzMRs5Gtcdfb2kc97TdfFcncGcjkG8nncnWQiQSqZpG6CpkmKiIiIiMihevO9PNn5RE36kXhTiCUiAOxs7yBdlxx+R0KYs33rPnZsaaMulSA1gvCqnETCSKST7NjaRtveTk5YdtSIgjCA/sFB2rp7mNXUMKa+R6Kzr58tbft5ancruzo6OdDbR38uB3BwmiTQmE4zvSHLwhnTWTx7BvOntZBUsCUiIiIiMiGyiSzH1aASK6tKrNhTiCUiAHT1DzCSmiZ3Z/PTe9iz+wCp9MhCr+GkUkkGBnI8/vBmTj7taDKZ4SvCcu7s6+ye8BDL3dm0bz+3PrWR7fsPkPM8DakUqWTymVs5bV097Njfwe1PbaAhnWbZkXO5YMlCmjJjW2NMREREREREnk0hlogAkMvlqRvBQutbN7WGACs1MQFWQSJheN554tEtnLJ8IanU0F9PCTP6BgcnrH9354HN27l7w2Y6evtoTKdoyqRHfLyZUZ9OUU8I4B7ZtoOVW7czf1oLr1x2ArOaGidsrCIiIiIih5PefC9Pdmg6oSjEEpGIM/yi7h3t3ezc3jbiKX+jZQnDc3nWr93JicuOGnKNLDNjYDA/If0e6O3lZytXsbujk8Z0mpYxXqGxWH0qhFltXd184877Oe/Yhbzg+GNJjGLdLxERERERgUyivibTCTOJ+qr3IeOjEEtEAIYNV3K5POvX7Ry2Qmrc40gm6OzoZe/uA8yZO63ifu5Oqm586065O/dt2sIf1m0gnUxWZepfMpGgOZPh3o1bWLNzD6874xRmqypLRERERGTE+nI9rKtBJVZfrqfqfcj4KMQSEYBhFyLfumkvuZxTV1f9SqJUKsnmDXuZObuZZIUpju5Otm50V1MslnfnFw8/zrpde2megMqr4TSm0/QNDvLNO+/nDWeeypLZs6rep4iIiIjIc0EmWc9xzSfXpB+JN4VYIpNgMJdnX2c3W/fuZ8OuNrr7+snlwnS+RMJIJRPMnzWNxXNnMmdaE9kqTd8r1phJ09nbV3YKXz6fp3VPB3XjrHwaDXdnz8525h01o+z2hCWY1Ty2Rd1z+Tw/efAxNrftr0mAVZBMJGhKp/nxA4/x6uUns3TeETXrW0RERERkqurL9bKuY3VN+pF4U4glUiMDuRxPbN7FA09tpb2rl8FcHjOoT6cOqYJyd3a2dXDPmk0kkwka0ilOPGoO5564kKb66oQu86c3s2pbN5m6Q78WWvd0kM85FS7MVxWpVJJdO9uZO3962WAtXZdkesPoL4HrUQXWlrZ2GtMjX7h9opgZzZk0v3j4cd54Vh1LZs+s+RhERERERKaSTDLL8TWpxBr97wupLYVYIlV2oLuXO57YwLpte+nP5WjKpKlPDz0NzszIplNki/Z7ZOMOVj69nfkzW3j+ycewcE75cGesFs+dxQMbt5UNsXbvaCeVrmGCFenvG6Cnu5+GxkODu5b67JjO/w9PbmD9nn00juLKgxMtBFkZfvrgY7zzonOZVq9/WYqIiIiIVNKb62XtgepXYvWqEiv2FGKJVEk+79yxegP3rt1MKpkkk6ojM45F0QvBV1tnNz++/REWzpnG5ecuo2GCwpgjpzeTLBMKuTt9fQMV16aqJjOjo73nkBBrMJdnwYzRTyXc3dHJPRs201yFBdxHy8zI1CX52UOP8f/OO2tCA0kRERERkeeSbDLLCTWoxMqqEiv2FGKJVEHrgS5+cfcqDnT30jTBay4lEwla6jPs3t/Jv113D5ecfjynLJw37hCkMZOmpT5LLp9/Vlt9fQPkc/lJCbHq6pK0t3czd/70Zz3e3T/A8mPmj6qtXD7Pzx5aRUNq7IvBT7RUMsmeji7u27SFc49ZONnDERERERGJpd5cL2trsCaWKrHiTyGWyAR78Kmt3PzIUzRkUhNWJVVOuq4Od+f6B9eyduseXrXiFOrGGTSdvWQBN6966lnBW09XP+7jHe3YmEF/78CzHnN3muszLJw1vcJR5d3y5NN09PZNyjpYQ2nKZPj9ug2cNPcITSsUGQN3Z3tHBw9s3cquzi46+/vJu5M0oymTYUFLC+ccvYBZDWO7EISIiIhMvkwyywnNS2vSj8SbQiyRCXTHExu5a/VGmqu0+HopM6Mpm2HT7v38160P8caLlpOqG/vaVc87+khufeLpZz2WGwwL0E+WfEmC1t3Xz4UnHjuqyrP+wUEe3Lw9dgFWQTqZ5Ka1T/Ga5adM9lBEppSnW/fx23XraOvuoT5VRyqZJGn2zNTo7v5+Htu5kwe2bWNuUxOXn7yUuU1NkzxqERERGa2+XC9rOtbUpB+JN4VYIhPk7jUbuWtN7QKsYg2ZFHs7uviv2x7mTS84fcwVWem6JMfNm83Tu1tJRwu8O5NUhlWBJRKccexRozrm4a07GczH6zyKpZNJ1u/ZR//g4DPPu4hUNpjP85vVq1m1azfN6TQtQ0zbztTVkamro7Ovj2/d9wDnLTyai5cs1jp0IiIiU0gmmeVEVWIJCrFEJsSjG3dw++MbaJ7E6WDZVIo97V384u7HeN0FzxvzD7SLT17C2h17SEffDnV1yUmbTgiQKDqP7r5+Tls0f1QL5Ls7923aQtMwV4ScbLm889DW7VobS2QYg/k831/5ELs7O2kZxUUakokEzZk092zZyoG+Xl518skKskRERKaI3lwvqw9UvxJLa2LFn0IskXHq6OnjxoeenPAF3MeiIZNi4642Htmwg+WLR7fweUFzfYaLly3h5lXracqmqW9MT9oPPXdIZ0P4lMvnacikufjkJaNqY1v7AQ709NEcg9dnKI3pFPdt3MY5i47WD2uRIfzi8cfZ3dlJ/Rgv0tCUTvHE7j1My27g4iWLJ3h0IiIiUg3ZZJaTalCJpasTxp9CLJFxcHd+dc8qUslEbIKHpmyamx95kuOOnEXTGKc2nnnMUazasov93T1kMikSyck5t8HBHC3T6gHo7hvgiovOHPVUyYc2byc7isqtyWJmdPT10dHXR0tW//IUKWfN7j2s3bN3VBVY5TSl09y1eTOnzj2COVojS0REJPZ6c301qsTqq3ofMj7x/2UnEmMPP72dnW0dsajCKjAzUskkv7rncd78wtPHFK6ZGa85+xS+fvM91KUTpDMpcoO5Kox2aO5Oy7QGuvr6OePYo5g3vXnUbezu7CKVHPti97WUzzvb2zsqhlj9uRx7O7vY0LqPja376ejtJ5/Pk4vmeyYMkokkzdk0i2ZO59hZM5nT3Eh6ipy/yFDy7ly3bh3NE3SBhoa6On61eg1/cfZZE9KeiIiIVE82meGklpNq0o/Em0IskTEaGMxxy6r1NGbid8W7VF2Sba3trN/ZynFHzh5TG831GV5/7qn8+O5HOWJuC5s37CWVrm0Yks6koM5YNHsGL1l23KiPd3fae3qnTIhVn06xfk8rJ82d88xj+3t6uWP9Rja2ttHZ389gLk8qkSCbSj1rvTCAvEM+l6O1s5utbe384ckNpBIJmjJpFs2YzoXHH8v0SVy3TWQ81re20tnfP+4qrIJkIsGujg7aenqYUV8/IW2KiIiISHUpxBIZo8c27WAwlyebisc0wlJN2Qx3PrFxzCEWwDFzZvLqs5fx8/tWYWO74OGYDQzkmD1/GkdOb+G155xCIjH65/lAbx+9g4NTJsRKJ5PsPNBJ3p31e1q5ff1Gdh3oJJVMkqmrozGVhhEsA2RmNKTTNER/u8Pa3Xt5bMcu5jY3ceFxx3DcnFmHhGAicXbv1q00TVAVVkEqkeSBrdu45PjRh+QiIiJSO725Pp44sLYm/Ui8KcQSGQN35/4nt8ayCqsgkTB2tXfS3tXLtMaxV9+ccOQc3njeaXxhyz4693WTTtUmEBogzxlLF/L6855HMjG2BK2tu4fBXH6CR1Y97s5Te1q59g930TUwQFM6TdMEVZ1k6urI1NXR2dfPz1Y+RkM6zQuPX8zyBfNis56byFD2dfdMePCaqUuyub19QtsUERGRiZdNZllak+mEmrUQd1M2xDKzBcAngUuBWcAO4FfANe7eNop2ZgJXA68CjgRageuBq919a5n9PwucBZwAzAZ6gE1R319199YR9Plt4M+iP49396dGOl6Jhx37Omjv6qV5jAun10o6meSu1Rt5+Vnj+8JffMRMPvbmS/j4v/0fvblBMlWsbMq509s/wB+ffwqvP/+0cQUsvYODU6baqH8wx+pdu9nf08ORLc0TNmWqVMKMlmwWd+e6x9fy6PYdvGb5KTTFOJAVyeXzdPX30zDGKxJWYmZ09un/uIqIiMRdT66XJ9qrX4nVk+uteh8yPlMyxDKzJcBdwBHAr4E1wDnAe4FLzeyCEYZJs6J2TgB+D/wYOAm4EniFmZ3n7k+XHPY+YCVwI7AbaARWAJ8ArjKzFe6+ZYg+LyMEWJ2ALok0Rd27bjP16Yn9MVUNmVQdT+7Yy6Xu4662mTejhY++6SV87Ye3sDPfS8oSEx4Q9efzZElw8ZIlvP6SM8c95oFcjrhnWO7Oro5O1u/ZhxmkknU1qYwyM5qzGfZ2dPO1W+/mpUtPUFWWxNZgPk/eq9N23qvUsIiIiEyYbCLL0pYTa9KPxNuUDLGAfyUEWO9x968UHjSzLxBCpk8D7xhBO58hBFhfdPf3F7XzHuDLUT+XlhzT4u6HxLNm9mngI8DfAe8s15mZzQG+BfwEmAe8YARjlBjae6CLumSNF4kao76BQTp7+yekamzhojlc/vxTufe+J9noPewf6CcBpMY43Q/CD8h+z5NJJFmcbWZBtpFXv+acCQlTUskkcf59mnfniZ27aOvqJZVMYGY1/UHdP5ijo6+Pzr4+vnHHvcxqauCCJYuor0vRkK7jqOnTOKKpkXTdVP1XhTxX1CUSmFXns6HgVkREJP568708fmBdTfqReJtyv0zMbDHwUmAj8LWSzR8HrgLeamYfcPeuIdppBN4KdEXHFfsqIQx7mZktLq7GKhdgRX5KCLGOH2L434zu3wX89xD7SYwN5vJ09PRNiUosgFze2dnWMWFTH8+76EQG+gdpeGwryeY6NnZ3sLOvm37Pk8SoMxv2R2HOnQHPk8BorktxSkMLLRYqkF771guob5i4daDyxDPFyuXzPLptJ10DA6TrDk7PrNYPaneno6+P3Z1ddPSFBe8Hc3ncnUT0mu3t7mZzWzsnzzsCLFS/1EVXN5yezbJo1gxOmz+PabrCodRYMpGgITXxU17dfcLWnRMREZHqySaynKxKLGEKhljAxdH9De7+rBWb3b3DzO4khFwrgJuHaOc8oD5qp6OknbyZ3UAIxF4ElE4pLOey6P7RchvN7ArCult/4u6t+j+/U9e+zm4GcjnqR3KZuBioT6dYv7OV4+eP/SqFxcyMi16yjIbGDA/e/RTH1bdwfOM02gf62DfQz/7BPnpyg/Tlo4+nAe54FHBlE0lmpFLMTmeZkcqQSSTp6uqjYUaWy153No1NE/cvjmn12XFViVVLLp/n4a07wpUTS8aXmuAKv1w+z84DnWxtb6d3cJC6RIKkGQkz0iVrm9UlEgzkcqzZtYflC458ZkF997Co9rb2A9y+fiNHNjdx4ZJjWKIrHEoNzcjWs6+ne0Lfc/25HEe1tExYeyIiIlIdvbleHm+vQSWW1sSKvakYYhXi10rv4CcJIdYJDB1ijaQdonYOYWYfJKxpNY2w0PvzCQHWP5XZdxFheuIP3f1XQ4xpzKxa8yzkENta20kwdX64p+uS7N7fOaFtmhlnn388S06Yx3W/WklHezfTGzPMSB8MoHLu5N3J4SSABEaypEprYCBHV3cfZ563hDNXLCExwYHTjPrss6qc4iDvzqPbdz0TKJVua0xPTLVJ7+AgT7fuo627h7w7qUSC7AimBdYlEvQNDvLotp2ctuDIZwIDM6MhGtuBvn5++vAqGtIplh91JBcsXnRIICYy0c5aMJ9fPP7EhF70oC+X46yj5k9YeyIiIlId2WSNKrF0dcLYm4oh1rTovtI1sQuPT69yOx8E5hb9fT1whbvvKd7JzBLA9wgLub9nmDHJFNDR0zdl1sMqGMzlqtLuzNnNvPHKC1n96BYeeWADB9p7yGZT1KWSJC2EVqX1avm809PTRzKZZNHiOay48ERapjdUZXxmRksmS+/gYFXaH4u1u/bQ1ddftuJqIJdjZuP4ngt3Z2t7Oxv37SdpdkhQNhJ1iQTdAwOs2bmbk4+ce8j2hBnNUZBwz8YtPLx1B68+bRmLZg73tSsydifNmUN9XR0+AReqgBAaz2lsZE6TrrEiIiISdz25PlbVoBKrJ6erFsfdVAyxhlP4L9vxViYN2Y67zwMws7nA+YQKrIfM7JXuvrJo1/cRFnB/hbu3jXNMFbl72f+iN7MHgTOq1e/haGAwN+UWAs5V67JeQDKZ4JTTF7Fs+UJ279zPqoe3sG9PBx0HehjozwEOZiQSRn1DmpkzGznmuMWcuGwB6XT1v4LmNDeyYW9bLILH1q4u9nR2k6lQHWZmNGfGXonVOzjI4zt30d0/MO7KqFQiQWt3D61dXcxqbKy4X2M6Td6dHz7wMKfMm8vLl52gqiypimQiwYuXLOH/1q4b1+ekoKt/gNedesoEjExERESqrT6ZYdm0spOkJrwfibepGGIVKqSmVdjeUrJfVdtx913AL81sJf+fvfuOjuw87zz/fW+sCBQy0I1G59zNZmhmSVSyJEuUlezVjmVpJDmsx/axz3o858x4Zx3GY+/Mmdlgj2f2eLy2PJKcZAXbkiyKpKhAskk2yQ7sHNFooJFRCJVvevePArrR6AZQCQWAfD/n8OAQuHXfi0IBjfvD8zxvsTXxi8ABACHEToo7JX5BSvlPy1yPsk6IddRKeFMdLlkIQUdXEx1dTUCxIqhQcPG9AF3XMEwdYxVa++7Z2MmZoREaVrk02AsCLoxMYC0Rppm6Tsgs/8dysfpqhmvJSQyh1SxEsnSdCyPjPLQlvGRF11xl1vnRUa5OJPnEvfvpaVJVWUrt3buhi1MjIwynUthV7JqZcRzu29BFd+NivwIoiqIoirKWFCuxLi1/YA3WUda29RhiXZh9u1gMO7c74HK1hrU6DwBSyj4hxFngXiFEq5RyomSWaQAAIABJREFUHNgP2MDnhBCfW+Shl2arej62UvOylNqyTJ1Arq8RZLqofxWSEIJQqPa7iZVra0vTmth97MLIGFIGCO3uAZPr+2xqSpQdkkopuTw+wXAqXfMKqLkruTAyxv67tBUuFDaLVVlfevUEnzi0nz0dbTW9HkURQvDJew7y56+9TqpQqCjIyrouPYkEP7575f+aqyiKoihKbahKLGXOegyxvj/79n1CCG3+DoVCiDjwOJADXl7mPC/PHve4ECI+f4fC2TlW71uwXinmpsPODSC6BvzZIsd+COgE/g6YmT1WWQeaouHZGVPrY3dCKeWaG25eT0II7t+0gRev9tVsaHq5kpksE5ncom2EAAjYkIiXdV4pJefHxhlPZ1ashc/QNJIltBXO0YQgZll87cRpnjywh0Mbu1bkupS3Ltsw+PzhB/jKG6e5NjVJ3LJKavGWUpJyHO7p7OTJPbvVzpqKoiiKso7k/AKnptRMLGUdhlhSyitCiKcphky/DPyXeR/+XSAK/ImUMjP3TiHEntnHnp93nrQQ4kvALwC/A/zLeef5FWAL8F0p5dUF55mSUg7Pv6bZ0Ov3gHbgyNzsKynlCeDn7vZ5CCF+QDHE+k0p5eXSnwFltXU1N8A6uvkpuD4bWt7aW8g/uLmbI719q7b+1YnJJdsIAylJhMOYi1Rp3Y2UkotjEysaYM2xdJ3e8cmSQiyYne0VCvGtM+exDUNVZCk1ZxsGP3PfId4YHuaZy1fIOQ4xy7prMOUFAVnXpcEO8TP3HmJrc/MqXLGiKIqiKNUI6TYH6lCJFVKVWGveuguxZv0ScAT4IyHEe4BzwMPAuyi2//1vC44/N/t24W+3vwm8E/h1IcS9wFFgL/ARYJRiSDbfB4D/JIT4EXAFmKC4Q+ETwDZgGPj5Kj83ZY1LRMPrqrKp4Hls62hZ7ctYVSHT4EBXJ6cHR4hY9a2gSxcKZB13ySosLwjY1tJU1nmvTU4xmq59C+HdCIotWOlCoazWzJhl89UTZ/j0g4fY3Fze56coyxFCcKiri4OdnVwan+DVgQEm83nShQISiYZGzLboiMd4rKeHTY2N625TDkVRFEVRivJ+gVN1mImVV5VYa966DLFmq7EOA/+OYrD0QWAI+CPgd6WUyRLPMyGEeBT4beCjwNspBlNfAH5LSjmw4CHPAv+dYsviISABZCgGZ18C/qjUtZX1S9MEDeEQBddb7Uspia5pdDSpLeTft3cHl8cmCKSsaxvR9eQ0hrb4eq7v051oJFJGq2MqX6B/cnrp9sQaM3WdvuRUSbOx5szttvi1k2f5lbc/jFXFIG5FWYwmBLvbWtnd1goUqxTnvs9VaKUoiqIobw6qEkuZs27vKKSU/cBiw9IXHrvob7GzodOvzf633HlOc2d1VkWklO+sxXmU1dHVFOfi4Ni6uCmPhkzCda4+WotMXefj9+7ji6+coCFUn3+cvCBgMpdbdGc/KSW2YbClpfSd/AIpOTs6tmR74krQhWAql8cLgiV3KlxICIHreXz77EU+ds++FbxCRSkSQqCr8EpRFEVRFOVNae3fgSvKGvTIns2c7huuKMSSUlLwfdKOw0Q+S8Z1CaREIkEWb8AExZkvzaEQjXaIsGFWVD2UcxwO79xU9uPerDY1Jbi3u4tTg8N1GfI+PD2z5E6Wju9zeGNnWTsSXplI4no+Zp1DLCi+doemZ9jUVHroBhA2Tc4Mj3Dvxk62tqh5RIqiKIqiKEp5cn6BN6ZWvp1QDXZf+1SIpSgVaIqFaUtEyeTdksIlPwgYyaQZyqTJ+x5eEBBIialp6AurWmYzj3zBYzyXASnQNEFIN0iEQvTEGwmVGJ5JCYd3qBBrvg/s28lEJsvwTJqwubI/AicyOcxFqpYKnsfujray2wiHplN1bSOcz9R1JjK5skMsgLhl8403zqm2QkVRFEVRFKVsYd3mYOPOuqyjrG3qTkJRKvTYni38/ctniIcX/0GX81yuTU0xkc/iz4ZWmhAlDePWhMDWb32L+nI2CEuniFkWWxuaaAqFFp354vkBm9oSRGzVSjifrmn8s8P38MVXjjORyRI2V+75yXvuXb8+Bc9nR1sLHfHyZpVdGBuvexvhfAIoeG5ljxUCx3N5/so13rN7R20vTFEURVEURXlTK1ZiXa7LOsrapkIsRanQjq5WIraJlPKOoCLtFDifnCDtFtCFhqFp1KJ2Zm4WUd7zeGNsBEvX2NLQRFcsdsc1ZB2Xd+zbVoNV33xMXeczD9/H37z2BjemZojatW8tdH0f1w8w5wWWEnA8j90drXTE42WdL+s4y+5yWA+uH+D6/m2fV6kipsXJwRHetWt7XYfrK4qiKIqiKOtbWA9xMFGPSqzQiq+hVEeFWIpSIU0TfOjBPXzlhVM0zFZjBVJydWqSgdQMpq7dVklV07WFwDZ0pJRcnBpnOJtmf2vbzfVyjsv+ng46m8sLSt5KTF3npx88xPcvXuVo3wAR07yztbMK6YJz2zwsNwgwNMGh7i4aQ+X/43htcmrJXQ7rJZCSdMGhKRKu6PFZ1+XC6Dh7O9pqfGWKoiiKoijKm1XOz3OyLjOx8iu+hlIdFWIpShW2drSwp7uNq0MTeAScHh/D8b26VcuI2ZbDrOvwyuAA2xPNdEVjmIbO++9b+S1o1ztd03jvnh0c3NjJV4+fJpV3iC6yk6PreGTTOWYm0jgFDykluq4RiYeJJyKEojbavBBsMptDF9rN6qvOhjjb25rRRflBmR8EJLOLz9eqJ0PTSGazFYdYMcvixSvXVIilKIqiKIqilCysh7hHVWIpqBBLUar2gft28ZuXn2YgPYOpaRW1WVVL1zQ0Kbk0NcH1mWn+7ZPvwlzltrP1pCMe4xff9hA/vHyN4/2DFDyPmGXhez6jA0nGbiRxndngytCLrXBCIKVkYniquKukLohEQ2zY1k5DS4yZfAFP+tiazt4Kq6/mDM3MIJfY5bCedE0j61Q2FwuKVYQj6QzJbI7mCoMwRVEURVEU5a0l6+U5ObnyM7GynqrEWutUiKUoVZBS8tzlXuyYgZgpthiuFiEEmhQ0REM8d+Uqm1oTKzq0/M1G1zTevWsb79ixhVPXh/jr77xE//AkAknYMDHvUqElhECzblVHZXMOp070YlsG4Y0JDvd0Ey1j98HFjKQya6IKC4rD3asN1AxN40T/IO/evb02F6UoiqIoiqK8qYWNOlViGaoSa61TIZaiVEhKyTdPn+fs0AhdiQZs3eBs/wi2YRTv9OvM9XxaG6Ls3NBKpuDwZy+9xs8+elgFWWUavTbGyb86yk5HsjXcwljgkMQlI30cAgKgOKJd3HxrACF0mnWLNiNOCwav9E4wmHLZur8bo4qqOCklec9fE/Ow5lRbFBYyDK5PT9fmYhRFURRFUZQ3vZxX4EQ9dif01O6Ea50KsRSlQk+fu8SZoVFidnGoe3M8wt5N7ZzvH8My9LoGWa7n09oYY2dXC0IILMOg4Hr8xcvH+NlHH8Ay1Ld6KU786Bwv/dNJIg1hQpHi1zWmG2yd/bgvJQUCAiQS0AAdgY12x+6Qlm2Rmsxw+shF9j64HTtcWUVW3vPwZYBRk/0ta6TK17YQgumcKtVWFEVRFEVRShM2bA4ldqz4Ol837BVfQ6mOurNVlAq82tfP6/2DxEO3/5BriUc5sFnj3MAYBMXB3ytJSonrB3S3NNLTlrgtSLEMg3ShwF+//gafeei+O0IW5XbHfnCWo0+fIpaILHqMLgSREsMkTQh0Q0cGAWeOXmb/wzuwQ+UHWal8gbUxDeuWWrySso5LznVVpaCiKIqiKIqyrJxX4MTklbqso6xtKsRSlDJN5/I8e+HqHQHWnMZomMM7urk0NM7ETGbF2gtdP8AydO7t6SS6SDgSMk0Gp2d47foNHtzcXfuLeJO4eqqfV777BrHGxQOscukIAinRNA0tCDj/6lUOPLar7GBzIpvDqmBHw5UiKc4Pq5bnB4zMpNnS0lT9RSmKoiiKoihvamG9PpVY39BVJdZap0IsRSmDlJKvHj+NvcyMI0PX2NvdTjKV5dLgOJ4X1KzF0PMDpJR0NTewub2puFPeEmK2zfcuXGF3RysNVeyQ92aVzxb4/tdeIdpQ253ymjEZw8FGIDQNz/PoO3+Dbfs3lXWenOuu6oYBC7m+TyJc/evI0nWuT06rEEtRFEVRFEVZVtYvcLweuxP6qhJrrVMhlqKU4WhfP6Pp9M05WMtpjkd4cOcmxmYy3JiYJldwMHS97FBCyuLcK0PX6GyK093SiFnGsHDL0Pnq8dN87pEHVFvhAs/+7UuAqPnz0qyZtw1ANwyD5Mg0rRuaaGiKlXyeoNop6jUmpaSxBiGWrgmyrlODK1IURVEURVHe7CK6zb1NK1+J9Q+qEqssQojecg6XUm4RQmwFnpv7/3LXVCGWopRoJp/nuYu9JQdYczRN0JGI0ZGIkck79I9Pkck7OJ6PHwRICXfLT+RsK5qp64Qsg+2dzTTHIxWFLaauM5JKq7bCBZIj0wxcHCZaZhuhlBIpJUIsHn5F0e+YnmVaBtfPD3Hg0dK3B5ZrLMTSNI2IVf0cKyEErhfU4IreeqSUN8NNbYnXoKIoiqIoyptFsRJr5WdiqUqssm2i9H6juePM2cdVNKNEhViKUqJnzl8ptgRWIWKbbGxpYCKdZTqbJ5V3yOYdXN9HBrM73gmBrgnClkXENomHbZpjYRLRcFU3qzHb5oUrfTzQs3HZFsS3ilefPYW1zK6BQSCZSaaZHJshlyngFDwCfzZ8EQLD1LFsk3giTEtngtDs+TQhCKHj3zaWXZDPFcim80Ri67O109J1tBrM6BKI2T0eleXkPZdTE8Ncmp5gIp8h5RZuC7Hipk1LKMrOxhYOtnQSMtSwfEVRFEVR3lzqVYn1j6oSq1zbKnjMlQofB6gQS1FK4ngeV8YnKt5JzfMDhqdSDE7NUHA9dE3D0DQsQ8eKLT6LyZcByXSWkekUhqbRHIvQ05ogXGElTNZ1uTg6zp6OtooeX09SSma8DMO5JIO5UQqBiyE0usKtdIVaabIaqgr1nIJL/4Vh7MjdQyzX8Ri+Ps7EyAy+56MbOpqu3fxvvkLeITuQZ6hvgkgsRGdPC01tcaJCYyrwbgsNdUNn8OoIO+7ZXNJ1CiFgjVRjBVISq0EV1ty5TL26UPjNbiST4ns3rtA7k8SXAVHDQhOCsH7718ALAoYyM1yeHueZ/ktsaWjmPRu30xGNr9KVK4qiKIqiKG8FUsrrFTzGB8p+3BwVYilKCU4MDOMF5QcJnh9wZXSCiVQWOXvTbpvlfdtpmsDWio9JZrKMzWSIhix2dbUStZeuIlooZlu8cOXamg6xAhlwfuYaJ6YuMeNmEEhszUYTGhLJ1fQgkoCwHmJvw1bubdqFqZX/o2y0fwLHce8IsaSUTAxP0X95FAkYpn5HaLWQmK3IwtRxXY+rZ28Qidl0HuhgzHCw5zUWappGZiZf8nWupVYx1/fZmGioybkCKbFUiHVXbuDzT30XODk2SNgwiJRQWTVXkQVwIz3Fn5x5hUNtG/jg5t2YmnqeFUVRFEVZ3yT1+bvu2vjT8ZuPEKIFiEkp+6o9lwqxFGUZUkqO9vWXXYGSTGe5ODROICXGMiFIqXRNQ9eKlWHHewfpbmmkpzVRcnugJgSj6QyT2RxNkdruxlcLU4UU3xk5woybIaKFiBkLr1EQMW614Z2YusDZmau8p+MhuiPtZa3Vf2kYy15Q0eL5XDk9QHo6h2HpFQVIQgiskInjeAwe7cc9EMeKh247l+d6eJ6PUUJ7qqnruJ63JsIs0zBq9rpxfI/ORlUptNBgepq/unQSx/eIW5WVsxuaTtzSOTMxzKWpcX565yE2xBprfKWKoiiKoiiKUrL/BHyWCudgzadCLEVZxo3pGWbyBeKh0m4o/SDg0tA4Y6kMtmGUvRNhKYQQWKbOjeQ046kM+za2EymxKsvWdZ6/co2fOLi35tdVjb7MME8NHyGk2UT10oKSsB5CSsm3Bp/n0ZaDHGraVfJ6I33jWKFbIZbnepw7dg3P9THt6n80apqGZmkYfWmSm3xaErd2JAyCgGwqV9Iuhc1hm+lcbtWrlhzfp6cpQelzG5emaYINDSrEmq93JsmXLxwjYliEazDXKmyYBFLyZ+de5Wd238/WhuYaXKWiKIqiKEr95fwCxyev1mUdpTxCiEPAR4CNwGI3pY/PHvuFuYdJKT9byXoqxFKUZZwaHMEucaC76/ucvDaE4/tltw1WwjR0/CDg2LVBDnR3kIguH/5YhkH/5PSKX1s5hvPjPDV0hIgeKrviSAhBzIjwUvIUpm6yr2FrSY/L55yba/lewLnjfXhegF7l8P6F2h2ds6k8KSGIN0ZvXnMh50DT8o9PhMNIOVXTa6qMYEOidqGTpeskwutzuP1KuJGe5ssXjhE1rJpW3WlCEDUsvnzhGJ/f+yAbVUWWoiiKoijrUFi3ua9p+4qv80012L0sQoiPA1+hWGFVSjfmZ+YeSrEyq2wqxFKUZQzPpEoaQO16PsevDeLXsH2wFEIILF3n9MAI+za20xyLLPuYVMHB8f1Vr+4BcAOP7w69QriCAGu+mB7hhbET9EQ6iBnLPwdy3oyza+cH8Qoeuln750OX0OBqTE/nsEPWbAujIPBL67gPm+aq7ybpS0lTJIRRw9lKjaHqvt5vJgXf468uniBS4wBrjhCCiGHxVxdP8KuHHsfW1T/9iqIoiqKsLzm/wLHklbqso5TlNyiGV/8ncA7wFjnu54C3AZ+rdkH1m6yiLEFKyVQut+zNu+cHnOgbIpASfQXaB5clipUtZwdGOdjTSWNk6QoXz/eZSGfpWgMziY6Mv4ErXUJa9X/1sITJM8NH+Vj3O5c9dq7Nc2o8xdRECtOuza57d7NxClKdguRYio6NTYBE00t7neizu1jKVdqhUAJBINnZ1lKzc7q+z+ZmVRE051u953ADvyYthIvRhKDge3yr9xyf2HFwxdZRFEVRFEVZCfWqxPqWqsQq137ge1LKf7XUQUKIdwJvk1J+sdoFVYilKEvIOA551yNmLx5iSSk5e2MEz/fR61iBdQcBlqFzpn+EB7d3Yy7RFmdoGtcmJlc9xHIDj8vpgZoEWFAcaD2aTzLtpmk0l543ZYZMctkC1y4MYVgr+6PQCGDDtGCgIWBmMkM4amGHSt9ZstG2mchm0bX6v74c32d7azOWUbvnKOe67O/qqNn51rP+1BSnk8M0WCvfWhk2TE4nh3kotYlN8cSKr6coiqIoilIrWc/h9eTKz8TKes6Kr/EmEwMG6rmgCrEUZQlD0ym8YOkKmJHpNDO5AlaNZylVRIDQ4PzgKAc2dS7amhS2THqTkzy6rafOF3i7y6l+vMDH1mpXgWJpJseS53lXx+Elj+vobuHyyX58z1+yCisIJJ7jUcgVKORcfN+f7faWCARCCAzLwI5YWCET3bj7roZNGZiMaGSyBSIxi0i89NCipznBSCaNXv1mHmXxpSRuW2yocdgZs222tazMkPGc4zI2lebqcJLRyTR+ECClRNMEhq6zoaWBLR1NtDZEseowt245z924TNQoPdCsVtSweO7GZf75nqW/PxRFURRFUdaSiGFzf/PKV2L9k6EqsSpQ15aR1f8NXlHWsKGZFPYSc6Mcz+fq6MSamC01R9c0prMFRmfSdCwSPmhCkC6sfr93b2aQsF7bG3hTMxgpJJc9btPuLgb/2zOLVmE5BZfUZAYn5yCDACEEQhO3BVQSiZSSQrZALpMHQNc1Ig1hoo0RtHmVUwLYPAFn2ySFgodRRoASNk2ilo3reXWbIzXXRrivs51a7UgIkHUcHt3aU9M5X2NTaV48e43BiRlyBRc/CLAMA9u8PVCUUtI/OsULp3sxdI2IbbG5o4nH9m2hMVr/IfNpt0B/apqoWb8QS9c0rqemSbsFYqb6JU1RFEVRlPUh6xV4faIelVirf4+0znwOuFjCcf8f8INaLKhCLEVZguP5N2cn3c35wVGE0Gp5j18TlqFzZSRJcyyy6FD6YJkKs3qYdtNoovbVRVkvTyCDJc8djtr4boBp3/7FK+QcpsZn8F0PTWjFr/8yM9GEJtBnXwRSSlKTGdKTGULREI2tcbTZNlMjgO5JyXh0sXmHi+tJNHJuZKzknTKr5fgeu9rbatpGCIAQPLS5u+rTeH7A+f5RXj7XRzKdJWyamIZOdIk2TSEEYdskzK3Ku4sDY5y5Nkx7IsZj+7ewrbNlye/5Wnp99EZd1llIzK79xMZtq7K+oiiKoihKuYqVWCv/u8t3VCVWWUqdcSWlPAIcqcWaKsRSlCU4vr9o5Uum4DCTza+JlqQ7zF5y/8Q029rv3rYVrNKg8Pk86SNWIAH0pY8nA6wlQqyJkWmijSFcx0fTNYJAMjM+QzaVR9MFeoXVdUII9Nmh7flMnnyuQKKtgfBspU9sJqA91MhMPk9DqPTqn9ZoBLNOM9cc32drSzMd8aXnipXLCwI2NzUSNqtrH70xMc0/HDlDNu8QDVk0hCuvorJNA9s0SOccvv7CKVoaInzs8YM0x5ff4bJaV2eSRFZwmPtiIobJ1ZmkCrEURVEURVk3sl6hTjOxVCVWJYQQUeAXgB8HNgE+cBX4DvAFKWW+VmutwbtvRVk7lgpYro9PYayhNsKFDF1jdDrNlramu7durYHqMTk7V6r2513+0+u/NEzPno1ceL0XPJ/J4Sn8QKIbtQuKNF1DSsnk8DS5aJ6m9gZCEZMtRoTN7Z28PjpCQ6i0v/YIIehONHItObmi7asF32dTUyPdidrvHph3XX5s946KH+/6Pt87fplTvUNEQxaxcO3+UqZpgoZIiHzB48+/e5RH9mzmsX1bVrQqa6qQq1t76HxCCKYKubqvqyiKoiiKUqmIYfNAHSqxnlKVWGUTQhwAvk0xvJpfKbEXeBL4V0KIj0gpT9ViPRViKcoSzNkKHRZkBn4QMJnJYazmboQl8PyAZCpLa0P0jo+tRBtfuWzNwg3Kb61bjqHpGGLpoGdqIk1DUxTD1Bm6Noqua+grEFgIIdANQT7nMHJ9nAffexApJTutGC174zx97hJx2y4pzOhubGA0ncFxvZqHK5Ji++y21iY2rkCAlXEcHtnSQ2vsztdiKZKpLH/7gxPkXI94DcOrhXRdI6bbvHL+OpdujPHJJ+4lUsZOkqVyfJ+s5xCp41D3+TKeg+P7a2qen6IoiqIoymKyXoHXJnrrso5SOiFEHPgm0A38BfB3wL+gGF795Ox/nwSeEkIclFIuP7x4GSrEUpQlRCyTQAZ3vH94KoVcA+14yzENnf7k1F1DrJUIbOaTUpLPOfh+cSi6ZRuYC1ovm60GBnPj6EsEaoGUuF5AIEETxQqz5a49bkSXDYV81yc1lcHJOwhY8YoYARimwXDfOD17ushnHR48sJ3GkM0/njqPlBBapjVVCMH+znZevT6AtTBZrYIXFF/je7vaaI1WFjItxQ8CGkMhntixpaLHj0ym+cvvH8M2dCJWfdrvoiGLVM7hC0+/ymfe+wDxSG0Hv7uBz2qOpZOyeA0qxFIURVEUZT2IGDYPtKx8JdZ3VSVWuX4J6AF+T0r5OwBCiJ8CkFJ+Hfi6EOIE8H8Avw7822oXVCGWoiyhO9GI698ZYo2nMosOTF9LhICc4xEE8rbKHT8ISITDNV1LSsnw4CSnj19nciJNaiaH5wVIKW8GRJGoTUNDiC07Oth7sJudsR6upAeIGbfmDwVSMpYqMJDMkS745F0fKYuNhyDRhMA2NCK2TldjiA2J8G2hVt4vsCO+adnrLeQdLh7rxQ5bJNoamBpL1bSVcD4pi1+L5s5GcpkCfecGeduT9wGwq6ONX25O8I9vnOfK2AQx21oyUAsZBluaE1xLTlUdQMxVXzVFwuzpbMVYZoB9pXKuy88cPoSulf/8jk2l+cvnXidkmnUbuD7HMnQ8P+AvnnmNz7//oSWHxpdL1ncn4jV7DYqiKIqiKKXIeAVeq8PuhBlViVWujwCTwH9c4pj/DPwy8BOoEEtRVlZ7PIZxlxvvvOutyiybSvhBQNZxic27Ac86Ljta7z7wvezz+wGnj/dx6lgfqZkcobCJYehYloG18J5fSqansrz8owscfeESG7c0I7ZokCi2Pl4Zy9CfzOF4AZahoQlx12HmgYRUzmM8PcO5oRSdjSF2dcQImToBcCixc8lrllJy+eR1NE1DCEEkHsZzfdLTWfQat4hKCTIIaOtuRtM0NA2mJ1KMDkzCw8VjwqbJJx84yPnhUb55+gJSyiWHn3c3NjKWzpJz3bu+PkvhBgEC2NvZVnGLXykyToFHtvbQVsGQ+FSuwF9+/zi2adQ9wJpj6Bqu5/OlZ1/n8+9/sGYbORiavupj6cwVCi1Xg+f53BidpndwgtFkmkBKdE1jY1sDmzc009XauGqvIUVRFEVRqlevmVjfNWwmVnyVN5W9wHEp5aIDV6WUgRDiJPCuWiyoQixFWULINO5oX3I9H88PMI31cQMohGA6m7stxJJAd1P1c4/GR6d56h+Ok0nlCUcsYvHlW66EEESixTLdkYEpRs7mme6eYKzZIAAsXSNkLv/cCiEIzX4NhqbyDE7m2NEZ5rHurYT1pcuAT790CbfgIObd1DY0F4Oc9FS2ZhVZMijWurRubMaYF36YlsHZo5d57McPEYrcutY9ne1sbmni+cvXOD00Qs7xiN+lMksIwT1dnbw+cAM/CEqucJqrvDJ0jc54jM0tiRWrvgLIOg672tp4146tZT9WSsk3XjyNLkRFFVy1ZBo6ecflqdcv8hOP7KvJOUO6QXgVdiacEzFMbH39/wqQy7v88NhlLl8fp+B42JaOZeoIIYrVoePTvPTGNSJhi/3bOnn00NZ187NbURRFUZRb1EysNSsEDJdwnEaNthZb/7/BKsoKawyHmM7lbwYJ6bxDsA7mYc1S5P5yAAAgAElEQVQxdZ2pTJ6NzbdCK0vXaI5U3k4opeToixc5/spVwhHrZihVLt3UyQc6V47OoEc04o8m7hiiXwpT1whkwIWhDN26TqHNwzbu/uMtl87z8lMnadvQTP/lYcybIaWgoTmGZRtMjqUAqqrc8P0AyzZp7mhEW1DdpRs6obDFD752lA98+u23fSxsmrxv707es3s7F0bGePHqdUZTGUxNI2QaN1+Hhq5xX/cGjg0MLhlkSYozrwIpiZgm2zqbaI0uPzOsWlnHZXNzEx87tK+itY5dvsHoZKqmOxBWI2SZnO8f5Z6tnWzpqE0VY8IOMV3I172qU0pJo1XbGV+r4VzvMM+8fKHYYmwZ2NbtPzyEEIRsk5Bd/B4/ceEGZ64O8+Tb97Ops2k1LllRFEVRlArVaybW06oSq1xjQMdSBwghEhR7UC7UYsHV355MUda4nqYEee/WDnqpfAFtlStDyiEEFLzbdwCMh+yKq1uklPzw6dOcONpLNBaq+LkIpOSN4WHGs1k6E80EbkDq+Un8TCW7FUo86bO/aTMT2Rx/+vpr5D33rkeeeP48APGmKHeZ2U8oGqKjpwU7bOL7QXF3yjL4fnEOWKI1TuuGxB0BFkhM28AOW1y/NEw2lb/reXRNY19XBz//+IP8i7c/xMNbN5GYDR4zjsN0Po/j+ezraAMg73m4QYDj++Q9D8f3kRSrCTcmGjjcs5EHejbSFouteGiScRx2tDXzyfsPolWwViqb54dvXFkzAdaceMjimy+fw3Frs6PmxliCgl/73TmXU/A9uuOJuq9bSz96/TLfeeEcIcvAtkr7e1w4VJyr9nfPnODU5cEVvkJFURRFUZS3hH5g0bYLIUQX8BWgBfhSLRZUlViKsoxD3V281HsdZgt2/CBYd+nv/MKxrOPw0OblB58v5sXvn+PCmcGKq6+gWB10ZmyUmULh5syr9kgTY/kpUkemaHhHM5pdfL/0AqQbgF/cnlAYGsK69RUIZEBAwO745psD4jOOw58fO8bPP3D4tgH8QRBw4Vgvodlrt0LG7NXcHrRomkZzRwLf90lPZsll8sUwS0o0XbstBAoCiQyKOzDqpk5TcwOhqL1oUOQ6Hhu3tgPFgPHE8+d57IP3Lvl8NUcjPLHz1r8NBc9jdCbN4MwMBc/nvk0b+OHlXtKOQ8yyiNs2UduqexuelJKUU+C+jRv44L5dFYdl3z56fk1unCCEwPc9vn/yCu8/vLvq8z3cvomjw9epd02UGwQ83F75z4DVdvR0H6+fGyBewc8gTQjiUZtnXrpA2DLZ0dO2AleoKIqiKEqtZb0Cr42rdsI16Gngt4UQB6SUp+d/QAhxGthJMXd6FvjjWiyoQixFWUZzJExbPErWcdGEKLYSrrP5wPN3IZMSHt5S2Q1sf984p471lTT7aimDMzMkcznseUGFoel0hJtJJpMkv34du9lAZnxk3i/mTHNZkwBh6YioDi0mDdua2bd1B5Z+a76QbRhM5/N8++JFPrp37833Xzt3g1zWIdZQrGhq727mxtUxzEUqOXRdp7E1TmNrDM/1cQouhaxDMLdjpRCYloEdsTAto6SqNCE02jYW29FCEZuLx3t5+P33lDVQ3jYMNjUn2NR8q5rmx/ft4khvHz+80kdIN+oeYOVcF1PX+dQDh9jaUnm7XTpX4Mb4dE13AqylkGVyfmCUd9+3o+qgrSkUpjMaJ+UUKqpYq0QgJZ3ROE2h2u5OWi/J6QwvnuglFqnu9RGLWDx15Bw/15G42W6oKIqiKMraFdFtDreUP2e1XM/oqp2wTF8D3g8cBuaHWJLi0Pdx4L8CfyClrEkLggqxFKUEb9++ha+fODPbhidYbzvTz1XEeEFAT3PijmH1pXAcj2e/dYJorLoWr4LncWUyeVuAJQNJYTxNrn8Kcg5mECDzEtkgkLZAIhGIeTVTEj0niQ1oGH0z9B09TfOBTpr2tqPNDoUPmyZvjAxzX1cXmxPFsOf0y5eJxG4FcG0bmhnsHSvhqgWGaWCYBpFY5Tf/vueTaI3dNjg+lymQHJ66GWxVSgjB49u2sK+zg787foqJbI7oHdtD1t5c9dWBzg4+uG8X1iKzyEr1yvnrxe+xNcz1fM5dH+WerV1Vn+uJDdv4m0snabDq0zqZdh2e3LJ3+QPXqG8/f5awXf2vLnM/E5995SJPvmN/1edTFEVRFGVlZf0Cr9ZjsLuvKrHKMVt99diCd/8H4E+BMSnl5VqvqUIsRSnBrvZWwrPVOrqmFbdvX+VrKsdcJJB1HJ6oYKc4gCM/OI/nBRhV7ux1bnwcXdwKcfJjaTJXxpFegLA0NNvAlCCTkpbWZgItwA18AhmgCYEhjDt21JNSMnq0n/HXb9By3wZaDnUhhCBmWXzj3Fl+7ZFHEUKQmkzfNqxdNzS6d3TQf2n+gPeVUkw+e3bdHnzousbAleGqQ6w5TZEwP//Yg7x07Tov9faTdz1id9nhsFpeEJB1XRLhUNXVV3P8IODs9RFCK/61qE7Utjh64XpNQqxdiVa2xJsYzs5grfBugQXfY3M8wa5E64qus1LGJ9OMTaWJR2oT+NmWwdWBcfKOu+Zfc4qiKIryVlevSqxnVSVW1aSUF1fy/CrEUpQSaEJwsKuTY/2DNITtdbU7oZSSkGkUh42Hw2xojJd9Dtf1uHx+iFCouhu9vOcxU8hj6TqB65O6OIo7lUOYGpp9K5gSAgIgN5kj0hpBX6ZtSwiBESle29hrA8xcHqf7/bux4japQoGrk5P0xBrIpPKEF9wAt29sZmJomkLeWdGB/fl0gbYNCfrO9pNP5XEdDyklQQB9Z65z9sWLNLTG2bSzk407u2jpSlR8PUIIHtu6mYc3b+LCyBgvXO1jNJ0lZBpYVbbAZV0HKWFzc4IndmxlQ0O8ZgHZ5RvjjE9l8PyAXMHBDySaEIQsg0Q8TDRkYayBWVlCCCZTOZKpLM3xSNXn+skdB/l/Tr6AKeWKDd2XUuJLyU/uOFj33RBr5ZXTfYRKHOJeqkBKTl8Z4vDenpqeV1EURVGU2sp6Dq+OX6vLOsrapkIsRSnR23ds4eTgMBHNqtv8mlpwvYCmaJhUweGfPXBPRTewF88M4nk+dpVtPNenp9CEwE3lmTk9hAQ06+6hhKYL8sk8kZZIWTPIjIiJm3W58jcn2fCubcS3t/CD3l4+3LEF3/Xv+pgd92zizCtXkDUOEaSU5GZyTI9PY+gaWuCjWzqapiE0gUCg6eAWfIIgIDk8xeDVEfxvHScSD7H3we0cemLfzUH05Zrb4XBfVwcTmSw/vNzLjekZ0gUHLwgwNZ2waSz6Ofuz1VYAhqbRYNs8vHkTD2/eVFFL6mJrXLg+ytEzfZy4OsjYTAbT0NE1DSGKM9ymUpIbY9NoQmCaOl0tDXQ0x+s+92uhgbHpqkMsgKhp8bHtB/i7S2+sWFthynX4qR0HiZlra8fHcowm05hVVoIuFAmZ9A4kVYilKIqiKGtcxLB4sA6VWN8zLFWJVQYhxBfKOVxK+dlq11QhlqKUyDYMPnJwL3977I2a30itNNs02NfZzrbWylq+Tp/sI1LlIGWA8WyWYCbPzJnhYvXVEoGREOC7AV7BwwiV96NKMzSELhh87gqdrk9uexNJK4UM7l5BZ1oGex7YyrnXrqJpAlFlOCKlJD2ZJp1M4bk+4ViIlo7GRY/3vWK4pmnitpldb7xwnhM/OseW/d288ycfwQ5X/jVoiUb4+KHi7B/H8xhNZ7g6Mcn15CRpxyUIAoLZPQs0TaALjUQkxPaWZjY1NdISjdQ0NJJScvziDY680UvB9YiFLRzPJ2yZt4VqQhQDTWN28L2Ukr7hSfpHpuhoidPT0bQqoXLEMrk6PME926pvKQTY19TOh7fs5ZvXzhE3a9f+KaUkNTsHa19zR03OuRqCQJLOFWpeiSWEYCaTr+k5FUVRFEWpvYzncLQOuxNmVCVWuT6zzMfnfqmdG2/82WoXVCGWopRhR1sLezrauDQ0ju+vj10KhYBoyOJD+3dV9HgpJemZPFaVN4+O75OfzpGdDbBKuUkXAtyMW3aIVXyswIhZDD9/jQbpM7KzBbHEwPBQxGL/Q9u5cPwaruNhmJV9vp7rkbwxgVfwkALiiSiNLbElHyMXaU+dq8DqvzDIl//gG7z7f36Mrfsr21lyPssw6E400p1ohO1bqj5fudLZAv/wo9OMTKaIhS0sU0dKScHzlw2kBAJrNkQenpghOZ1l9+b2uu9mqOsayVS2pue8v30jIV3n671nMIVedeun4/u40uenth9gX0tnja5ydfhBQLBICF0tz7t7haaiKIqiKGtH1LB4sHXlK7GeMyySK77Km8rnFnm/BnRT3LnwMeC/Aa/WYkEVYilKmZ48sIejvQP0jiSxa1wVUGtSSgxD4+OH9le8a9zMdBbX8asOsaYyWVJnhjFKDLAAhK7hZFzCLZXvCKhHTZLP93E50rTsrpJWyOTAwzvovzLC2EASw9TLqsrKTKWZHp1GymLI0dTegFXKHLFlng87bCGl5Kn/8UO2HezhvT/9NnS9NlVRUkoGRqc4e3WETN5B0wSNsRD37eomEa/8eV/M0Pg0X/neCQxdIzavssz1A7wgKCu4MXWdIAg4eekGu3raaG1cOiystVS2UPMW1H0tnWyKN/GVyye5kZ4hVkFVlpSStOuwMdbA/7TjEPE67Xy4klby7wXrqDtcURRFUd6yMp7D0bFrdVlHKZ2U8ovLHPLvhRC/Bvw+8P/WYs21fQeuKGuQbRj83OMP8q+//lTNb2BrLeu4fPjQnorbCAGSY2mCIKj6Wq4c70UIynq+hIBgkTlWpZ9DYIctTn7/HLubln8ehCbo2dlJa1eCgcsjpKeyIMAwdZa6lZ4Zn2Z6dBrTMgnHQ8QTkdLDupKq0gSxxgh95wb45p88y5M//+6Kq8UAXM/nxIUBjl28QSZbwLYMDF1DSrg+PMnxCzdoS0R55MAWdmxqrcnrfGh8mr9+5hiR0J1z5Tw/WLQibSlCCCxT5+L1MeQmaEvUL8jyggDPD2reXhy3bD6/90FOJ0f40WAv47kMYcPA1JZexw18cp5HazjKj/Xs4kBzx5r++VQOXdewzJVp41Y7EyqKoijK2lesxNqy4ut8X1Vi1ZyU8g+FEJ8Hfg/4RLXnW7chlhCiG/h3wAeAFmAI+Hvgd6WUk2Wcpxn4LeCjQBcwATwF/JaUcuAux/9H4DCwC2gFckDf7Np/LKWcWHD8JuDfAA8Am4Gm2TWuAH8OfFlK6Zb8iStrws72Ft67ezvPXrhSswHXteb6Pu2xKP/8kfurOk9+tjqnGpPDk6THZtAquNmvxUaQwtDwcy5jg5M0NJcWckRiIXbduxnP9RjpT5KazFDIu3iuf+uiZkO5QjpPbiZL64ZmQhXMDjPLqHILR0OM3UjyrT99jg//L++tqCIrk3P4m6ePkcrmCdsmsXk7NgrBzblDmbzDPz5/ml097Xzw8b1VzcRK5wp85Xsn7hpgAQQyWLZSbjGCYpB1qX+MkG0SD9ep8kgWgyyT2ocrQggOtnRysKWT0WyKF4b6GM2lmSrkcAP/5lMlAFPTSdhhNoebeFvXZtoj5e9AutYJIWiIhsjkCjUN5jw/YENbtGbnUxRFURRlZRRnYl2ryzrKijgDvLcWJ1qXIZYQYjtwBGgH/gE4DzwE/BrwASHE4wvDpEXO0zJ7nl3Ac8DfAHso9nV+SAjxqJTy6oKH/a/AMeAZYBSIAo8AvwP8ghDiESll/7zjtwOfAl6hGHQlKYZuP04xxPqMEOLHpJReuc+Dsrp++uF7uTg0xo10GksvvUWuHlzfJ6ybfOrwvWhVDuMWVTbyyEDSd+o6umWAt3ovczsRJnlmnJ5dXWWFRoZpsHFb+83/91wPzw2AYhVeNpXj8rFe2rtbKrquIJCEY+WFLqGIxdjABD/86su8+5OPlfXYvOPyl995Ddf3iSwzR0oTgnjE5sqNcb75/Bk+8o4DFb3OpZR88/nTxR0aF318da+zuVlZF/tGuW9397raQXQ57ZE4H99+ACg+lzNuAdcvViiauk6Daa+pnz8rZdfmNl480Uu0ig0OFsrlXfZvr81wfkVRFEVRVk7UsHioDjOxfqAqsVbKTqAms0rWZYhFcShYO/CrUsr/MvdOIcT/RTFk+n3gF0s4zx9QDLD+bynlr887z68Cfzi7zgcWPKZBSnnHVkZCiN8HfpNi1dUvzfvQEaBJShksON4EngbeCXwc+EoJ16usIY2REG/fsZWjvf1cnZ5aM0GW4/u0hMPsbG7h4Z3VDwG3w2ZVA5WnRqbwHA9d04rtYmU+RzV5SiXoQiBsk5HrE3TvqHyXNsM0MGaL73zP59qp65h25T9KPdejsaX8yplQ1ObC61fZc3g7G7aX/vn8449OU/A87DJaESO2Se+NCV4/18/hfT1lX+vJy4MMJ1NLDl/XhKh68JEQAtf3uTaUZNuGykLF8hYEo4Y7Npa0pBA0WqHlD3wTumfnRl5641rNzielJBq22dzVVLNzKoqiKIqyMjKewytqJta6I4RIAP+aYmfai7U457oLsYQQ24D3AdeA/7rgw78N/ALwaSHEv5RSZpY4TxT4NJCZfdx8f0wxDHu/EGLb/GqsuwVYs75CMcTaOf+dUsq7fhdIKV0hxN9TDLF23u0YZe370H176BufImKZnJuYABmUfVMbBBI/CG5rD9KEKLZulXFTH0iJFwRsjDfQFo7wkcP7Kh7mPl9La7yqdsKhS0OYtknIE0wX8pTzSUkJeg3m4HhBQEPIhtY4o/3jbNzeXpPA8drpfqSUaKLyIEMgiCUiFT02Eg/z9Jef51P/5qMlVZdNp3MMjE7fNlC95LVCJq+fH+CBvZvKeu6CQHLkjV4i9tJtt0aNQmBT1xmdTLGpI4FZ5e5+y9E1DaNGA/aV5dmWwf5tXZy+MrhsFWEpMjmH9zy0a0388UFRFEVRlKVFDIuH6jATa71VYgkh/gz4/Oz/7pRSXq7z+r1LfDhGsQsNwAH+91qsue5CLODds2+fXljdJKVMCSFepBhyPQJ8b4nzPEqxnO1pKWVqwXkCIcTTFAOxdwELWwrv5sOzb98o4ViEEDrwwXIeo6w9tmnw4fv38tVXTvFIdzcXxscYz+YWr8qSUPA8so6L4/p4gU8QyDtGAQmKFRe6rmHpOmHLIGyZi95sub6Ppevc09mFITS2dzSzrb02lSjxxnBZ7XfzFbIFsqkcVsjC0vWyWxOlH2DFqp855ktJUyhMaEuUoWtjTI+nSLQ1VHXO1GSG5PBUabsPLkJKCMds7ArPoWmCfN7lyDdf44lPPLLs8a+c7sM0KgtdhBCksgX6R6bo6Sy9cuXKjXFyjrvsnCrL0GsaCA2Nz9DTsbIVNg2Rt0Yb31ryxOEdXBkYJwiCqlqlHdens7WBgzs31PDqFEVRFEVZKdk67U6YXUeVWEKID1MMsNIUA6PVsImlqxQkcAn4OSnlC7VYcD2GWLtn315c5OOXKIZYu1g6xCrlPMye5w5CiN+g+EJppDjo/W0Uw6j/sMjxrcCvUPwCtwE/BuwA/gr41hLXqaxxOzpb2LOhlcsjSfa3dzCezXB+fAIZ+DcrQYJAks4XSOUd/CBAE6K4Ux8CfYkqJxlIcr5LpuCgCUHIMkhEwjdv9ueqrzbEG9jW1ISUEiklH7xvT80+PyEE8YYw+ZxT8g173vOYKRS4fvEG49kshu9i6zoSiZSi5BZBKcGsYFD6QpoQxCwLXdOIJaKMD01VHWLduDSIUWG4N8dzPHp2dVZ1jlDE4tLxPh778OElw0YpJZf7x8pqI1woGjJ55XRfWSHWy2f6lmwjnC9kGBRcr+pgyNR1RpLFaqxqZ7otxvcDWluqew0p5TN0jZ987yG+9O3XCFlUFGS5no+ha3z0nfeoEFJRFEVR1omIYfFQ25YVX2e9VGIJIdqAPwX+FugEnlilS9m2yPs1oBt4P8Uc5D7gLRtiNc6+nV7k43PvT6zweX4DmD+I5ings1LKsUWOb+X2tkUJ/GfgN2Ul+8ovIISowR5uSqWevH8fX37hOMlUhtZIlEe6w/ROTjKSTjOezuLM3phrFczQ0YS4OaQ673gMFmbQdUEiEqY1GmF7UzMx2yaQkoLr8fl3Hq5JG+F8B+7fzI+ePk00tvgsHgmMZTJcn5oi67hIIDc8ha+B73nkXJe86+EFAZZhEDIMlupSnGslNOzqWsKklDcDLIDuHR30nr1R1TndgktmKlt1iKWbetVhGhTnal18/Sr7H71r5l48xg9wPB+jgh0i52iaRrZQ+l+nPD8gOZ0hvEwr4ZxYyCLruBg1CBY836fgeIRWaPfQrOOwrat5Rc6tLK25McqnPniYrzx9DNf1sMuYSZfNu8QjNp98//2Eq6iiVBRFURRFWWX/ffbtLwNfW62LkFJeX+LD14AXhBDfAX4khOiVUlZdwLMeQ6zlzN39VBvqLHkeKWUngBCiA3iMYgXWcSHEk1LKY3c5/nzxcKEDG4GPAf8OeJsQ4kNSyvUQ+CqLMHSNTz1+L196/hiTmRwhy6TFDjM2kSamGaT0ADcIkFIwt6tduaSUBEh0TRDRTAxX0KDZRCwLPwjIux6ffvt9tMRrv138rr0bePkHFxb9eMH3OTMySqpQwNZ1rNmgJFNwb36uuhBETJMZp7izmuv7hA3j5rELBX5ArD1a9bBvxw/Y03ori060xtE0Dc/1MCqsShruHa36ulzHo2dXV1XzxuZE4iGO/+DssiFWLZJu3w+WP2jW5EwWr4zjm2MRhiZTNRmWLiWksoUVC7EEgu7W5f5WoqyU1kSUn/3oozz90nkuXR/DtnSsJb6f8wUXP5Dcu2sjb39g+81QW1EURVGU9SHrObwy1leXddY6IcRngY8CH5NSTqz1ynIp5REhxAsUC4HekiHWXIVU4yIfb1hw3IqeR0o5AnxDCHGMYmviF4EDSxzvA9eBPxRCjAB/TTHM+pVlrndJspiO3EEI8TpwfzXnVkpjGjqffsf9fPn547x0qY9U3sHSNRr1EI1WiILvkfYcCoGP6wfF1jpAm01D5n8BJcVWQcTskHchsHSDmGES0W/NxhqanGF0Js3OzlZ+8b2P0NZQ+wALwDB0duzt4sKZAUILWsPSjsOJoSE0BPa8QEpKiXR8xLwZTJoQGELDlxIBZF0XTwZEzNuDhuImhoJQorpd2KSEkKHTFL61m6sQgp5dnaQmszS1V1YFNTU6U3EABsWALhIP07axNpU8QggyUxkyMzmiDXffudYy9SUr30pVTiXXwNhUWbtLJqKhimd2LWQaOlOpHG2J2o8HkFLSFI+QiNVkl2ClQrZl8OEnDjCVynHkZC/9w5PkCu7N4FQIMDSdSNjkwI4NPHigh9gys9kURVEURVmbIoZdl3bCHxo2Sbh/sU6nxe6760UIsRn4Q+DLUsq/X81rKdMw8IFanGg9hlhz5SCLlRzM7fS32KyrWp8HACllnxDiLHCvEKJVSjlewsO+M/v2naWsoaw9UkoCPyAIJJqu4Xo+OcclYluk8w6SW+GUrRvYunHzcb6UOIFH3vfxZbFKphhqFYMtUzcI6TqmuPuQ+LnjbdNACEE6X1ixEAvg0Sd2c/XiSPFznU1Dcq7H8cEhDE27M6wIivO5Fr47YpqknAJQbJN0fB+BIDwvFAq8gPiGOEKv7t8Ix/e5r6vrjvdbIZOdhzZz9XQ/4SVaJO8mCAKcgotRxa6JQSDZcc+msgKeUq5rfDBJtGHjXT+uaxrRkI0fBBXPAXJcr6xh6RPT2SWrYxYSQtDWEGN4KlX1zoKaEBRcr6pzLCaTd3jHPYu1/yv1loiH+eDb9gGQy7tMpXP4foChazQ1RLCrbPtVFEVRFGX1ZdwCr4xeq8s6a5UQQgP+B8VB7r+6ypdTMiGEBTxE1b0sRevxN7vvz759nxBCm79DoRAiDjwO5ICXlznPy7PHPS6EiM/foXD2xfG+BeuVYm6bI7/E4+fuNlfmTkupueToDH0XhrhxdZSZZJpMOg9BMaR3kBz3ZjBsg1gszJaGGDfyWRzPv6NlTgiBIQSGZhGp4LvQDySBlOzobKGjMYaU8LcvvcEnHt7Pzs62WnyqdzBNg/f9/+y9aZBc13mm+Zxzt1xrX1HYCoWVAEGABBeRkkmJ1ELttmVbttt2tOT29Iw97XB7fvS4Y9zjmJiY7uiZiRh7psOeDlnt9liyxLYlWVKTomRKIgmu2EgQ+1IoFGpfsqpyv8s58yMLa2VVZWZlFYrCfSIQADJv3nNzu3nOe9/v/T7zAN/5xlskkhE0cGp8DEOWD2rXqnzxmhSCiGmR97xS3heCou9jGRJTSnSgsWIWkcaVOSa8QNGdTNDglNmPhr2PbSc9k2VqZAYnWnl4fCFbRAUKahSxvKJP794e7ApzoirFjtgMnB5iy+7yIhbAg7s38tPjlyoOWr8T11d8YP/Wirf3A0W1vqqNrY2MzKSX37ACViso0LZMdm3sWKW9h6yEaMQKs65C1hzPD5hMZ7kykWJwcgZv3g0ohaCzMcG2zlY6GhPE6nzeDwkJCbmXiFsOj66BE+tlyyEFx7TWD63G/oUQV4AtVTzkb7TW/2T+339AKcD9U1rrVL2PrRbmnWGLkaBkDvoDYCvw3XqM+b4TsbTWl4QQL1ISmX4X+LNb7v4TIA78hdY6e/1GIcTu+ceevWU/GSHEXwO/A/zPwB/esp/fo/Qi/0BrffmO/cxorUdvPaZ50et/ATqA1279QAkhHgVOaq1zdzwmQckGCPD9Kl6CkDXG9wLOnxjgxGvnmZvKYhgCJ+YgpSAaKwkknlYczU6DURJhZqcz+GMzGFISxAyKcROMUklgre4bTWmiLISgOR5le2cr1rw4JkQpFPvv3zrFFx9/gC1tlbtlqmHDplYOPrqNd97uZ9ovkvM87EUcM2KJ2jXHMPCVwlclF+GadigAACAASURBVJYUgpzrkbBthBQ0bEyuSKcPlCZmWWxvaS2/gQDTMvnMl57iW3/+j8xOpSsWsjKpbM3voVv02Lyzm9au+mcpWY7JxNDS0Xr7+rp59Z3+mvavtKa9OUFjFSV0piGpPBGrhG0aNMYiZPLFFecWrYbXO++63L91w40Oofc6gVLM5AsUgwApBE2RCJEVlNqGhLxf0FozNpvh5TP9DE7O4PoBliGJ2NaNZixaa0Zm0rx5cRBTStoaYjy+cys7utvqkocYEhISci+R9Yq8OXFlTcZZZS4BhSq2HwYQQuwA/lfgq1rr/7oaB1Yjl1l+2q2BEW7XXGrm/TrT/O+A14A/FUI8DZwBHgU+TKn871/fsf2Z+b/vfHH/iFIp378UQhwA3gL2AJ8DximJZLfyCeDfCyFepvThm6LUofBJSq0lR4F/dsdj/kfgKSHETyllYeWATcCzlDofvgb8b5U/9ZC1QmvNO4fPc+zls/hFn2gyQqJx4QJea82J3AyeVpiitLCVhsQ2SqKI6Sq8TJ7AMRBtEleXyuxM08BYRg3RGrwgQGuNY5lsbmtiQ3ND2cW9EIK4Y/PcG+/y3z7zgZrdNsvx6Ad34rk+f/38W9hLdQ6UgqXUnrhlkfXAUyW3jlIaXyk6trciV5CL5CtFxDQ50NV1YyFxJ0IILNvEtEw+/988zff+8idMDKeIxpd3fxXzRWSV4orWpY6GW/f00N6zOgKjEAKv6C25jW2ZPLhrI0dOXyVWhftMa00u7/LpD+6t6piak1E8b6ETcTl2drdy5NIQUtfWBAFKopu9gpLPRfdpmjx5j5cSukHAO8MjHL02zGyhgKcC5iP8kFISsyw2Nzfy5LZeWmOxu324ISF1Zyqd5dtvnWYynSVqWzf+3IkQgohl3hB2s0WPb799iqht8ezBnezoXh3ndEhISMjPImvsxFo1tNZP1/jQvYAD/FMhxD9dZJsL83Pnn1/DvKwrLC5iFSiJcD8G/kO93GPvSxFr3o11iFIg+ieAT1JS9v4U+JNKO/3NJ/l/APg3lNL9P0RJmPoq8Mda62t3PORHlFpZPgE8QEmEylISzv4a+NMyY//H+W0epiSYxYAUcBT4JvCXWuuwnHCdkZ7J8YOvv8b02ByxZGTJ0q8hr8Bs4OHIxRxJEjtio5XGH86yoaeZeEeSmVyebNGl6AUordC3BGiJ+ZD0iGPRHI/SGIsQdyyWE7mFEBjS4NtHTvFrTxyoWQBYboxthzaSuHCW7IVZDMco67oSQiBt40a5ZTnilkXO83BdH2kb6G4HadUoYGlwVUBTJMLejs4lBULLMmloKeWHWbbJZ//ZR3jzB+/y7uFzROMOsozLRitNPu8yPZkhPZdHSImQAtsysBwTYxFnTuAFSFOy+9A2kk2ru6BXannf0xMP9JJK57l0bZJYBWVXWmuyeZePPbabjR1NC+6bnZxjtH+cgdPXSI3N4rnejVLSvOdzeXCC9i3tJHqaiLYkMMq4dDSaHEXmyDEtshQsl0KnS2qsiGEKBAITiaNtIljYmIhlvgueH9CcrO/rnS24fPGpAzcckJXiuj7T0xmGhlPkc6WsPMsy6OpspKOjgVjs/RE2rrXm+NAIP7xwCV8FxG2biGURYeHn6NJkitOjE/S1tvD5fXuIWmEZVcj7H6U0h89d4bXzA8Rsi2SVjQKkECSjDlpr/u7NU2zvbOXTh/aE7sWQkJCQCsh6Rd4YX/3uhOs4E+sK8JVF7vsU0AU8B8zNb7smaK371mqs67xvfzW11oPAYgrkndsuutqZF51+f/7Pcvt5j4XurOUe833CcsH3FeffucpPvnUEO2IRS5YP/fa1IqsDcoHP8XwKicBXutRJEIGFWCAgCSmwHIvU8AzZ6Sy7Dm7Fsk1Ao+YzrjSlSa4UCx9fKbZpcG16lpODo+zfvDDUvB6cnZykY3cbYmsL114fws24GBFjwTEbEQs/W1z0uWiliQiDZHecdCSgqANuS8OvkOvdyHa3tdERX74bXbwxepubyjAkj3/yADsPbuEHf3OYdCpLLBlBa5gan2NsZAa34BMoRWYiTSFbRBoSrSGjS4KNlALbMUk2xrAsA6U0gRfQ2t3Ell3dZYWxeiPF8mMIIfj0B+/jpSMXePfCEFJKZvwiY/nsfOg7WNJgYzyJrSVSSj7zoX3s2HzTMVDMFzn5yllOHT5Lbi6P1ppoIoJxh7jjGAZ+tsDYu1cZOd6PNA0S3U10P9hLrC2Ji8egmGKCWTwCEGBhIhHEmw2ycxKvqJASXAIKIssMpcYHUWwadRyLRcRjIeraiS7veezd0smm9spKQYtFj5MnBzl9ZphsrojvK0xT3hA7ldJ4XoBhSCKOyaZNrTzySB+Ni3SXvNt4QcA33zlJf2qGhG0TEUtPH2zTwDYNBmdn+dPDb/BrB/azqWmxZsAllFYEWiGFxKjgsxwSspb4geLv3jjJ1ckZkpGVnVuEEDREHa5OzfCVf3yL33zyoaoFsZCQkJB7jbjp8Gh7NVFStfGKubpOrFrRWp8AfrvcfUKIn1ASsf5Ia31xtY5BCPFvgBNa6++s1hiV8L4VsUJCVoOTb1zgteffJZaM3BBetNbMKI/JwGVGe+RVQFErFJoxv4CvFQJRKqcRJf1FUAput5DEMIiJUpdBANM28b2AU29d4r6Ht2E7FlKKqgOwlyLu2Lz47gW2d7YSc+pfVjg4M0vENBFJQd8zvcwMzDJ5fgo342HY8kY5oN0UxZvJI+ybQoPWoIJShzwnaRPviGNYkgatmc7nCbTCDxSWNJbMDNFa4wYKy5B0JZNsaWrCqqDMLwgU7e0NZe9r627mi3/wSc4cucz3/7/DXLs0DlLgRC1My8DEoGhbFEVpEVL6iNw8RrfoMzaUwjQlG7a2sevg1kWF0NVAVtjNUQjB4w/0MitdfnzqMuMzaaSG689Fo5kq5Nja2cInD+ymb2MbUBKvXn7uDfpPDqCVJpqMEm9c3O0khSDm2HhmgDnv1smNpzn9nbdIN3jwRCv2xiQWBs4d3wAhBG0bI4xeKolkQgiMW7bJ45ITRSxMWnUS+46fM9OQROrUlc71AxKOzUcfXKyZ7S3buj4/ffksly6Po7QmFrWJLlK6GbnFCdd/ZYLzF0Zpb0/y0af30dy8ep1Gq8VXir8+doLxTJZkuUYJS+CYJkpr/urocX7roYO3CVlaa4bzY5yaPc+cnybnF4DSidSRDkkzzp6GPjbHN4aiVshdRSnNc6+/y3Bqrq6l+hHLxAsCvvrjI3z5Iw+vWgxASEhIyM8CWb/Im2vhxPLXrRNrPfDHwH8GQhErJGQ9cPZYP4eff5fEvBPC1YorXpbhoEBRK0xEqYhJCBwhKaqAQGksUd4JojQUdEAWv7QoE5JmLOLCKLl4lOL0kcvsfbhv3pFVP64LcK9duMoz+7bXdd9QcmVcH0NIQXNvE829TeRn8qT6ZyikCng5HyMeKXXy8yUIkKbEdAycpE2k0bmtDFEKQdJ2eKCrG18pRjNpsq5HwfdR801IrzvVHMMkZlt0JhK0RqNVudYK2SI7Dy5+FSc1leH4iUGae7to3NjGyIUR8uk8bsFFKYU0JIEflBRLXVqIM++eMyyDeFOcWGOMQqAYG51lS8ypWFxaCUGgSLYs70IDmM7m+Ks3juMHATs3tbN9Yxv5oksQlARZ05REHQut4ZVLV7gyPcMjTiMv/+1htIZIvHJhrqejkfNXJ3AsE40maxZJWVkIQLxwDd3XhPnBDYgyZaSGKejY6jDaX0TK2/Ox5LxcHKAYFSmSOkoTcQQCLwjoam2oSzmt5weYhuSfPP3QsmWEA1cnefGH76GUvk2gqgTbNrFtk3S6wNf+9nUOPdTLw4e2rYvw5xfOXmA8nSFq17bAlkIQt22+fuJd/sUTHyBimVzNDvP61FFyQZ6ojCCFJGLcLpClvQw/mXgDe9LmUMv97ExuW5US6ZCQ5fjBO+cZmp5ble6ClmHg+QFfe/U4X/rIwytuaBESEhLys0rcdHi0Y+uqj7NenVghNwlFrJAQYGI4xU++fZR4Q5SZwOWin2VGeaDBFpJoGaEqFXjLBrMLURK+oORmGBVFTA0JYdIsbITSnD8xwH0P139xFrUtTg2O8uH7ttV/UrzIsUabokQPlkRArTUq0FyWPoEbIA1ZUVe/6x3OmiI3hZJAKRQgKQVHr+SVisRstuzuKXvf2ZODvPT8SeJxh2jMBmy2HyqVeQd+QCFTYHZiFi9fRJolp5hhGthRq1RKd8sTNEyjlJ81m2fP/k1Ydn1Dxu+kkCmwedeGZbfLFIp89fWjGEISmc8pkkIQL1MeIwTETIu3v/k6bw6neWjH5qpD7Vsa4pjmNArFuJjFxS85qgQQl7gDc3hDGRIf24LVsbCUznIMOrc6jA8UQesF+WsCMJCkRZ4cLh26ARD0tC1dulYJRc/HsUx+45mHiC6xeNVa88YbFzl6fIB43F7Rd9kwJPG4w9GjV7h6dYrPf+4hrDoH1FfD0Owsx4dHaFhh+ZQUAg1869RpurqzXMkOEjOixI3FnXxCCOJGDK01r08d41JmgGe6PoQtw3ytkLVjcHKGdwaGaYiunqvWMg1msgVeOXOFp/be240jQkJCQhYj67u8MbYWTix31ceoN1rrp+72MawloYgVcs8T+AE/+PrrWAmb4+4sk0ERG4kt5KLZTApNXgdUI6eUBK0Ss8pnTvi0CptoLmDkyiQbeuvfpSjv+VwYmWR3T0dd9xuzLDLF4qLd/2A+ZN4UdDy8iaGXLmLEl3dxCCGwygTkG1IuknxUHYVckfse6Ssbwn7qxFVe/uEpEreUkt52DGbJZRVvijNxaXTJzovXMS2DIFCcOnGVvQc2r6qQpYHu3uXf528eew+BqEjYVF7A5W8dpzidRVmSq6lZtrZW111RCkFnW4JjkwMIwW0lgQDSMdBKk/7uZZIf34K1caGbzI4YdG2LMD5QQHkaaZZ5f5BoNENMs7thQ9Xh63eSyRfpaErwS08+ULbr2K28evg8J9+7RiJRv0ybWNxmOpXlv/z923zhFx6+a0LW8+cukKhTSbJjGPzjwDs8HHForqJroRCCmBFl2p3lu0M/5DM9Hw2FrJA1wQsCvv32KRIrFHErIR6xeeviVfZt6qStYf2UE4eEhISsF+KmzWMdq5+J9apph06sdU4oYoXc87z+4kmuptNcNAtoXQoaX465wLtRRlYLhhBorZnExbEkwcA4LZ0NROrcpSzu2Lx2YaDuIlZvczMDqRTxCsqLGrY0M2qbN3KNlsKSEmsVA9CV0hz8ufsW3D54ZZKfvvgeyQpDtSPJKLm5XEWuJGlIVKA4c3KQfQe3rFp5mO1YNC6S9XWdqUyO0bl0RblGKlAlAWsmhxEtxaePzqXZ0tJUldPI0wHjcgZjvoS2nO4rpICoQfqFAZKf2orVvXABZ1qS7r4oM2MumZSPNCgvNgrJrJMhH7hEjerFFz9QFD2fD+3fxiM7Ny37XE+8M8DJk4PE4vVf5DqOydxcnu//1xN87rMPrnkp3WyhwFg6Q6LKHKzFGMwN4+oiA2MGzb3VP96WFvmgyI9GX+XZ7qfC0sKQVefkwCg511txkHulRCyLH528wBefOLAm44WEhIS8n8j67tp0J3wfOrHuNUIRK+SeZnx8lq+/fpy0DY6WiArDg7MqWLaUcDlKQdXgKsWwEaBO9vPYI7vqujCTQjCdyeMHCrOO4tCu9jb+8dKlirYVhqTriS0M//gS5hJuLKX0qi4U8tki+x/fuSBo3XV9fvTdEySqCGDv3tXD+VdOIysUHaUh8dyAwf5JtvTV33HnFj223rexrMPsVn56oR/HrOy0P/iDUxRTWYxbsp3cIGA6l6c1XpmLRqN5L3WNQGk6mhKMTM0hRXmDoxACYgbp56/Q+Ms7MBILPytCCJq7HOJNJpPXiviuxjBvilmBUnQ0JTGlwbupazzUugWzAlH6+mOzBZfOpgS/+dGHaIgt/3lIpwu89trFVRGwruM4JsPDKU6dHmLf3o2rNk45zoxNUHWr0EXIeFmm3RkilsVUxqt5P7a0GC9Ocj59mV0Na97ROeQeQmvNWxcH6+ZErATTkAxOzZIrequSvxUSEhLyfiZu2jwaOrFCCEWskHuYTKHIv/vmD0kbioiobrLoa0W9FndSlDobDhWynBkaY09PZ12FLC8ImMrk6GysLPS7ElpjMTricfKeV9GxNva1MXt+ktx4BmORkjpPKbY2V1eqVglKa6ZzWWaVi7m5yJkzb2FISU+ske2NrZx96QpBEGBWUa6VbE1ixeyK3GXXMS2D8dFZWjuSVQlmleAVfR7+2P4lt9Fac3lyuiIRa+biOOn+Kcw7hCTbMLmWmq1YxBrMTpMPXCxZGrO9Kc74TAZjkVwzIQRYkswPB2n4/OI5cXbEoLsvSj4dMDfp4RUVAYq2xjjOfJMEpQMupsfZ3dC95DHmXQ+lNN0tDXz6sfvY3F6Z00xrzfefP4HtrH6ZXzRm8+qr5+nd2k58FQWzO+mfThFbppSyUgbzw1ii9N4UPYUfaMwaGx5EZYQj0yfZnuwNuxaGrBpjsxlmsgUa6uyQXg4JHLk0yM/dF2ZjrRZ53+NKZoqLcxNMF7MgwBIGmxMt9CXb6IrWpzFISEhIfQkzsUKuE4pYIfckM7k8X3n5CFMTczfCrSsl0JpA6xU7sW7lehZU//AEwpTs7myv2wTKkJKByem6ilgAT2/v4+vvvEtDhaVGGz7cx8Wvnygr/ARKk3BsEjV2PyuHpwIuzU0zmc9QTBfp/ZW9zHpFxHwp50hmjh9fOM/Y4WF62prYardgVhhaLoSgY1sXQ6evYVXRhc6yDQYujbP3wOZan9YCVKBo626moTW55HZeEOArxXLvll/wGHrpLEZ84fMSotSgoBJyvstgbhpb3vyZcWyLjqYEEzNZpBBlq3GFKQmmCxROThLdv7hrTQhBrMEkmjTI5TxiKorwJa4boBRICZNBmmknSYuTKDUa0JqC65e6ipqSZNRhf283j+7eTCJa3UJ1YGCSyYkMieTqL3Cv58u98uo5PvHxpcXKepIuukvm3lVK3s9TDFys+RwrhabgBSSM2qYgQghc5XI1e43eRP2+SyHL43sBqck0xbyL5/oYpsSyLRpb4sQSqxd8fjc4OzSOba69SBp1bC6PTYci1iowXcjy4vAZrmSm8XVAzLBv5HBqrRnMpvjJyHka7CgPt23h0fatdTkHhoSE1Ie1ysQ6HDqx1j2hiBVyz5EpFPnq4aOkxmahsvX4bRR1gK7/YSGEwC/4TGSySCHZ2dFaFyEraln0j6d4pK++i73tra3sbm/n8vQ0kQrcPWbEYtOzuxj43hnMmHXjuWkNgVbs7ahfbtdYLs352clS8H4+YMvTfTR23syLEkIQMS2mzk1i2QYj2TnGchn2tHTQ4lTmMurc1sX4pdGq3FhCCHJZl0LeIxKtj8MlnynwyS9/eNntAlXZp3bopbMgyudNQcnZthwazbm5EawypXyObdHVmmRyJosfKIwyGWEiZpJ7exzZm8SI2xhClD0ezw8wTYODu3tIzpf/BYEin/WZmSmSni0yGEyxNd6KZRhYpsGm9iY2tTfRkoytqMT2rbcvE6ugWUG9sCyTgatTuK6Pba/VT3d9znSTxdTtjild+t6vhJiMcmbuUihirTJKKa6cG+HMsSvMTmXJZQr4XoBSGiG4kQ1pGAaRmEWiIcbGbe3c/2jf+17UujY1i2PdnWnybK5Q1W9LyNIorXll7CKHxy4TNUziZvly9dj87Vprfjp6gePTg/zy1gdpi9T3ImBISEhIyMoIRayQe4qC5/PVw8cAmJlIY9VQClTUatUmlkprRKAZz2QwDUFfW+uK9ymlIFNcHVvsZ/fs5j++fYSc61ZUphbvamDTx3cx+INzmDELELhBwO729oqzmpbj8tw0g5kZHGngZz06Ht9I896FApnyFenBDIYtkZSW6+9OjrCjqY2eeOOy40hDsu2RHZx75TRWpHIxwzAlQ1en6NvVVcWzKk8+W2DfE7tp71n+c2Kbd/YFXIifd0kPTGHGFn8+lZRvZf0iOb+IbZQX6kzDoKslyVyuSDpXQOvS51RrTdHz8XyFLvpkXruGfLCtJDraJo2xCJZp4PkBAO1NCXq7W24L2DcMSaLBJtFgA0kyfoH7N3Rwf3P98qTS6QITk+k1Le2DUifV02eGOPDA6l+FBCp2Ji5HLsgjbxE0haDmUsKb+xBkguxKDy1kEXKZAideu8D5d69SyLnEEg5SSiJLnBtKj8tz8s2LvPvGRTo3tvDwh/fQtak+F2TWmrl88a4dt+v7pAtFGqLvbyFwPaC05rn+Y1xOT5K0KjtnCyGImzZu4PP/njvMr207xNbkyudjISEhKyPrubwxdnVNxglZ36yKiCWEiACHgA3Aor/AWuv/vBrjh4QsxndOnKbo+0Qsk2LBRdSQa6W0rlMa1kKkEBRyLg0tcYZm07TEYjTHKuuYtxQqWA3vGDimyZcPPcR/OnacVC5XUbfC5KYmtnxqDwPPn8VHs2dDJx2J+rQTH8zMMJiZwUbiZz26ntpSVsACyE8VUF6AYZcW6gKIGCYXZiexpUF7dPkrr8mWJG2b25kanMJ0KjudGoYkky5U/JwWQwWKWCLKBz51sLJxpaQhGsH1g0UXZhPHB5fsuOkHqqI8rKvZKUy5jEAsBA3xCMmYQyZfZDyVKR0bpQWEcEzE1SziYBsA2bxLJlck6ljct6WLjR2NmMbyInTccHhz8lJdRayLl8ZqbUy6IqIxm/Pnx9ZMxOpKJpgam8A2V5b75WufWzMETUMSsVYukBX8IoFWYS5WHdFac+LwBY789ExJPI7ZJCrs2grzLtf5DKnURJrvfPUVure08dEvPPy+c2YVPf+uObGU0uRdnype+pBF+M7Vd+jPTpGoUMC6FUNIEqbN1/qP8Ns7HqcjunTZfkhIyOoSt2we61yDckIrLCdcgivA5N0+iLr/Ogsh/gD4Y2DpXu8lQhErZM04OzLO5YlpkhEH3wvwvQCrhgnq6shBJYQUuMVS5y7bkJwdm+CRLRsxVuiIULXUTVZI1LL4nYcP8ePL/bx5dRBTSiJLvK5Ka4IWh72//hCtpzOkR+bqUjaR8z3656YxXY1MmGz+/G4iLYuvALIjWWSZvBNHmpydmaDJid7IyliKzfu3kk1lKeYKGBV+nnw3IPAVRo15K0ppCjmXX/qDT2JUITA81ruZ5987R6JMF0itNDNnRjCXKHPUWrO5eWmXmqcDZtzcjTD3SsgVPRzbJOJYBErh+wqlFboQYIzkcfoaidgWtmWggcl0lp725d1yUFpUT7kZxvNzdEQr+VlansHBKSJVuO/qhRCCdGblAmil7Gxv48jQ8IpFrPmKs/l/a2K2rIvDRaNRoYhVN9IzWZ7/2zeYmUzXRXAyTEmiMcr0+Cxf+79e5Iln97P74Jb3jStLr7TmdQUoSqWcISvj3MwYZ2bGKnZglUMIQcyweO7Kcf757g+G55uQkLtI1nN5Y3QNgt1DJ9aiaK3XRWvouopYQogvAf/H/H/PAGeBuXqOERJSC3nX43vvnrvRKruQK6ICBTXEEpWJ8KkjgsAvTVyFEARKcWFiit2diwdcV4KsYtLlKY+UmyYfFBBIEmaURju55MTNkJJntvfxyMYeXh0Y4Mz4BHnfJ1DqRqmeEAJTSjriCT61exfbW1sRH4ZzR/s5/P3jeJ5PLBGpeYFzcngIEShaDnbT/kgPYpk3KjeeR9oLn5OY/3NuZoJ9LcuX/ElDsvtDezjz8incvFuRkKW0IpcrkqzhMrtSinymyM//7sdp6Wyq6rH7NnTyo7MXy95XmMrgFzysRPnJvtaaZMTBXqbsM1XMVrH400zOZvGCADn/fpmGvJFVpR0LY6hAw8HbOwwGSnPyyggH+noqCt2NCIvjqQE+Hr2/wuNamtnZ/I3jXWuKRY9crkhsDTqm9bY0E6+y8UU5bv1Ou55m78bKcueW3S8iXFDWicFLYzz/tdexo1bdHVOWbWJampe/d4LBS+M884uHbisBXq/czWOUQqz44tW9jq8U37v2Hoky+VfVYghJ2svz2vhlPtS5vQ5HFxISUguhE2v9IoT4RUpGpt3ANPBfgH+ltc7O358AGoExrbW/0vHq7cT6F5TWq7+htf5anfcdElIz33nnDHBzMeW5Qc37EohSKPEqrWFvFQBMQzKeydKZTKyorHC5BberPE7PXub03GVyfgFfBeh5z5khJIY0aLKSHGzezdZ496KiWEMkwid37eLZnTvJuC7jmSxZ18WQgrZ4nNZYbEHOzu5D2+jbv5lzx/p559VzpKczWBELyzaXFbR816eQd9ERA3NfE70PbsKIVHZaC9zFy+pMIZku5PBUUJEby7BM9jy5j7OvnKaQzmM6Sy/8BYJC3qtaxPKKHkGg+MLvP0vbhpaqHgulz9OHtm/lpXOXbwi618kMpRYX/jR4gWLfhuUzQVJudvlSwnlcL6DgeYsu1oQQqPTCq2GGFBRcn/GZDF3Ny5d3WNJgLD9b0TFVQrHoYVorcyfVigoU2ezaiFiGlDywoYu3B4eJ2bWLWZYw8ZSHBkxT0NVUn2OPmZGqBPqQ8lw5N8IPvvEGsWTtFxGWQwhBvCHCwPkRXvjbN/jEFx9b90KWYxkrbkBQK1IIEnfB7fmzxJnZUQqBj70CF9atxE2Ho1ODPNHRF3YsDAm5S2Q9l9fHQifWekMI8XPAN2+5qQP4XWAz8Ln5254Evgv8FvDXKx2z3iLWTuC1UMAKWU9cnZrh0vg0jdGbExmlag9njwg5L/Cs1iTm9lmzbUjOT0zyyOaNNR1zoBRNiwhgWmvOpQc4PPkOSgdEpUPUcKDM+jznF/jR6BvEzSgf6/oA7ZHmRccUQpB0HJJOZZNHyzbZ99gO9j66nfFrU1w4+5o4OQAAIABJREFUcZXxa1OkUzkK+eL8S3L9VRfYEZNkc4LWrka279/M22KCzNw0RgUZSbc+9+UYyc6xObn487wVwzTY8+Rehk5fY+zSCKZtLRSFtCbwAwq5AsOXx5gdm0Hr+TDyphiJphiRmLNAdNRak5sr0LmljY//5s8RS9YuaD7au4mZXIGjg0O3vT+ZqymMSBmhQoMb+NzX3blA+CpHzncrFhZms4Vlt1U5H+0GCPv299YyDYYmZysSsYQQzHr5unX7qqRD42qhKXVgXCue3NbLe6PjKK1rXrglzBiTxRSeD4f6GuriYlNa0WRX50QMWcjwlUl+8I03V1XAupVo3GGof4If/d0RPvqFh9d1aWFTLMp0JndXjjFim8QqON+GLM6b4/0k6+DCupWMW+RaNsXmRPUXkUJCQlbOWjmxXgudWNXyx5QWxn8E/D/ARuAbwGeEEAe01ieAFyg5tJ5lHYpYOWD1WwaEhFTBTy/0k7jDGSOluOE0qhZbGqs8qb1930IIXC9gJl+oyY2Vcz36OhZOuAKt+OHo61zNjhEzHIRc2mkhhSBuRlFa8a1rL/Fw614ONu+u+niWQghB56Y2Oje13bitkC1SLLioQCMNgWmbC8oOv/teP3YVAtb1sZb6DFjSYLqYq1jEglL5yaZ9m2nd2MrFty7g5gqYERvlKzLTGfLZAjpQBIFC+AkINIhSHlVqopQNJqUk3hBlw7YOEo0xirkiIHjyFx9l9yN9CCHwA8VUNsfYXJqCH+D6JVeubZpETIOOZILWRAxrkdfk43t3kIzYvHLxynxGkY03m79NdNMa3CDAkpL9Pd00VtAlS6MpKr+iEi+lFEXPX17Q8DX+dAGr6/bwfyGg4HlkCsWyGV93Ugh8sn6RhLXyUqlaGkLUC8HaljlZhsEv79/HV94+StJxajr3tTktXMtOsqklRldTfUrV8qrA3sZdddnXvUqx4PLC375OLFnb+1or0bhD/5lhTh+9wt5DvWs2brVsbm9iaHp2zcUkrTWNsfdXCP56Q2vNjFfAqdAVXCmOYXB+biIUsUJC7hKlTKywO+E65AHgHa31v53//xkhxL+i5Lx6GjihtQ6EEGeBB+sxYL1FrNeAfXXeZ0hIzWSLLsOpOeJ3TEIN06g5od1AYKziIrbcnk1TMjA9U3NJ4cbW20OwlVa8MHKY0fwUcbO6ybIUkrgZ5e2p0wB1F7LuJBJ3iMQXFym01sy5RSJGdaczacolSwoFUAhqKzuNNcXZ98x+JvrHOfPqabKpLIYpkaaJMA00Yr5ccn4sKZD2zePPZQq89/oFYskIz/zqEzz46QP0z87yjaMnSWXzpAtFvCAAITCEuOGQUVoTaA1aYxkGiYhDcyzCtrZmdnd13CZEPd63hYc293Bk4BrHBkfI54sI42anxphtsb29hdZ4rOIFrtKVB227flBy9yz3XRKgsuVL5w0pmZ7LVSRiKa3JBW5dRCzLNlndFg+LI6UgUs4xt4psaGzgVx64n2++8x5xx67akeX5sKmpgfu31KesR2tNzIiyIVK+82hIZbz0rSM38grXmljS4bUXT7J1Vzfx5PoUbPb0dPDqmStrPm626PHI9k1rPu7PErNeATfw6y9iSZOhbOjPCAm5W5ScWJtXfZzQiVU1DqU89Fs5Ov/31ltuGwT212PAeotYfwK8JoT4La31X9V53yEhVfPapatlF1yRmL2ikhZLCDytWI2SwnLd5qQQpN0iRd/HKROsrbWmkHfJZYv4Xkl4MS2DWNzBMAXN8dvFr2OpcwznJ4gZtS8e4maEt6ZO0RPtoCNy965KBloT6IBqT2eR1gjunIuwFn8Pa+0OpbVmeijF8KUxGje00tDZTHpiFq/g4rsBKgjQfoCbn3dizXdvE9LAjpjEEzF692/GbYrwdxcu8A9/M0JPXweJiIMUgqhtEa2gK0GgFOPpLANTM7x07jJdDUk+uH0LfW2tSClwLJMntm/l8b4t/PnzF8CUCASWIRd1cS1FNWV2QaUlvQK0V15MFELg+pUKjZpA16cMryEZYXbu7pQZmZZJ8i4s+ne2t/Glhx/kmyffI+e6xOzl3SmBUuRcj8e3bubApgP8w/APseow7cgFBZ7p+uC6LkVb71w+M8zVC+PEG+6OgCSEwDQlL37zTT7/pZ9bl+9lSyJGW0OcvOutaQaSKQUPbO1efsOQRSkGXs1u+6UQQuDX6XckJCSkekIn1rrlArDhjtsm5/++tZSigbKhNdVTbxErDvyfwF8KIT4JfJ9SeWHZM77W+uU6jx8ScoNAKU4NjRItE0hs2WZZsahSEtJk0ncx6jyv1VrPuzwWIhFcTc2yo70Urh0EismxWSZGZ3GLPkpp4Gbmj9YapSFumTw3+gq792/kvgc2kxdFjqXOEF+BgHWdmOHw4ugb/OqWT9y1LmECQFf/RiQ2xEmdSyGtJY67hvdXa83AyatMXpvGisyH05sGzRvbbtzvFzy27+oi8INSJpYpsSM2kbiDNgRXgjzvBHmKeg7HMchPZbiaLrL7UC+ygu6HtyKFuOFEnCsUee7oe8QciwMbN/D4tk3YZukYHcvErHLft6K0JjU2w0xqDhnML1Jtk3hjrOx3rfQ5rSxbbvHAeY1RsRgtlnd9VciGnmbGJ+bW3BEFJQHtbi34NzQ28HuPP8Yr/Vc4PjRCxi3iGBaOebPE2leKnOtiSsnGpkae3bmDtkRp/rK/cRcnZ88RXcG5pxAU6Y1vZFMsXOTXitaaV194h1hy9ZsDLIVlm4xem2b4yiQ9vSvrwLtaPLFrC995+zTJ6Nq8Vq4fsK2zddkusCFLI1ax6PtulpOHhNzrxC2bx7rCTKx1yFeAPxVCHNJaHwGYLx+E+Ym+EMIBDgFX6jFgvX8lf8LNVckX5v8shl6F8UNCbjA4PUu26NEYK7+AtiM2vltbh8+kYTId1F+lV0rhxMovjE1DMpXNsa2lmauXx5mZyqCUxrSMRTulFYKAnQ3NuK7PWy+f58irF8h0ZEgeMuuig0shyfh5rmSG6EvenfIHQ0oiNUz4Y21RhLn0ZLSSzoS3orWm/50BZkZT2NFFBA4NyeYETR2NC+6aDFxOunP4aGwhicy/SaZl4vsBp9++zH0P99XcGU8KQXK+9O7N/qscuzrEZ/fvYXtHK7JGUdfzfEYujTI1ksLzAtw2D2NeVCzmXTIzWeyIRbIlgXPLQvDOLpWLoZVeEOp+nUDpsiJ1OQRU3DVxObb1tnPkSH9d9lUNhYLH9r7ONR/3Vkwp+XDfNp7c1svgzCwXJqcYmp3FVxoBNMUibG9tpa+1hfgdbq2DzfvI+Dn6s4M1CVmFoEi708KHOh6t07O5Nxm6PE5urkCisfYGEfUinnB4+ydn1q2ItbO7nZZEbMkuqjDves1kmchkKXgevtJoNIaQOKZBcyzKhsaGJbMbtdb4geLp+7evxlO5p2iwV6dzaaAVCSsM3A8JuVuETqz1idb6PwghPgA8L4T4I+BvtNa5+buFECIK/BnQDny1HmPWW0R6mbsVFBIScgeXJqZwlnCWNLbEGR2cwqrBfSIQxKVBRgV1c3fAvLjmLL4oz+YKnHj7cmlBbhpL6lBaaxxD0mKVQnujcQelA/ovjGFelvQ+1U7z5tiKjzluRDiWOnfXRCyAJifCXLFYlUPFcAyibTHcmSKijKVOaU3SrM5pM3xhhNRoCmuJ99DzAvo2315+6WvFKT/NqHJxENhlJt/SkAR+wPnjV9jz8LYVu3Fito3Wmm8efZfdXe3YCQc/W92Pdj6T5+xbF2+IqY4jMXTxZtaXIQCB7wVMDU2TaE7Q0JoABLZpVBRQLg2J0VTeBSEEtDUmKjpWQwoarPos2tvbkjQ2RPGDxTPV6k3gB2QmZhFTDfzgP/0Er+gBYDkWrT3NbNq5gZbu5kWdnPVGCsGW5ia2NFfeIVAIwYfaHyFpJnhn9jQR6VS00NRakwvybE/28njbQ3fN9fmzwpGfniOWuLsurOtIQzI+lCKbLqzLbCwpBb/w6F6+8tIREpGF4oXSmoHpGYbn5lCqlEUoBPMO0Xkx3w+4NjPL4MwsLdEoOzvaypZrZ4ouH9u/Y81cXz/LRAyLWJ07E0Kp++625PoUXENC7gVKTqwwE2u9IYS4fmW3BfgL4M+FEKPcNDb9GiX7xCXg39VjzLrOdrXWT9VzfyEhK+Faag5nCXdJ56YWRq9O1bz/ZsMmHeSgXotYrbFtE2mUX6DNpXLMpnPErBjxCiZnrlZsizTctsjO+gWwNVIILr44RvvuJFueaF3RQlwIwZyfwVdB3Zwu1bK3uYsfXbtAwqpu8t+xv5UrLw5ixReeCj0d0BVLVryvfDrP6KUxrKVKzDQ4jknilsXaVODyrj9HgCayzOLcMA3yuQKjAxN0b115qLUQgmQkwqWJaQadPBtTLl12ZUJPIV/k9BsXMEyJOf+Z9dAECjypEJSaIJiI0qLOMsjMZAFNQ2sDCEEiYpPOFZfOpzMFsqHM4lFpGuPRG2MvR8KMVO2sWwwhBAcObOaVV84RW6LpwErxix6jZ66RGpigmCtiGZL3ZjLY0ZuZfkpprpwa5M3vHy/l4DVE2fFgLw88tZdofP2JAkIIDrbspTexiVcm3mLKTWFgYEtrwXnIUx6u9mi0kjzb+WG6IuHicaXkMgXGh1PrRsQCkIbgndcu8PjH77/bh1KW1mScJ3Zv4fDZK7c1kci7HidHxnADvyRKLXIqEoIbotVMocCbA9fY3dlGW/xmTEjB9djU2siBrXdGioTUSk+siSuZKew6zksEgp0NYUOJkJC7RdZzeWMkdGKtQzZxMx/kuqGpa/7fUWAC+Dvgf9Ja10UfDMv5Qn5mmc0XlgxjNS2TRFOMfKZQU8t6U0hihkm+Tm6sQCmamxvK3jcznSWfLWIaBnmhbkvIK4fWGlNINkVv33LOy2AgS46vuMnE+QyBp9j2VPuKhCxf+cx4c7Q5zTXvYyUcbO/hx8OXqn5ctD1KpMXBy3pI8/bPgCVNWpzKnGpaaS4d68dYxgXjuj7bdnbdeK2Hgjzv+RkcBFaF7hLLthjun6C5o5FIrD4L0Yhl0drewNGhfh5AspGl96vRnHv7UklwFYKsDkhpHxdNwdd4tkLMF5YLwNaSCBLDlGRSOZyogxNzSEYd0rnikmPJuF02E8vzA7Z0VOYCUlrT4iz3ramOPbs38Pbb/SilV9Qkohy5VIZrxy6TnpgDBFbEBMNg5309RO8QzQxDEE3cFKu00rz709O88+NTdPV18sTnHqZtw/prB99kN/CZnmdIe1nOpi8yVpgk5+dRuhT4HzUibIx1syfZR7NTudsrZGnGr00T1Nh1dbWIRG2Gr0zc7cNYksd3bmE2W+C9wVESEYes63L82giGFFU1wTClRGs4PTLBzg5FV0OSouvRGIvwS4/tX5cB9+9Xnuzazrlzo3UTsZTWdMUaabDX38WBkJB7hbi5Rk4sM3RiVcm2RW7XwJzWerbeA4YiVsjPJHnXI+d6JJylHUs9ve2cPdaPvcx2i9FhOlx1c1QaUL04GsMwsMu4eNKzefLZItKQpS6EKlg2z6qoAg42tGHeIYzkg+JtpTh21CB1OcdgNMXmx2pf6GoNaS9710QsxzB5sL2HI2PXiFeRVyGEYNOTPVz8h37kLWfDogrY3li5Qy01NkMxW8RaLAcLCHxFQ2OUlrZS+duVIMd5P4uDqHrhYpgGgxdG2fFA/cItk80JItLgDHl8NFtZfKI+N5XBzbsYEZNhVcRFY1ByXjmege/4iFtsCUWtKApFXBtYpmRuOkN7zEFKSUsyxlQ6WzZvRnsKc8NC8cn1AjZ1NBOPVCbi5YIi25OL/b7WhmkafOLj9/Otbx8lXidXi1KKoeP9jJ0bwnSsG64+31N0djQQr9D1FZl3YE0Pp3ju33+X+5/cwwc+cwijQtfaWpK04jzc8sDdPox7hisXRolE11+mT3o2h9Z63Yo4QgiePbgL05C8fekap8fGMaSoqWuhEGCbBufHJxEatnW08qsffABrBc1mQhbSFkmwOd7CWH4O21j5cifjF/lC78E6HFlISEitZP01ysTyQydWNWitV/9NuYNVEbGEEPcBvw88BfTM3zwE/Bj4M631qdUYNyTkOpOZLH4FV5sTjTEamhNk03mMKq6mXkci6DAdRv3iinLSVaBo6VwY9O25Pum5/I3FpxACf5nYOU8pOu0YrWWuFpZ7pBUzGDs5S2tfnHh7bYtxIUDd5TS8Z3p2cDY1gR8ESwbw3okVt+g42Mb4sQnMqEmgFUnLYUOsvCuuHCMXRzEji59O9fxr07erGyEE14J8ScCqMdtHGpL0TBbfC2oOeb8Ty7GJJiJ4rs8FUUAi2LyII2v44ijSNhlSLgEa4xYB1wwk4o5ukUII0JqsCIhhYBY9Ai/AsAxiERuNZjqdX9BpUHuK6MGbJWRag+v79LQ2sal94fdlMQwh2de0seLtK2XDhmZ2797A+YujRFfYqTCXynDx5dN4eRf7Fofd9byxTZtaq96nYRrEm2KcOnyO/pNXefbLH1mXrqyQtWNyZHbNctOqwXN9MnN5ko0rz2lcLYQQfOyBnZybmiAYUYBE1tiiWKNBQ1p7/MoT+8NuhKvEz289wP995qdYKxRIs77LgZaN9MRCV2hIyN0kdGKFXKful2WFEF8GjgG/DewAYvN/dgC/Axyd3yYkZNXIux6VOqO27e1B6/lJZQ3EpElcGihd2+O1UkRiDs6dV8c1pCYzCxf2Sxyn0hpDCPYky0+0bGmiyjzeihlc/NEYKqjxOQBR4+5mrBhS8hs7H8RVAYFSVT22ZVczjdsaKeY9hJDsa+mqeMJbyBYpZAqLbq81+J7Pzr09mJbBeFDgtJ+uWcC6lfFrtWe6laNnezde0cNBcpY8oyy8EuV5Ptm5HGN4BOgypbQCu2gu/JwKgdCQI0BJyMxmb9wVjzh0NiUwpSRQGqU1WmmMZgej0SFQGtcLsAzJnk0dbO1srrjNuacCehPtOMbKRKbFeOrJ3XR1NFAoeDXvY3YkxekXjqOD4LZMNaU0WsN9ezasqGQxmogQeAHP/e/f5eq54Zr3E/L+J5cp3O1DKIvvB8xMZu72YSzLeDrDrOfxyJ7NNCaieL7C8ysvzwyUwvUCbNPggR09tLckODI4tIpHfG8TN22+sPUgab+IrnGOlg88OqJJPt5zX52PLiQkJCSkVup66UcI8SilRHqA54C/pJRCL4Be4EvAL1FKrH9Pa/1mPccPCbmOGwRUuuYzLZOtu7q5fHpoyc6AS9FhOgzrAq5WVeZjaRCCpraFAeK5bAHfVxh3ZDUtNg3TWuNrzaON7QvKCK/TYCaYKM4s6O4lpMDLBYydmqV7f/VXGg0haXEqd8asFi2RGL9z36N89dwR8oFL1KisbEYIQfLBJqSADakoVhVOrtRoatFwf600vq/YvW8jiWQEVytO+mnsOlw/MC2Tmck0G3rrFzLb2N5QcmlojSMEp8jTjIlzy/H6RZ+iVhRQmIt81h3XpBjxF35YhUBoTUFooq5/2122ZdLRnCRQinSuQH6mQOKDndimSdQ22djeRNSu/vtZCDw+2L6j6sdVimFIPvuZB/mH7x1nZGSGWKy6Uq3ZkRQXXjqJFbNvE0KDoJQNtfe+Huw6OGekIYk1RPnen7/Ip//5x9i8KwyQvhcJAoW8Sw04lkIIgbsCIXitePnSFaK2iWkY7N7SgecHDE/MMjGTxQsCtALDKD0fQcmhHCiFQGAYksZ4lE2dTcTmOx1qrTk6OMwTvVvWbSnl+52+ZBtf3HaI5/qPYUmj4owsrTUZ32VTopkv9j6EWUN2akhISH0Jg93XJ0KIH6/k4bU0B6y3f/l/oCRY/arW+pt33HcR+KEQ4u+BbwB/CPxynccPCQEgUNVZx1s6G0nPZGsutRAIuq0Iw14eT5dzp5RDowJNW3dT2eDqzFwBo0ypQrmLiUprfDSHGtqIm4sv9BNWdNFjM6OS8VNpuu5vrHoyHTUiRCoUjFab1mic//7+J/jewBlOTY3iGCbOEnkYOd9DacUjnZt5+ssf4dzJaxx+6SymJbGs5T8L6akMpr1wUuy5AZGoxe77NxKJ2mitedcvBXXXa7Hi5j1Wnsd2EyEEPTu6uXLqGnbEQqB5lxyHiN9wPimlmBHBbSWEC/ajBXbRwHWCBaWFzJfELuZcNKSkMRqhpTHBrqf3lP1uVEox8NmSaKM9WnlpaC0YhuRzn3mQN9+8yLETA0QiVkX5U9npNBd+/N5tApbW4HkBDckIfX2dWHUqF4XS+xtvjPH9v/ghv/gvP03HxupLFEPe3+i7Xfe9CEIIfN9ffsO7iNaa/qkUzi2/C5ZpsKW7hS3dLXh+QDbvMpct4PoBoDGkpCHukIhGsC1jwblfCMFcochEJktHMrHGz+jeoS/Zxu/teZK/HzjBYDZFzLAW7aastSYbuEgh+VjPHh5q3RQKjCEh64S4tUblhFZYTlglH6L2xUhNj6u3iPVB4O0yAtYNtNbPCSH+kNKTDQlZFUodgKqbrG/e2U3gK1IT6ZqELIlggxVl1CtQUApjyUmPns/BasJyFo7luX5ZFxYsNP34WgGCRxraSSwhYAFY0iJqOHjKLzuZdrM+6dECDd3RJfdzK/mgyIGmnRVvvxY4hskvbrufp3u28/LwZS7NTZH1XAKt0Lr0XE0hiNsOD3X08ETXVhJWqRxy74EtbN3eyQ//4Tgj11JEovaSuVPFXPHmazkvQAgBGza10L2x+cZ9I6rAtPLqUkZ4nUAFuAUPO1I/AbF9YyuTQ9MUMgVM02AGn2FceubzsbQQFITGWsZNFi1YeJbiRpvCW9FQ0IuXfPo5j75fv39FApbWGo3mMxsP1LyPapBS8IEP7GDnrm5eeOFdUqks0Zi9qJilAsWll09jRSyEEDfEK8OQbOttp7U1sSoLJyEE0WSEF77yEr/+r38BIwyTvqeodyfNeqGVXpdZXbcyky/gqgBnkamzZRo0JaM0JSv//QSQAgZSM6GItcokLIff3P4oo7lZfjJ6kZH8LAXfI6A0L5BCYAqDpO3w4fadHGzdtOQFsJCQkLUn47m8vgZOrEzoxKqWj6z1gPU+O7cA/1jBdheBsMVHyKrhmGbVGVVCCHrv68E4P8rEcArLMSvO3bmOnHdkpQKPmcBDwoJ9aK3RWtPa1VS2GyFANlNcdLFx/VatNUWlaLZs7k+2VGyR74l2cCFzFVssHNt0DCbOpKsSsQSCfU2rV661EpqcKJ/t3QtA3veYLuYIlMKUBq2R2KIT1Hji/2fvPaMju84z3WefWBGFHBuxcw5sdmCTbGYqkBRFUomSOJJlaXTtsb0crsf2nWuP7eU7o/GMPZY9Y9ljSzZlZVGBSqRIibnZzc45ZwANNHKodNK+PwpodgAaqQoooM+zFha6qk6dvatRVeecd7/f+wV4/OnNdHcOsPPNk1w814llOWiaghkwrvnbOJYz5PwD3dCYV19CWWXsGvHCkh5HnEGMLDmmhpGexLZcstvxW7BgbSMHXjuCIiWGEBwjRSl6pqxQV8YpLgnCCZ3BiJXRsa6ZuIRRhEEnblO+uZZA6dQCngedNI/MW0NIm16HYElxhKc/tpm2tj52vHOaS229OI6HrquYpnZFmLq45zTphIVQFYQQmAGd2nnFFBdHci40KIpCMp7irR/u4u4nN+Z0LJ/8QtM1Ri9Kn1lCkax+kWWdrngC151Y3uJ4COo653t6ub0u+80nfG6kMhTjo023ZRxXjkWvlUQiMRSNYjOEnofltj4+PhkiusHmqtw7sd7WDXpzPsrcQUr5+nSPmW0RqxtYMI7t5g9t6+OTE6JBc5KtrwX1i6soKA5z7lgrUoI2QaeCQFCsGkQUlXY7jT0Utj5cPqibGsXlBSg3KTeyLWdUoUBB4EgPD1gWKaLKDE7IsVFgRCjQI8SdBKq49rUpmiDZPf5ckribYnPJqrwpJbwZQU2nRptYbldxaZSHH1+HY7t0dw3Scr6T5gtd2GkHT4KqCiIFQUrKo0SiQYyrRIqrOeoMICAnzprJhtXeDN3QaVxZx+n95zACBgLJUZKsIYwEjKCBm7THfD2qqxJIaaQCzjVlhUKIawLMh3FTNsHKMJV3T+0EJeGkWVpYzbLCmcl9EkJQVVXI4x+4Dcty6Owc4EJzN5cu9eI6HgMd/fSd66C8qpDCWIhw2MxK7tVECIYDHHrzGEs2zKe8tnRax/aZOaKFQfp74nlXHqVqKkVlN2ZD5hNp182JwKwIge1kXxzzuTlCCCK6ecWF7ePjk/8M2hZvt16clnF88ptsnzVvAx4XQjwhpfzeSBsIIR4HNgIjPu7jkw1KwiF0dfKraUVlBUQLw5w91kJf5yCarqJMMNTTECrzjCC9jk2Pk8aTUFQcIVwwtsvJdbwbRSwpcaSHKVSKNZMlkcJxu6+upzFcw+G+U3jSQ7muvM2KO8hxtKNOeRbVgVJWxOZPag6zCU1XKa+MUV4ZY+3Ga19v37n2IXfDyDjSo8Oz0LNYRng1uboYLa4swrVdzh25iGHqdAkbB4mqCCKFYXrjPYhxtJc30zqekO/mY0mJqqs3iMxuysYsCjL/YyumVEaYcNPUR0p5f82qSe8jmxiGRnV1EdXVRVfu+/4Xf8rKtQ3jys3KJaFogLd+sJMP/sZ7Z3QePtNHdUMZHa29BCbYgCDXhMImZhbLonOBqap4OcgU86REn+HvAh8fH5/ZgO/E8hkm2yLW/wA+AHxLCPEN4F+Bs2S8603AM8DHAG9oWx+fnKCpCtGAieNNfnVT01UWrqwjMZii9WwH/T1xpCfRDHVcZYae5+HYLoWmzsL6CvTCAC29/QymLKSU6Ko64qqu9DLlhgKBlBIPiQQ0ISjQDFaHSpg/xaBqTVFZWtDE0f4zuNK9xpHluRIn7aEHRhfIkm6aMrOI91ZvybsyELB/AAAgAElEQVQV/elGN/WbhiU3u0m8HJXvCCHQRwiVzxZltaUIReHsofNIQ+WCkqZOM9B1jUDYJJVIj0vcDaYMBDZp0wFPogcMzKscjk4848Ca/7EVKFN4PQNOisUFlTw6b80N4my+EO9P0H6+k9A4xOxco6gK7ec6iPcnCBdMrXzTZ3Ywr6mM3a8dm+lpXIOUkoKi8ExPY0xKwiHUHHSoS9o2dcUT7wrs4+Pjc6vhZ2LlB0KIr0zl6VLKT011DlkVsaSU24QQvwH8DfDxoZ+rEYAD/IaU8u1sju3jcz2FoQCX++NTtv+HIgEWrKzFdVzaW3robO2hvzdBMp7Gdbwr5VwCgVAyuUiRgiAFxWGqG0qJFoavCD1lsQiW49CfTNM9mCSetkjbDu5wyLUEz8sIVypgKCohVSOgqBiKQkp6lGUpAMlQdZbHFnA23kK/PYgutKGAaYl0RhZdHM8l7VmsKlrE7cXLUfNUKJhOCorD9FweGFmQlJILXgpjjBD0yaJq6qi5atmitKaYQMTk1L4znEsnaDRNCkMBlHKB29qDbdnjErICKQ2RBlGpgZBEAybS9XASNmUbaqi6px4xSTeC7blY0uH+ymWsK87vVvV7f3EQZYSGDTOFoins/cVB7vygn411K1BWVZTz74yJkhhIs37r0pmexpgUBgOYWvbLfj0pqS/yRSwfHx+fsfCdWHnDM6PcfyW6+Sb3C+BTU51A1o/GUsq/F0K8BfwWcDdQM/RQC/Aa8EUp5YFsj+vjcz1NZcWc6+olYk69RCE+mKL1XCcDfUk81yNUECRSGMqIWF7GKSVEJjBZURUc22VwIM3Z4+2UVsaoqCm60uHO0DRKoxql0czKs5QST0qkBIlEupID3c6InZpUBGElexcgmqKyMFpHnzVAa7KDpJvG9hxc4V2Zmy0dLM9GVzSqAmVsKllJkTk1J9hcoqapgrbznQTDN4qLPZ5NSrqYIjduKcPUmXxH2/ETiYVZfddyjh46T/P5AWqiEbrjCUpriuhp6yOVSCEUZVTxyHNdhKJQVVWGoqsMKnES/UkiRWEW/cpaghWTc2F4UjLopKgKxXi89jYK9Jl3N43FqX3nCITyJ4MlEDI5te+cL2JlGSklrec7ObjtFMlEOtPLQFdZuKqWhatqZ6wrpKarNC6u4uzx1qHvj5lHN1UWraqd6WmMiRCCuqIYF3v7phRXcD1R06Q88u53oOdJzpzv4NCxFizLBcA0NFYuq6GhtjRvO0z6+Pj45Jq4bbG9NfdOrLjvxBqLT49w30bg82Q0n+8A54bubwQ+BFQD/wBsz8YEcpIkOyRSfSYX+/bxGS/zy0p4+eipKe2jvyfO+VOXSSctNF1F0xS4ykWhGCM7Kgzz3Y9WW3M37c09FBSHaFhYeUXMGkYIMRT8nsETclRXT1BRr9k2W8SMKDEjiuXZdCl91BdWkhY2AijUo9SFKikPFGPOggD36aZpWQ07Xz444mPnvSR6jlxYju1QUjmxoPqpIBSFxSvqKVipUnU2zqlLnSTjaYoqY7h2hP7uQdKJdKYr6ND6ixCZbmgFJYUEIwGk65IYSHLXikUsun8RrbUWl9J9JB2L4AS6CKZdB0s6VJgFPFS9nAXRirx2Xw2TjKdIDabzopTwalKDaZLx1IhCrM/E8DyPA2+f5tCOUwz2JQlGzCvZZ1JKXnt+D9tePEDTsho2PbhiRgTN9fcs4cTBi3khYllpm8alNTMm6k2UrQsa+T9v78yaiBW3LLY0Ztyjtu2yY89Zjp64RDJlEQqaV84F+gdS/PjnBwkFDZYtqmLDuoYJN53x8fHxme2EdYNN0+DE2qYb9OR8lNmLlPLZq28LIdYAXwL+J/AfpZTOdY//R+ALZESuf8jGHKa3HZKPzzRSEglREg5hOe6EL3Bd1+P8yXa6L/ejm9o1otRE0YdEq4HeBAfeOUPDogqKy0Z3MimKQDe0GwJkLTzmG7l1QBmKTnVRGQ/WbMrpOHOJwrICispipFPWDe+zuHQn1SVzXEioqC3Jzb5HQVUU0orgvZ+5nzs+tIn//qXn6TlxGWF7FERDuKEAnieRSASZ97IYcicKVSCKwnzqNx/hno3Lruwz7qTZ0XGas/FOBqwklnQBiYoyVN4KrvSQQqILlageoD5SyR1lCyg0ZleOU2dLN47jzvQ0bsBxXDpbuqldNDPdHOcKtuXw0397i7YLXYSiASKxa8VKIcSVxh6nDzdz7lgrj//qPRSWTG9XvmhhmIqaYnq7Bm5YVJlubMtl/dYlMzqHiVARjbCorJRz3b0EbtLQYzx4UhLUNTY31JJIWHz7+V3EE2mCAYPIdYKyoggi4YzguffQBc5c6ODJR9YRzPMwfB8fH59s4jux8pY/BZqB35MjtE2XUtpCiN8FHhna9vGpDuiLWD5zms3z6/nJgWNEA+Nf7R4cSHLqUAueJ6ckXl2PqipI4MyxS3S19zN/aTXKKBlAgaBOfDB9jSiiIqjK8UW750mKSvI/YDffWHvPEn7x7R1EYu/+fTwpSUk3J10JpecRKQzdtCtirhhIpXFcj1hBmD/4rSf52vZ9dAwMErAg1TmANZDCs10UVUExNELlBciogSfg/auXsHJe5TX7C2sm91VlRC1PevRaSdqSvQzYKZyhpgMhzaAqWEixGZ7VOWwXjrZgBvPvotMMGlw42uKLWFPA8zx+8uybdLX3jasDrRkwcF2P7/3jK3zkPzxEODq6C643leRARxv96TQAMTPAyrIKCgOTd/Td/+R6vvG3L82oiJUcTLN68wKisdklRj+2Yil/98bbuJ43paD3wVSaZzasxXMl3/rhLizLGZcoFQwYxOMW3/7hLp5+YuOVhTIfHx+fuU5YN9hU7Tux8pAtwM9HErCGkVJKIcRO4KFsDDilKyAhxJfJFI78kZSyfej2eJFSSr/k0CenLKsq5+Uj4y8p7O+Jc/JQy1DpYPYdNAIwDI2BviRH919gySjZKEUlEfp6EldysRzpUa4HclJKeDWpeJqmzfNzOsZcpGl5LTuLD2NbzpXSobh0c9aV0HFc6hZV5WTfY2F7Hl3xBBUFEcKmwWfuWs+xSx1sO3Uey1QIEkNVFKSUONJD1XWWVZVxx4J6CoI3L1dThEKxGabYnJtCaldrT96FagMYAZ2uVv90bSrs/OUROlp7CUbGv2CiqgrSk/zsa2/x1Ofvv+YxKSUnujt57eJZ2uJxFAG6kjlW2J7LKxfOUBmOck9tIwuLSybsNo7GQmy8fzk7fnGY0ATmnC1c1yMcC3L7vcvG3jjPCOgaz9y+li/v2I2paRMWsqSUDKYt3rd8MXVFhfzoxQOk0jbmCDmYo6HrKomkxcuvH+W996+Y6Evw8fHxmZXELZvtLRenZRyfCREEysaxXTmQlZWrqS7jf4qMiPUFoJ2JJc1L/NwsnxyjqQpLq8o43Hp5TOv/QF+CE4ea0Q0t51HZmqZgpWyO7b/I0jV1NziyikujXDjbceW2i6QpkPswdVVTWLRiXs7HmWuoqsJDT2/muf/18hUXRo/MzQHQthyq6stmLBxcEYLz3b1UFESATInh8poKltdU0DkY53xnL/F0GkPTKA4HaSorQZtk18G5hud6eZndJYTAc72ZnsasxfM8ju45NyEBaxhNV+ls66W3c4DC0kxZoeN5fOvoAU72dBE1TKLGte4cYyiPacBK8/Wj+1lcXMqHlqxEm6CYsmrTfE4faqa3e2Ba87GklCTjFo98cssV0X+2UR6N8NlN6/nG3gMMpi1Cxvgclpbj4HqSD65axrLKcpIpiwstXZMqCzQNnbPnO0lbzoQEMB8fH5/ZStjQ2VSd+0Yg2wzdd2JNjCPAViHE7VLKnSNtIIS4HdgKZKXB31SPesPJ9Jeuu+3jkzfcubCBA81tN93GStucODg9AtYwqqqQTtmcOtLCwhXzrrm4VVSFouIIPd1xpAJFqklIye1Jqm071DWVY/gnw5OitLKI1Xcu5sBbJwhFA3R7VtZD3T3XIxA0qGoYz2JHbggZOue7etjQcKPYWRoJUxqZmy6qbHATl7XPLOb88Uuk4tYNGVjjxQwY7PzlER788EY8Kfm3w/toGegnZo7lXBTEzABn+3r42pH9fHL5mgll8AkheP8n7+A7X/oltjVyR9xsI6UkPpDigSdvp6R8+hpT5IKSSJjPb9nIKydOs6+lDdt1CZvGDUK1lJKU7eBJSUNJIY+tWEp4SPTae3BqjgKJ5ODRFtavrp/Sfnx8fHxmA3HLZnur78TKQ/438E/AS0KIvwb+DRgOL6sDPgn8Nhnt6e+yMeCUzliklP96s9s+PvlANGCydVEjrx4/Q2SEbCwpJacOt6KqyrQJWMNomkJ/b4Kuy/2UVlx7Ql9TX0JXRz8oKitDRTmfi5V22Hzv7AnYzUc2PrSS/u5Bzh1rxTFlVkPdPddDUQSL1zXMqJtHEQLLdcbe0OcGFFUgpcw7N5aUctR8Pp+xOfD2SUI3ybQaC93QuHCqDSklL549SctAHyF9/M6coKZzsb+Pl8+d4qHGhRMa2wwYPPnZe/neP71KOmXl1JE1LGDd+9g6FiyfG45fTVF4cMlCti5sYn/LJY62d9CbSJJyHCSgKwqxQIAVleVsbKglYl57DnL8VNuUwtkDps7hY62+iOXj43NL4Dux8hMp5VeEEGuBXwf+ZOhn2OI/fIIpgS9KKf8lG2P6lgufW4KNTbUcbm1nIGXdUNrU3tpDMp5GN2YmHDVzAXOZWFH4mpVww9Qprikk2u5gKrmdWzKeZv2WhbMuYDffEELwwEc28cvv7mDPwaNgKpAFwcK1HTRDY+n6phkJc79hPn7l2aQoKi+ko7l7Wku3xoOddii8ScdUn5uTStgoytQ+557jkUhZ7L98aUIC1jBhXWdv+yXuq58/4bLCYNjkqX9/Lz/66lt0t/cRikxekBsN23JwHY+HP7yJxiUzk+eXSwxV5fa6edxeNzFxzrZdtBFyMceLEALb9hcVfHx8fHxmFinlbwohXgB+C7gTGD6ZSAJvAn8tpXwhW+NN29WQEGIhsAo4L6XcNV3j+vhA5kTvqfUr+ftXtxNR310JtS2HlrOdMyZgQSbsXQg4e6Ltmjwqy3FZs7SWoBJnoC+Zs1IPx3aJFYVZt8kPdM8GiqJw/4c2sUcOcPTIxUyXvsm6XKTEshyKy2PUL6lCVfOjC9VoZXGeJ0mkLWzbRVUVgqaOPoULtLlG3bJ5HHzzaN6JWOlkmrplc8MZMxN4XnbKRPe3X8JyXUx1ct/1adfhUEcbayom3mXSDBg8+av3sH/bSd555ShGQJuSuDKMlJLEQJrKuhIefOr2nAhksxkvCyXG2Xr/+fj4+OQ7frB7fiOl/CnwUyGEAhSTucztklJmffk7q1fFQogngF8F/lRKueOq+/8T8J/JvBCEEN+QUn4im2P7+IxFLBjgnkVNvHL8DNGhssK25u5pLyEcCVVVGOhJYFkOhqFlOrt5Lk+uW4G6Gr79lTdwbDfr7dAdx0UIwaMf3YgyhVbhPtcihKCmsQytMMCZQ80M9iVQVGXETpQjIT2JbWfeC4vW1FNQFMnxjCfG9a6TZNpm97GLHDrVStJyMiVzCBRFUF0a445VjVSXFeRdGd10UzavOG+EyKtRFYXy2pKZnsasRdNVXMed2k6EYE9XG5FJuLCGiegG21ovTkrEykxBsGbLIuavmMfL332Hyy29GKaGbk78VNHzJInBFIGgwdbH1rJ4dd0t//kfiWw0vciG2Ojj4+MzGwjr01ROqPvlhFNhSLTqzOUY2bZ2fAK4Gzg4fIcQYgXwZ4ADbAeWAx8TQnxPSvm9LI/v43NTNjbV0j4wyPFLnQQNja72/qwLQ5NFKIJLF7qom1/OYNri6Y2rCZuZC5qnPnUnzz37JlbazpqLw0o7aLrCU8/cSSg8M53u5jKKUNB0jcXrGkknLVrPXqa3c+BKFzhVU6+IQRJwbRfP81BUhVAkQNPyeUSLQpAXMuu7SClRhy5GpZS8susUB063IoCgqRO+Lt+ls3eQb760h6JokCfuXU1hdHLh13OBUDSIGZq8SJErzLBJ6Bb+u0yVyroSTuw7jxmc3N9WSkm0MESfa6OKyYsaQgiS9tRXj6OxEB/8zD30dQ+y69WjnDvRhpWyMQI6hqmNKka5jksqYYOA4vIC7nzvKhoWV/kLJDehqDBMd0980l0aHcelwi8F9vHxuUWI29PkxMrCsdQnt2RbxFoL7JdSJq667xNkrtF+VUr5rBCiiUwbxs8CvojlM60IIXh01VLS9kH2n2jGsV2MSawy5wJNU+i63E9hdYwP376S+pJ3w9xDYZMPf/puXnp+L81nOwlFzUmvakspScTT1NQW8+Bj6wjk4UX1XKA0EqK5p5+ArmEGDRqXzUNKSTppEe9P0t89iGNlwn8VVSESC1FQFCYQNvP6os9yXUoiIaSUPP/6Ic5d6iZ0E2FVVRWiIZOU5fDsT97h6fesp7Tw1uxgKISgbmkNZw9exAjkR0mhlbJpXJn7Vc25zLq7l3B099lJPz8xkGLrY+s4b11gqkY9J4uO/VhxhPufuB3bcmhv7ubiqXbamrsZ7EtiWw6e6yGEQNEUwpEARbXF1C2ooLqhzM9XHCebbmvkuR/vIRKeXJllKm2z+famLM/Kx8cnV0gp6etL0trey/nznfT1J3GHFzdVhcJYmIa6EqqqCokV+ItL1xPWDTbV1OV8nG264TuxJoAQ4itTebqU8lMTfVK2r95LgJ3X3bcVGAS+DiClPCOEeBNYOpWBhBDzyDi83jM07iXgB2RKGcf9vhNCFAN/DDwOVAFdwAvAH0spm0fY/gvAemARUEomrOz80Nh/J6Xsum77hcATwMPAQqAC6CHjSvufUspXxv+qfbKBogieum0lB3ecQer543KRUpK0be6vr2dhRekNj5sBnUc+vIHTx1p59YVDOLZDMDx+MWtYvNJ1lfvet5qFy6r98o4cMr+0hHfONRO4KohdCEEgZBIImZRUFs7g7CZP2naYX1rCL3ae4GxrF6FxdtbSVAVFwLde2sOnH9047ufNNW5/eA0ndp3JGxHLTtvc/vCamZ7GrCYcDVBeU0Rf9+QcNbqpsXBVLeqeG045Jowhsu8s1g2NeU3lzGsqz/q+b3WqKwuJhAOT6loqpSRWEKKsJJqj2fn4+GQLx3E5eryVvfsv0D+QQkpJMGBcd8xwGRzs5uTpNgQQi4VYv66BhQsqs1J6PBeI2xbbm6fDiWXlfIw5xjNjPD58gBspxFEAn5rogNkWsUyuqn0RQhjAGuA1KeXV7VPagC2THUQIMR/YBpQDPwSOARvIpOG/Rwix5XoxaZT9lAztZxHwS+CbwBLg08D7hRCbpZRnrnvabwN7gJeAy0AY2EQm8+tzQohNUsqrP11/DnyEjPvsp0A3sBh4DHhMCPFbUsovTvg/wWdKKIpgsQxj6zatTgITZUYFHVdKXCS3mcXovTe3sM5fUk39/ApOHGlh344z9PcmMgfDsHnDBZTreiQTFgKIxoLc+cAyFq+Yh54HHe7mOpWx6JWyu7mEogiihsHBU5cIT7B8SlEUbMdh24GzPLBhcY5mmN8UlEQprSkm3p+ccke7qeJ5kpLqYgr8i+Apc9cja3nuS78kGJmYSzben2TzwytRNZUCw2TASqNM8nvDk5JYwA9On00IIdi6ZRE/+fmBCbux4ok0H3jPGn8xyscnj5FScuJUG6++fhzbdgmFDMKh0SM8VFW58l1g2y6/ePUob247yf33LKWp0V9ICOsGm2umIxPLd2JNkE+Pcr8C1AHvJaPVfBHYm40Bs30lewlYdtXtu8kIW29dt10E6J/COP+bjID1m1LKvx2+UwjxV2REpr8APj+O/fx/ZASsv5ZS/s5V+/lN4G+GxnnPdc8pkFKmrt+REOIvgD8C/hD4taseegH4gpRy73XbbyUjhP2lEOI7UspL45ivT5awLYdk3GJFuJAy2+Rwug8pPfQp5JFMBiklFh4FqsGaQBGGUGg938Vtd938eZqusmx1HctW19HbHaetpZsLpzvo643juRmRW1EFscIwtU1lVNUUESsO+ye700jI0Akac89tFNR1jpxqQ52kAGMaGsfPX+ae2xbesiuLmx+9jef//udEZrisMjGQ5IGP3zmjc5grlFTEePijm3nhG9sIRgLjEijjA0lWbJzPqs0LAbirtoFvHT1IzJxcRuGgbfHo/CWTeq7PzDG/voy7Ni3ije0nCIfGFkGllMQTae7dsoR6vyGDj0/ekkrZ/OylgzS3dBMOmRgT7DIuhCAcMpFS8pMXD9BYX8aD9y/HzFG38tlA3LamKRPLd2JNBCnls2Ns8mdCiP8X+F0ylWunpzpmtj8FrwGfEEL8Phnx5s/J2MZeuG67FcCkfPNDmVoPAeeA/3Xdw38CfA74pBDid6WU8ZvsJwx8EogPPe9q/o6MGPawEKLpajfWSALWEN8mI2ItvPpOKeW/jLSxlPI1IcSrwIPAHcBzo83VJ/v0dA7iDHWTqtCDFGsmh5I9dLjpaXNlDbuvlgQKmacFr4w50JcY45nXUlgcprA4zBI/1ybviAVN+pPpOSMeSimJBgyOnmsnMIUGA2nL5fj5yyxvqszi7GYP8xZVs2BNA+ePNGPeZEU2l6QTaRasrmfeosl1svO5kfrFlXzgV+7mhW9sJzmYJhgJ3iBmSSlJxtMoimDD/ctZe+e7jsTFxaWE9cl/riK6wYJiX9SYjaxdWUs4qPPLN49jOy6hoHHDcUNKSSJpoesq73tgJQt8V4aPT94yGE/zre/uwHG8SWfeDSOEIBIOcLGlm298ewcffvJ2QpNsJDLbyWRi5f565y3fiZV1pJR/LoR4how+9PRU95dtEesvyGRL/ZehHwG8LKW8kpMlhFgENAFfmuQY9w39/vlQ+8YrSCkHhBBvkRG5NgG/uMl+NgPBof0MXLcfTwjxczKC2L3A9SWFI/Ho0O8D49h2mOG6MeemW/lkncRgph59GF0orA2V0G4nOZLuw5YeJiIn4oMjPRwpKdIMVgWKMJVrM0wsy387zBUWlpXwxunzhOeIIytp2ywoKeGYdYnIFE6gwkH9lhaxAO756Ba++mffnVQWzlSRUqJqKvd8dNJV/T6jUFlXyjP/9/s4f/wSu187Ts/lPjwJSIlQBKFogA33r2DZ+sYbmoooQnDnvHpeOneKqDExcXPQTvNQw8JJlyL6zDyLFlTS1FDG8VNt7N5/gf6rzlMy+ThB7rh9CQvn+/k4Pj75TCKR5pvf2YGUcsLuq5sRMHUsy+Hbz73DR5/aSCBPsjWnEz8Ta9azk3e1nCmRVRFLSnlCCLEF+B0y5X7vAH953Wb3A/uBH09ymOFlyxOjPH6SjIi1iJuLWOPZD0P7uQEhxO+RKYuMkQl6v5OMgPVfbzLm1c+vJ/N/kQBeH89zxtjfSEFpPqNgW86I5R4VepByLcBlJ8UZa5AB10YTAm2KZYZSStJIVARlWoD5ZpSwMvLHb7gc0Gf2c1v9PN46fWGmp5FFBMsryjl6qHVqexEC27l1xFrP8+jvGuRySzfJgRTSkxhBnfUPreb157ZTUByZNiFLSslgb5wP/NrDGFNw0/mMjqIoNC6toXFpDamkRTppIT2JbuqExsjM2lRdS0cizv6ONiL6+ITiASvNuopqNlb7btzZjqapLF9Sw/IlNSSS1pVFLcPQblnnhY/PbMLzJN9/fg+elOhaDhpt6CrptMOPfrqPpz64fs44/cdLWDfYNM93Ys1iKoCsdLbKelGtlPIQ8Cs3efzvgb+fwhCxod99ozw+fP9Y/0FT3c/vkflDDPMC8CkpZccY4yKEMIGvkckL+/2JdFP0yQ6KoiBH0YqEEFToQSr0IAnP4XR6gE43jSMlHhKdsUUtKSU2EldKVCEICJUlRogaPYQ6xnNvtQPSXCagazSUFtHc04euZv9kZjpxPI+64hjhLK38zfX3ueu4nNh7joNvnWCgN46ddoYcUJnPv+dJpOsRH7A4ufcQ5XWlVDWUoecw6yIjYCW47+k7/TLCaSIQNAhMQHwQQvDogiUENZ1trRcwVRVTHfk9kXYc0q7LXbX13FvXlK0p++QJoaDhC1e3CFJKmvv72dPSwoBtZ5rxmCbrq6upikbn/PFyLrFzz1l6+xKEchgVYBga7R39HDzUzKpbLErEd2LNXoQQd5Ix/Yynwm1M5mIy3M1aOGZtP1LKSgAhRAWZTKv/CuwVQjwipdwz6k6FUIGvkunO+C3gv09xnsPzGfEIJ4TYDazLxhhzCcPUkN7Yb5GQorEyWJRxUkmPftemw03R7zqkpYMnwWPY7i8QgK4ohBSNItWkVDMIK9qEnFwz3bHMJ7tsXdjAP7+1a9aLWEnL5p5FjQQNY8rvUdtxKZhiRkS+4jouO17cz9GdZ7Ath1A0gBHQMUYR/yJFYYyQzqldp2m/0EmsJErD0ppRt58sUkrifQnue3oLSzcsHPsJPjOGEIIHGxewvqqG1y6e5VhXB0nbYfg6VspMg4WlpeVsrW2gMBCc2QlPkEsDA7zdfJHL8TgD6TSu56EogohhUBoMsaFmHvWFhf6Fu8+cx/E8drW08E5zM72pFCFNQ1My54ut/f3sv3SJomCQTbW1rKuu9suF85zevgQ7d5+9affBbBEOmbz59knmN5UTDs9MtuZM4Gdi5SdCiOvzxa+mAFgCPEymW+FXsjHmbBSxhh1SsVEeL7huu5zuR0rZDnxfCLGHTGnis2SC629gSMD6N+BDZILgPyHlaH4gn1wSKw6jTsDmK4bcVAFFpVy/9uL7SmZFlk4uQpFb52B0K1BREKUoFMJ23Vl7USalpDAUoLIgsyJcEguTSFmTPqFOWQ63L6vL8ixnnvbmLl786hukEhbBsDluIaqsthTd1Dmz9yz9XQMc3HaC2oWVlM0rzsp7Jp1MoygKjzS3x04AACAASURBVHz+IeoW+w6s2UJRIMjjC5dhN7m0Dg4wYKURQMQwqY5EZ50w3tzXxw+PH6MrmSCoaeiqiq4q6EP5TpbrcqG/j+NdnURNk/csWMiS0rIZnrWPT25IOw7P7ttH++AgEV2/oSupoaoYqorjuvzsxAkOtrXx9OrVmNpsvHS7NXjr7ZNZzcAaC0VR2LHrDPdtXTptY840cWuanFiW78SaIH9MxgAkedcINMyw1uEBf8uNUVOTYkqfNCHEGTITe0BKeXbo9niRUsr5kxj2+NDvEbOqeLc74GhZV9neDwBSyvNCiCPAGiFEqZSy8+rHhRAa8HUyAtbXgWeklO549u2TfaKFIQwzOxcA2RQmHMelqszvLjXXuHdRI8/tO0JBYHYKlINpiw+sXnblvb55RQM/euMQkUmsNkopKY2FKYmFsz3NGWX/60fZ9tN9hAoCBCexKlpYHmP1fSs5s/8cPW09nD/eSm/nAAtW16Eok8vkG3ZfNa2q576PbcEI+KVJsxFdVamPZSVCYkbwpORnJ0+wu7WViGlQYI7++dAUhaiZaSn/7UOHWFxayhNLl806wc7H52bYrsuX9+yhP5UiOkbjFyEEBabJ5XicZ/ft49Pr1l1xa/nkD2nL4cLFLgLTeJw1TY1TZy5z952Lb5lmD2FjmjKxDN+JNUH+MzeKV8OkgFbgFSllS7YGnKpc3EBGxNKvuj1eJutAemXo90NCCOXqDoVCiCiZMr0ksH2M/Wwf2m6LECJ6dYdCIYRCJhz+6vHGw/AS9zXilBDCIOO8+gAZp9anr++s6DO9CCGIFIRIp/JLaU8lLGoX+G2z5xpLqsppam6jpbdv1q2iph2HhtJClle/+76cP6+UgKnjeXLCpYWDSYut6xZke5ozyu5XDrPzpYNECkNT2o+qqyxcP5/ey31cPNpM96UejqZtlm5YMKH/Z9tySCctiiti3PvRLdQtqbnymCc9LiXbuJBspifdiyMdBGCoBhVmGfWhOorMoim9Dh+fYTwp+dahg5zt6ZmQiC+EoCBgcqanm3/dt5d/t2atL2T5zBl+fPw4fckkQX38ZeMBTaMzHufFkyd5/+LFYz/BZ1o5eqwVbwaKa9Jpm9NnLrN44a3R7TluWWy/6Dux8g0p5Z9P95hTvZpqHPrdct3tnCGlPC2E+DkZkenXydjShvlTIAz8g5QyPnynEGLJ0HOPXbWfQSHEV4HPkVEPf/eq/fwHMoLci1LKM9ftp1dK2Xb1nIZErz8n05Fx29VB7UMh7t8D3gf8M/A5X8DKDypqijh1pDmvOnQJIaiqLZ7pafjkgMdXL+XvXn0bKeWsKSuUUiKl5PHVy6+5X1EET9y7iq+/uIdwQB/360mkLJY2VLC4fu4Itcd2nWHnzw8SjmUvl6iwPEZheYzkYIpzB89zYs9ZahdVIl2PQCSIpl97Me/YLqnBJEJV0A2d2sXVbHjfWorK362W96THgd5DnBw8xcBgErsb7EHwHA8EqLrgQlE7+2KHKArFWFu4mrrwrRUY65N9Xjx1krM9PYSMyR1ng7pORyLOc0cO89GVq7I8Ox+f6Sdl2xzr6CA0AQFrmJCuc6i9nQcXLMDwRd284sy5DoIz4HYOBgxOnW6/ZUQs34nlM8yURCwp5fmb3c4hvwZsA74ohLgfOApsBO4lU/73/1y3/dGh39dfaf0RcA/wO0KINcA7wFIyjqnLZESyq3kP8JdCiNeB00AXmQ6FW4EmoA347HXP+RIZAauTjNj3xyNc8L0qpXx1jNfsk2XW3rGAo/suYORJhZfnSYrKokRjU3Nz+OQnQUPnkZVL+N4sKiscTFs8tnrpiBeglSUFfOi+1Tz3yn4MTUW7ScaclJJ4ymZxXRnv2bx01oh4YxHvT/Lm87sJFeQmpD4YCbB082IGexNseu9qIgVBLhxppr9rAG+oMYWiCKLFEeqX11JRV0q0OHLD/29nopvvbn+Ri0d6cePgpCXSkwiVd8PCvcyPogr00AC7o82s2VjP+1fcj6n6ZYg+E6elv59drS1Eb1I+OB6Cus6J7i6OXL7MsvK5I4D73JrsammZkmPH9jwOtLWxvqZm7I19bmBwIEUinsZ1PXRDo7AodNPzl/HS35+ckXMbVVXo6U1M+7g+PiMhhCglE8nULaU8Ptb2U2F21bUMMeTGWg/8GRlh6X3AJeCLwJ9KKbvHuZ8uIcRm4E+Ax4G7yAhTXwH+WErZfN1TXgb+kUzJ4mqgEIiTEc6+CnxxhLGH3WmlZELPRuPV8czZJ3tEC0OUVhYQH0hOOnMmmyTjabY8NGJPAJ85wtKqcha0tHO+u2dCZQQzQcq2aSwtYkV1xajb1FYW8alHNvDqnlNcaOvB9SQh811nluN6JNM2hdEgD29sYllT5ZwRsKSU/Pxrb6FoSs5fUzgWZPcvD/OJ33+MhWvHb3geHEjx8mu7ef3AHrAFZkhFUQVGaIz5SrB6JG98/xS7XrrA43ffy9q189F1f+XfZ/z86MQxwmPk/YyXqGHwwqmTLCkr8zu0+cxq9ra1EZ7C8T+i6+xsafFFrAnguh6nT7az+50z9PTEr7jMhRCYhkZDUzkbNs+nYJKLyJblkEhaM+LEAhiMp64sbM11/GD3/EUI8UUyRiNl6PYh4Ekp5cmh24uB9WQq3TpH3dE4yaqIJYSoJeOG2jGa+jb0AjYCvxxBJBo3UsqLwKfHue2oZzxDotNvDf2MtZ9D3OjOGus590xke5/p5bY7F/HCd3YSyZGTYrxIKTGDBg2Lbg078K3ME2uX8a9v76EnkSKg5+c6QtpxiAUDPLl2bFG1MBri8a2rSFk2+0+0cK61G8f1UFSFSNBgw/J6Koqj0zDr6aXlzGXazndMOQdrPAghQMC2n+zlvg9vGnN7KSV73znDW28e4UzyLAHTQEzw3FpRBcGIjuM5fPfFX3B410UefnQt1X65s8846IjHuRyP3zTEfSIIIRiw0pzp6WZBsd/8xGf2krLtKZUCCiFI2XYWZzS3aW/r48c/2EMqbREJm0RG6AB+7uxlThy/xMJFldz30ArUCYakW7aLnEERyXUljnNr9ArzywnzEyHEZ8hoJA6wn0xO+HLgh0KIlUPN7ArImH5+G/ibqY6Z7Suo3wR+B1g2xnb/AnwB+MMsj+/jMyHqFpRTVBohlUijZsFOPFkSg2nufHjlhA+cPrMPXVX55Ma1fPnt3cTTVt4Fvacdh5Cu88ymdRgT+EwEDJ2NKxrYuKIhd5PLI3b+/ACh6PSJ32bA4OyRZhzbQbuJ+Nnfm+BnP9xDd9cg7bIN09SZinFFU1Rs0+Vy+jI//M47LFxSxdYHV/iuLJ+bsqu1BTPLmT0Rw2B780VfxPKZ1WQj/NudgQDx2UjLxS6e/95ugiGDaGT047VhaBiGxtnT7fzo+2ke/eBtEzofl56cdLey7CBvKSfWDj/YPR/5v8g07XtASvmGyJQoPEem0u0+4CUp5U4hxEUyVXR5J2I9BBy+WQ2klPL4kL3sYXwRy2eGURSF93zodr75D68SjszMRZltOZRXF7FsXf2MjO8z/Zi6xqc338az2/fQl0wRyJPSwpRtEwsGeGbTurx1ieUD8b4Ely92ESrIXpj7eLAth+O7z7J808IRH2+92M2PvrsTw9RI6wnSyTS6MvW/o66odNpdlBWUcvZkO+2Xenny6c0Egn5Wls/ItAz0Z12gV4SgJ5nK6j59fKYbLQvirh/qPjZ9fQl+9IM9hMLGuEv+A0GD9rZeXnnpMA+8Z+W4x1IUgbghdnn6EEKgabfGIvh0ObHe9J1YE2UR8JqU8g0AKaUUQvw3Mjnjm4GXhrY7BWSlvWq2r1JqGV+202ky+VM+PjNOrDjC+rsWsfuNE4RGsBnnEikljuPx0FPr50xWkM/4COgan9p8G9/cdYCWnj4iMxz2PphOUxUr4OnbV2HkmTss3zhzKPergCMRigY4vufciCJWy8UufvTtnVe+wzoSHWgie39HgeByqoOaUDXppM23n93Gh5+5Y84KWa7r4boeUkoURUGbhuyzucRA2pqSA3DU/VppPCn9XCyfWUthIEB/KjXp97DjeZRHIlme1dxj2+vH0XV1wt/bwaDByeOX2HznQsI3cW9d/xxVm7nvJEPXbplKjrhlsd13YuUjNpkGd1dzbOj31Vk57cAd2Rgw21cqATIvYizSQDjLY/v4TJq1dyyg5Wwnl1t7CISm76IsMZDm/g+uG/eB0mduYWgqn9y4hp3nW3jl+Gl0VUWf5hVW23WxXZf7FjexoaHWv1AfBxdPthEIT7/oKIRgoGfwhvsvt/Xx4+/uIhQxM3kpbhrLs7PiwhpGU1T67H5qqEY3NGzL4bmvvc2HntmCYcxu0VNKScflfs6d6aCluZuBviTJpIWUQCaODEVRiEQDFJeEqW8oo76pjOAcFfCygSs9NJH9iyopwfU8FN+J4jNLubu+nm8eOEBBYHLnfXHb5p7G8Tf4uBWx0g4XL3QRCEzO5a7pCrveOcPW+8ZKx8mgKIJIOIBtz0wuVTQauGXO3cKGwaZa34mVh+wGrrcv9g39vvrLroKMDjRlsn3m2QKsG8d2t3GjWufjM2MoisL7P7aRHz77Ft2dAzl3F0gpiQ+kuPPhVSxY5neYuZURQrChYR5LKkr57p5DtPcPTpsrazCdpjwa4al1K4gFfSF1vPR1DaDM0KpnKpEmOZgiOCR8O47LT7+/m0Dw3ZKJuBOHHCR0uNLB8Rw0RUM3NJKJNK/+/BAPPbIm62NNB5blcPRQMwf2XqC/P4muKZiBTHfNkQQq23JovdjD6ZPtqKpKVU0RGzYvoKKy4Ja5gBgvag4ELAAhQM2DbsI+PpNlfkkJ4Uk2PJBSUhgIMK+gIMuzmlscPtSM63qTfr5p6pw63sadW5eM2+FUXBSm9VLvtDuiLNuhoezWyQmMWxbbL/hOrDzkL4EXhBCfllJ+Ba6UFEJmLRAhRAWZ0sL92Rgw2yLWK8BnhBCfklL+y0gbCCH+HTAf+EqWx/bxmRKqpvLYJ7fwk29up72lh1COnBae55EYSHPPI2tYsqYuJ2P4zD4KggE+fcdt7DzfwhunzpK2XSLm+LMcxouUksG0hampvvtqkqQSFtoMBZt7rmSwL3FFxHrjF0ewLeca4X3QGcxqKeGVsZGkvDSRIYeXGTA4c7yNiys6qG0oy/p4ucLzJLvfOc2+3edwHI9QyCA6zpB+VVOIDP3fd13u53vf3E5hUZiH3r+a0rK514VzskRNg8EcXAREdNMvJfSZ1ShCcG9jIz85cYKoMbEF0wHL4slly/xj9hhcONtBaIpVFZbtEo+nKRhn9uWK5TWcPnP5yvFhurDSDsuX3jqL4WF9mpxYuu/EmiAngP8C/JMQ4gHg38jkXwFEh+77b2RcWV/NxoDZPsv9K+AZ4B+FEAuBf5ZSngEQQjQCvwr8HpmSw7/K8tg+PlNG01Ue/fhmdr12nL3bTxEIGlldVUnGLQJBnQ9++i4qaoqytl+fucGwK+u2umqOXupg25nzdAwmCOjalINcbdcladuURcLct7iJJZXlaLdIhkK28VwPZkjEkpIrK8yXmns4dqiZSPTak2xXujm7yHGlc83tUMTk5Z8c4OOf3Torygp7e+L89Ef76OtNEA4ZTNIQAQwJWtEAlmXzna+/zaq19WzasvCWySa5GTXRAg60t2U13N2TktJwIONktiwu9Q9wqqObzngcV0o8z0MRCqoiqI4VsKC0mPJohGCeNM7w8RlmbXU1Xckk2y9cIDrOL6H+dJp7GhtZVlGR49nNfhzXm/IxUHoSK+2MveEQtTXFhKc5ZkBKSSwWorzs1nHmxe1pysSyfSfWBDlDxnElgY8N/QzzxNCPBH4AfCkbA2b1jFNKeUwI8Tngn4A/AP5ACDH8DTA8lgd8Vkp5OJtj+/hkC0VR2HDvUhaunMcL39lJf0/8StbMZHFsl1TSZunaOrY8uGLGXBw+swNVUVhRU8GKmgo6B+O8fvIczT19JCwb1/MwNA1TGz2wVEpJ2nGxHAdVVQjqOvXFhdy9sIGyqB8IO1WEMpNdiLjiRHn9F4cJhUda9RVISdaDtaUEwbUCjRAC23HYt/MsG7aM3DUxXzi0/wJvvHqMQEAnnMXsQ0VRCIdNDu27wLkzHXzwQ7fnzMk7W1hfXcOu1pasiVhSStr6B1As+OtX3yLlOLieJKhr6Oq134VSSi71D7Lt7AU0RSFk6NQVxdg6v5HicCgr8/HxmSoPzJ9PWNf55ZkzKEIQGkFslVKSsG2kELxv0SJuq7l1HDdTQcnCMVoIJtTxTwjBymXzeGfPWULTlJcYT1jcc3dWGr3NGnwnVt5yDkZs0SmBfjJOre9KKb+TrQGzvmwqpXxWCHEE+E/A/bwb4J4AXgb+Qkq5M9vj+vhkm6LSKB/59/dw6nALe7edomcoK2u8ApSUkmTCQlEU5jWWsuGeJRTfQqslPtmhNBLmibXLAUhYNu39g5zp7Ka5p4/BdKZTlysz+UeqEChCEAkEmFdYQFNZCRUFEUKG70TIJro+c44joQjMkEFvd5yezkHCI5TBhdQgcSeOJrIrlitCYCo3npwHgyZHDlxk/eYFWbl4yAV7dp5lx1snCeewA20wZJBKWnzzq9v48Mc3ExlnieJcpCwcpiwcJmHbUyr/cz2P9r5BWnr6SFoOFfURdFUlchNnqhCCgK4RuOpzerqzhyNtHZRFwtzVVM/iijK/LNFnxtlcV8fqqireuXiRPZcuEbesK2mGAoiaJnc3NLC+poaA7ygcN7FYiO6ugSkdqxVFEJzgYsfaNXUcOHwRz5M5Pxa6rkesIMCyxdU5HSffiFsWOy40T8s4PuNHSjl/usfMyZm4lHIX8LgQQgFKh+7ulFJOPmXPx2cGUBSFRStrWbSylq7L/ex58ySdbb3EB9I4joMQAkURCCHwPInreqiqgm5qRGNBVqxvZPltDRimf/LhM3VChk5jaRGNpX4p6kwSiYUY6IvPSC6JbmhEi8K8/JP9GObIh/CoHqE9fRnItoilYIwgYgEkEmkunu+kvjH/srH27z3Pjm25FbCG0XUV1/H4ztff5iOfuOOWdmQ9umgJX967e9zlUtfTE09wvK0L1/XwpGR5RcWku7camoqhqSRsm+cOHKE4FOTDa1ZQGvEbZfvMLCFd556mJrY2NtKVSJByMgUsAV2nJBj0868mwbrbGzl+tHXSIpbrepRXxTAneO6uaSrve3gV3/3BLiIjuqSzRypt88HH1t1y5ethw2DjNDix3vC7E+Y9OV1OHhKtLudyDB+f6aKkvIAHn7gNANdx6ekapONSH1bKxnHczMVlQZDSqhiRAv/Ew8dnrlLZWEbX9l7MaSoZuJpoYQgp4cK5zlFPsINqEDXLLiwpJUF19Dbe4XCAPTvO5J2Idamlh7dePTatrihVU7Btl+e/t4sPf/yOvHWn5ZqaggLWV9ew99KlCblBXc/jZHsXnQPxTKmgIigxg5SFpy44KUJQEDBJOw7/+PZONtXXsXVBg9/x0GfGEUJQmoX3uA8UFoUpKgmTStqT+v5NJiweeu/qSY1dVVnIyuXzOHK0dcQut9kgkUizfl0jJcW3XjxExonldyf0yaGIJYRYTqaNYhlwWEr5/ND9CqBJKf13h8+sRdVUSitilFbEZnoqPj4+00zTinnsf+3otItY6ZRF/eJqenviOLY7qoilCIWYXkCf3Zc1McuWDnWB0bupKoqgryeelbGyheO4vPiT/TPihtJ1ld6eOHveOcP6TdPuss8bHl6wkN5UirM9PeMSsuKpNAdbLuMNZf85rktIN1hWll1xVFUUIqbJjgsXOdHRwSfWryViTr8o7ePjkxs2bFrAz368b9ydZ4dxHY+CWIjqeZN3vG+9czHJpMXZc51ZF7ISyTRLFlexcX1jVvc7WwgbBhvrfCdWviCEuAt4W0o5/i4IWSLrIpYQog74F2DrVXf/K/D80L9/A/grIcRDUspfZHt8Hx8fHx+fXFI+r4RwYQgp5bQ6Ll3LZe29yzh/rnPMbasClfRafUimHvDuSY+QGiKs3jwUO520ScTTeVNC9/orR7Esh0Bg/C4gKSUJxyHtubiehwdoQqAqCiF1Yl1CgyGTt98+SWl9IQWFQTRFJWIYt1QWkyIEH1mxkp+dPMHu1lYi5uivvz+R4mBzO5qqoCkKacehNBRiaWkZSo6cUmHDIG7Z/J+3d/IrG28jFrx1c8x8fOYSTQsqWLGqlqOHWgiFxyckua6Hbbs88dGNUzq2CyF4z4MrefmVIxw7cYlwaGrNoSBzbIrH06xaWcvdWxbdstUecctix3nfiZVHvArEhRCvAXsAT0r5p9MxcFZFLCFEKfA6UAccAN4Efu26zb4N/A/gA4AvYvn4+Pj4zCqEEKy8YxE7XjxAaJrK1DzPo7SmmGhhmItnj40ZOKspGjXBai4mWzDE5A/1UoIrPerDdWOeNHueR0d7H/VN5ZMeL1t0dw9y7HArkTFysDwp6UwmaU8lMuKV6+JKD0+CQIIY7sqYyT7UFYWgqhIzTapDEULXdd9Luy7N/X30plOkHQfbdjn0zz9j3t01CCHQFIVYwKQ8FGHzvFqqItE5fzGiCMH7Fy1mdUUlPzx+jK5kgqCmXZNvNZBKc6C5HVURWK6LqWksLy+nNJT78ipdVXE8j3/evpvP3bGeyCQzvHx8fPKLu+9diiIE+/edJxIJ3LS0MJWyUVWFp57eRCw29S6mQggevG85DQ2l/PKVowAYxuSOxem0jaIoPPb+tdTXlUx5brOZsGGwyXdi5RMPAF8A3ge8f+i+2SdiAX9IRsD6AvBHUkophLhGxJJSXhJCHAXuzPLYPj4+Pj4+08KyjQvY88qRaXNjJfvTPPR05rA5MJAalzOlyCgk4SbotnrQlYkf7qUER9o0hOsxlLHdTLqh0dbamxci1jvbTmGOEnwPGbHp/EA/7akEjuehKwqKEGiKQBsjED/lOgzGbS4ODhDWdOojBehCcK6vlwErjUJGrFKEwDQ07EEH0xYYkYzwmHZczvR0c7ijnaJAkDtq61hbWT3nHVrzYjF+fcNGLg0M8HbzRS7H4wyk08SHBKyQrhE2DGqiBRQGAlO3EE4ATVGwXZcv79jD5++4HUObuQ6kPj4+2UEIwV33LqW+sZR3tp+m43I/mqpcaYrieZku4oGgwbIVNdy2YX7Wy/8WNlVQ+/+zd9/hcV33nf/f3zu9oRAEQIIFINgpkqJYRFJUb5atKFJs2bLjuG3i7tjZJL/nl7ZJnGySzbO7yS+214njxEnsxE0uUuxVtaplSmKTSLH3XgACIMr0uff8/piBBIFtANwpAL6v5+EDEXPnnAOSEO585nu+p2UKz724h6PHu3Ach1DQf9X7BmMMyWQGj9diXnsTN924kMAoQzClSsUY8xxwvYi8F/gNoGwnmbn93XAfcIRCgHWF604AK12eWymllCoLf8DHHQ+t47F/e5GoC+/aXkkqkWbByjamt+X7Ajn2lX68vkVEmBFqwSMeOtPn8Yqn6MDNNg4Gw5xIGzFfrLj5LCGTKXtbhItkMjmOHz1/yW2EWcdhT08XXekUVqGyyjPCE+9EBF/hzzFt5/j5mZP5rW+BADVe30V/xpbPomNPFzPXTH/zcz6P580KoMcO7OeVkyd5/zXLaAiX9t9SNZgei/HuxUuA/Au1r7+8mZrWIN4KN1f3eTyksll+unsf715+TUXXopRyz+y2Rma3NdLfl2TH68fzfSVth2DAS/u8ZtrnNZf0lL9g0Mc7715OOp1lx86T7N1/hng8TTZnA4KnUCFm2w4igsdrEY0EWXntbJYsnjHqCq6JSBu7VydjzMPAw+Wc0+3vilnAT68SYAH0AXpGvFJKqXGrddEM2q+ZyYkDZ0vW5N22HXwBHzc9sPrNzxmKC7EgH7hMD02j1lfDscQJsk4Gr3gvG2blt9I5RL0RZodn4R1BBZeI4NhO0deXyq43TmI7F6/jXCLB3t4eBENghMHVpaRzNufiA0C+b1ZXJk08l6MpGMQrb70g8ngt+s/EcWwHa9gLJREhFgiQzGX56tZXubt9HmtnlH6rRLXYeOQY5+PJqmmqHvT52HOuk0Pnu5g7dXJv21FqoonVhNhw88KKzR8I+Fizag5rVs0hZzv09MTp7Owjmc5hCQSCPpqn1lBXFy5pqDaelaux+4u6nbDquR1iJYG6Iq5rBS64PLdSSilVVnc8tJ4f//3P6OsewD+CBuLFcGyHbDLLg1+4B9+Qd2JHc2R42BtmUWwB8Vycc+lO0naanLGhEIgJgtfyUuOL0RxoIuAZeV8g4xh8vsq/Y3xo/1nCQ0LFnOOwq6eL7nQKv2UhMvYXB6lcjrMDA3gsi8G/DS9CxnE4kUww1R8g5n3r34Ods0n2pIhMvXSlldeyiPkDPHXoIIlMltvmtI95jdWuO57ghUPHqibAGhT1+3n0jT187qZ1uq1QKVUSXo9F49QYjVOLq3RWedrYXQ1y+6fzTmCViNQaY3ovdYGIzACuBV5weW6llFKqrLw+L/d/8g5+8o/P0t3RS9Clk/ly2Rx2zuHdn7ub+saatz3m93lJp7Ij7sUlIkR9UaK+aH4OJ4dt7HzDcfFijTHcyWZz1NZVfjvcQH/qzaAvY9tsOd9BzrFdqb4aHPNs/O0B1iBLBGMMnekUWcdhij//78HyWsQ7k5cNsQbFAgFeOnGMoM/L+pmzXVlvtfrPXXsJVkHoOZyIkLMdnt5/iHuXVK5qQyml1NtpJZYa5Pbdw7eBrwJfE5EPG2PeFmNK/u3PLwEB4N9dnlsppZQqO3/AxwOfvpMXH9nC3i2HCdeERlUtNSjRn2LKtDre+eGbLtlvq7mljq43+gkExlb55bW8eF2+DZg2s7KdApKJDKlkhnAkQMa22dx5Dsc4rvVbMgbOxeN45OIAa5CI4EXozWYwo2ePZAAAIABJREFUQIM/gMdnkTifAK6+RS0WCPCzI4eYV99AY6T0p/NVQm8yxakLfcSC1XkSYNDvY/eZDu5eOO9tpygqpZSqHK3EUoPcDrH+Cfgg8D5gjYj838Lnl4rIXwMPAPOB58kHXkoppdS45/F6uO3BtSxa3c7T3/4FAxcSIwqzjDGk4hksS1h3z7Usv3HhZSutZrVN5bUtR8YcYrnN4/VQV1/Z0KWrqx/bdsg6DlvOd+AYB4+LDcO7UwkcY/AUUQXnEYvebAYLqPcHyCSKb3of9vr43u43+MzqtRPy1MKfHzqG31vd4VDWsdl1toMVM6Zf/WKllKqQZC7D+XQfaTtb6aWUXMTvZ22rVmIpl0MsY0xORN4FfJ18kPW5wkOrC78AHgE+UkTzd6WUUmpcmd7WyK/93i9zdM9ptj6zk+6zF3AcQzDsxzvshCHHdkjG02AMkZowa+5cyjXr5l+1t1Zjcw3eKmz6Go0GKt6MNhHPICLs7Oki59iunniXdRz605kRjekVi55shpDHQ0CKD208lsWFVIqNJ49z46zW0Sy3auUch70dHQSqvN9UxO9n45HjGmIppapOMpdhW/chdvedJJFLYxub3myi0ssquXg6w6tHy1CJldZKrGrn+h2EMaYfeL+IfBF4J9AOeIATwOPGmNfcnlMppZSqFpZl0X7NTNqvmUm8N8HZ4+c5vvcM3ed6yeVsMAbLYxGrCzNrwXSmz2mivqmm6B5XwZCf2voImfTI+2KVSiqZYfnKtkovg2zW5nw6xYVMyrUeWIMuJJOjqoryIJxLp5jhG1m/sIjPx+ZTJ9kwc3bV/D274WBnF6lcruqbposIPYkk3YkkU8KhSi9HKaUwxrCt5zAbO/cCEPL4iXgLvRcn0M+Jy4kEylSJFdBKrGrn6h2EiNQAxhjTb4zZA+xxc3yllFJqPInUhpm7bDZzl7nbpHvl9e088/gOorGgq+OOlmPg2tVtlV4GOeNwMN5LKOBuQOIYQyKXK2ob4XAigo2hK5ce8fN60ymOXuhhTv2UEc9brQ50dhHyVddW2MsznLxwQUMspVTF5RybH594ldOpbiKjOEF4IkikM2wqQ0+shFZiVT233wa7AGwG1ro8rlJKKaUK5i6YxkvPVsf7RLbt0DKjnmDIX+ml8MrRk4BxvXIpkc1ijIFRjutBGLBt+gfSxKLFv/iI+vxsPHliQoVY5/r7x02z9JDfz4HObpa36JZCpVTl2Mbh4eMv05Xum7QBFkC4TJVYLwT8dJd8FjUWbodY/cABl8dUSiml1BAer8WiZTPY+doxQuHK3tAmEmneddOqiq4BoC+RoiuRxGe5H5Akczk8Yzhx0mDw+yxOnOphycJpRT/PY1l0JSdOnxNjDBdSafzjJMTyWhZd8Ynz56+UGp9+3rGbztQFwt7JG2BBvkKqHD2xtBLr6kTkFrfGMsa8MNLnuB1i7QFmujymUkoppYZZu2EBh/aexXFM0acgui2VzLBoyUyaptVWZP6hXt53jGDQh1WC5vJZ20YYQ4jlGPwxHwMDabJZG5+v+BCnP50mY9vjJvi5kv50mkwuN66+lr5UCmPcr+5TSqlidKZ6eb3nCFFvdbQPqKRwwM/aNq3EqhLPwhhujN5uxDdubodYXwe+JiKrjDFbXR5bKaWUUgUer8U77r+OH337ZSLR8t/cOo7B6/dy051Lyj73cDnbYe/JTkIBH4Ggl1zWdnd842CNJcSyDf6oDxE4dbaXtlnFbw+0jeFCKklTJDrq+atFKpvDGWdnU2cdB9sYvBpiKaUq4IWOXYQ8ld+uXw20EqvqnAX2juH5i4Diy9OHcDXEMsb8s4hcCzwtIn8N/Bg4ZowZWTdTpZRSSl1V07Ralq5oZdeO44TLuK3QGEN8IMUDD60dUVVRqew/1Uk6m8Pv9VBTE6LjbC9eN9dlGNP7jWIJ3qAXy2PR1R2ndWZ90ZU9BkPWcUY/eRXJOQ6Y8ZViCYLtOHgt9yv8JhNjDPG+JMl4GjvnYHmEUDhAtC6sVW5KXUYyl+F0spvwJO6DNVTYX6aeWH6txCrSk8aYj432ySLyDeCjo3mu26cTDn3r8y8Lvy73w8kYY6r7fGWllFKqym24bRED/UmOHzlPKFyed2vjAyluv2cZLSOoKCqlPSc7CAfyJ95Nm17HubO9FV7RW4wx+MJvbXPM5WzS6RzB4Hg5oc894zGsMIyv0K2apJMZdm46xJE9pxnoTZJOZXAckw8yRfBYgs/vI1oXZtb8Zq5dP59QRF+sKzXoZKIrH/5X/r2iqpDIlKkSK6OVWNXO7RBpJHcn4+9ORimllKoyIsLd913Hk4++xvFjnSWtyDLGEO9Pc/Od17BoafW0wOwZSOIpVMr4/F6i0SDJZMa1XmFjyV6cnCEyNfTm7w0wEE8XHWJZyLjqIXUlHpGx/WFWiEersEak/0KClx57nVOHOzCOIRQN4vV58PpCl7w+ncyw85WD7Nh4gGmzGtjwrmtpaK58nz2lKu1ovIOQZ/K94XE5YX+ZemJpJVYx3gscHeMYXwF+Oponur2dUH/KK6WUUmVmWcI9D1zHz5/Zza7tJwhHAq43e89mcuRsh7vvu5a5C6e7OvZY2I5DXzJFyP/Wjf6MWVPYs+sUgYA7tzkesXBGsQ3OYPD4Lbyht9bh83jo6U0ytaG4Hldej0V98NIv/sebsN9Hhc4gGDWf5cmHb2WSTGXoPD9Ad0+cTCYLIvh9XhqmRGhsiFV1BZ8xhu0bD7L5mZ34Al6CIwjUB6/t6eznB//wDNfeMJ81t1+DpwQHNSg1XvRm4/gs3bikqo8x5kcujLEN2Daa5+p3hVJKKTUBiAg333kNC5a08OSjr5FMZgmFxr690BhDIp5m2ox63nHfdWXbslis7v4EWdsmxFsv7qOxILW1YeLxlCsvgv0eD4lsFmuEYYaTM9TNjr3tZEPLEpLJbNFjxHyBCdOPKeL3E/SOr1vP2lCgpNsgHcdw+Fgnr79xnL7+FMlkFtu28Xg8bwbRjmMKn7MIBv3U1oa4buls5rROrdjJpMNlUll++q2X6DpzgdAYDprweC0isRBvvHyQI3vPcP/HbiZcgYMrlKoG46yFYMklMhk2HdHthEpDLKWUUmpCmdZSzwc/fgsbn9/LgT1nyGZzhCMjfyHuOA6JeIZwJMCtdy9j4TUtVdnT6EIihRnW91xEaJ/fzI7Xjg223xmTkM/HQCYzohDLsR2CtQH84YsrZ3K54hq15xyH6TXj/1TCQSJCXSjIQKb4EK+SsrZNa31ptrUlUxm2vHaUvQfOkkpnCYfy1ZPhIkLi3t4kjz39BqGQj8ULprPq2taKVmilkxl+9PXnSAykxxRgDRWKBkkn0jz81Z/x4KfvJBLTIEtNPiGvn+5MPx6ZGG9kjFXY7+f6MmwnfF63E1a9koRYIuIH3gPcCswk3wLiNPA88EM9rVAppZQqHa/Xw813XsP6Wxaxd+dJdmw9Sn9vEoMhGPLj9V7cY8kYQyaTI5PO4fV4mNpcwx33LGdGa0NVhleDMtncJUMqn89Da3sjRw524PePradU2OsbUYBljEEsi1hz5NKPF9ksPJHNcuPstqLnHQ+m1cTYfbYT/yX+DVabZCbLvMapro5pjGH3vtO8sPEAAgSDPqIj/LOwLCEazW+/277zBDt2n+S2GxeycN60sn+v2jmbR7/xIsmBNAGXgzSvz0sua/Ojf3yWhz57F/4q3kqpVCm0RZo40Hea2GX6yU02iUyGTdrYveqIyEfG8nxjzL+N9Dmuh1gicgPwbWAWFzdv/3Xgr0Tkg8aYl9yeWymllFJv8fk8LLuulaUrZtPfm+Tc2V6OH+mkq7OfTDqH4zgIIJZFOBpgXks9s9qm0thcQ9CFrYjlYNvmsi/cGxqiDPQlOd/Zj883+tBEBMI+X1FbCg0GYxvq59Qgl9vqVcQeEWMMdcEgLdHYaJZctRY2TWXbidPjIsSyLGFWXY1r4yUSGR5/5g1On+slEvK7EjiFQn6MMTz93G527zvDO+9cSihYvu/dl596g97ugZKdKuj1ecikMjz74y3c84H1JZlDqWo1K1zdbyKVm1ZiVa1vMLZD+yobYonINcBTQBg4DHyHt7rWtwEPAfOAJ0RkrTFml5vzK6WUUupiIkJNXZiaujDzF1VPU3Y3+LwezGVCIRGhdU4jtu3Q0x0fU5BVFwwSz2a40n2aMfkAq7a1Bu+VmsoX8aJkIJPhXfMXTrgXMG1T6t/WhL9aOcYwNRKhJujONrbungG+/+hWLBGiLp8gKiJEo0E6z/fzre+9wvseWE1dbdjVOS6l80wPuzYdJlJT2ioRf9DP0X1nOLbvDK1VdKiEUqVW648wxR8lbWcn3M+C0dCeWFXrXxlbiDVibldi/Rn5AOuvgP9mzNu7VIjInxSu+QPgi8CDLs+vlFJKqUkk5Pde8eRAEaF9XjPHjnTSea4Pn987qh5ZXsuiJhCkL526ZH8Sx8mvoa6tFl/wyrdXV2vGnbVtpkVjrJw28V6weyyL5S3T2HbidFWHWfF0hncsmufKWOe7B/j+jzcTCHixStik3+/3YtsO3/7hJj7wnjXU1156O6tbnvnB5pJVYA0XiQV5/tFtfOh331nSP0Olqs36xkX89ORm3VJI/nCQtWWoxHpBK7FGxBjz6+We0+0Q6xZgnzHmDy/1YCHU+iMRGeyXpZRSSik1alNiYbxXOYFQRGhrb6K2LsKRQ+cwBrzekb8QrgsGSWQzOMa8eeKgweDkDP6Ij5qWKNZV1mKMIeC//O2XMYa0bfPQNUsn7Dvv69tms+X4qUov44pCPi8LXOiH1T+Q4uFHtpQ8wBrk8Vj4ge//eAsffO86oiUKmTrP9HDhfD/RMlR8Qf57OJVIcvzAOdq0GktNIvOi05gZmUpnqhe/NbnPZIuny1OJFU9XfyWW5G8QPgx8DFgOhICzwGbgj4wx+yu4vJJz+zshBGwr4rptwP0uz62UUkqpSSYWChDwFXc7Uz8lQqymlSOHOujpieP1ePB4ig+KBGiKRDnd349gwDZgCTXTowRriwsLbNu82ZT7UvozGd69aDE1gYl7Gls04KetoY5TF/qrsjdWMpNhzexZeMYYOhlj+M8ntuP1WmWtHvJ4LIwx/PSp7Tz0wJqShKFbnttN0OVtkVcTiobY+vweDbHUpCIi3Nuyin89/Cy2cSb1SYWRgJ/r55ShJ1aguiuxRCQIPAz8ErCPfD/yfqAFuAlYAGiINQL7gGJ+skwHDrg8t1JKKaUmGRGhJhQkmckWdb3X62H+wukkkxlOneim90ICxzH4fJ6ithlaRpjqD3E+kyTSFCZcF7x8A/dLsB2Hukv0EDLG0J9Jc+/8hSxtmlb0eOPVfdcs5is/fxmfsaqq4swxhpDfz83z2sY81pbXj3GhN0G4AockeL0ezp+Ps33XSVYsdfdFn+M4nDnWhT9Q3u2gliV0neslncwQGCcHTyjlhrA3wK+23sR3jr2EbXKTtiIrns6w6bBWYgH/m3yA9Vfkq66Gt3CqyF59EVkOfJ787rwZhU+fAl4E/s4Ys8Otudz+DvgH4KsissEY84tLXSAiG4Cbgc+5PLdSSimlJqFp9TH2neocUVVPKORn3oJp2LZDV2c/PT1x0qks2axd6G9lMCbfg10QLI+FP+AlVuOnqbkWT9BiR8c50raNn+LntSwhHH77C/C0ncN2DO9dvIzFjY1FjzWeRQN+3rFoAY/t2UcsUN6KniuJp9N8bO0qvGOsnOrtS/DqtsNEQpX72iIRP7949SBz2xqJRd2r7OvrSZBJ58oeYgHYOYeuc720tE2O7xOlBtUFony0/XYeO72VY4lOop7gVU/LnWi0EgtEZC7wKfLbBv/QXOJkG2NMce/qubuuTwN/Rz5fGrqm9sKvD4nIF4wxf+/GfK6GWMaYfxSRReRPH/wq8B/AkcLDbcAHgc+QT+L+wc25lVJKKTU5rWyfwfYjp0e1Nc3jsWiaVkvTtFogX2WSTuVwHAdj8qGTx2vh93svqhha0zKTIxd6ONnXi9eyrrrNwxhDMODDV1inYwz96TRz6up5z+JriPgnV3XJihnTeOPMGTr6E1WxrTCZybBq1gxaamvGPNYLG/cXvc21lHxeDz9/+QDvumuZa2N2nOzOb6etAF/Ay8lDHRpiqUkp5PXzntnrOTxwlpfP76Mz1YeDIWRNjp8d2hMLgA8AFvBvQI2I3AfMArqAZ40xB8u9IBFZD3wZcIBvAd8EDpPvwjAH+Ehh3V8Wke3GmI1jndPVn64iYg/57e8Wfl3Kb4nIbw37nDHGVP6nvVJKKaXGlaa6KFNiYbI5e8xb0yzLIhQu7gWBJcLc+im0RGMcudBDVzKJYxx8lueS75BnczazZ9aTyGaxHYeWWA33L1xMe119VW2pKxcR4d3Ll/K1jZuwHWfMPajGIp3LUR8Oc+fCsZ9ImEplOXmqh1AVbHnz+TwcO9lFOpO74oECI3Hm2Hn8wcp8bYGgj45TPRWZW6lq0R6dRnt0GgPZFGeS3RyJd+CVyr8RUGplrsRaKSKXTOuNMZX8gb2m8LEWOAQ0DHnMiMjfA583xtgXPbN0fod8sPagMeaRYY8dBp4RkUfI9/H6HaC6QixgLH+hk+/uTSmllFKuWLtgNk9s20esAtu3Qj4fSxqbyDkOHfEBupJJktksaTv35pZEyFd9zZ0+lTlT6lg5rWVCN28vVjTg52PXr+Trr2zBDxUJsjK5HCGfjw+vuW7M2wgBduw+WVV3tbZt2LX3FCuXt7oyXi5rY42gD5ybRAQ751z9QqUmgagvyHxfC/NrWqjzRyq9nJLTnlgANBU+/hnwM/JFQ0eB64Gvkd/11gn8aRnXtAF49RIB1puMMT8Skc3ADW5M6PZ2wsl7XIJSSimlKmbJrGae3VH2Kvq38VoWLbEaWmL57WjGGJxCu4p0NsfythbuWjG/kkusSlMiYX5j3Wr+5dVtGOPg9ZTvdjKdzREJ+Pno9SsJurT9b9e+04QqVKl0KeGQjzd2uRdiVTqgm4RFi0opyl6Jtc0Ys6oU44vIUWAk/0P+D2PMrxX+e7Dk7gzwK8aYZOH3z4rIg8A24LdF5C+NMeVK46YAzxRx3UHgOjcmHLfb90RkJvkE8h7yZXRngEeALxpjiq4zFpEpwB8DD5A/NbELeAL4Y2PMyUtc/9fAavJHV04FksCxwtxfMcZ0DbveRz4RXUH+L20J4AM+boz5pxF8yUoppZS6DK/HYsPiVp7feZhosDoahYsIHhGMMXgsi5uWzKn0kqpWQyTMJ25Yzb9v2U5fKkW4xP3BjDHEMxlm1tXy/uuW4fe6c0ucTmdJJDIVOZHwckSEeCJNNmvj8419y1EoHCCXc/BUoI+ZYzsVaSivlKq8eDrD5olRiXUISI3g+tND/nsw53hiSIAFgDFmu4gcAeYCi4HtY1pl8bqBYvbizytcO2bjMsQqdOXfSL6c7lFgL/kSui8A9xROR+y6whCD4zQUxlkAPAt8F1gEfAy4V0TWG2MOD3vafyWfcD4NdAARYB35kr1PiMg6Y8zQ764I8P8V/vsccJZ88zWllFJKuWj1vFnsOt5BXzKFz1M9/UEGUhkeWHcNQZd6Ek1UNcEgn7xhDc8fPMIrR08Q9vtKsr0wY9vkbId7Fi3gupnTXe1Hdr57gFwVbnfL2Q7dPXGam8betH5GexPbNx4gECx/mJRKZJjRrk3dlZqMIgE/17eX/mX0cyU+ndAYc8cYnr4PuBu4cJnHB0Ou0BjmGKmfAw+KyHuNMQ9f6gIReR/5fl4/cmPC8br976vkA6zPG2MeMMb8njHmduBvgYXAXxQ5zl+SD7D+1hhzR2GcB8iHYU2FeYarMcasM8b8l8L1v2mMWVMYqwX4/WHXJ4B3AS3GmGnAN0b4tSqllFKqCCLCu9cvJZMtZz/TK0tnc8yd1sCCFn3hXQyPZXHHgrl8fP1qAl4v/an0m1syxyrnOPSl0kwNh/ncTWtZOavF9Yb6x0504fdXT4A6yOuxOH7KnZdljS11eLyVeQlhjNGTCZVSk9ngtr2lwx8QkQAw2LPgaLkWBPxvwAa+IyLfEZG7RGSeiMwVkbtF5LvAtwED/C83Jhx3bwmKSDv59PEo8H+GPfwnwCeAD4nI7xhj4lcYJwJ8CIgXnjfUV8hXXL1DRNqHVmMZYy5X+vd94A946x/O4PUZ4PGrfFlKKaWUckFNOMity9p5bschohVo8j6U7TiICPeuXlzRdYxHTbEon9pwPYfPd/PioaOc6evH5/GMqm9VIpPBAK31ddwybw4tNbGSnQbZ0dnv2imAbgoEfJzt6HVlrHA0SKBCPb+8Pg91U6MVmVspVVnxdIZNhybEdsKxeJz8iX/vEJG7jDFPD3nsv5E/tfAFY8zZci3IGPOqiHyKfDbzUOHX2y4BssBnjTGvujFn9f2UvbrbCx+fMsa8rV7bGNMvIr8gH3Kt48oNxtaTL7N7yhjTP2wcR0SeIh+I3Ub+H8rV3Ff4uKOIa5VSSilVIqvnzaJnIMn2I2eIVOjFtmMM6WyOj96xWrcRjpIlwrzGBuY1NtCbTPHSkWMc775AXzpNznYQgaDPhyWCkL9Lth2HZDaLJYLP8lATDLC8bTbXz55J2F/67W+27ZQsIBsLyxJyOfcqFOcvn8XuLUcIhsv3/ZXL5pg5t7kivbiUUpUX8ZdpO6G/tNsJx8IYkxGRjwBPAY+LyI/J9+deA9xM/mTCT1RgXf8sIq8Av0k+P5lReOgU8Dzwd8aY3W7NNx7vqhYWPu6/zOMHyIdYC7hyiFXMOBTGuYiI/C4QJZ92rgZuJB9g/Y8rzFkyIuJOrb1SSik1Adx57XxytsPO42ev2ujdGMOFdJLOVJz+TIa0ncM2DoZ83wWf5SHk9VEfCNEYjhL0XPn2yXYc0tkcH7xlJQ2xiX/seTnUhoLcuyR/65ZzHLriCY73XOBUbx9Z2yZnG7weIej10Tqllhm1tdSHQ1hlDpSq+WbMpV2ZAKy4cQE7Nx1yb8AipJJZVt+mVY1KTVbxTIZN5WjsnqnqSiyMMS+JyGryu8luA+rI997+R+DPL3U4XZnWtQv4VDnmGo8hVm3h4+Vqogc/X1ficX4XaB7y+yeAjxpjOq8yr1JKKaVKTES4Z+VCgn4vm/afIBYKXFQhk3NsTgz0cibeT8ax8YrgEQsR8MpbPX8cDAPZDD3pJIf7uon5A7TF6qkPhC4aM53NIQgfvWO1Blgl4rUsmmNRmmNR1lR6McNUYREWkA9qLReb5IejQWbPm8bpo534y9DgPZezaZxeR0Nz7dUvLhPbdujujnPqZDddXQMARGMBZs5soLExhl8rMJVylVZivaVQ1TR8296kMRH/7zp4+zDW95uuOE6hSTsi0gzcQL4C6zUR+SVjzLYxzj1ixphL3jaJyFZgZZmXo5RSSlWciHDbsnm0Ntbzk817cByHYGFL2blEP/svnMcAPssicJXTDEXAX7gmlcuyo+sstf4AS6Y0E/B4McYwkMowp2kKv3T9YkJl2Lqmqk80EqS7J463yra85XIOsZi7PeJu/ZVV/MffPI4xpuRbKDOpLHd/fG1J5yjWwECKTa8e4tChDjLpHJZH3gyscjmHza8exuvz0DKjnnXr5tHkwomQSintiTUeiEgbcBP5A+8gv53wRWPMcTfnGY8h1mCF1OXeiqkZdl1JxzHGnAN+LCLbyG9N/CaXOC1AKaWUUpXRPq2BT92zjse37mXPyQ6OJnrozaTwW5633rIaAREh4PEQz2Z59dwJ2sJ1NEdi3L92CQtnNLn/Bahxo212A/sPnSVaZSFWKp2lbdZUV8cMhvzcev8qnn54E9Ha0p3mHu9Pcf0dS4jVV7ay0RjD9u3HeXnjAbxeD36/96JqK6/XQ7BQmXa+s5+Hv/cqCxdP55ZbFuPzVde/CaXGm0igTJVYgeqvxKo0EfEYY+whv58B/D1wLxffWRkR+QnwaWPMGTfmH48h1r7Cx0v2quKt0wEv1+vK7XEAMMYcE5HdwAoRmWqMOV/M85RSSilVegGfl3esXsjO7FlSR3KIDbZx8HhHt8XKGIOdc/B4LM57E9x57XwNsBTNjTVV29i9udH9iqC5S2dy6mgne7cdJRwNuj5+Mp6mdcE0Vmy43O16eTiO4fHHtnPs2HkikeIq2jwei2gsyKGDHZw5fYEH33s9oVBlDppQaiKIpzNs1kqsihORPwP+SETWGGO2ikgj8BLQSv4Uwo3km81L4XPrgV8GrhWR691ovzQeQ6znCh/vFhFr6AmFIhIDNgBJ4JWrjPNK4boNIhIbekKhiFjkm8MPna8Yg2Vz7h3/opRSSqkxS9s5/nn3ZrxhDyuWt5BO5zh1qpfu7iS2bQOCz2ddNoAwxuA4+eBKRAgGfcyaVU9DQxjLEn5x7hiRQIB102aX9wtTVaW2JlSVFTcBv5dIiU4SvOneFTg5h/3bjxOOuRdkJQbSzGxv5K73ra1oMGiM4aknd3DiRFfRAdZQwaCPdDrLD76/iYc+sE57ZSk1SpGAnzVzS1+J9axWYl3NncBxY8zWwu+/CMwGfgp81hjztqRRRGYBXyVfpfVF4DNjXcC4+7+oMeaQiDxFPmT6LPDlIQ9/EYgAXzPGxAc/KSKLCs/dO2ScARH5FvkjKP8U+J0h43wOaAOeNMYcHjbOBWPM2aFrKoRefw40ARuNMT1j/0qVUkop5ZZHD++iL5smVDhZMBDw0t7eQFubIZXK0d+Xorc3RTKVxc4ZHGPAGMQSLMn3vIlG/dTWhohG/RcFFTF/gCeP76ctVs+0SKwSX6KqAiLC9KZazp3vx+txr5H6WGRzNjOm1ZUsCBIRbrl/JbH6CFuf30Mg5B91hSOA4zgk+lMsXTuXG+5Z7mpD+tHYt/cMhw93jirAGuTzeUkk0zz/3B7ufscyF1en1OQRT2fYfFArsapAO/AGMOZHAAAgAElEQVT6kN//MtABPGSMSQ6/2BhzQkTeBxwG7mMyhlgFnyFfpvYlEbkD2AOsJX/E5H7gD4ddv6fwcfhP7z8AbgV+W0RWAJuAxcD95P8iPjvs+nuA/ykiLwKHgC7yJxTeQv4v8yzw8eGLFZHfAxYVfrui8PFjInJj4b9fMsb801W/aqWUUkqN2P6e8+zp6aTGf/GLUMsSwmEf4bCP5mljC58iPh/fPbid31x2A54Kv/BWlbN2dTvff2Qz0Yj72+tGI53OsXZ1e0nnEBFW3bKIuUtn8OS3X6HnfB/hWAjLKj44M8aQ6E8RqQnx7k/eTlNLfQlXXJxMJseLL+4l7EIVWzDo58D+syxbNovpLVc7RF0pNVwk4Od6rcSqBnXA0KKdBuA/LxVgDTLGJAsZyi+7sYBxGWIVqrFWA39GPlh6F3AG+BLwRWNMUf/ujDFdIrIe+BPgAfKd9LuAfwH+2BhzcthTfgb8I/kti9eS/wuMkw/OvgV86TJz30M+6BrqhsKvQRpiKaWUUi7LOQ6PHNlJzFf6XjQesRjIpnn+1GHumDWv5POp6tQ0NUZNLIxt2xXvj2WMob4uzNQp0bLMV9cQ472fvYOje8+w9YW9dJ3rxWMJgXDgkoGWMYZUIoNtO9Q1RLn1/pXMWzYLT5U0xn/jjRPYtnunL4bCfl555SC/8u7Vroyn1GSSKFNPrIRWYl1NGhiaJp4CiilVDQDD85VRGZchFuTL0oCPFXntZX/yFEKnLxR+XW2cnVxcnVXM/LeO9DlKKaWUGrudXWdI2TlivtFvBRqJiNfP1s6T3DqjfcJVY6USac6f7aXrXC+5TA7L8hCpCdI0cwo19eGKb/uqFiLCmutaefbFvWPaguaGeCLDjevmX/1CF1mWRfuSGbQvmUFfT5zDu09z6vA5ervjZFJZHCffV84f8FE7JULLqkbmLJnBlCb3G8+P1a6dpwiFfK6N5/FYnD3bSzKZ0SbvSo1QuEw9sZ4J+Okq+Szj2n5gpYjMKmQy3wc+KyLNxphzl3qCiDSR3zX35Us9PlLjNsRSSimllLqal84cI+ot74vFlJ1jT08HSxumlXXeUrjQNcCW5/Zw+th5Uok0tu3g9XoKlSkGO2eDCD6/l5opEVZsWMicRdPxVEk/qEpZvGA6O/ec4kJfEl+Fqooy2Rwt02qZ3165UzNr6iOs2DCfFRvKG6S5IZ3OEo+nXdlKOFQuZ3PuXC9tbY2ujqvURJcoU08srcS6qh8CK4Fvi8iHyO+OuwV4WkT+H2PMk0MvFpF7gP8FvAH8dzcWoCGWUkoppSak7lSC7nSibFVYg6JePxvPHhvXIdb5sxd47pGtdHf04Q/48Pm9hKNX7vGU6E/zzA82EQz7WbZuHis2zJ+01VkiwrvuWs43v7sRr+fyp16WijEGxza8885lFd/SOF6dPz9ALuf+gePBoI/jx7o0xFJqhLQSq2p8CXg/+dZIB4CjQCdwDfC4iHQDxwCL/KmFgw0ONwJPkO9JPiYaYimllFJqQjra140xpuzzigg96STGuNdLp1xs2+HVp3eyY9MhwuEAkVio6OdalhCpyV+/5YU9HHjjBPe8fx21ZerHVG2ikQA33bCA51/aW/Ym7/FEmrtuWUJYt6yNWjyeLsn3r8/noa//sv2PlVKXoZVY1cEYkxCRm4A/Bz5C/oC7wdNDDPnQaujJHIM3Yuu5+KC9UdEQSymllFIT0sHebiJl3ko4KGPb9GZS1AWKD4EqLZXM8Og3XqDvQoLoCMKrSwlHgqTiab77lae568HraV8yw6VVji/LFs+gtzfB6ztPEAmXpyIwnkizekUbixZML8t8E1Wp4mdjSje2UhNZOOBnzTytxKoGxph+4LeA3xKRecBUoGw3XBpiKaWUUmpC6k7HK9Zc3TYO5xID4ybESiUz/OBrz5JN5Qi5FLZ4vB7C0SBPPfwqd75nDfOWlv7FRzXasHYejjHs2HWy5EFWPJ7muuWzWb9mbknnmQxqakK8VUDgnmw2R11dxPVxlZrotBKrOhljDgIHyzmnhlhKKaWUmpCyjlOxuQUhkRsfN8K27fDov7xINpXDF3D31lBEiMRC/OyHW4jUhJg+e6qr448HIsLN6xcQCQd4efNhAgEvXpcb3+dyNulMjpvWz+faSRoWuq1+SgSPx/2m/JlMjtmtk+/7QKmx0kosNWhydttUSimllCohAZwK9OMajVef2UVfT9z1AGuQiOQrsr6/iWwmV5I5xoNV17bywfdeTyjoI55wL+CMJ9NEwgE+/NB6DbBc5Pd7qakJud5Xz+f10txc4+qYSik1mWglllJKKaUmJI9Y5KhMNZZjDEFv9d9mnT93gTdeOTiiBu6jYVlCKpnl549t5/YHVpV0rmpWXxvhV9+zlq3bj/H6G8dJpnNEw/4RNxB3HEM8mSYc9LN+9VxWLp897g4RGA+uXTGLF1/YRyTizjbQXM6mZWY9fn/1/79BqWqTSGXYcqAM2wlT46OKejwTkXVAM/CSMWbEhW/6f1CllFJKTUg1/gDnElmsCry4t0RoDFX/qXwv/OdrrvXAuppgyM/+7cdZc9tiYrXhssxZjSxLWHNdGyuXz+bwsU42v3aU7p44tu0QDPrweT0XBVLGGLLZ/JZBy7JoqI9wy4YFtLc2YlkaXpXKokUtvPrKIddOGk2ncmzYMN+FlSk1+YSD5dlO+LOgbiccCRFxgE3GmHUjeNqvAZ8Bfh34l5HOqSGWUkoppSaktpp6jvb1EPGV/4RCn8dDQ6C6g5r+C3HOn+klEguWbU5/wMvWF/Zy6y+vLNuc1crjsZjf3sz89mbiiTQdnX0cPdFNR2cfyVT2zW1sliUEA36am2pondVA89QawuHKnLo52Xi9Hu5+xzIefWQbsTF+nySTGa5dMZspU6o/3FaqGiVSZWrsrpVYI2UY+SkYWwrPWYOGWEoppZRSeXNrGnjGlPXAHCBfNVPjC1TsZMRibXlhHz6/+42rr8Qf8HF4z2luete1eLzlnbuaRcIB5rQ2Mqe1sdJLUcPMmtXAtStm88aOE6PeVphOZ2loiLJu/TyXV6fU5KGVWBPKtsLH1aN5soZYSimllJqQmsMxIr7ybJUbKp7Lsqa5+htsnzrSgT/gK/u8mWSGjtM9k/KkQjU+3XjjAoxj2LHjONFocERbC+PxNE3NNdx//yo8Lp9KqdRkkkhl2Kw9sarVTBH5kxFcP5hDLR/NZBpiKaWUUmpCskRY1TiDjWePEfaWL6yxRKo+xMqkcyTj6bL1wxrK6/Ny6nCnhlhq3BARbr5lEa2tU3n66Z1kMjnCV2nIn0plcRyHtWvnct3KNu1dptQYhQNlqsQKaCXWKMwARhJiQX474aj2xmuIpZRSSqkJa9202Ww8c7Rs86XtHAtqpxL0VPctVk9HH7lMDioQYgVCPk4d7WQ1i8s+t1Jj0do2lY989CZ27TrJju3HGYincWwnv3VYwLYNlgUBv4/Fi1tYtXqOaycbKjXZJdJlqsRKayXWKJwDHi7XZNV9h6WUUkopNQYhr4+bZ7TzwqnDJW/wbozBYLi3bVFJ53FD9/k+rAr17BIRUgl9kaDGJ5/Pw4oVraxY0UoymaGzs5/eCwkc4xCNBmlqqhnxlkOl1NWFA37WzNdKrCp11Bjz+XJNpiGWUkoppSa0G6e3sbPrLAPZNF6rdM3EB7IZ3j13KeEKnIY4Utl0Dqng9ibHHulBRkpVn1DIz+zZDTC7odJLUWrCS6QzbNmvlVhKQyyllFJKTXAiwgfmr+CrO1/GEgurBBUSiWyWxVOaWNowzfWxS8Hr82BM5YIky6NVKkoppYoXDvhZXYZKrKe1Emukyn4zoSGWUkoppSa8umCIX1+yhn/avZmA5cn3sHFJIptlTm0975m7zLUxSy1SE8JxKhdieb2lq4hTSik18STSGbZoT6xqdDvQV84JNcRSSiml1KTQHI7xyWvW8q97t5LMZQmN8cRCYwz92TTLp07n/jnXlKTCq1SmTqvD46lMT6xsJses+c0VmVsppdT4VK6eWFqJNTLGmBfLPaeGWEoppZSaNKaGInx++QYeO7aH7efPEvH58MjIw5ykncMrFr+64Drm100twUpLKxwNEAiNLcQbrXQqS+s8DbGUUkoVT3tiqUEaYimllFJqUvF7PDzQvpSVjTN5+sR+Tsf78YoQvEplljGGgVyGoMfLiqnTuX3mPIKe8XkrJSLUN9bQ09lf9oosj8eieZY2wlZKKVW8cvXEekorscZMRGJAGxAj3zNrgPwJhv1ujD8+77yUUkoppcZodqyOX19yPb2ZFL84c5Tj/Re4kEmSzdkYwMEgCF7LIuLzMyUQ5q6m+Sypb3a1p1alrLp5ET/55s+J1oTLNqdtOzTNrCcUCZRtTqWUUuOfVmJVNxGZCfwWcD8w9zLXHAIeBf7WGHNqtHNpiKWUUkqpSa3WH+RdrYuAfLVVXyZNys5iG4PXsqjxB8dtxdWVtLRNJVITwhiDlKmfVzKe5s4H15RlLqWUUhNHOOBn9QKtxKpGIvJ+4J+BEPnKKwNcIF+BJUAEqAPagd8GPi0iv2GM+c5o5hv/byMqpZRSSrlERKgNBGkOx2iJ1NAUik7IAAvyX+vqWxaRGEiVZT47Z1PfGKOldfz1EFNKKaXUxURkDfBNwA98A7gTqDXGNBhjWo0xs40xDUBt4bF/Llz7TRFZO5o5J+ZdmVJKKaWUuqpF17Wx7/Xj9HT24/OX9rYwlcpy/3+5uWxVX0oppSaORCrDln1l2E6Y0u2EI/T7gAd4wBjzk8tdZIwZAJ4DnhORR8lvK/w94FdGOqGGWEoppZRSk5SIcNd71/LtLz2J1+cpWcAU70+x+tZF1DXESjK+UkqpiS0cLNN2wqBuJxyhDcBLVwqwhjPG/FREfgHcMJoJNcRSSimllJrEIrEg7/rVG/jJN18iEgu6HmQl4mnmLJ7OypsWujquUkqpySORKlNjd63EGqka4PQonncKGFWTTA2xlFJKKaUmuRlzGrn31zbw2H/8gmA4gMfjTtvUxECStkUt3PmeNbqNcJxKJjOk01kc2yCW4PVaRCJBLEv/PpVS5aOVWFXrBLBORDzGGLuYJ4iIBawrPHfENMRSSimllFLMmtvEg5+8nSe+9wqJ/hShcGDUY+VyNplUljW3XsOKG+drgDWOdJ3vZ8/uU5w900t/f4p0KosxBmMMICDg83mIxYLU10don9dEe3szHq+eF6WUKp1EKsNW7YlVjX4I/L/Av4vIJ4wx/Ve6WERiwNeAVuB/jmZCDbGUUkoppRQADc21vP+zd7Hp2d288epBBCEUKT7MsnM2yUSGhuYa3vPxW4nVRUq4WuUWO+dw4MAZtm4+yoULcXw+D36/F8sSQmH/JZ+TTuc4dbKHgwfPEQzuZd78ZlatbicaC5Z59UqpySCilVjV6i/JN2d/CHiniPwE2ES+ymqgcE0UmE1+++B95LcgHgD+YjQTaoillFJKKaXe5PFYrL9rKStumM/2lw+w7/XjJONpLBGCYT/WkK2Gxhgy6SyZdA6v12Ja61TW3LqY5plTtPpqnDh8uIPnnt5FOpMlHA4QjRYfQnm81pvX79t7ht07T7Fw8XRuvnUxXq+nVEtWSk1C8TKdThjXSqwRMcb0i8iNwD+QD7N+Ffjg5S4vfPwx8EljTN9o5tQQSymllFJKXSQUCbDuzqVcf/sSejr7OXPsPCcOdRDvT+E4DhaC1++heeYUZs1rprGljkDw0lU7qvqk01meeWonR450EokEiPjGVkEVCPggAAf3n+PY0S7ueedyps+od2m1SqnJLhL0s3qhVmJVI2PMeeBBEVlEPsi6DmgDBo8k7geOAK8Bjxhj9o5lPg2xlFJKKaXUZVmWRUNzLQ3NtSy9fm6ll6Nc0HW+nx//cAuO44yo8qoYgaAPxzH86IebWb1mDmvXz3d1fKXU5BRPZdiyVyuxqlkhnPqrUs+jIZZSSimllFKTRMe5Xn748GaCQS8+n68kc1iWEI0G2bblKJlMjhtvXqTbS5VSYxIJ+Fm9cGbJ53kqoJVY1U5DLKWUUkoppSaB7u4BfvSDzQSDPiyr9KFSOBJg5xsn8Xo9rN+woOTzKaUmrkS6TKcTprUSq9ppiKWUUkoppdQEl8vZPPqjLQQC3rIEWIPC4QDbthylZcYUWtumlm1epdTEEg74WVWGnlhPBrS349WIyBeA3caYp8cwxl3AEmPM3430uRpiKaWUUkopNcG9+PweMpkcwQo0349EAzz95A4+/NGb8Qf05YdSauQS6Qxby9ATSyuxivI3wDeBUYdYwAeAjwIaYimllFJKKaXecuZUD3t2n3a9iXuxRATHMTzzs528894VFVmDUmp8CwfKczqhVmJVPw2xlFJKKaWUmqCMMfzsZ7sIhwMVXUcg4OPwoQ46O/tobKyp6FqUUuNPIlWmSiw9nbBYD4rIrWN4/qj3l2uIpZRSSiml1AR17lwffb2JilVhDRUM+tj0yiHuve+6Si9FKTXOhIN+Vi0qfSXWExXYcj0OGSAMzK7E5BpiKaWUUkopNUFteuUgoSp5Ueb1ejh5oot0Oksg4Kv0cpRSSo1OeyUn1xBLKaWUUkqpCSiVynLmdE9FmrlfjnEMO3ecYNWair4GUkqNM4lUhq17dDthNTDGHK/k/BpiKaWUUkopNQGdPXOBbNYmWPmdhG8KhvwcOdyhIZZSakR0O6EapCGWUkoppZRSE9Cxo51Vt21PROjvT1V6GUqpcUYbu6tBGmIppZRSSik1AXV09OHzeSq9jIukUlmSiQyhsFY8KKWKo5VYapCGWEoppZRSSk1A8YE0IlLpZVwkl3Po7h5gRnhKpZeilBonEqkM27QnlmIch1giMhP4M+AeoAE4AzwCfNEY0zOCcaYAfww8AEwHuoAngD82xpy8xPV/DawGFgBTgSRwrDD3V4wxXZeZ5wbgj4B1QBA4CHwD+LIxxi52vUoppZRSShUjl7Px+arvdl+AdCpX6WUopcaRclViPa6VWFWv+n6qFUFE5gIbgSbgUWAvcD3wBeAeEdlwuTBp2DgNhXEWAM8C3wUWAR8D7hWR9caYw8Oe9l+BbcDTQAcQIR9M/SnwCRFZZ4x5W0QsIvcDPwRSwPeAbuA+4G+BDcB7R/hHoJRSSiml1BUZU+kVXJqIkM1piKWUKp6eTqgGjcsQC/gq+QDr88aYLw9+UkT+hnzI9BfAp4oY5y/JB1h/a4z57SHjfB74u8I89wx7To0x5qJulCLyF8AfAL8PfGbI52uArwM2cKsxZkvh8/+NfHD2oIi83xjz3SLWq8bIztkc2nGM117YQyqews7aeHxeIjUhVt+5jNmLWrAsq9LLVEoppZQasyrcSQiAYww+73h9GaKUqoRw0M+qxVqJpcZhiCUi7cDdwFHg/wx7+E+ATwAfEpHfMcbErzBOBPgQEC88b6ivkA/D3iEi7UOrsS4VYBV8n3yINX/Y5x8EGoFvDgZYg+OIyB8BzwCfJl8FpkrEGMOmJ7az65X9pJNZIjUhLLGw/PnAKt6b5Il/fYFgNMB1t17D8psWVWUPCaWUUkqpYvm81dfUHfLbCYPh6jo1USlV3RKpDFt3ayWWGochFnB74eNTxhhn6APGmH4R+QX5kGsd+YDoctYDocI4/cPGcUTkKfKB2G3A8C2Fl3Jf4eOOy6z3iUs850UgAdwgIgFjTLqIedQI2bbD4//6PKcOniMcDeLzX3zTZFlCpDYMwCuPvUbXmR5ue996DbKUUkopNW5FokEGBlJVdz/j8VrU10cqvQyl1DiilVhq0HgMsRYWPu6/zOMHyIdYC7hyiFXMOBTGuYiI/C4QBWrJN3q/kXyA9T+KnccYkxORI8A1QDuw5wrrvSIRqdKuB5VljOGZb/+CM4fyAVYxwrEQB18/RjAc4Ib7VpV4hUoppZRSpTFtWi17dsfxB6rrlj8Y9BMK6QtFpVTx9HRCNai6fqIVp7bwsfcyjw9+vq7E4/wu0Dzk908AHzXGdLo8jxqDU4fOcXDHUWJ10RE9LxQNsv3FPSxZN5+6xpoSrU4ppZRSqnRa5zTy+mvHqirEMsZQU1vcG4tKKTUoHPSzsgyVWI9pJVbVq56faO4ZrJcea2XSFccxxkwDEJFm4AbyFVivicgvGWO2uTVPsYwxl6wTF5GtwMqxjD2ebX5qO5FYeFTPDYT8bH5qB3d98EaXV6WUUkopVXrNzbX4/dV1u59MZlhxXWull6GUGmcSyQzbytETK6mVWNWuun6qFWewcqn2Mo/XDLuupOMYY84BPxaRbeS3DH4TWOr2PGrk4n1JOo53EY6FRvV8X8DHsT2nyGZy+KrsBlAppZRS6mr8AS8zZk3h7JkLeKukybtlWSxZOrPSy1BKjTPl6omllVjVbzy+Mt9X+HjJXlW8dTrg5XpduT0OAMaYYyKyG1ghIlONMeeHzLO6MM/Woc8RES8wB8hRXPN4NQKHdxzDjLEeL53KcObIOWYvnOHOopRSSimlymjt+nl879svE41WPsTKZm1a26ZWXXWYUqr6JVJlqsTSnlhVbzz+BHmu8PFuEbGGnlAoIjFgA5AEXrnKOK8UrtsgIrGhJxSKiEW+OfzQ+YrRUvhoD/ncs8AHgXuA7wy7/mYgDLyoJxO6r78njs8/ths2j2UxcCHp0oqUUkoppcpr6tQY9fUR0ukslmVVdC3pdIa16+ZWdA1KqfEpHPSzcknpK7H+r1ZiVb3K/iQbBWPMIeApoA347LCHvwhEgG8aY+KDnxSRRSKyaNg4A8C3Ctf/6bBxPlcY/0ljzJsVUoVxpg1fk4hYIvIXQBOw0RjTM+ThHwDngfeLyOohzwkC/73w27+/8letKkZg7O3VlFJKKaUq5653LCNZ4T4v6XSWRYtmUD9lZIftKKWUUkONx0osgM8AG4EvicgdwB5gLXAb+e1/fzjs+j2Fj8Obn///7N15fNTVvf/x15ktM9kDRNkEgiwuFFkEN0SU1traVq14rfdXrbTWerXW22pvfbTXvXrbq6ittlXb29bae6+97laraEVlVSmroKwJIIsYIAlZZjLb+f0xmZiQAFlm+U54Px8PHoGZ73LCzCTzfc/nfM6PgRnAD4wxE4D3gOOBC4BP6BiSnQfca4yZD2wG9pJYofAsYCTwMfDttjtYa/cbY75NIsx6yxjzJLAP+AowtuX2v3Tv25euKCjNJxqJ4evFAjixaJzC0p41hhcRERFxgvKjihl/0jDWvv8R/kBexs8fj1u8XjfTzz4+4+cWkb6hKRRm+VpNJ5QcDbGstZtbqpruJBEsfRHYBfwSuMNau6+Lx9lrjDkNuA24EDiTRDD1B+BWa+32A3b5O/AYiSmLJwGlQCOJ4OwJ4Jedndta+7wx5iwS4drFgB/YBPygZR+V+qTBqPHDWfJSdxaK7Cgv4GPgiKNSNCIRERGR7DjtjDFUbv6EaCSO25PZyRhNjSEuungqXm/2+3KJSG7SdEJJyskQC8Ba+xEwu4vbHliB1fa+fcANLX8Od5w1dKzO6hJr7SISYZtkSEFJPkcNG0D93gZc7u6/WYuEoww7bjC+PG8aRiciIiKSOW63i4tmTeHJPy/BuMhYf6zGxmZOmzaGwUPLMnI+EembVIklSTkbYol0xdRzx/PiY29QVFrQ7X1DTc1MPfekNIxKREREJPOKigLM+tpUnnryXTyeRLCVTg2NIU6eMpJJkyvSeh4R6ftUiSVJCrGkTxsyaiAjTzyGbRt2EijoenOspoYgnzl9LKVHFadxdCIiIiKZVVZWyKX/fBrPPbWU5lCEPH/qK85jsTihUJgzzhjDhEkjUn58ETnyBENhVmSgEiuoSizHU4glfZoxhs9dfiZ/+6832bXlky4FWU31QY49aRjTLjz5sNuKiIiI5JqSknz+3zemMf/ND1m/bieB/DxcroN23+iWYFOYwqI8LrjoNPoPKErJMUVEAn4fk05MfyXWS6rEcjyFWNLnud0uzr/qbN7520rWvbeJcChCfnEAYz59sxaPW4L1QfLyfUz9/ElMmHFCu/tFRERE+hKv183Mc8dx/AlDeG3uahoaQuT3IswKBpsBw/gJx3DKaaPTPlVRRI4swWCY5Wu2ZeQ84mwKseSI4HK5OP1LkzjlvJPYsLyKVQvWEWoMEYvE8Xjd5BcHmHbBZEaceIzedImIiMgRY/DQMq6YPZ2Ptu3hvXc3U/3Jfoxx4fd7DxloWWuJRGJEwlHyC/KYfPJIPnPSMPxpmJ4oIhIIeDNTiRXQzzCnU4glRxS3x83xU0dx/NRR2R6KiIiIiCO4XIbhI8oZPqKc+vogmzbu5qNte6mrbaKpsZm4ta3bGmPI83koKvZz9MBSKkaWM2RoP1Wwi0haBYORjKxOGAxG0n4O6R2FWCIiIiIiAiRWMJw4aQQTWxqyx6JxwuEosVgc4zJ4PC7y8lSpICKZpUosSVKIJSIiIiIinXJ7XAQ8anQsItkVDEZYsUaVWKIQS0REREREREQcLBDwMjEDlVh/VSWW4ynEEhERERERERHHUiWWJCnEEhERERERERHHyg94mTROPbFEIZaIiIiIiIiIOFhTMMKK99NfidWkSizHU4glIiIiIiIiIo6V7/cyMQOVWH/1qxLL6VzZHoCIiIiIiIiIiMjhqBJLRERERERERByrKRRmxZptGTmPOJtCLBERERERERFxrHy/j4njhqX9PH/1+9J+DukdhVgiIiIiIiIi4lhNoTAr3lcllijEEhEREREREREHUyWWJCnEEhERERERERHHagqGWZmJSqygKrGcTiGWiIiIiIiIiDhWfsDHxM+kvxLrxYAqsZxOIZaIiIiIiIiIOFZTMMyK1arEEoVYIiIiIiIiIuJgiUqsY9J+HlViOZ9CLJE+KB6PU1dXRzAYxBiT7eGIiIiIiEgXWWsJBAKUlJTgcrmyPRxHUCWWJJrxmXgAACAASURBVCnEEulj4vE4u3btorS0lNLSUoVYIiIiIiI5xFpLU1MTu3btYtCgQQqyUE8s+ZRCLJE+pq6ujtLSUgoKCrI9FBERERER6SZjTOt7+bq6OsrKyrI8ouxTJZYkKcQS6WOCwSClpaXZHoaIiIiIiPRCfn6+QqwW+QEfE8erEksUYon0OcYYTSEUEREREclxel//qaZgmJWrVIklCrFERERERERExMHyAz4mZKAS6wVVYjmeQiwRERERERERcaymYJiV6omFMSYPuAr4BjAS8AMfAa8Dc6y1W7M4vIxQiCUiIiIiInKEi9kYoViIqI0St/HW2w0Gt3GT587Da7ya3iZZkameWE6uxDLGeIA3gDOAdcD/As3AFOB64ApjzOnW2g+yN8r0U4glIiIiIiJyhIrZGA3RBqLxKIaOPZgslqiNEo6EcRkX+e58fC6fwiyRzLuIRID1BnCutZ+mzcaYO4BbgZuAb2ZneJmhEEtEREREROQIFIwFaYo24TIuXMZ1yG2T9zdEG/C6vBR6Cg+7j0iqBJsy09g92OTo6YQjW76+3DbAavECiRCrPLNDyjyFWCIiIiIiIkeYxmgjoVio20GUy7iI2ij7I/sp9hYryJKMCGSosfvzDp5OCKxt+foFY8wvDgiyvtTy9e8ZHlPGKcQSERERERHJkriNE7cWiwUSPahcxqQ1HArFQj0KsJIMhjhx6qP1FHuKNbVQ0i4YDLNyVfp7lged3dj9ZeBZ4KvA+8aYvwNhYDIwDXgIeDh7w8sMhVgiIiIiIiIZYK0lHI8SjkeIxGPESQRYtARYn2oJsnDhdbnJc3nxujwpCYviNt46hbA3DIZoPEooHiLgDvR6XCKHEgj4mHBSBiqxnvEBTDLGHPiiBMBam7XE1lprjTGzSEwbvAU4oc3dbwD/Y62NZWVwGaQQS0REREREJI1iNk5TNEQoFsESb9dA3WUM0Pl1cZw4oViMYKwZY1wEXD7yPXm9CqAao40HO123uYyLYCyI3+VXNZakVTAYZuXKDPTESnMlljFmCzC8G7v8t7X26y37+oE/AV8AriPRB6uJRLP3XwLzjTGXWGtfSOmgHUYhloiIiIiISBrEbJz6SBPheBQDGGMwdC+ASuyTCIiaYs00xZrxuTwUe/O7HWbFbZyIjbQeLxWstTTHm/G7/Sk7psiBMlaJ9awPYLm1dnKaTrEZCHVj+51t/n4zcAlwg7X20Ta3v9JSobUS+AWJcKvPUoglIiIiIiKSQtZagrEwDdEghmS1Ve8ljxOJR9nbvJ9CTwC/29flKqhwPIy1NqVVUy7jojmmEEvSKxgMs6oPVGJZa2f2Yvdk8/Y3OznuKmPMPmC4Maa/tXZvL87jaAqxREREREREUiRm49SFG4naaNqasydDqPpoE6F4mBJvQZfOFYlH0jKmGLGUh2MibWWqEuu55xy9OmFey9fyA+8wxuQBxS3/dHR3+t5SiCUiIiIiIpIC0XiUmnADQFpXF0xyGRfReIy9zfsp8xXhcbkPuX3cxtMyDmstceK4OfT5RXoq0RPriF+dcAEwDvixMWaRtba5zX23k8h3llpr67MxuExRiCUiIiIiItJLkZYAK1VTB7sqWf1UE64/bJBlO6yCmEKWlDWMFzlQIOBjwoTu9EPvmeeed3Ql1t3Al4GZwDpjzKtAkERj96ktf78he8PLDIVYIiIiIiIivRCNx6jNQoDVljGGmnA9/XzFuF2dV4EZTPqCLAVYkkbBpjArV2SgEqvJuZVY1todxphJwI+A84HZgAvYBfwR+Lm1dl32RpgZCrFERERERER6yFpLbaTBMf2gaiOJIKuz8biMKy1TCo0xuLq56qJIdwTyfUyYkIGeWC84uhILa201cFPLnyOSQiwREREREZEeqo8GiVub1SqsJGMMMWtpjAYp9OZ3uN/r8hKOhlPer8uFyzEhnvRNwaYM9cRycCWWJCjEEhEREZFOXXnllTz++ONUVVUxYsSIbA9HxHHC8SihWHNGmrh3lcsYmmLN5Ll9eF3tL/d8Lh9NNKX0fHEbJ9/dMTATSaVEJVYGemI5vBJLFGKJiIiItBMMBikrK+Paa6/l/vvvB+Dqq6/mySefZN++fXg8evskIolphPsjjY4KsJJcxsX+SGOHaYUu48Lj8hCzsZSdy2DIc+el7HginQk2hVl1hPfEkoScfRdmjBkK3AmcB/Qn0czseeAOa21NN47TD7gVuBAYBOwFXgVutdZuP2Db/sBFJJqofQYYAoSB94E/AH+wtuMkc2NMIYnma7OACiAELAPmWGv/1vXvWkRERNJt0aJFNDc3c84557Te9sYbbzB9+nQFWCLSKhQLE7dxR4ZYADEbJxyPkOduX1lS4CmgNlybknFba/G7/Y79P5C+I5Dv46QMVGI9+6IqsZwuJ3/aGGOOJRECzQbeAx4AKkksJ7mkJWzqynH6A0ta9tvccpz3Wo67zBgz8oBdLgF+C5wCvAs8CDwDjAN+B/yfOWAyuDGmtOUc/w7EgEeBp0mEYC8bY77Xne9dRERE0mvevHm43W6mT58OwJYtW6isrGwXaomINDlsGuGBXMZFYzTU4Xa3cZPvzu91g3eLxWVcBNyBXh1HRKQ7cvXjxF8DRwHfs9Y+lLzRGHM/8H3gbuCaLhznHmAM8IC19gdtjvM94Bct5zmvzfYbgK8AL7etuDLG/JhE+HUx8FUSwVbS7SRCrmeBS6210ZZ9ylv2uc8Y84q1dmOXvnMRERFJqfr6enbv3t3679dee43jjz+eTz75hE8++YRnn30WgIqKCjZt2gTAkCFDCAS6duG2bt06fvazn/HGG2/wySefUFpaysyZM7ntttsYO3Zsu22TPag2b97M888/z2OPPcaWLVsYMGAAl1xyCXfccQfFxcUdzrFs2TLuueceFixYQF1dHQMHDuT888/nlltuYdCgQR22b2pq4qGHHuKpp55i/fr1WGs55phj+NznPsdPfvITjj766A77PProozz88MNs3LiRkpISLrjgAu69915KSkrabbd69Wr+4z/+gyVLlrBr1y6Ki4s55phjmD59Ovfeey9er7dL/28iThaNR4nZmKNDLICojRGNx/C43O1u97v9xIjR3MMgzmIxGIq9na+CKJJqmk4oSTkXYrVUR50LbAF+dcDdtwFXA5cbY2601jYe4jgFwOVAY8t+bT1MIgz7vDFmpLW2EsBaO6+zY1lrPzbGPEIiPJtB+xDrqy1fb00GWC37VBtj5gAPkQjcbjzYWEVERCR9nnnmGWbPnt3h9tGjR7f791e/+tXWv7/55pvMmDHjsMd+9dVX+epXv0okEuHLX/4yo0aNYvv27Tz77LO8/PLLvPnmm0yaNKnDft///veZP38+//RP/8QFF1zA3LlzefDBB1mwYAELFy7E7/e3bvvSSy9x8cUXY61l1qxZDB8+nGXLlvGb3/yGF154gUWLFrVryl5TU8PZZ5/NqlWrGDt2LN/85jfx+Xxs3ryZ3//+93z1q1/tEGL927/9G3PnzuXLX/4y5557Lm+++Sa//e1v2bRpE/Pmffr2aPXq1ZxyyikYY/jKV75CRUUF+/fvZ9OmTfz617/mpz/9qUIs6RMaoyEMzg9vDIamWIhiV0H7242hwF2ACxfBWLBbQVbcxvEYD0XeIseHeNJ3BAKZaeyu6YTOl3MhFpCs5X/twP5T1tp6Y8wiEiHXqcAbhzjOaUCg5Tj1Bxwnbox5jUQgdjaJqYqHE2n5Gj3g9oEtXzs7RvK2mV04vgBN9UHq9tTTHAzjzfNSVFZAcb/CbA9LRERy2Nlnn81TTz0FwOLFi3nggQe48847Of744wH4xje+wSmnnMK1117bus+JJ5542OPW1NRw2WWXkZ+fz/z58znhhBNa71u7di2nnHIKV111FcuXL++w76JFi1i5ciXDhyfesP/Hf/wHl1xyCc8++yz33nsvt9xyCwANDQ1ceeWVRKNR3nrrLc4888zWY/z85z/n5ptv5uqrr+a1115rvf26665j1apVXHPNNfzqV7/C5fr0IrS+vp54vOMUo3feeYf333+fYcOGARCNRjnnnHN48803ee+995g6dSoAjz/+OKFQiOeff54LLrigw/9Hfr5WMJO+IWJjOVGBZIwhEj/w8uTT+/I9+fhcPhqiDcRavqeDhXNxG0/s487H7/bnxPcvfUcw2MyqFVsych5xtlwMsZJ19xsOcv9GEiHWGA4dYnXlOLQc55CMMR7gipZ/vnrA3XtINIyvAD444L5kz63jDneOLozB9vYYTmWtZWflJyydu4rd2/YQjcbAJr5dt8dN2VElTJo5jpGfGYbbrU+DRESke4YPH94aFi1ZsgSv18sPfvADCgoK2LBhA01NTVxyySXMmjWrW8f905/+RG1tLQ8//HC7AAsSIdi3v/1tHnzwQT744IMO999www2tYwJwuVzce++9PP/88/z+979vDbFeeOEF9u7dy2WXXdYuwAK48cYbeeSRR3j99dfZtm0bw4YN45NPPuEvf/kLgwYN4r777msXYAEUFRV1+r3ceuutrQEWgMfjYfbs2SxYsKBdiJXU2VTLsrKyg/1XieSUuI0TtxZXjoQ4MWux1h40dPK4PJT6SonGo4RiIaI2Spw4tFxduIwLl3GR704EXgqvJBsCgTxOmpj+Sqxn/qqVNp0uF0OsZOOFuoPcn7y9NEPHAfgZib5Xf7PWzj3gvpeAbwO3G2Muszaxnm1LU/lkH648Y0zAWhvswrmOKKGmZl585O/s3VVDoMhPfpG/wzbNwWb+/j8LyS8KcME1n6O0vGOvEBERka6YN28eU6ZMoaAgMfXm7bffBuCss87q9rGWLFkCwKpVq7j99ts73L9hQ+JztA8//LBDiNXZ+UaOHMkxxxzDli1bqK2tpbS0tLWKq7Om8x6Ph+nTp7NlyxZWrFjBsGHDWLp0KfF4nOnTp7d+j11x8sknd7jtmGOOARIVVkmXXnopv/jFL7jwwguZNWsWn/3sZznjjDM49thju3wuEaeLxmMkEp5cCXMsURvDaw596edxeSh0fTrDwbZ8aKzQSpwgGGxm1fIM9MRSJZbj5WKIdTjJn7K9rUzq0nFamsDfCKwj0WPrQLeSqAy7BDjeGPMGkA9cANQDTS3/jvVmsNbaTn+7GGOWAR2bbeSAUFMz/3f/y0TCEQpLDz79wBhDYUk+sVic/7v/ZS6+4Tz6D9SnvSIicnhvvfUWb731FpC4YFu9ejWTJ09uDZ3+9re/4Xa7+ctf/tJ6IddZINWZvXv3AvDb3/72kNs1NDR0uK2zxuoAAwcOZOvWrdTV1VFaWkpdXeIzt86at7e9vba2tt3XIUOGdOE7+FRpacfP9DyexNvIWOzTtzBTp05lwYIF3H333Tz99NM88cQTAIwdO5bbbruNyy67rFvnFXGicDyaE/2wPmUIx6N4Xd279FN4JU6SsUqsl1SJ5XS5GGIlK6RKDnJ/8QHbpe04xpjrSKxi+AEw01q778BtWpq+TwH+HfgycC1QQ6JC6y4SfbHqrLVaBqENay0vPvI6kXAEX17XGsC63S7yAl5e+PXr/PPNF+DP1w8gERE5tLfeeos77rij3W1Lly5l6dKl7W678847W//e1RAruWrfqlWrGD9+fLfGtXv37g4rFwJ8/PHH7Y6d/Jq8/UC7du1qt10yjNqxY0e3xtMdp512Gi+99BLNzc0sW7aMV199lYceeoh//ud/pry8nM9+9rNpO7dIJkRzpB9WkssYogfpiyWSK4JNGarEalIlltPlYoi1vuXrwXpVJZcSOlivq5Qcxxjzr8ADwBoSAdYnBzuRtbYauKHlT9tjnE2i4mtpZ/sdybZv2MXeXbWHrMDqjMvtojkYYfX8D5l63oQ0jU5ERPqK22+/vTWUuvHGG/n1r39NbW0teXl5rdP8HnnkEb7zne90+9innnoqzzzzDAsWLOh2iPX2228zffr0drdVVlby0UcfMWLEiNYwauLEiUAijPvWt77VbvtoNMrChQsBWldAnDp1Ki6Xi/nz59PY2NitKYXdlZeXx+mnn87pp5/O6NGjueKKK3jhhRcUYknOs72e8JF5uTdikfYC+XmcNCkDlVgvqxDC6XIxxHqz5eu5xhhX2xUKjTFFwBlAEHjnMMd5p2W7M4wxRW1XKDTGuEhMAWx7Ptrc/yMSfbBWAp+z1u7p4ffy7Zav/93D/fus9+au6rT/VVcECvP48L1NnHzu+A4Na0VERA7mzTff5NRTTyUvL/EGNjnNcMaMGT063uzZs7n77ru54447mDJlSofm5/F4nPnz53d6/F/84hdcccUVrc3d4/E4P/zhD4nH48yePbt1uwsvvJB+/frxv//7v1x33XWceuqprfc9+OCDVFZW8tnPfra1KXt5eTlf+9rX+J//+R9uuummDqsTNjQ0EIvFWiu3uisZ2B24/+7duwG0OqFIluRi8CbSVrCpmVXLtmTkPOJsORdiWWs3G2NeIxEyXQc81ObuO4AC4FFrbWPyRmPMcS37rmtznAZjzBPA1cDtJPpaJX0XGAHMtdZWtj2/MeYW4E5gGXBuZ1MID9jeBeRbaxsOuP0q4DISQZhCrDYa65qo3rGPguKOKxt1VVN9iI827GL4cd3r+SEiIkemmpoaVq1axa233tp621tvvcXAgQM7ndbXFf379+fpp5/moosu4tRTT2XmzJmceOKJuFwutm3bxpIlS9i7dy+hUKjDvmeccQYTJkzg0ksvpaSkhLlz57Jq1SomT57Mv/3bv7VuV1hYyO9//3suueQSzjrrLC655BKGDRvGsmXLeO211xg4cCCPPvpou2M//PDDrFmzhkceeYS33nqLz3/+8/h8Pqqqqpg7dy4vvvhij4O7OXPm8NprrzFjxgxGjhxJYWEha9eu5ZVXXqGsrIyrr766R8cVkd7JrR5eIh0F8n2ZqcT6my/t55DeybkQq8W1wGLgl8aYmcCHwCnA2SSm//3kgO0/bPl64E/vHwMzgB8YYyYA7wHHk2i6/gmJkKyVMeYbJAKsGLAA+F4n8+G3WGv/2Obf+cBuY8zrwKaW284EpgKbgYustZGufNNHin2764hFetXnHp/fw46NHyvEEhGRLnn77beJx+PtwpuDVUl1x8yZM1m9ejX33Xcfc+fOZcGCBfh8PgYPHsw555zDxRdf3Ol+DzzwAM899xy//e1v2bJlC/379+eGG27gzjvvxO9vX6l8wQUXsGjRIu655x7mzp1LXV0dAwcO5JprruGWW25h8ODB7bYvKytj8eLFPPjgg/zlL3/hsccew+12c8wxx/DNb36zw0qJ3XHttddSVlbGu+++y6JFi4hGowwdOpRrr72WG2+8sbWyTCSX5WIglItjFmkr2BTOUCWWWlU7nUkunZprjDHHkAiUzgP6A7uA54E7DqyOMsZY6HwFP2NMP+A24EJgELAXeAW41Vq7/YBtb2/Z9lDettbOaLOPF3gEmAYMbbl5M/A0cP+BFVqpZoxZNmnSpEnLli1L52lSasPyKt7434UUlva8T0c0EmPYcYOZ+bUzUjiy3LBr166DrlIlIiLOduWVV/L4449TVVXFiBEjsj0cEelEYzRIU7Q5Z5q7x22cQk+AfE/PWnVIdnXlvf3kyZNZvnz5cmvt5AwNK6OMMcuGDxs96c6fPJb2c91699Vs3baxz/5f9gW5WomFtfYjYPZhN6Tz8KrNffvopOn6Qba9ncTUwy5rqbL61mE3lFY+vxcb7124Go/H8eWpFFREREREUstrPFhCOVXd5HXl7GWfCJCokFqtSiwhh0Ms6buK+xVietmQPdIcpXxovxSNSEREREQkweNy07FLiXMZDB7jzvYwRHolP9/HSZNHpP08z7yqQginU4gljlN2dAnF/YuIRaM9LtP2ej0ce5L6boiIiIhIarmMC1eOTCWExHhzZeqjyME0NYVZ9Y+qjJxHnE0hljiOMYaJM05g/nPv9WiFwnBzhGEnDMHr09NbRERyyx//+Ef++Mc/ZnsYInIYPpeH5ljE8eGQtZY8tzfbwxDptYxVYs1VJZbT9W7OlkiajJlcgdfnJR6Pd2s/ay3hYISp556UppGJiIiIyJEu3+3H4vwFsiyoobuI9CkqVRFH8ng9fPk7M3n2l68SKPLjch3+Uy5rLQ11QWZ+7XRKBhRlYJQiIiIiciTyuNx4jJu4w4Msr8uN26huQXJfU1OYVRlo7K7phM6nEEsc66ih/bnou5/nxUdfx2DIyz94aWckHKW5KczMr53O2MkjMzhKERERETkSFXj81EUacTk0JIrbOMWe/GwPQyQlNJ1QkhRiiaMdPWwA/+/mC1n29/fZtHIroWAzeQEfbreLeDxOczCC2+Nm2HGDOeW8CZSWF2d7yCIiIiJyBPC5vLiN25HTCq21eIwbr0uXe9I3NDVmqLF7oyqxnE4/1cTx8osCnHnRVE7/yslUvr+NnZs/JtTYjNfv5aih/RkzeSS+PDWsFBEREZHMMcZQ4i1gX3i/46qxLFDiLXB843mRrkpUYlWk/TzPzPXB3rSfRnpBIZbkDLfbxegJIxg9YUS2hyIiIiIigsflJt/tpynWjMshgVHcxinwBHC73NkeikjKNDWFWbU0A5VY6onleAqxREREREREeqjA46c5HiFu41mvfEpOI8x352V1HCKplp/v46STR6T9PM+8rkosp1OIJSIiIiIi0kPGGEp9hewL14O1kKUgy1rbOpZsh2kiqRZsamZ1BnpiBZua034O6R2FWCIiIiIiIr3gNi76eT8NsjIdIiUDrDJfkeP6c4mkQiA/j/Enp78nVuD1PFViOZxCLBERERERkV5yu9z0yyumJlzfGiplgrUWlwIs6eOCTc2sXlqZkfOIsynEEhERERERSQG3cdHPV0R9pInmeCTtoVLcxvG7fRR68lPWWD4SiRAOh4nFYsTjcVwuF263G5/Ph9erFcElOzJWifV3VWI5nUIsERERERGRFHEZFyW+QppjEfZHGgFSXpVlrQUMpd5CfO6eB0uRSISamhqqq6upq6sjEokA4HK5MMZgjMFai7WWeDwOgMfjobS0lPLycsrKyhRsSUYkKrHUE0sUYomIiIiIiKRcnttLf1cxDdEgzbEIFtvryqy4jWMw+N0+CjyBHlVfWWvZs2cPGzduJBwO4/V68fv9FBUVdSlss9YSDAbZuHEjkUgEn8/H6NGjGTBggBrKS9oE8vMYPyUDlVhvqBLL6RRiiYiIiIiIpIHLuCj2FmA9lmAsTDAWImbjrfd1Rbxle7dxU+TJx+/29SgsikQiVFVVsXPnTjweD0VFRRQXF3f7OMYY/H4/fr8fgFgsxvr161m7di2DBw+moqJC1VmScsHGZla/l4GeWI2qxHI6hVgiIiIiIiJpZIwh35NHwO0jZuOE41Ei8QhRGydu49gDtycRcnmMC5/bj9flweNy9+jc1lq2bt1KZWUlhYWF9O/fP6UVU263m9LSUqy11NbWsmDBAkaOHMnw4cNVmSUpEyjIY/yUkek/z7w82Jf200gvKMQSERERETkCWWsJhUIdGnj7/X6FD2lijMFj3C2BVB6QeBwSIVYyyjIYUtNHKxQKsWzZMowxlJeXp/VxNcaQn59PIBBg586d7Nixg8mTJ7dWbIn0hiqxJEkhloiIiIjIESAUCrF7926qq6sJBoPEYrGDNvB2u90EAgHKy8s5+uijFUSkkTGJ0ApSFzC1rb7KdPN1YwzFxcVEIhEWL17MyJEjGTFiRMbOL31ToCCPkzLQE+vpN1WJ5XQKsURERERE+qhkE+9NmzbR3Nzc2suorKzssPtGo1F27dpFVVUVeXl5jB49OuVT0ST1rLUsX76cUCiU9uqrQ/F6vZSXl7Njxw727t3LpEmT9NwRkV5TiCUiIiIiHTQ3NxOJRFqnmXm9XvLy8rI9LOmGbdu2UVlZ2drEu6ioqFv7ezweCgsLKSwsJBaLsW7dOqLRKCNHjmTYsGFpGrX0RjweZ+nSpQCUlJRkeTSJqqySkhIaGxt57733mDJlCi5X71ZolCOTphNKkkIsERERkSNcslpnx44dNDQ0EIvFMMa0u9iMx+NYa3G73RQWFjJkyBAGDBigygoHam5uZvny5cTj8ZRVTrVt3v3RRx+xY8cOJk2apGDTQay1LF26tLU3lZMUFBTQ1NTE0qVLmTp1qn5uSLcFCvIYPzUDjd3f0nRCp1OIJSIiInKEikQiVFZWsmvXLjweD/n5+ZSUlBzyAtNaSyQSYf369axdu5bBgwdTUVGR0Z47cnDbtm1j06ZNlJaW4vP5Un78ZGVNOBxm0aJFjBo1SlVZDpCcQgg4LsBKys/Pp7GxkeXLlzN58uRsD0dyTLCxmdXvbs7IecTZFGKJiIiIHGGstWzcuJEdO3aQn5/frWodYww+nw+fz4e1lpqaGnbs2MGQIUMYPXq0KiyyxFrLmjVrqKury0gfJJ/PR3l5Odu2baOuro5x48bpsc+irVu3EgqFHDGF8FAKCgqoq6tj69atDB8+PNvDkRwSyPdlqBLLp0osh1OIJSIiIl0WCoWoqamhurqapqam1n5J+fn5lJeXU1ZWplXMHC5ZCZFsutwbxhgKCgooKChg3759LFy4kEmTJlFQUJCi0UpXWGtZuXIloVCI0tLSjJ3XGENpaSn19fWsXLmSCRMmKMjKglAoRGVlZa9fz5lSXFzM5s2bteqldEuwKczqdzPQE6spnPZzSO8oxBIREZFDikQiVFVVsXPnTowxeL1e/H5/u6Aiuc2GDRuw1mqKmUPt2LGD9evX069fPzye1L4NLCwsJBqN8u677zJ27FiGDBmS0uPLwa1Zs4ZQKNTtxu2pUlRURH19PWvWrOEzn/lMVsZwpEpOIywrK8uZANEYQ1lZGcuXL+f000/P9nAkR2SsEuttVWI5nUIsERER6VQ8HueDDz6gurqajcBvywAAIABJREFUgoKCQ045S04vg8RFVW1tLQsXLqS8vJwTTjhBq1E5wNatW9myZUtap5p5PB7Ky8vZtGkT0WhU04UyIDmdL5MVWJ0pKiqitraWbdu2qUdWBm3duhUg5z4w8Hq9BINBTSuULlNPLElSiCUiIiId1NbWsnLlytZpgt2RXBkr2cR3/vz5TJgwIesX2UeyHTt2sGXLFvr165f2ag1jDP369WPLli14PB5VZKVRc3MzmzZtcsw0spKSEjZt2sTRRx+tVQszILkwg1Me/+4qKipi8+bNDB48OOdCOMm8QEEe40/JQCXW/DyoSftppBcUYomIiEg727dvZ+PGjfTv37/XFVQFBQX4/X5WrFjB6NGjGTp0aIpGKV3V2NjI+vXrM9LsOykZZK1fv57S0lL1yEqT5cuXU1pa6phpZMkeWcuXL+e0007L9nD6vKqqKgoLCx3z+HeXMYbCwkKqqqoYM2ZMtocjDhdsbGb1O6rEEoVYIiKSAtZa9uzZQ1VVFc3NzcRisdb73G43eXl5VFRUMGDAgJx9s32k2L59O5s2bUrpY+V2uxkwYACbNm3CGKPKnAyy1rJs2bKMVGAdKBlkLV++nGnTpum1n2Lbtm0jHo+3TuN1Cp/PRzAY1LTCNLPWsnPnTvr375/tofRKIBBg586dWtlUDitRiXVs+s+zQJVYTqcQS0REemXXrl2sX78ej8dDUVER+fn5HbaJxWKsX7+etWvXMnbsWAYNGpSFkcrh1NbWsnHjxrSEjcYY+vfvz4YNGygoKNDUwgzZuHEjPp8v5U3cu8rj8eD1etm0aROjR4/Oyhj6ImstlZWVjg0wiouLqays5JhjjlEwkSZ79uzB4/Hk/P+vMQaPx8OePXtydlqkZEaiEmtTRs4jzqYQS0QyxlpLdXU1tbW1RCIRvF4vpaWlGZ3iIqkTi8VYvnw5oVDokA2/IVGJU1pairWWTZs2sX37diZNmoTb7c7giOVQ4vE4K1euPOxj2RvJypyVK1cyffp0NXtPs0gkwo4dO7J+YVhYWMj27dsZMWKE+t6kyN69ex0dYCSDib179zJgwIBsD6dP2rhxY9ZWo0y1oqIiR/V2E2fKWCXWQlViOZ3ePYpI2oXDYT788EPefvttNm7cSF1dHaFQiLq6OjZu3Mjbb7/NunXrCIfD2R4qTz/9NNdffz1nnnkmxcXFGGP4+te/3um2W7ZswRhz0D9f+9rXMjz6zInFYixZsgSgW8t6J5fVBliyZEm7aYeZsnfvXn73u99x0UUXMWrUKAKBACUlJUybNo3/+q//Ih6Pt9v+SHmcP/jgA/Lz89MeLLndbgKBAB988EFazzNixIiDPmYDBw7sdJ/FixfzxS9+kX79+pGfn8/48eN58MEHs/I8TYWqqqpOKyOzoaCggKqqqpQe849//OMhX5vGmHZBeV96LedCgFFUVMTGjRt7fZzu/F5O6slr+aWXXmLGjBmUlJRQWFjIKaecwuOPP97r8adDJBIhHA73mQ+C3G43zc3NRCKRbu2Xqfdsjz/+OFOnTqWwsJCSkhJmzJjBSy+91K2xSu8lK7HS/UeVWM6nSiwRSauamhpWrFhBYWFhpxUegUAAay379+9n4cKFTJw4sTXkyIaf/vSnrFq1isLCQoYOHcq6desOu89JJ53EhRde2OH2cePGpWOIjrB8+XLy8vIIBAI92j+534oVKzj55JNTObTDeuqpp/iXf/kXBg0axNlnn82wYcPYvXs3zz77LFdddRWvvPIKTz31VIfnal9+nCORCNXV1Rn7FLygoIDq6urWisx0KSkp4V//9V873F5YWNjhthdeeIGLL74Yv9/PpZdeSr9+/fjrX//K97//fRYtWsRTTz2VtnGmg9P65aSj782ECRO47bbbOr1vwYIFzJs3jy984Qsd7sv113IoFKK5udnxIVYymAiFQvj9/h4fp7u/l3vyWn744Ye5/vrr6d+/P1//+tfx+Xw8/fTTXHnllbz//vvcd999PR5/OtTU1PS5qkafz0dNTQ1HHXVUl/fJxHu2m266iTlz5jB06FC+/e1vEw6HefLJJ/nyl7/MQw89xHe/+90uj1d6J1CQx/hTM1CJtUiVWE6nEEtE0iYZYA0YMOCQ1R3GGPLz81tXMMtmkPXAAw8wdOhQRo0axdtvv83ZZ5992H0mTJjA7bffnv7BOcSuXbsIhUK9fowCgQA1NTV8/PHHB62MSYcxY8bw4osvcv7557d7Xt5zzz1MnTqVZ555hmeffZaLL7643X59+XGuqqrK+OpxycqcdK5IVVpa2qXHbP/+/Xz729/G7Xbz1ltvtQard911F+eccw5PP/00Tz75ZE5V6jitX046+t5MmDCBCRMmdHpfcmW8q6++utP9cvm1vHv37l6FQpnk9/vZvXs3w4cP7/ExuvN7uSev5S1btnDTTTfRr18//vGPfzBixAgAbr31VqZMmcKcOXO4+OKLHbXaYnV1dc48B7rK5/NRXV3drRAr3e/ZFi9ezJw5czj22GNZunRp6/ueH/7wh0yePJmbbrqJL33pS63PGUmvYGMzq5eoJ5YoxBKRNAmHw10KsNpyuVwMGDCAFStWMG3atKysuNSVN0BHMmst69atS1mPk9LSUj788EOOPvrojF1sn3POOZ3ePnDgQK655hp+8pOf8NZbb3UIsfqybFTsJCtznLCs+tNPP011dTVXXHFFu8pAv9/PT3/6U2bOnMlvfvObnAqxduzY4ZiphEn5+fns3Lkz7RV/a9as4Z133mHIkCGcf/75aT1XNuRSgOH3+9mzZ0+vQqzu/F7uyWv597//Pc3NzfzoRz9qF0aUlZXx4x//mG9961s88sgjjgqx6urqHF+J1115eXnU1dV1a590v2d75JFHAPjJT37S7oO7ESNGcN1113HXXXfxhz/8gTvuuCOt45CERCXWqPSfZ7EqsZxOIZaIpMXmzZspLCzsdn8dl8tFYWEhlZWVHHfccWkaXWrt3LmTRx99lL1799K/f39OO+00xo8fn+1hpcWePXvwer0pC5yMMXi9XsesSpScntHZSm599XEOhUKtPUEyyRiDy+WiubmZvLy8tJyjubmZP//5z2zbto2CggLGjx/P9OnTO/SRmTdvHgDnnXdeh2NMnz6d/Px8Fi9enNaxplpDQwMlJSXZHkY7Xq+32xepPfHoo48C8K1vfavTnkG5/loOBoNZnXbfHR6Ph/r6+oydryev5UPtk5yOmtzGKaLRqGOqLFPFGEM0Gk37ebrz+j/cc+Ouu+5i3rx5CrFEMkwhloiknLWW3bt397iyIxAI8PHHHzN27NiceJP2+uuv8/rrr7e7bcaMGTz++OMMGzYsS6NKj6qqqpR/+ltUVERVVVXWQ6xoNMqf/vQnoPM3rH31cc5mbxWPx8O+ffsYNGhQWo7/8ccfc/nll7e7raKigj/84Q+cddZZrbetX78eoNOqMI/HQ0VFBWvXrqWyspLjjz8+LWNNtVgs5rifn8aYtDfJDwaD/PnPf8blcnHVVVd1uk0uv5attTm30EAsFsNam5HnY09ey4faZ9CgQRQUFLB9+3aampocUd0YiUSw1mZ7GGlhrU17r8Suvv4bGxvZsWMHhYWFnf6OGj16NAAbNmxI21ilvWBDiNVLer9YRFfOI86mEEtEUq66urpX1TrJ6pzu9kbItPz8fG655RYuvPBCRo4cCcDq1au5/fbbefPNN5k5cyYrV67MeK+hdGpubk75m/hk899su/nmm1mzZg1f/OIX+fznP996e19/nLM5Ncnv91NdXZ2WEGv27NmceeaZnHjiiRQVFVFZWcnDDz/MY489xhe+8AWWLFnCSSedBNBaHXSwyqXk7bW1tSkfZzo0Nzc7LsBKMsaktaLt//7v/6itreX888/nmGOOaXdfX3gth0KhtK8gmmrJistM/JzpyWu5K/s0NjZSV1fniBArHA7n3HOgq1wuF+FwOC0hVndf/33t90JfECj0Z2Y64RI/6GF1NIVYIpJytbW1ve5n5fP5qKurc3SIddRRR3HnnXe2u2369Om89tprTJs2jXfffZff/e533HDDDVkaYeqlqwIg25UFv/zlL5kzZw7HHXccTzzxRLv7+vrj3NTUlLWLdq/XS2NjY1qOfeCqdePGjeORRx6hsLCQOXPmcPvtt/Pcc8916VjJqgenBkMHikQijr3INcYQiUTSFmI99thjAHznO9/pcF9feC07scLucDJRgddVPXktO+31n4vPga5K53MlXa//vvpYOFGiEisDjd1VieV4CrFEJOVScQGV/DQuF3k8Hq666ireffdd5s+f7+gLIoFf/epX3HDDDZxwwgm88cYb9OvXr0v79ZXHOR6PH1Hnv+aaa5gzZw7z589vvS35ifrB+jXt37+/3XZOl+3H9FCMMWkb3wcffMDixYsZOnQoX/ziF7u8Xy69luPxeM5dNGcyxOrJa7mkpIQ9e/ZQV1fXaRuE5D7FxcWpHm6P5OJzoDsy/fPrYK//wz2XDlepJakXKPQz/rQMVGK9o0osp1OIJSIp5/V6aWpq6tUx4vF4VlYnTJVkf6d0VZlIajz44IN8//vfZ9y4cbzxxhvdrvzrC49ztit2Mn3+5GPc9jEbO3Ys//jHP9iwYQOTJ09ut300GqWqqgqPx9M6BcXpsv2YHoq1Nm3jO1xD90PJldeyy+XKuX5I1tpuPx491ZPX8tixY9mzZw8bNmzosALhrl27aGxsZOjQoY6YSgi5+Rzojmz8/Ors9V9QUMCQIUPYsWMHu3bt6jDtfePGRG8mJ6ywe6QINoRYvVg9sUQhloikQWlpKdXV1QQCgR4fIxwO5/SnW++88w5Azlz0dlW6LkQydYHT1s9//nNuvvlmJkyYwOuvv86AAQO6fYy+8Djn5+cTiUSyEhpHIpGMXxguWbIEaP+YnXPOOfz3f/83r776Kpdddlm77efPn09TUxPTp0/PmZUJvV6vY6uxrLVp6XcTCoV44okncLlcfOtb3+r2/rnyWna73TkXYGQyxOrJa/mcc85h0aJFvPrqqx1CrFdeeaV1G6fIxedAV2XyudLWwV7/55xzDk888QSvvvoqs2fPbnefE58bfV2gIC9DlVh5qsRyOIVYIpJy5eXlfPDBBz1ejSi5Ok22V6s7nHfffZeJEyd2uPifN28eDzzwAABf//rXszG0tMnLyyMWi6X0TWYsFst4OHDXXXdx6623MnnyZF577bVDTiHs649zeXk5VVVVWQmxQqEQFRUVKT/u2rVrGTRoUIfHdevWrXz3u98F2j9ms2bN4kc/+hFPPvkk119/PSeffHLr+P793/8dgH/5l39J+TjTJS8vz7EXudbatLzen3rqKWpqavjSl77UoaF7Ul94Lfv9fscGlAcTj8cz9jO+J6/l2bNn85//+Z88/PDDzJ49mxEjRgCJlVvvueceIDEN2Sl8Pl/OPQe6Kp1V+D15/V9zzTU88cQT3H333Vx44YWUlZUBsGXLFn71q1+Rl5fXIdyS9Ak2NmdmdcLG7C82JIemEEtEUs4Yw9FHH019fX2PqiyCwSADBw7MSs+H559/nueffx6Ajz/+GEhUblx55ZUADBgwgPvuuw+AH/3oR6xdu5YZM2YwdOhQILHSzbx584BEUHL66adn+DtIr4qKCtavX09paWnKjllfX8/YsWNTdrzDefzxx7n11ltxu92ceeaZ/PKXv+ywzYgRI1of877+OJeVlWVtifBoNNrlHmTd8dRTT/Gzn/2Ms88+m4qKCoqKiti8eTMvv/wyoVCIL37xi9x0002t2xcXF/Pb3/6WWbNmMWPGDL72ta/Rr18/XnzxRdavX8+sWbO49NJLUz7OdEpWazipd046qyySDd2vvvrqg27TF17LxpisVKr0htvt7tXzsDu/l3vyWq6oqODee+/le9/7HieffDKXXnopPp+Pp59+mu3bt3PjjTd2qNDKpt6s/ux0ydWpuyrd79lOP/10fvCDH3D//fczfvx4Zs2aRTgc5i9/+Qv79u3joYceag09Jf0SlVij038eVWI5nkIsEUmLY489loULF+L3+7vV3yAej9PQ0MCECRPSOLqDW7lyJY8//ni72yorK6msrARg+PDhrW+ILr/8cp577jmWLl3KK6+8QiQS4eijj+af/umf+O53v8uZZ56Z8fGn24ABA1izZk3KLo6TVXc9mcrXU1VVVUCiAuzBBx/sdJuzzjqr9U1wX3+c/X4/1tqMBx7W2rRVaJx99tmsX7+eFStWsGTJEhobGyktLWXatGlcfvnlXH755R2+1wsvvJC3336bu+++m2eeeYZQKMSoUaO4//77+d73vpdzF42FhYVZmyZ6MJFIhKKiopQf98MPP2ThwoWHbejeV17LgUCAaDSKx+P8t/HRaLTXU4a783sZevZavv766xkxYgT33Xcff/rTn4jH45xwwgn89Kc/5Rvf+Eavxp8OHo/HcSF1b1lru/2czsR7tjlz5jB+/HgefvhhHnvsMVwuF5MmTeKHP/whX/rSl3rwnUpPBRubWb04/R+6qRLL+YxTy80PxxgzFLgTOA/oD+wCngfusNbWdOM4/YBbgQuBQcBe4FXgVmvt9gO27Q9cBJwPfAYYAoSB94E/AH+w1nao7zXG5AFXAd8ARgJ+4CPgdWCOtXZrl7/xbjLGLJs0adKkZcuWpesU4jCdNZ/MlpqaGlasWMGAAQO6FGTF43H27NnDxIkTW0u2xXl27drFpk2bUvIY1dTUMHr0aAYOHJiCkUlPbdiwgdra2oz2p2pqaqK0tFRNcdOkuro65VWTvVVbW8vYsWMdP1Xc6bZu3cquXbsoLCzM9lAOq6GhgUGDBjF8+PBsD6VPWbt2LaFQKGf69HVFKBQiEAhw4oknZnsojtOV9/aTJ09m+fLly621kw+5YY4yxiwbMeTYSXf96y/Sfq5bHryBLTs299n/y77A+R/hdMIYcyywGDgKeAFYB0wFbgDOM8acYa3d24Xj9G85zhhgHvAkcBwwGzjfGHOatbayzS6XAL8hEZi9CWwDjga+CvwO+IIx5hLbJhk0xniAN4AzWsb5v0AzMAW4HrjCGHO6tfaDHv53iDhWWVkZEydOZMWKFRQWFhIIBDr91NBaSzAYpKGhQQFWDhg0aBDbt28nGAz2qnl/MBjE7/crwHKAiooKFi5cmNEQq7GxkYkTJ2bsfEeaAQMGsHbtWsdUa1hriUajGa267KuOPvpoqqqqciLECoVC+hmfBuXl5WzcuLFPhVjhcJhhw4ZlexjiYMGGZlYvykAlVoMqsZwuJ0Ms4NckAqzvWWsfSt5ojLkf+D5wN9CVDoz3kAiwHrDW/qDNcb4H/KLlPOe12X4D8BXg5bYVV8aYHwPvAReTCLSeabPPRSQCrDeAcw/Y7w4SVWA3Ad/swnhFck5ZWRnTpk2jsrKSjz/+GK/Xi8/nw+VyEY/HCYfDRCIRBg4cyIQJExw19UUObtKkSa2rvPUkyAoGgzQ3Nzuqz8iRzOv1Ul5eTmNjIwUFBWk/X2NjI+Xl5WlZpU4SjDEMHjyYmpqajDymhxMMBhk8eLAjArVc5/f707LIRqolF+3oS0GLU5SVlRGJRLI9jJQKh8P6EFMOKVCYx/jTM9ATa2ke1KX9NNILORdiGWNGAucCW4BfHXD3bcDVwOXGmButtY2HOE4BcDnQ2LJfWw+TCMM+b4wZmazGstbO6+xY1tqPjTGPkAjPZtA+xEqu1fpyJ1MNXyARYqmuXvo0n8/Hcccdx9ixY6murqauro5wOIzP56OkpITy8nJd2OQYt9vNaaedxvLly6mpqaG0tLRLj6G1ltraWvx+P6eddpqjL8CONCeccALz588nEAh0q49dd8ViMYLBIFOmTEnbOSShoqKCHTt2OCLEUuVdao0ePZp169Y5arrogerr6znuuOOyPYw+KfmBoNODzK5KBp76YEMOJdiQoZ5YqsRyvJwLsYBzWr6+dmAoZK2tN8YsIhFynUqi+ulgTgMCLcepP+A4cWPMayQCsbOByk72P1Dy45DoAbevbfn6BWPMLw4Yc7Ib4N+7cHyRnGeM4aijjuKoo47K9lAkBdxuN1OmTGHXrl2sW7cOr9dLUVFRp2+oY7EY9fX1RCIRjj/+eE0vcSCXy8WECRNa+9ilI1i21rJv3z4mTpyY1qBMErxeL0OGDGHfvn1ZnXrW0NDA0KFDdYGaQv379ycajTpmuuiBktNH+/fvn+2h9FmjR492XN+7nsr0KsWSmxKVWOnvo6lKLOfLxRAr+RPuYDHsRhIh1hgOHWJ15Ti0HOeQWvpeXdHyz1cPuPtl4FkS0wzfN8b8nUQz+MnANOAhEpVfvWKMyc0O/SKS8wYNGsTAgQPZs2cPVVVVNDc3E4vFWu93u93k5eUxduzYtIUjkhqlpaWMHj2aTZs20b9//5Q+VtZa9u7dy5gxY/rERVeuGD16NAsWLMjaanbRaJRIJMKoUaMyfu6+zBjDyJEj+eijjygpKcn2cDrYv38/I0eO1M/7NHJa37ueUr886apgQ4jVi9Zn5DzibLkYYiV/Ux8sH03efrh3yKk6DsDPgHHA36y1c9veYa21xphZJKYN3gKc0ObuN4D/sdbGEBHJYcYYysvLtepYHzB06FAANm7cSL9+/VIyVSUWi7Fv3z7GjBnDkCFDen086TpjDJMnT+bdd9/N+NTtZOXdKaecktMX2U41bNgwduzY0To93ynC4TAul0tNutMs2fcu0yvLppr65UlXBQr9manE+odflVgOl4sh1uEkfwL2tjKpS8dpaQJ/I4mVBy/v5H4/8CfgC8B1JPpgNZFo9v5LYH7LioYv9Gaw1tpOf/IbY5YBk3pzbBERObIMHTqUwsJCVq5cSSAQ6FVPpcbGRoLBIBMnTlQFVpYUFBQwduxYNm3aRL9+/TJysZgMsMaOHeuInlx91aRJk1i0aJFjeksm+x6eccYZ2R7KEaGiooIFCxYcdPVnp7PWtq5MLXI4wYYQqxaqEktyM8RK5qIHq50uPmC7tB3HGHMdiVUMPwBmWmv3dbLZzcAlwA3W2kfb3P5KS4XWypZj9CrEEhERSaXS0lKmT5/OBx98QHV1NQUFBV2+ULLWEgwGW1chnDJlinpgZdmQIUOIRqNs2bIl7UFWMsCqqKhQ5V2a5eXlMWrUKLZt2+aIkLiuro5Ro0ZpRcIM8Xq9jBw5kp07d1JcXHz4HRymvr6eY489Vv3ypEsChX7Gn5GBSqzlqsRyulwMsZLx68Gewcl1Nw+3dEGvjmOM+VfgAWANiQDrk4McJ9m8/c0D77DWrjLG7AOGG2P6W2v3HmbMIiIiGeNyuRg3bhyRSISqqip27tyJy+XC4/Hg9/vbXXhEIhFCoRDRaJR4PM7gwYOZOHGiLk4cZPjw4Xg8HtavX0+/fv3S0iMrGo22VmApwMqMYcOGUVdXR319PUVFRVkbx/79+ykpKdE0wgwbPnw4O3fuJBKJ5NTP20gksSbW8OHDszwSEck1uRhiJcOgc40xrrar/RljikhM0wsC7xzmOO+0bHeGMaao7QqFxhgXiebwbc9Hm/t/RKIP1krgc9baPYc4T/KjqA6NYowxeXxa8RU+zHhFRESywuv1MmbMGMaMGUMoFKKmpobq6moaGxuJx+O4XC7y8/OpqKigrKwMv9+f7SHLQQwZMoTS0lKWL1+O1+tN6aqFDQ0NRCIRTjnlFE0hzLBx48axcuXKrAVZ+/fvJxAIMG7cuIyf+0hnjGHSpEksXrzYMdNKD8daS01NDaeffnq2hyI5JNgQYrWmEwo5GGJZazcbY14jETJdR2J1v6Q7gALgUWttY/JGY8xxLfuua3OcBmPME8DVwO0k+lolfRcYAcy11la2Pb8x5hbgTmAZcO5BphC2tYBE0/cfG2MWWWub29x3O4nHYGnbEE1ERMSp/H4/gwYNYtCgQdkeivRQQUEB06ZNY+PGjWzfvp3CwsIe99RpO3V06NChjBo1KicuovsaYwwTJkxgzZo11NbWUlJSkrHeZ3V1dZSUlDBu3Dg99lni9/sZOXIkO3bscORqlQfav38/xx57rD7wkG4JFPgZf8bY9J9H0wkdL+dCrBbXAouBXxpjZgIfAqcAZ5OY/veTA7b/sOXrgb9ZfwzMAH5gjJkAvAccD1wAfEIiJGtljPkGiQArRiKc+v/t3XuQXnV9x/H3dzfLXpKQhNy5hHBJAqLQgKAJICIzFi9UqujYKt5tsVq0NjNWrRec0qJWq0IH21pEkala0Gor3kYUUKrQhKuABpMQMAGTAMsmu5tNNt/+cc6Gx7AbErL7PGefvF8zOyfP75zznN9OvnP22c/+fr9z4TA/rNdk5pU1ry8GzgHOAu6LiO9RjgADTin//e49+7YlSZL2XUSwcOFCjjjiiJ1TRSdMmEBXVxdtbW27DSMyk23bttHb28v27dudOloREcFznvMc1q5dy/3338/UqVPH9KmFAwMDPP744xx99NFOIayAww8/nE2bNrFly5ZKj4TcsmULHR0dTiPUXitGYt339AeOwnVUbeMyxCpHYz2XIlA6G3gpsJ7iaX8X7cHoqKH32RQRS4CPAOcCpwObgC8CH87Mh3Y55Yhy2wq8Z4S3vQG4suYav42IE4H3AS8D3gy0lP29Evh47QgxSZKkehmaKrpgwQI2btzIunXr6O7uZnBwkIjY+ZWZO79aW1uZPHkyixYtYsaMGY6+qZh58+Yxe/ZsVqxYQV9fHwceeOCo/h9lJk888QQtLS2ceuqpLuJeEUPTCm+55RZ6e3vp6upqdJeeore3Fyieqintrc5J7Rx/Wh1GYt3WDk+M+WW0D8ZliAWQmQ9SBEJ7cuyIP7nLwOvd7MFoqMz8KMUUwL2SmRuAZeWXJElSpUQEM2fOZObMJ5fw3Lp1K9u2bdu57llbW5uBxTjR3t7OkiVLWLt2LatWrWLChAkIHLA8AAARHklEQVRMnjyZ1tbWZ/yeg4OD9PT0sH37do488khHX1VQRHDyySdz6623Vm5E1pYtxUovJ598ssG3npG+zVu586Z6jMTa+vQHqaHGbYglSZKksdPe3m5oNc7NmzePww47jE2bNrFy5Uq2bt1KR0cHHR0de/R0yu3bt9Pf309/fz/t7e0cc8wxTJ8+3RCiwlpaWjjllFNYsWIF3d3doz4Sb28Njdzr7Oxk8eLF1o6esbqNxLrdkVhVZ4glSZIkNamIYMaMGcyYMYP+/n4eeeQRNmzYQE9PD4ODg7S0tDxl2uiOHTtobW2lq6uLuXPnMnv2bBfhHkcigpNOOok1a9awatUqpk2b1pA167Zt28Zjjz3GUUcd5RpY2md9m/vrNBLLNbGqzhBLkiRJ2g8MLag9FChkJlu3bmVwcJDBwUFaW1tpbW2lvb3dETNNYP78+cyZM4fly5cTEUyePLluT63s6SkevL506VIDUI2KzkkdHH/aMWN/nTs6HIlVcYZYkiRJ0n4oIgwYmlxHRwdLly7lgQceYNWqVUyaNInOzs4xCbMyk76+PjZv3uzoK426YiTWvXW5jqrNEEuSJEmSmlREMH/+fA455BBWr17NunXrRmWx/yG1i/4ffPDBLF68uCHTF9Xc6jYS605HYlWdIZYkSZIkNbm2tjYWLlzIggUL2LhxIytXrmRgYGDnk0f3dBrp0DTUgYEBBgYGaG9vZ9GiRcyYMcNpqBozvZv7uaMOI7F6HYlVeYZYkiRJkrSfiAhmzpzJzJkzdy6+vmHDBrq7u9m2bRuZSWtr6+8FUpnJ4OAgEUFbWxtTp05l3rx5DVs0XvufrkkdnHD62I/EuvYuR2JVnSGWJEmSJO2H2tramDVrFrNmzdrZtm3bNgYGBhgcHGTHjh20tLTQ2trKAQccYGClhunt6eeOG+swEqvHkVhVZ4glNZmhx2M7nFuSJEl7q62tzbCqIoY+1wu6JndwwunHjvl1rr27A3rG/DLaB4ZYUpPp7Oykt7eXiRMnNrorkiRJkp6h3t5eOjs7G92NSujt6eeOG+6py3VUbYZYUpOZMmUK69evB6Crq8sRWZIkSdI4kpn09vby+OOPM3fu3EZ3pxK6JnVwwgvqMBLrl47EqjpDLKnJtLS0MHfuXLq7u+nu7jbEkiRJksaRzKSzs5O5c+fS0tLS6O5IlWKIJTWhlpYWpk2bxrRp0xrdFUmSJEnaJ72b++oznXBz35hfQ/vGEEuSJEmSJFVW56QOjq/DdMLOe5xOWHWGWJIkSZIkqbL6NtdnYfe+zS7sXnWGWJIkSZIkqbI6J3Vwwhl1WNj9XkdiVZ0hliRJkiRJqqy+nn7u+EkdRmL1OBKr6gyxJEmSJElSZXVO7uCEM5415te59r4O2Dzml9E+MMSSJEmSJEmVVYzE+mVdrqNqM8SSJEmSJEmVVbeRWL9yJFbVRWY2ug8aQxGxqbOz86Bjjx37RfAkSZIkSfV177330tfX92hmTm90X8ZCRCw/YMIBJ86deciYX2v9ht8ysH1gRWaeNOYX0zNiiNXkImLoP3hFQzsi7bsTy621rPHMOlazsJbVDKxjNYsTATIzGt2RsRARVwPH1PGS92Xm6+p4Pe0FQ6wmNxRiNesNTfsPa1nNwDpWs7CW1QysYzULa1n7k5ZGd0CSJEmSJEl6OoZYkiRJkiRJqjxDLEmSJEmSJFWeIZYkSZIkSZIqzxBLkiRJkiRJlefTCSVJkiRJklR5jsSSJEmSJElS5RliSZIkSZIkqfIMsSRJkiRJklR5hliSJEmSJEmqPEMsSZIkSZIkVZ4hliRJkiRJkirPEEuSJEmSJEmVZ4glSZIkSZKkyjPEkiRJkiRJUuUZYkmSJEmSJKnyDLEkSZIkSZJUeYZYkiRJkiRJqjxDLEmSJEmSJFWeIZYkSZIkSZIqzxCrSUXEoRFxRUSsi4itEbEmIj4TEdMa3TepVkScFxGXRsRNEfFERGREfOVpzlkaEddFxKMR0RsRd0bEeyKitV79lmpFxPSIeFtEfDMi7o+IvojojoifRsRbI2LYn7fWsqomIj4eET+KiAfLOn40Im6LiI9ExPQRzrGOVXkRcX75GSMj4m0jHPPyiPhJef/eHBG/iIg31ruvUq3y97gc4evhEc7xvqymFZnZ6D5olEXEUcDNwCzgW8B9wCnAmcCvgFMzc1Pjeig9KSJuB04ANgMPAccAV2fm60c4/hXAtUA/8DXgUeAcYBFwTWa+uh79lmpFxAXA5cB64MfAWmA28EpgCkXNvjprfuhay6qiiBgAVgD3AL8DJgLPB54LrAOen5kP1hxvHavyIuIw4C6gFZgEvD0zv7DLMe8CLgU2UdTyAHAecCjwqcxcVtdOS6WIWANMBT4zzO7NmfmPuxzvfVlNzRCrCUXE94EXAxdm5qU17Z8G/gr4l8y8oFH9k2pFxJkU4dX9wBkUAcCwIVZEHFgeN4UijP2/sr0DuB5YAvxJZn61Tt2XAIiIF1H8sv+dzNxR0z4HuAU4DDgvM68t261lVVJEdGRm/zDtFwMfAC7PzL8o26xjVV5EBPBD4AjgG8AydgmxImI+xR99twAnZeaasn0acCtwFLA0M/+3nn2XYGeIRWbO34NjvS+r6TmdsMlExJEUAdYa4J932f0Rih/O50fExDp3TRpWZv44M1fmniXq5wEzga8O/VAu36Mf+Nvy5TvGoJvSbmXm9Zn537UBVtn+MPD58uULa3ZZy6qk4QKs0tfL7YKaNutY48GFwIuAN1N8Dh7OW4B24LKhAAsgMx8D/r586R+ANR54X1bTM8RqPi8qtz8Y5pepHuBnQBfF1ABpvBmq7+8Ns+9GoBdYGhHt9euS9LS2ldvtNW3Wssabc8rtnTVt1rEqLSKOBS4BPpuZN+7m0N3V8nd3OUZqhPaIeH1EfCAi3h0RZ46wvpX3ZTW9CY3ugEbdonL76xH2r6QYqbUQ+FFdeiSNnhHrOzO3R8Rq4DjgSODeenZMGk5ETADeUL6s/UBpLavSImIZxdpBUyjWwzqNIsC6pOYw61iVVd5/r6JYo/ADT3P47mp5fURsAQ6NiK7M7B3dnkp7ZA5FPddaHRFvzswbatq8L6vpGWI1nynltnuE/UPtU+vQF2m0Wd8aby4Bng1cl5nfr2m3llV1yygeTjDke8CbMnNDTZt1rCr7MLAYOC0z+57m2D2p5YnlcYZYqrcvAjcBvwR6KAKodwF/Bnw3IpZk5h3lsd6X1fScTrj/iXLriv5qRta3KiMiLgT+mmKx4PP39vRyay2rITJzTmYGxV//X0nxS9NtEXHiXryNdayGiIhTKEZffWqUFmO3ltUwmXlRufbmI5nZm5l3lw/p+jTQCXx0L97OWta4Z4jVfIbS9Skj7D9wl+Ok8cT61rgQEe8EPgvcA5yZmY/ucoi1rHGh/KXpmxRLEUwHvlyz2zpW5dRMI/w18KE9PG1Pa/mJfeiaNNqGHhzzgpo278tqeoZYzedX5XbhCPuHnio00ppZUpWNWN/lh9YjKBbPXlXPTkm1IuI9wGXA3RQB1sPDHGYta1zJzAcoQtnjImJG2Wwdq4omUdTksUB/ROTQF8WTugH+rWz7TPl6d7U8l2Iq4UOuh6WK+V25rX3qvPdlNT1DrObz43L74oj4vf/fiJgMnAr0AT+vd8ekUXB9uT17mH0voHjy5s2ZubV+XZKeFBHvA/4JuJ0iwPrdCIdayxqPDi63g+XWOlYVbQX+fYSv28pjflq+HppquLtafskux0hVsaTc1gZS3pfV9Ayxmkxm/gb4ATAfeOcuuy+iSOq/nJlb6tw1aTRcA2wEXhsRzx1qjIgO4O/Kl5c3omNSRHyIYiH35cBZmblxN4dby6qciDgmIuYM094SERcDsyh++Xms3GUdq3Iysy8z3zbcF/Dt8rAvlW1fK19/kSL8eldEzB96r4iYxpNPNhyauiXVTUQcFxEHDdN+OMWob4Cv1OzyvqymF5mu6dZsIuIo4GaKD5vfonh86vOAMymmES7NzE2N66H0pIg4Fzi3fDkH+EOKvyjdVLZtzMxluxx/DdAPfBV4FPgjikcKXwO8Jr2xqc4i4o3AlRQjVC5l+LUm1mTmlTXnWMuqlHIq7CeBG4HfAJsonlB4BsXC7g9TBLT31JxjHWvciIiPUkwpfHtmfmGXfX8JfI6i7r8GDADnAYdSLBC/DKnOypr9G4rZNqspnk54FPAyoAO4DvjjzByoOcf7spqaIVaTiojDgI9RDCWdDqwH/gu4aJgFhqWGqflAOZIHMnP+LuecCnyQYhh1B3A/cAXwucwcfMo7SGNsD+oY4IbMfOEu51nLqoyIeDbwDoqlBw6leAT7Foo/gH2Hoi6f8hnCOtZ4sbsQq9x/DrAMOJFixso9wGWZ+aV69lMaEhFnABcAiyn+2DsReJxi2YKrgKuGC6S8L6uZGWJJkiRJkiSp8lwTS5IkSZIkSZVniCVJkiRJkqTKM8SSJEmSJElS5RliSZIkSZIkqfIMsSRJkiRJklR5hliSJEmSJEmqPEMsSZIkSZIkVZ4hliRJkiRJkirPEEuSJEmSJEmVZ4glSZIkSZKkyjPEkiRJkiRJUuUZYkmSJAER8aaIyIi4stF9kSRJ0lMZYkmSJEmSJKnyDLEkSZIkSZJUeYZYkiRJkiRJqjxDLEmS1BDl+lNZ/vutEfGLiHiibJ9atj8rIj4WETdHxLqIGIiIDRFxXUScPcL77lzbKiImR8QnI2J1RGyNiN9GxOURcdBe9vW4iFhbvu8H9/27lyRJ0t4yxJIkSQ0VEZcC/wpsBf4HWA5kufu9wIeAqcAdwDeBNcBLgO9GxHt389ZTgJ8BbwFuB34AdAEXAD+MiLY97N+Z5fvMAd6QmRfvxbcnSZKkUTKh0R2QJEn7vfOBJZl5yzD7rgL+LjPX1DZGxPMoQqlLIuLrmfnQMOeeC1wHLM3MzeV5BwM/B04EXgNcvbuORcTrgCuAfuAlmfmjvfnGJEmSNHociSVJkhrtEyMEWGTmDbsGWGX7L4DLgDbgFSO872bgrUMBVnneuvI8gLN216mIeD9FiLYBOM0AS5IkqbEciSVJkhrtG7vbGRGTgZcBfwAcBBxQ7lpQbheOcOryzHx4mPb7yu3BI5zXGhGfB/4cuAt46QgjvSRJklRHhliSJKnRHhhpR0S8gmI63+4WYj9whPa1I7Q/UW47Rtj/WorPSOuB0zOzezfXliRJUp04nVCSJDVUZvYN1x4RhwL/QRFgXQIcTxFYtWZmUIyUAogR3nrHM+zSTRSLx88F/iEiRnp/SZIk1ZEhliRJqqqXA53AtZn5/sy8KzN7MnMonDp6jK67Fjgd+DXwDuCKiPAzkyRJUoP5gUySJFXV0BTCB3fdERHtwKvG6sLlGlgvAO4G3gRcHREuwyBJktRAhliSJKmqhhZgf1VEzB5qjIgDgEuBI8fy4pn5CPBCYDnFOln/WV5bkiRJDWCIJUmSqurbwG3AYcDKiPh2RHwdWA38KfC5se5AZm4CzgJuBs4FvhURnWN9XUmSJD2VIZYkSaqkzNwOnAF8guJJgS+mWKvqRuAkioCrHv3oLq99PXA28J2ImFiPa0uSJOlJkZmN7oMkSZIkSZK0W47EkiRJkiRJUuUZYkmSJEmSJKnyDLEkSZIkSZJUeYZYkiRJkiRJqjxDLEmSJEmSJFWeIZYkSZIkSZIqzxBLkiRJkiRJlWeIJUmSJEmSpMozxJIkSZIkSVLlGWJJkiRJkiSp8gyxJEmSJEmSVHmGWJIkSZIkSao8QyxJkiRJkiRVniGWJEmSJEmSKs8QS5IkSZIkSZVniCVJkiRJkqTKM8SSJEmSJElS5f0/527jICCosWcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "image/png": { + "height": 335, + "width": 600 + }, + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.figure(figsize=(10, 5))\n", + "# limit plot display region\n", + "plt.xlim(0, 50)\n", + "plt.ylim(0.8*study.best_value, study.best_value + 0.002)\n", + "# plot points\n", + "marker_scale = 5\n", + "sc = plt.scatter(\n", + " trials_df['rank'], trials_df[target_metric],\n", + " # circle size indicates the number of epochs \n", + " s=trials_df['epochs']*marker_scale,\n", + " # color encodes the value of item_alpha (logscale)\n", + " c=np.log10(trials_df['item_alpha']),\n", + " alpha=0.5, cmap=\"viridis\", marker='o'\n", + ")\n", + "# prepare legend handles for each number of epochs\n", + "legend_labels = n_trials.keys()\n", + "legend_handles = [\n", + " plt.scatter([], [], s=n_epochs*marker_scale, marker='o',\n", + " color='lightgrey', edgecolors='darkgrey')\n", + " for n_epochs in legend_labels\n", + "]\n", + "# add legend, ensuring 1:1 scale with the main plot\n", + "plt.legend(\n", + " legend_handles, legend_labels,\n", + " scatterpoints=1, ncol=len(legend_labels),\n", + " title='# epochs', borderpad=1.1,\n", + " labelspacing=1.5, markerscale=1,\n", + " loc='lower right'\n", + ") \n", + "# add colorbar for item_alpha values\n", + "clb = plt.colorbar(sc)\n", + "clb.set_label('item_alpha, [log10]', rotation=-90, labelpad=20)\n", + "# annotate plot\n", + "plt.title('LightFM hyper-parameters tuning')\n", + "plt.xlabel('rank')\n", + "plt.ylabel(target_metric);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that even lower values of the number of epochs produce a relatively good score, whcih also supports the assumption that we have probably found nearly optimal values. **How good is this result?** To answer that question we need to introduce the baselines. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PureSVD" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This will be our first baseline. We will simply reuse the scaled version of PureSVD which has shown to perform very competitively with other models. For more details, see the \"[Reproducing EIGENREC results](https://github.com/evfro/polara/blob/master/examples/Reproducing_EIGENREC_results.ipynb)\" tutorial. The only difference is that the model is additionally adopted for the cold start evaluation scenario. In this modification the latent representation $v$ of a cold item can be obtained by solving the following linear system:\n", + "$$\n", + "W^\\top v = f,\n", + "$$\n", + "where $f$ is a one-hot vector of real item features (tags in our case), and $W=V^\\top F$ is a precomputed linear mapping between the learned latent space represented by the right singular vectors $V$ and the feature matrix $F$ (the one-hot encoding of tags for known items). Everything here can be calculated efficiently. For more details I invite you to check [our paper](https://arxiv.org/abs/1802.06398). After you find the latent representation of a cold item, you can utilize the standard notion of a scalar product to find relevant users. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The model" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.coldstart.models import ScaledSVDItemColdStart" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the cold start scenario the model depends on side features (needed to construct the matrix $F$). This informaion can be provided via an input argument during instantiation of the model itself. An alternative and more robust way used here is to provide features in the data model constructor. Hence, the required `item_tags` variable will be taken from the `data_model` object." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "svd = ScaledSVDItemColdStart(data_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Computing and evaluating SVD is going to be blazingly fast as the dataset is small and the model is computed only once for every scaling value due to a simple rank truncation procedure. Hence, we can create a very dense parameter grid, which can still be greedily explored with the grid-search in a reasonable amount of time." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.evaluation.pipelines import find_optimal_config # generic routine for grid-search" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "def fine_tune_scaledsvd(model, ranks, scale_params, target_metric):\n", + " 'Efficiently tuning SVD rank for different scaling parameter values.'\n", + " # descending order helps avoiding model recomputation\n", + " rev_ranks = sorted(ranks, key=lambda x: -x)\n", + " param_grid = [(s, r) for s in scale_params for r in rev_ranks]\n", + " param_names = ('col_scaling', 'rank')\n", + " config, scores = find_optimal_config(\n", + " model, param_grid, param_names, target_metric,\n", + " return_scores=True, force_build=False, iterator=track\n", + " )\n", + " return config, scores" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "# define the hyper-parameters grid\n", + "rank_grid = [1,] + list(range(5, max_rank+1, 5)) # 1, 5, 10, ..., max_rank\n", + "scaling_grid = [-0.8, -0.6, -0.4, -0.2, 0.0, 0.2, 0.4, 0.6, 0.8] # 1.0 is for PureSVD" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "369/369\n", + "[01:06<00:00, 0.18s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 369/369 [01:06<00:00, 0.18s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# perform tuning\n", + "svd_best_config, svd_scores = fine_tune_scaledsvd(\n", + " svd, rank_grid, scaling_grid, target_metric\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The best value of precision=0.0298 was achieved with rank=45 and scaling parameter=0.6.\n" + ] + } + ], + "source": [ + "print(f'The best value of {target_metric}={svd_scores.max():.4f} was achieved with '\n", + " f'rank={svd_best_config[\"rank\"]} and scaling parameter={svd_best_config[\"col_scaling\"]}.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Recall, the result is fully deterministic and reproducible in this case." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualizing tuning results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The resulting hyper-parameter search grid can be conveniently represented as a heatmap:" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
col_scaling -0.8 -0.6 -0.4 -0.2 0.0 0.2 0.4 0.6 0.8
rank
10.02240.02310.02310.02310.02310.02310.02310.02310.0231
150.02550.02530.02570.02610.02590.02590.02600.02630.0269
300.02660.02670.02690.02680.02700.02680.02820.02820.0275
450.02940.02890.02900.02900.02910.02900.02900.02980.0288
600.02860.02860.02870.02890.02890.02920.02860.02850.0287
750.02790.02790.02810.02830.02850.02870.02860.02860.0291
900.02790.02760.02760.02770.02780.02820.02790.02830.0285
1050.02790.02790.02840.02820.02840.02830.02850.02870.0286
1200.02790.02790.02760.02760.02810.02820.02820.02810.0282
1350.02700.02710.02720.02740.02730.02730.02760.02760.0281
1500.02660.02680.02720.02730.02730.02760.02760.02760.0278
1650.02640.02690.02660.02670.02680.02750.02730.02740.0270
1800.02560.02570.02610.02630.02610.02620.02660.02680.0274
1950.02520.02510.02530.02560.02540.02580.02630.02600.0261
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(svd_scores.sort_index()\n", + " .unstack(level='col_scaling')\n", + " .iloc[::3] # don't display all rank values\n", + " .style\n", + " .format(\"{:.4f}\")\n", + " .background_gradient(cmap='viridis', high=0.2, axis=None))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, the optimal rank values sit somewhere around the rank 50. The prediction quality is not impressive, though. Surprisingly, unlike [many other examples](https://arxiv.org/abs/1802.06398), this time the scaling does not help too much and the model performs poorly. It may indicate that properly handling side information plays more critical role than data debiasing. Seemingly, *SVD is unable to reliably connect tag-based description with the latent representation*. **We can try to fix this problem with the help of the hybrid version of SVD**. This is going to be the second and the main baseline. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HybridSVD" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
HybridSVD allows to impose the desired structure on the latent feature space using side information.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hopefully, this structure will be more appropriate for discovering relationships between the real and the latent features, and will improve predictions quality in the cold start. The hybrid functionality of SVD is enabled by the use of additional information about similarities between users and/or items. These similarities are computed beforehand and utilize side features (item tags in our case). The **model still uses SVD for computations**, even though internally it is applied to a special auxiliary matrix instead of the original one. It, therefore, remains the \"real\" SVD-based model **with all its advantages**. Once again, you can look up the details in [our paper](https://arxiv.org/abs/1802.06398). The process of computing the model and generating recommendations remains largely the same, except for the additional handling of similarity information. Polara already provides support for this. We only need to invoke the necessary methods and classes. See the related data pre-processing procedure below. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Preparing similarity data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We start by computing the tag-based similarity between items using the Polara's built-in functions. You can use your own favorite methods to compute similarities, there is a lot of freedom here." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.coldstart.data import ItemColdStartSimilarityData\n", + "from polara.lib.similarity import combine_similarity_data" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Similarity matrix density is 8.4%.\n" + ] + } + ], + "source": [ + "training_tags = item_tags.reindex(training_data['items'].unique(), fill_value=[])\n", + "tag_similarity = combine_similarity_data(training_tags, similarity_type='cosine')\n", + "print('Similarity matrix density is '\n", + " f'{tag_similarity.nnz / np.prod(tag_similarity.shape):.1%}.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This similairty matrix will be feed into the special data model instance, which will take care of providing a consistent on-demand access to similarity information for all dependent recommender models. The input should be in the format of a dictionary, specifiyng types of entities, similarities between them and the corresponding indexing information to decode rows (or columns) of the similarity matrices:" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "similarities = {'users': None, # we have no user features\n", + " 'items': tag_similarity}\n", + "sim_indices = {'users': None,\n", + " 'items': training_tags.index}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All other functionality remains the same as in the standard data model:" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ItemColdStartSimilarityData with Fields(userid='users', itemid='items', feedback=None)\n" + ] + } + ], + "source": [ + "data_model_sim = ItemColdStartSimilarityData(\n", + " training_data,\n", + " *training_data.columns,\n", + " relations_matrices=similarities, # new input args\n", + " relations_indices=sim_indices, # new input args\n", + " item_features=item_tags,\n", + " seed=seed\n", + ")\n", + "print(data_model_sim)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also use the the same configuration of the new data model to ensure the same data pre-processing and the same overall setup." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing data...\n", + "1 unique users entities within 2 holdout interactions were filtered. Reason: not in the training data.\n", + "Done.\n", + "There are 54975 events in the training and 2853 events in the holdout.\n" + ] + } + ], + "source": [ + "data_model_sim.test_ratio = data_model.test_ratio\n", + "data_model_sim.prepare()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to generate the latent representation of a cold item the model uses the same linear system as in the standard PureSVD case. The main difference is how the linear transformatin matrix $W$ is computed:\n", + "$$\n", + "W=V^\\top SF,\n", + "$$\n", + "where $S$ is an item similarity matrix and $V$ is now $S$-orthogonal, e.g., $V^\\top S V = I$. The model also introduces an additional weighting hyper-parameter $\\alpha \\in [0, 1]$:\n", + "$$\n", + "S=(1-\\alpha)I + \\alpha Z,\n", + "$$\n", + "where $Z$ is the actual similarity matrix computed above (see `similarities` variable). Higher values of $\\alpha$ will make the model more sensitive to side information, while setting $\\alpha=0$ will turn the model back into `PureSVD`. To simplify the process, we will not tune its value. We will just make it relatively high to emphasize the importance of tag information for the model. This is going to help us **verify an assumption that tag information is critical for building an adequate latent representation with SVD**." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.recommender.coldstart.models import ScaledHybridSVDItemColdStart" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "hsvd = ScaledHybridSVDItemColdStart(data_model_sim)\n", + "hsvd.features_weight = 0.9 # the value of alpha" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On the technical side, the model takes as an input the product of several sparse matrices, which forms an auxiliary matrix. As these computations are based on matrix-vector products (due to the Lanczos procedure used in the truncated SVD), we are presented with the choice:\n", + "* we can either avoid explicitly forming the auxiliary matrix and just consequently compute matrix-vector products with all involved components, or\n", + "* we can precompute and store the auxiliary matrix first and then perform a single matrix-vector multiplication.\n", + "\n", + "Computational efficiency of the aforementioned approaches strongly depends on the sparsity structure of the input matrices. Therefore, the optimal choice should be decided case by case. There is a special attribute to control this behavior in the `HybridSVD` model:" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "hsvd.precompute_auxiliary_matrix = True # faster in this case" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have already spent some time tuning `PureSVD`. Assuming that the main contribution of `HybridSVD` is related to tag handling, we can simply reuse the the scaling values obtained earlier.\n", + "The only parameter left for tuning is the rank of decomposition. Conveniently, it **requires computing the model only once!** We can use the Polara's built-in `find_optimal_svd_rank` routine for that." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Note that some calculations with the tag similarity matrix may eat up to 10Gb of RAM due to its relatively high density.
" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.evaluation.pipelines import find_optimal_svd_rank" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updating items relations matrix\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "100%\n", + "41/41\n", + "[00:08<00:00, 0.20s/it]
" + ], + "text/plain": [ + "\u001b[A\u001b[2K\r", + " [████████████████████████████████████████████████████████████] 41/41 [00:08<00:00, 0.20s/it]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "hsvd.col_scaling = svd_best_config['col_scaling'] # reuse PureSVD config\n", + "# perform rank tuning (will compute the model only once for the max_rank value)\n", + "hsvd_best_rank, hsvd_rank_scores = find_optimal_svd_rank(\n", + " hsvd, rank_grid, target_metric, return_scores=True, iterator=track\n", + ")\n", + "# restore item features embeddings with the maximal rank value\n", + "hsvd.update_item_features_transform()" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The best precision=0.0458 is achieved at rank=135\n" + ] + } + ], + "source": [ + "print(f'The best {target_metric}={hsvd_rank_scores.loc[hsvd_best_rank]:.4f} '\n", + " f'is achieved at rank={hsvd_best_rank}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result is much better than for PureSVD! Let us compare all three models together." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Comparing tuning results for all models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that this is still a preliminary result, awaiting for the confirmation on the test data, provided later in this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAAIyCAYAAAAHYG9QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3xddf3H8dcno0mb1ZG26aS0SQeUTjYtUFBARUCGgALuVRBUHIjKUlF+iIIMRRSRjQJK2SizLbuTlo6ke6Vp0mY3+/v745zc3iT3Zt7kJrfv5+NxHzfne77nez4Jpb2ffL/fzzHnHCIiIiIiIrEgLtoBiIiIiIiIRIoSHBERERERiRlKcEREREREJGYowRERERERkZihBEdERERERGKGEhwREREREYkZSnBERERERCRmKMEREREREZGYoQRHRERERERihhIcERERERGJGUpwREREREQkZijBERERERGRmKEER0REREREYoYSHBERkXYyM+e/xnXi2gf8a2+IeGAiIhKgBEdEpA8K+rD8Rohz55jZDWZ2cs9H1nPMc46Z/cPM1ptZiZlVm9kuM3vFzH5oZsOjHWekmNmpZvaomW00s/1mVm5mG8zsTTP7jZmdbmb9ml0z0P+zcEOUwg6O5Qb/NTDasYhIbEuIdgAiIhJx5wBf8r9+I4pxdBszmwg8DswMaq4BKoDhQBbwSeAmM7vWOXd7z0cZGWYWD9wLfC2ouQ6oBA4BxgMnAtfg/TyWB/UbCFzvf31Dd8fahsY4HgCKoxiHiMQ4zeCIiEifYmbTgXfxPszvw/tgP8E5l+ScGwwkA/OAvwP98BK+vuzHHEhu/gRMAZKcc0OA/sBReMnL5mgEJyLS22gGR0RE+gwzSwGeBAYBG4FTnXObg/s452rwZq7eMLM7gCt6OMyIMTMDvusf3u2ca/K9OOdqgQ+BD83sl+jfdRERzeCIiMQKMzvZzBwHlqddH7Qp3vnnml8TZ2aXmtl/zWyPmdWY2U4ze8LMjglznxv88R7w98FcbmbLzKzC3//yDzMbHdQ/x2/bbmZVZrbKzL7RyW/z20A20ABc1Dy5ac45twL4Zpjv41wze8n/vqv9+B4xs1mdjA0zO8bMnjWzvf4emeVmdpWZdfbf20xghP/1c611dM41+MldYyxvAJuCjl2z1w1B51LN7AL/+19lZsX+Pp88M/uLmeW08j0HCi+Y2RT/v/U2M6s1s/807hcLumRTszge6NBPRESkDfpNj4hI7KgBdgMZeMu0KoDycJ3NLA14GviE3+SAMrwP1J8Hzjezq5xzd7Vyz8eAC/171+LtfbkMmOsnSBOAF/H2gpTgLRk7HPiLmQ10zt3awe/xW/77y865D9pzgXOuSWLnJxt/9+MEqMf7vkcBXwAuMrMrnHN/6khgZnYR8DAQ7zcV432vtwNz8f57dMWoDvbfCxTiJUng/dkIFvxn48vAnUHHZXi/BJ3gv75gZuc45/7Xyv3mAn8GBvjX1/ntJf69Gws+FOL9zAk6LyISMZrBERGJEc65t51zWcATftPvnHNZwa9mlzyIl9ysBD4DpDjnMvCWf12L9wH1DjM7Icwtz/GvuwRI818nAvnAocAv8QoBLMLbIzMQL9H5s3/9TWY2pL3fn5mNAhpnEha097oQfoyX3DjgF8Ag59wgYDTwL7x/G+8ysxM7ENsEvKQpHngF7/sdhJdsXo33szq7o4E65/YAW/zDX5jZER249ly8/TmNx1nNXr8L6l6El+AcDwx0zqXjJclTgEeAFOBRf4lgOPcAHwBH+NcPAK52zl3V7M/eUc3iuKq935OISHsowREROQiZ2SfwPnRvBuY5515wzu0HcM4VO+d+g/fhPw74aZhhMoArnHOPOOdqnGchXgIB3mxLDfA559xGf+xS4HIgD+8D9Gc6EPaUoK9XdOC6AP8DeuP3c4tz7lfOuTI/th3AxXgJWRzwqw4MfS3e97MOODvo+610zv0erwhARmdiBm703w8BVprZEjO7w8wuMbPsTo7ZhHPuMefclc65d5xzJX6bc86tBS4F/gcMBc5vZZgC4FPOuVVB12+IRHwiIh2hBEdE5ODUuE/nAefc3jB9HvXf5/mlipvbDjwUoj14GdOtzrm64JPOuQbgdf9wajvjBQie7QkXc1tOA9LxEq//a37SOVePN/ME3jK75rNeLfiFAM71D//gnKsK0e12vLLOHeac+zteFbU9ftMs4Eq8n32umW0ys5+1MbvSaf4Sv+f9w3CzeQB3NSbJIiLRpARHROTgdLz//n0zyw/1wqvOBd5So1BLyT72k5XmCoK+XhXm/o37QQZ1OPKuaSwgsMI5ty9Mn7c4sH+kPQUHxuMtvQN4M1QH51w5sKS9QYa4/n68GZwL8Jb4LcNL0gDG4c02fWBdeLCpmY02s1v8GaJiM6sPKk7xB7/byFaGeKez9xYRiSQVGRAROTg1VubKoH1LpwaEaNsVqqNzrt6b1AjfhwObzBPbce9GRUFfD+7AdcGG+u87wnVwzlWZWRHepvih4fqFGBNgZyv9wt6zPfzZkSf9F2Y2ADgV7zlAx+Mt4fsz8LmOjm1mJ+FVaUsNai4BGmej+uPNfLU2S7SnlXMiIj1GMzgiIgenxr//z3bOWTtem6MZrG9N0NfTuzhWUhev7wxru0v7+ft7ngXmAP/1m8/uSOEGADNLxKv+loq3vPBEoL9zbmBQcYofNHZvZaj6Vs6JiPQYJTgiIgenxiVih0U1ig7wiwDk+odndXKYxlmGQ8J1MLNkDizJa8+sRHCf1pZwjWjlXKf5e2T+7h8a3nOCOuI4vApye/ES3oUh9hF1eumbiEhPU4IjIhJ7GvfFtPbb9sb9Eud1cyyR9hf//XQzO6rVnj4LWi8HLPXfc/yy06GcyIEl3EvD9Am2Ee+ZN43XhoohBTiyHWN1VvAzdmqCvg7skWr2cwjW+FDW9c65cIUQPhGmvSMan0cU0ZksEZHmlOCIiMSeUv99YCt9HvDfjzSzy1rph5n1dCGA1vwZL6GIAx43s3GtdTazacC9QU2v4P18EoEfhegfj1ceG2Chcy6/rYD8GZSn/MPvmVmo5W9XEnofU6vMrJ+/P6YtX/Df9+OVqm5UGvR1uD8PjQ/azPFnr5rHcBowrx0xtKU9fy5FRLpMCY6ISOxZ7b+fYWYhl0U5514CnvYP7zezG4P7mtkgMzvbzJ4Bft+94bafX43sfLwZk/HAEjP7iZkd2tinMSkws/vxZmAmBl1fAdzsH17pl1dO9a8bBTyGt6elAfh5B0L7Dd6G/CnAfxrjMbP+ZvY9vNLTJa1cH04/4A0ze8fM5pvZxMaZGDNLNLMjzexfwIV+/78Gz8I454o5UPjgK2HusRivhPUQ4MHGPwd+7F/FS96KwlzbEY1/Li8LU3ZcRCQilOCIiMSef+Ptp5gIbDezXWa22cw2N+t3GfAfIB64Dtjplwcu8a//D53f69JtnHPL8PaNrMCrpvZbYKOZVZnZXrxE4w28D/T7gX82G+J3wIN4S6V+BRT7123DK8PcAHzXOfdWB2La4N+vHjjDj2cf3qzFH4Bn/FdHNfhjHgvcjTc7U+1XeasGPuDAwzf/zYGHrAb7q/9+m5mVN/5Z8BOvxiSo8eGnF+D/OfBj/xveQ1lvbD5oJzTG8T2g3My2+HH8LgJji4gEKMEREYkxzrlCvCVFT+NtgB+Kt6n+kGb9KpxznwPO9PvuwCsH3A/vQ+2jeB+e5/dY8O3knFsLzMR7wObDwAagFq+M8W68qmI/AA51zt3T7Np659yX8L63V/Bmg1LxSlo/Bhzd/Jp2xvQ43oMwn/fH7Ad8jPeB/gIO7EHpyJiVeMUJvoqXlH2El7Rl4O27WYf3wM8znHPnhnnI6E3AT4CVeEld45+FwFIx59wf8X6WjbM5CcBa4Hq8EtRlHY09xPfyd+AbwPt4zxka48eR2dWxRUSCmbd0WEREREREpO/TDI6IiIiIiMQMJTgiIiIiIhIzlOCIiIiIiEjMUIIjIiIiIiIxQwmOiIiIiIjEDCU4IiIiIiISM5TgiIiIiIhIzFCCIyIiIiIiMUMJjoiIiIiIxAwlOCIiIiIiEjMSoh2AdC8z2wSkA5ujHIqIiIiIxLZxQKlz7tBoBqEEJ/al9+/ff/CUKVMGRzsQEREREYlda9asYf/+/dEOQwnOQWDzlClTBi9ZsiTacYiIiIhIDJs9ezZLly7dHO04tAdHRERERERihhIcERERERGJGUpwREREREQkZijBERERERGRmKEER0REREREYoYSHBERERERiRlKcEREREREJGYowRERERERkZihBEdERERERGKGEhwREREREYkZSnBERERERCRmKMEREREREZGYoQRHRERERERihhIcERERERGJGQnRDkBERESkuyxYsZN7Xs8jt6CcnGGpzJ+XzVnTR0Y7LBHpRkpwREREJCYtWLGTKx9bFjhem18WOFaSIxK7tERNREREYoZzjk2FFTz0zmauffqjkH3uei23Z4MSkR6lGRwRERHp0/ZV1LB4QyGLcgtZmFvIjuL9rfZfv7ucyx9dygWzRzM3ZyjxcdbtMWqpnEjPUYIjIiIifUpVbT1LtuxjYW4hi/MKWbWzBOc6NsbzK3fx/MpdDE9P4txZozlv1miyh6VGPFbnHH9btIlfPb8m0Na4VK6wrIovHHMIyYnxEb+vyMHMXEf/RpA+xcyWzJo1a9aSJUuiHYqIiEiHBM96jBnUn+ljBrK3ooYPNu+lqrahzeuTE+Pa1a/RzLEDuWD2GM6cPoL05MROxbyvoobl24pZtq2Y5duKWbGtmJL9ta1eM2hAIsPTkxmRkUxWRjJZ6f3JykgiK6M/Wele25vrCrjnjQ2aAZJebfbs2SxdunSpc252NOPQDI6IiIj0Os0LBGwuqmRzUWWr1yTGGzPHDmJudiZzcjI5YlQGL6zK557X88grKGfC0FSOnzCETUUVvLV+Dw3Nfse7bGsxy7YWc+Ozqzn98CwuOHI0x0/IDLuErbquno93lrLcT2aWbytmSxsxhrKvspZ9lbWszS9rV38VSxBpnRIcERER6VUaGhy/fv7jdvWdODyVOdlDmZuTydGHDiYlqelHm7OmjwyZBOwureLfy3bwrw+3sWFPRZNz1XUNLFixkwUrdjIiI5kjRqWTV1DB5qIKhqcnkzM8lZL9dazZWUpNfftniCLtntfzlOCIhKAER0RERHqNDXvK+elTH7G7tDpsn8/NHMUcf5ZmeHpyp+4zPD2Zb580gW+dOJ7l24p5csl2FqzYSVlVXZN+u0qq2FVSFfa4NWMG92fmmEEkxBlPL9vR4vzFR48hK70/+aX7yffH3V1axb7K1pe0NVqXX0ZJZS0ZAzq3nK4zFqzYyd3+jJiWyklvpQRHREREoq62voG/vLWRO17NpaYu/KzI5Kw0/nDhjIjd18xb1jZz7CB+ceZhvPLxbp5csp2FuXs6VLggLTmBGWMGNnkNSU0KnD958rDAUrnsNhKDqtp6dpceSHh+9fwa9pS1TPgccPbdi/jLZUcycXhaR7/1DluwfAdXPr48cKylctJbqchAjFORARER6e0+2l7CT55ayce7Stvs+8eLZ/bIh+ldJft5eukObn15Xdg+lxw7lhljBjFjzEDGZ6YQ103lppvvR2oupV88v79wBqcfntUt9wfYtreS029/i8qa+hbnxg9N4bWrT+62e4eistuR9eSS7dz1Wi5b91YyZtAAzps9mrk5mR0aY2FuIT/70meoqWugOj+v+2uvt6LPJjhmNhq4CTgDGALsAv4D3Oic29eBcQYD1wHnACOAIuAl4Drn3PZ2XH8p8KB/+A3n3F+bnT8ZeL2VIW5xzl3T3ng7SgmOiIj0Vvtr6rn9f+v566JN1Dfb8X/0uMF88vDhPLVke7tmPbrL6be/xboQm/8nZ6Xx0vdO7LE4Gj/Q5xWUM3pQf0oqa9nXrDrbVafmcNWpORFNtOobHH9fvInbXlnP/tqWyU2jH50+iW+eOJ7E+O5/hny4hK+nkt++rKSylrw9ZeTuLievoJy8PeWs3F7M3or2LYtsy64HrgJQgtMZZjYBeBsYBjwDrAWOBuYB64ATnHNF7RhniD/OROA14ANgMnA2UAAc55zb2Mr1Y4CPgHggldYTnDeBN0IMs8g597+2Yu0sJTgiItJRPfHb8Xc2FPHTp1e2qIyWmpTANZ+azBeOHtttMyId0Vs/TBeWVzP/kaW8v2lvk/ZPHjac339+OmmdLHMdbF1+GT95aiXLtxW3q/+UEencct4RTBs9sMv3Dqeiuo6Tf/dGyCV7Kf3iufCosWQPSyV7WCo5w1IZlNKv22LpLbNILeI4eQLHjh9CboGfxPiv3IJyCsvD722LBCU4XWBmLwOnAVc65+4Mav898H3gXufct9sxzr3AN4E/OOd+ENR+JXAH8LJz7oww1xrwX+BQ4Gngh7Se4NzonLuhA99mRCjBERGRjujuD/Ql+2v57YtreOz9bS3OfWLKMH55zlRGZPTv8n0iKXj2JFozSaHU1jfwy+c+5sF3tjRpzx6Wyn2XHcmhmSmdGre6rp67X9/An97Io7a+Y58T4wy+MXc83/vERPr3i9wDTEv21/Lg25v52+JNFLezCAPAkJR+TAhKeBqTn6z0ZJ5duavVBMU5R3VdA+XVdZRX1Xnv/tdv5e5p8XMH+PLx4zhp0lDSkhJITU4gNcl7pSQlhJzdak+S1NDgqKipo6K6nvLqWsr8WCqq61iYW8gj723twE+yeynB6SQzGw9sADYDE5xzDUHn0vCWqhkwzDlXEXIQr28KsAdoAEY458qCzsX59xjn36PFLI6ZXQX8ATgZOAW4HiU4IiLSR1XW1PHmuj38+KmVLSqJASTEGZ88bHhgA/0RozMY0K9jtYpeXp3PL/6zioJmv30fktKPG846nDOnjcD7/aF0xOPvb+UXz6xqkoykJSdw58UzOXnSsA6NtWTLPq55aiW5BeVN2vsnxnP1aRPJTE3iz29uCCR708dk8MLKfMqqm/6ZOWTIAH5z7hEcP6Fj+zia21tRw/2LNvGPtze3uEdXJCXEUR2imMXIjGTi4iyQyNQ1f1hSFyQnxpGalEhqUjypyQlU1daTV9Dyo+rYIQPoFx/XJKnqDglxxiFDBrC7tDrkPcYM6s9N50xt11i/+M8qtu/b32sSnL5YRe0U//2V4OQGwDlXZmaL8WZ3jgVebWWc44D+/jhNFtc65xrM7BW82Z15QJMEx8ymAL8F7nDOvWVmp9C2bDO7AkgH8oGFzrncdlwnIiLSLUqranltTQEvrtrFm+v3UFUbvnpZXYPjxVX5vLgqH4D4OGPS8DRmjPUSnpljBjJhaGrIZWV7yqq5YcFqnv9oV4tz584cxS/OPKxblxLFuouOHkvO8DS+8/CSQPJYVlXHVx74gB+fPplvnzS+zcSxorqOW19exz/e2dyietyc7Ex+c+4RjBk8AIBzZo5qcv77n5jEz/+ziv+t2R1o21JUyRfue4+LjhrDTz89hYz+HVsyV1BaxX0LN/Lwu1tb3fsTLCsjiaLymnbNOoVKbgB2trMEeGdU1TZQVVtNYXnr/bZ24mGxrUlOjGN8Zio5w1PJHurPZg1PZezgFPolxIWdtf3RGZOZ184E+cdnTG61EEZP64sJziT/fX2Y87l4Cc5EWk9w2jMO/jgBZpYAPARsBa5tK9ggX/RfwWM9hTfr0+6iCOGYWd+aihMRkZC6e11/UXk1/1uzmxdX5bM4r7DDS5Aa1Tc4Pt5Vyse7SnnUXyKTlpTAtDEZzBgzkJo6x+vrCti4x/s01/wX4aMG9ufXn5va4RkGCW32IYN49rtz+NZDSwJ7ZpyDW15ay+qdJfzf+dPCzri9uX4P1z79ETuK9zdpz+ifyM8/M4XzZ49uNUHKykjmvstm8/xHu7hhwWoKy2sC5x7/YBuvrS3gprOncsbUtqu8bd9Xyb1vbuSJD7eFLBc+PD2Jb504gfT+ifx14cYWywbr6hvYurcysP9kg7+RPq+gPGQFuFg1LC2J3553BDnD0hg1sH+r+9ka/37pyjLMxr6XPhJPVTsT0u7UFxOcDP+9JMz5xva2drh1dpzrgJnAHOfcftq2B7gGeB5vWV0ycCRwM3AekGVmJzafjRIRkYNPuOeMvLByFydPGsrwjGRGZCSTlZ5MRv/Edi/nyi+p4uXV+by0Kp/3NhW1SDaCxRuEynmmjkxnV0kVRRU1LU/6yqrrWJxXxOK88HV+zOBLx43jR6dPIiWpL34M6b2GpyfzxLeO5Rf/WcU/PzxQCPa5lbvYsKeCv1w6OzALA7CvooZfPvdxyIeQfuaIEVx/1mEMS2vfg1TNjDOnjeSECZn88vmPeXrpgTELyqr59sNL+PQRWdxw1uEhx9xUWMGf3sjj6aU7Qi4LGz2oP985eQLnzx5NUoK3t+f82aNb9EuIj2P80FTGD03l9MMPtDc0OHaVVgU23P/x1VxK9rfcy5OalMBphw0nNdnbN5OalEBa8oF9NI17a97btJdfP7+mxfU//fRkjjxkcGCPTHlVHWX+e0VNXWD/THlVLe9u3BtydiqlXzwnTRra4p5N40kkJSme9zfu5cbnPm4xxs/PPIxTJg9v0R7OWdNHdvkXKWdNH0n2sFSWLl26tEsDRUAs/s3S+Ld9V2c0WoxjZkfjzdrc5px7pz2DOOdWA6uDmsqBl8zsbWA5cALwWbxqcJ3mnAv5r5yZLQFmdWVsERHpfluKKrjm6Y9CnntpdT4vrc5v0pacGEdWejJZfsKTldGfERnJDE9PJregjKeX7mBzUQVJCXGtLj1rHOukiUP51NQRzJs8jDfX7wn521znHNv37WfZtmKWby1m+bZ9rNpZ2uqDOYP1S4jjsW8cy+xDBrXvhyIdlpQQzy3nTWPqqAxuevbjQLKwZlcpZ921iLu/OIvjxg/huZXebEvzhHVYWhK/PGdqp5+pMyilH7///AzOnjGqxazQCx/lsziviJ99ZgoX+LNC63eXcffreTy7YmfIxHt8Zgrz52Vz9oyRXSpBHRdnjBrYn1ED+3PSxKEMTUsKuaTq5nOPaNcH/WmjBzI8PblLsx7hlob95rxp7R7n8JEZDElL6pVFMKKpLyY4jTMrGWHOpzfrF5FxgpamrQd+0XaYrXPOlZrZo8DPgBPpYoIjIiJ9U119A/cv3sTv/7u+zUQkWFVtA5uLKluUWQ7VL5TUpAROnTKMMw7P4qRJQ5ssXwr321wzY8zgAYwZPCBwvqaugbX5pSwPJD3FbCwMXeOnocEpuekBZsZlx41j4vA05j+ylL1+ErOvspYv3Pde2OsuPnoM13yq4/tlQjlp4lBe+f6J/O6VdTzw9oF9PSX7a/nxkyu5YcFq9tfUh/1t9OSsNC6fl82njxhBfDeUC4/UsqyuJBKRiCESccSivpjgND5SeGKY8zn+e7i9NZ0dJzWob1WYZQH3mdl9eMUHvtfG/cFbvgbQuTqOIiLSp328s5Rrnl7Jyu1t/U4uMgYNSOSThw3njKlZnJCdGVjq0xX9EuKYNnog00YP5LLjvLZP/v7NFlW4wCtfLD3n2PFDePa7c/jmgx+yemdp2H7jhgzg5ghUPGsuJSmB6z97OGdOG9miMlu4/TDTRmfw3VNyOHXysG5/DlJvSAx6QwyxqC8mOK/776eZWVyIMtEnAPuBd9sY512/3wlmlhaiTPRpze5XDfwtzFiz8PblLMJLnNq1fA2v0hs0q9ImIiKxraq2njtfy+XeNze2qwztbRdM56hxg8kvrWJXyX7yS6rIL61q8l5QVk19K2PFm/HBzz5BQg88af67p+aEXHozf152t99bmho1sD9Pfvt4jrn5f5SGKP89OKUfL33vRJITI/fMmuZmHzKI566cwz2vb+COV0MXkO2fGM+9l85mbk6mSoVLl/W5BMc5t8Ev4XwacDlwZ9DpG/FmQ+4NfgaOmU32r10bNE65mT2EVwr6BuDqoHGuwHsGzsuNz8DxCwp8PVRMZnYDXoLzjxDPwTkBeKd5EQEzuwS4EKgB/tm+715ERPq6Dzbv5SdPrWTjnqbLuAb0i+fHp09i4IB+TZ4zErxkZeyQAaGGBLyqZoXl1Xz+3nfYEmLZWs7w1B5JbiByS28kMvr3i6cizIxJ6f7abk1uGiUlxPP9T07kztdyQ+61qa1v4MSJQ7s9Djk49LkExzcfeBv4o5mdCqwBjsF7Zs16vH0twRrLXDT/lcC1eA/q/IGZzQDeB6YAZwMFeAlUVz0CxPlFBbbjVVE7CjgaqAO+5ZzbHIH7iIhIL1ZWVcv/vbSOh95t+fTzkyYO5defm8roQaGfM9Ie8XHG8PRkrj5tUq+YPdHSm94lZ1gqa/PLWrT39LLBicPTekUcEtt65lc5Eeac24BXavkBvMTmamAC8EfgOOdc+PqUTccpwnvg5x+BbH+cY4C/A7P9+3TVn/CWrZ2AlzB9Hcj0Yz/SOfdABO4hIiK92Gtrd3PaH95qkdwMHJDIHy6czgNfOSqQ3HTVWdNH8seLZzI5K42EOGNyVhp/vHimko2DXLgEt6cT394Sh8Q2c80fVysxxcyWzJo1a9aSJUuiHYqIyEGnqLyam577mGeW72xx7rPTR3L9Zw8jMzUpCpHJwajxIbLRXjbYW+KQyJs9ezZLly5d6pybHc04+uoSNRERkV7LOcczy3dy47Or2VfZ9GGCWenJ/OqcqXzisPY/hE8kEnrLssHeEofELiU4IiIiEbJgxU7u+N96NuwJ/RyYLx4zlp98ajLpyV1/zoiIiISmBEdERCQC7l+0kZueWxPy3PjMFH5z7hEcM35ID0clInLwUYIjIiLSSc45FucV8cDbm/jfmoKQfYak9OOFq+b2SCleERFRgiMiItJhlTV1/HvZDh5YvLnJ09lDKemh54yIiIhHCY6IiEg7bdtbyUPvbuHx97eGfCp8KHq+h4hIz1KCIyIi0grnHO9sLOKBxZv535rdIZ/CbgZTR2bw0Y6SFuf0fA8RkZ6lBEdERCSE/TX1PLN8Bw+8vTnkk9cB0pITuOioMVx67DjGDhmg53uIiPQCSnBEREQ48PDB3N3lZAxIpJA/h+gAACAASURBVKq2nsqa+pB9JwxN4csnHMq5M0eRknTgn1I930NEJPqU4IiIyEHNOcf9izfxy6ASz3sralr0M4NTJg3jyyeMY052JmbWk2GKiEg7KcEREZGDzt6KGhbnFbI4r5CFuYXsKN4ftm9aUgIXHDmGy447hHGZKT0YpYiIdIYSHBERiXlVtfUs2bKPhbmFLMrbw+qdpbgQxQKaizN459pTSU3SP5ciIn2F/sYWEZE+L7B/pqCcnGGpfOekCWQPT2VRbiGL8gp5f9NequsaOjzuxOFpSm5ERPoY/a0tIiJ92oIVO7nysWWB47X5ZVz1xPJ2XZsYb8wcO4ihqf14/qP8FudV4llEpO9RgiMiIn3aPa/ndaj/xOGpzMkeytycTI4+dHCgCtrpKvEsIhITlOCIiEifVVVbz7owz6hpNDQtibnZmZyQncmcnEyGpyeH7KcSzyIisUEJjoiI9EmbCyu4/NGlhKsVMCwtiYe+dgwTh6eqpLOIyEFECY6IiPQ5z6/cxU+eWkl5dV3YPj8/8zAmZaX1YFQiItIbKMEREZE+o7qunl8/v4YH39nSpL1/YjxDUvuRX1Kl/TMiIgc5JTgiItInbCnylqSt2lHapH3m2IHc9YVZjBrYP0qRiYhIb6IER0REer0XP9rFj59cSVmzJWnfmHsoPz5jMonxcVGKTEREehslOCIi0mtV19XzmxfW8sDbm5u0pycncNvnZ/DJw4ZHJzAREem1lOCIiEivtG1vJVc8upQV20uatE8fM5C7Lp7JmMEDohSZiIj0ZkpwRESk13l5dT4/+tcKSquaLkn76gmHcs2nJtMvQUvSREQkNCU4IiLSa9TUNXDLS2v526JNTdrTkhO49fzpnDE1K0qRiYhIX6EER0REomrBip3c83oeubvLSUwwqmobmpyfNjqDu78wS0vSRESkXZTgiIhI1Dy9dDs/+OeKwHF9rWty/svHj+Onn55MUkJ8T4cmIiJ9lBIcERHpEc45thRVsnxbMcu3FbNsWzErtxWH7BtncNcXZvHpI0b0cJQiItLXKcERETmIBZaHFZSTMyyV+fOyOWv6yIiMXVxZE0hmlm8rZsW2YvZV1rbrWjNTciMiIp2iBEdE5CBUUlnLHa+u5/7FmwNta/PLuPKxZfzlzQ1kD0slNTmB1KREUpPiSU1KIDU50XtPSvDPea+3cvfw14UbySsoZ2haEmMHD6CwvIZNhRWdji9nWGoEvksRETkYKcERETkI1NQ1sHTrPhblFrIwr5CPthfT4EL3XbWzlFU7Szt1n92l1ewurW6zX0KcMWVEOhn9E1mUV9ji/Px52Z26v4iIiBIcEZEoWLBiJ3e+msuGPeVkD0vlilNyIrY0DLz9Lut3l7Mwdw+L8gp5b+Ne9tfWR2z8jho1sD8zxg5k5piBzBw7kMNHZpCc6BUOaFwml1fg/SwiuUxOREQOPuZcmF/hSUwwsyWzZs2atWTJkmiHIhIR3blnpLs1zqLcv2gTr3y8u8X5rPQkcoankZWezIiMZLIy+pOVkURWen+yMpIZNCARMwv0b/6zuOSYsQxISmBRbiGL8gopKGt7JiWUpIQ4hqcnU15dR1lVLbX1Hf93woD58yYwY8wgZowZyNC0pE7FIiIifcfs2bNZunTpUufc7GjGoRkcEekzFqzYyZWPLQscN+4ZAXplkuOcI7egnIW5hSzK3cN7m/ZSWRN+FiW/tJr8VpZ39UuIIys9mayMZOobHEu27AucW5tfxs+fWd2uuMYPTWFudib9EuK5b+HGFudvvWB6k59ndV09FdX1lFfVUVZdS3lVHRU1dZRV1XHzC2tCLkmblJXGj06f3K54REREIkkJjoj0CbX1Dfz6+Y9DnrvumVVU1dYzc8xAJgxNJS7OQvbrCQWlVSzKK+zyLEooNXUNbN1byda9lR26bkhKP07IzmROdiZzcjIZObB/4NwRozPaXB6WlBBPUkI8g1P6tRjbzJoknY20h0ZERKJFCY6ItEu0loY553jho3x+98q6sJvXiytr+fGTKwFIS0pg2pgMZowZ2G3Lo4J/FhOGpnDKpGHUNjgW5RaybndZp8dNTUrg0MwUdpVUUVRRTWdXECclxHH0oYMDCc2UrPSwSd9Z00d26b9j47XaQyMiIr2FEhwRaVO0loa9nVfILS+tZcX2knZfU1Zdx+K8IhbnFQXamm9w31xYyX0LN7ZI1mrrG6iorqO88VVVR5n/3tj+4eZ9vLQ6PzD2+t3lrN9d3mZcg/1ZlLnZmeyvref6BS2Xk9187hGBn2dNXQMFZVXsLq1iV0kV+Y2vUu99+bZi6kKUQRs7eACvfP/EwAb+ntDVJElERCSSlOCISJvueT0vZPsvn13N7EMGMSpoyVMkrN5Zwi0vreOt9Xva1T8+zqgPV/MY2FG8nx3F+3l+5a4W5xqTtav/ubxTm+nDaWsWZVBKv1ZnPfolxDF60ABGDxoQcvzmSWejH54+qUeTGxERkd5GCY6ItGl9mGVXe8prmHPLa5wwIZMLjhzNaYdl0b9f5z9cby2q5Lb/ruOZ5TtbnEuMN754zCHkDE/loXe2NEkMzjg8i7X5pSzfVszyrcUs21bc4YdMRiK5mToqnTnZQ5mbk8nsQwa1mmhoaZiIiEj3UIIjIq2qb3D0i4+jqq4h5Hnn8DbV5xWSlpTAmdNHcP7sMcwaO7BJSePWFJVXc+dreTzy3paQicbZM0Zy9ScnMXaIN5vxxWMOadFn2uiBTBs9kMuO846LK2u8hMd/rdhWzL7K2nZ+1x2XMyyV5747t9vGD0VLw0RERFpSgiMirXrs/a1hk5vmyqrreOz9bTz2/jbGZ6Zw3uzRnDdrNFkZySH7V1TX8deFm/jLWxuoCFE+eW5OJj85YzJTR2V0OO6BA/px8qRhnDxpGOAVK9hSVMnF973LrpKqFv0Hp/TjS8eNIzU5gbSkBFKTE0hNSiAlKYE0/+vU5AReW1PA955Y3uL6756a0+EYRUREJPKU4IhIWHsrarj15XWBYwPi4oycYal856QJDE1L4l9LtvPiql1U1TZNgjYWVnDry+u47ZV1zMkZyvmzR1Nb1xDY3J+Z2o/K6jrKqlsmNkeMyuCaT03mhOzMiH0vZsa4zBR++ukpIfeu3HDW4e2aDTln5iji4kxLw0RERHopJTgiEtb/vbSWkv0HlnVdfdpErjil6UzF8dmZ3HT24bzw0S6eXLKdDzbva3K+wcFb6/e0KBgQquTzuCED+OHpk/j01BHd9iybSOxd0dIwERGR3ksJjoiEtGzrPp74cFvgeNyQAXzjxPEh+6YlJ3LhUWO58KixbCqs4Kkl23lq6faQS8FCyUxN4qpP5HDRUWNIjI+LSPytUYIiIiISu5TgiEgL9Q2O655Z3eRBkzecdThJCW1XSDs0M4Ufnj6J739yIm9vKOTJJdt5aVU+1WH28cQZvPmjk0lJ0l9HIiIi0nX6RCEiLTz+wVY+2nHg4ZqnHz48sFm/veLjjLk5Q5mbM5SS/bWc/oe3yC9tOaMzcXiakhsRERGJmO5fCyIifcreihr+76UDhQWSE+P4xZmHdWnMjP6JXPuZKSHPzZ+X3aWxRURERILp16Yi0kTzwgJXzMtm9KABXR5XD6YUERGRnqAER0QCOlJYoDO0uV9ERES6m5aoiQjQtcICIiIiIr2FEhwRASJTWEBEREQk2pTgiEi3FBYQERERiQYlOCLCrS93T2EBERERkZ6mBEfkILd8WzGPf9B9hQVEREREepISHJGDmFdYYJUKC4iIiEjMUIIjchB7/IOtrNyuwgIiIiISO/QcHJFutmDFTu55PY/cgnJyetHDLVVYQERERGKREhyRbrRgxU6ufGxZ4HhtfhlXPraMDQXlfOn4cQxO6Re12FRYQERERGKREhyRbnTLi2tCtt/xai53vJrL4JR+ZA9L9V5DU8kZ7n2dlZ6MmXVbXCosICIiIrFKCY5IN1mbX8qO4qpW++ytqOH9TXt5f9PeJu2pSQlMGJpC9rA0auobWLplH7tK9pMzPI3Lu7jETYUFREREJJYpwRHpBvsqavjGgx92+vry6jpWbC9hRVABAIB1/hK39zcW8d1TcxientzhsVVYQERERGKZEhyRCKurb+CKx5aybe/+sH2+PudQBqf2I293OXl7yskrKKeypr7d93j4va08/N5WJg5PZU72UObmZHL0oYNJSWr9f+m9FTXc+rIKC4iIiEjsUoIjEmG/eXEti/OKAseDUxIZkpLEpsIKssNUUXPOsbOkiryC8qBXGR9s3tfqvdbvLmf97nLuX7yJxHhj5thBzM3OZE5OJtNGDyQ+ruk+nltfXktxpQoLiIiISOxSgiMSQU8t2c7fFm0KHPdPjOfhrx3LYSPTW73OzBg1sD+jBvbnpIlDA+1n3P4Wa/PL2nXv2noX2M9z23/Xk56cwHEThjAnZyg1dfU8+M4WthRVBvqrsICIiIjEIiU4IhGyYlsxP/33R03abr1gWpvJTWvmz8tuUma60RXzsql3jkW5hazaWdKkYECj0qo6Xl69m5dX7w459hlTs1RYQERERGKOEhyRCCgoq+JbDy2hpq4h0Db/5AmcOa1rD/RsXMp2z+t55BWUt1ji9pMzvH01b28oZFFuIQtzC9lRHH7vT7A31u3hmk9N6VJ8IiIiIr2NEhyRLqqpa2D+w0vJLz1QEnrepKFcfdqkiIx/1vSRrZaFHpzSjzOnjeTMaSNxzrGlqJKFeYUsyt3D2xuKKKuqC3ldXkF5ROITERER6U2U4Ih00fULVvPhlgPFAMZnpnDHxTNbbPDvCWbGuMwUxmWmcOmxh1BX38Cpt73Jlr2VLfpmD0vt8fhEREREultctAMQ6csefncLj72/NXCclpTAXy47kvTkxChGdUBCfBxXnx56Jmn+vOwejkZERESk+2kGR6ST3t+0lxsWrA4cm8HtF83odTMjbe3jEREREYklSnBEOmFn8X7mP7KEuoYD5cuu/uRETp0yPIpRhdfWPh4RERGRWKElaiIdVFVbzzcf+pDC8ppA26ePyOJyLfkSERERibo+m+CY2Wgzu9/MdppZtZltNrPbzWxQB8cZ7F+32R9npz/u6HZef6mZOf/19Vb6nWlmb5hZiZmVm9l7ZvaljsQq0eec45qnVrJqR2mgbXJWGreePx2zni8qICIiIiJN9ckEx8wmAEuArwDvA38ANgJXAe+Y2ZB2jjMEeMe/boM/zvv+uEvMrNXHvJvZGOBOoNV6u2Z2BfAsMBV4GLgPGAk8YGa/a0+s0jv8deEm/rN8Z+B44IBE7rvsSFKStNpTREREpDfokwkOcA8wDLjSOXeOc+4a59wpeAnKJODX7RznZmAi8Afn3Kn+OOfgJTzD/PuEZN6v6/8OFAF/bqXfOOB3wF7gSOfc5c657wPT8JKqq83suHbGK1H01vo9/ObFNYHj+Djj7i/MYszgAVGMSkRERESC9bkEx59VOQ3YDNzd7PT1QAVwqZmltDFOCnCp3//6Zqfv8sc/vZVZnCuBU/BmeypaudVXgSTgLufc5sZG59w+vAQL4NutxSrRt6Wogu8+toygmgJc++kpnJCdGb2gRERERKSFPpfg4CUVAK845xqCTzjnyoDFwADg2DbGOQ7oDyz2rwsepwF4xT+c1/xCM5sC/Ba4wzn3VjvjfSnEuReb9ZFeqLy6jm88+CEl+2sDbefNGs1XTxgXvaBEREREJKS+mOA0PrVwfZjzuf77xO4Yx8wSgIeArcC1bdyj1fs453bhzf6MNrMurXMKKnTQ5AXM6sq4B7tnlu3g2JtfZf3uA9uspo/O4Nefm6qiAiIiIiK9UF/cGZ3hv5eEOd/YPrCbxrkOmAnMcc7tb+Me7b1Pit+vsh3jSQ9ZsGInVz2xvEX7ebNHk5wYH4WIRERERKQtfXEGpy2Nv1Z3rfbqxDhmdjTerM1tzrl3ujh+2Pt0hnPOQr2ApV0P8eD0y+c+Dtn+6HtbezgSEREREWmvvpjgNM6EZIQ5n96sX0TGCVqath74Rdthdvg+pWHOSxT8deFG9pRVhzyXV9BqVXARERERiaK+mOCs89/D7bHJ8d/D7a3p7Dipft8pQFWzfS6NVdju89tub899zGwE3vK07c45LU/rJf7x9mZ+9fyasOezh6X2YDQiIiIi0hF9cQ/O6/77aWYWF1xJzczSgBOA/cC7bYzzrt/vBDNLC66kZmZxeKWog+9XDfwtzFiz8PblLMJLaIKXr73mx3RGs3aATwX1kV7gkfe2cP2C1a32mT8vu4eiEREREZGO6nMzOM65DXglnMcBlzc7fSPejMiDzrnAs2nMbLKZTW42TjnekrMU4IZm41zhj/+yc26j33+/c+7roV7AAv+6f/htTwSN9Xe85OgK/6GfjTEN4kAVtrAPCpWe888PtvGzf69q0vaJKcOYnJVGQpwxOSuNP148k7Omj4xShCIiIiLSlr44gwMwH3gb+KOZnQqsAY7Be2bNeuBnzfo3rjdqXtf3WuBk4AdmNgN4H28J2tlAAS0TqA5zzm0ysx8BfwQ+NLMngBrgfGA0kS1YIJ309NLt/OTplU3avjH3UK799BSVgxYRERHpQ/rcDA4EZnGOBB7AS2yuBibgJRHHOeeK2jlOEd4DP/8IZPvjHIM36zLbv08k4r0TOAtYDVwGfBPIB77snPthJO4hnffM8h388F8rcEF17L58/DglNyIiIiJ9UF+dwcE5tw34Sjv7hv2U6pzbC1zlvzobyw20XObWvM+zwLOdvYd0j+dX7uIH/1xBQ1Byc8mxY7n+s4cpuRERERHpg/rkDI5IJLy8Op+rHl9GfVB2c9FRY7jprKlKbkRERET6KCU4clB6dc1urnh0KXVByc15s0Zz8+eOIC5OyY2IiIhIX6UERw46b67fw3ceXkpt/YHk5uwZI/m/86cpuRERERHp45TgyEFlcV4h33zwQ2rqA49P4jPTRnDbBdOJV3IjIiIi0ucpwZGDxrsbi/jaPz6guu5AcnP64cO5/cIZJMTrfwURERGRWKBPdXJQ+GDzXr76wAdU1R5Ibj4xZRh3XjyLRCU3IiIiIjFDn+wk5i3duo8v3/8+lTX1gbaTJw3l7i/Ool+C/hcQERERiSV99jk4Im1ZsGInt72yji1FlU3a5+Zk8udLZpOUEB+lyERERESkuyjBkZi0YMVOrnxsWYv2nGGp/OXSI0lOVHIjIiIiEou0Pkdi0p2v5oZsjzPo30/JjYiIiEisUoIjMae+wZFbUB7y3IY9FT0cjYiIiIj0JCU4EnPufC307A1A9rDUHoxERERERHqaEhyJKW+sK+COMMvTAObPy+7BaERERESkp6nIgMSMbXsr+d4Ty3HuQNuIjGT2lFWTPSyV+fOyOWv6yOgFKCIiIiLdTgmOxISq2nrmP7KU4sraQNuFR47hlvOnRTEqEREREelpWqImMeGm5z7mox0lgePDR6Zz49mHRzEiEREREYkGJTjS5z25ZDuPvrc1cJyenMCfL5mtZ92IiIiIHISU4Eif9vHOUn7274+atN1+0QzGDB4QpYhEREREJJqU4EifVbK/lu88soTquoZA25WnZHPK5OFRjEpEREREokkJjvRJDQ2Oq/+5gi1FlYG2uTmZXPWJiVGMSkRERESiTQmO9El/fmsD/1uzO3A8MiOZOy6aSXycRTEqEREREYk2JTjS5yzOK+R3L68LHCfGG/dcMpvBKf2iGJWIiIiI9AZKcKRP2VWynysfW0ZD0MM8r/vs4cwYMzB6QYmIiIhIr6EER/qMmroGLn9kKUUVNYG2c2eO4pJjxkYxKhERERHpTZTgSJ9x8wtrWLq1OHA8OSuNX3/uCMy070ZEREREPEpwpE94ZvkOHnh7c+A4LSmBP10ym/799DBPERERETlACY70eut3l3HNU00f5vm7z0/n0MyUKEUkIiIiIr2VEhzp1cqqavn2w0vYX1sfaPvWSeM5/fCsKEYlIiIiIr2VEhzptZxz/OSplWzcUxFoO3b8YH502qQoRiUiIiIivVlCtAMQCWXBip386rmPKSirDrQNS0vizotnkRCvvFxEREREQlOCI73OghU7ufKxZS3av3DMWIamJUUhIhERERHpK/SrcOl1bntlXcj2l1bl93AkIiIiItLXKMGRXiWvoIwtRZVhzpX3cDQiIiIi0tcowZFeY/u+Si756/thz2cPS+3BaERERESkL1KCI73CnrJqLv3b++SXVoXtM39edg9GJCIiIiJ9kYoMSNSV7K/lS/e/z6bCA+Wgh6cnkZ6cyKbCCrKHpTJ/XjZnTR8ZxShFREREpC9QgiNRtb+mnq//4wM+3lUaaBs1sD9Pfuc4RmT0j2JkIiIiItIXaYmaRE1NXQPfeWQJH2zeF2jLTO3Hw18/RsmNiIiIiHSKEhyJivoGx9X/WsEb6/YE2tKSE/jHV4/m0MyUKEYmIiIiIn2ZEhzpcc45rntmFc+u2BloS06M4/4vH8XhIzOiGJmIiIiI9HVKcKTH/e6VdTzy3tbAcUKc8adLZnPUuMFRjEpEREREYoESHOlRf3lrA3e/viFwbAa/v3AG8yYNi2JUIiIiIhIrlOBIj3nig63c/MLaJm2/PHuqyj+LiIiISMR0S5loM0sGjgRGAsnh+jnnHuyO+0vv8+JHu/jp0x81afvR6ZO45NhDohSRiIiIiMSiiCc4ZvZ94DogvR3dleAcBBbm7uGqx5fT4A60ffPE8cw/eUL0ghIRERGRmBTRBMfMvgrc5h+uAdYCpeGvkFi3dOs+vvXQEmrqGwJtFx45hp9+ajJmFsXIRERERCQWRXoG50rAAZc65x6N8NjSx6zLL+Mrf/+Aypr6QNunpmZx87lHKLkRERERkW4R6QRnIvC2kpuD24IVO7n9v+vZWFjRpH1Odia3XzSD+DglNyIiIiLSPSKd4FQCW9vsJTFrwYqdXPnYshbthwwZwL2XziYpIT4KUYmIiIjIwSLSZaLfBqZGeEzpQ+55PS9ke1JCHClJ3VK0T0REREQkINIJzo3AZDP7UoTHlT4id3d5yPaNeypCtouIiIiIRFKkf6WeAvweuN/MPg08j7dkrSFUZ+fcWxG+v0TZoJRECstrWrRnD0uNQjQiIiIicrCJdILzBl4VNQPO91/huG64v0RRZU0d+4MqpgWbPy+7h6MRERERkYNRpBOMt/ASFzkIPfb+NiqCEpw4g4nD05g/L5uzpo+MYmQiIiIicrCIaILjnDs5kuNJ31FdV899b20MHA8akMjia05hQD9N0omIiIhIz4l0kQE5SP176Q7yS6sCx1894VAlNyIiIiLS45TgSJfV1Tfwpzc3BI5TkxK47Lhx0QtIRERERA5a3ZLgmNlhZnavma0zs3L/tc7M/mxmh3fHPSV6XliVz5aiysDxJcceQsaAxChGJCIiIiIHq4ivITKzrwF3A4l41dQa5fivL5vZ5c65v0X63tLznHNNHu6ZlBDH1+YcGsWIRERERORgFtEZHDM7BrgXL3H6F3AGXlIzETgdeMI/92e/r/Rxr60tYG1+WeD4wqPGMDQtKYoRiYiIiMjBLNIzOD/Em7W52Dn3z2bn8oD/mtnTeInO1cDnI3x/6UHOOe4Kmr1JiDO+eeL4KEYkIiIiIge7SO/BmQN8ECK5CXDO/Qt4H5gb4XtLD3tnYxHLthYHjs+eMYrRgwZEMSIREREROdhFOsEZjDdT05Y8v6/0Yfe8fqBymhl852TN3oiIiIhIdEU6wdkLZLej3wS/r/RRy7cVsyivMHB8xuFZZA9Li2JEIiIiIiKRT3DeBo4ys3PDdTCzc4BjgMURvrf0oODKaQDzT25PXisiIiIi0r0ineDcBjjgCTN70MxONbPxZnao//U/gH8CDX5f6YPW7y7jlY93B45PnDiUI0ZnRDEiERERERFPRKuoOefeNrPvAncAX/RfwQyoA77rnHsnkveWnvOnNzY0Ob785AlRikREREREpKlIz+DgnPsTcCTwALARqPZfG4H7gSP9PtIHbS2qZMGKnYHjIw8ZxNGHql6EiIiIiPQOkX4ODgDOuZXA17pjbImue9/aQH2DCxxfPi8bM4tiRCIiIiIiB0R8BkdiV0FpFf/6cHvg+LAR6Zw8aWgUIxIRERERaUoJjrTbXxdtoqa+IXCs2RsRERER6W26tETNzO7Hq5p2rXNut3/cXs45p2VsfURxZQ0Pv7slcDw+M4UzpmZFMSIRERERkZa6ugfny3gJzi3Abv+4vRxd2KdjZqOBm4AzgCHALuA/wI3OuX0dGGcwcB1wDjACKAJeAq5zzm0P0f8WvCIKE4FMYD+wxb/3Xc65omb9xwGbWgnhCefcRe2NN1oeeHszlTX1geNvnzyB+DjN3oiIiIhI79LVBOcr/vuuZsfdyswm4D1UdBjwDLAWOBq4CjjDzE5onmiEGWeIP85E4DXgcWAy3vfxGTM7zjm3sdll3weWAv8FCoAU4FjgBuCbZnasc25biNutwEuCmlvVVpzRVl5dx98Xbw4cj8xI5pwZo6IXkIiIiIhIGF1KcJxz/2jtuBvdg5fcXOmcu7Ox0cx+j5eA/Br4djvGuRkvufmDc+4HQeNcifcsn3vwZoiCpTvnqpoPZGa/Bq4FfgrMD3Gv5c65G9oRU6/z2HtbKdlfGzj+5onj6Zeg7VsiIiIi0vv0uU+pZjYeOA3YDNzd7PT1QAVwqZmltDFOCnCp3//6Zqfv8sc/3b9fQKjkxvdP/z2n9e+gb6mqree+hQcmsYak9OPCo8ZGMSIRERERkfB6LMExsxwzO8/MjuziUKf476845xqCTzjnyoDFwAC8ZWOtOQ7oDyz2rwsepwF4xT+c1864Puu/rwxzfqSZfcvMrvXfp7Vz3Kh6aul2CsqqA8dfnXMo/fvFRzEiEREREZHwIvqgTzM7F/g63kb/94Laf463R8X848ecc5d0wEbnFAAAIABJREFU8jaT/Pf1Yc7n4s3wTARe7eI4+OO0YGY/BFKBDLyiA3Pwkpvfhhnvk/4reIw3gC8557a2Eme7mJlru1fH1NU38Oc3NwSO05ISuPS4QyJ9GxERERGRiIloggNcApwIfNTYYGZT8aqd1QHvAocDF5vZ0865pztxjwz/vSTM+cb2gd08zg+B4UHHLwFfds7tadavEvglXoGBxrVe0/ASvnnAq2Y2wzlX0Ua8Pe7ZlTvZtnd/4Piy4w8hPTkxihGJiIiIiLQu0kvUZgIrnHOVQW2X4JWE/rpz7kTgKKAW+EaE792osXZxV2c0Wh3HOZflnDMgCzgXGA8sM7NZzfoVOOeuc84tdc4V+6+38GaZ3gOy8Wa9usQ5Z6FeeBXfOqyhwXHP6wdmb5IT4/jKCYd2NUwRERERkW4V6QRnCLCjWdtJQDnwKIBfdnkRMKWT92icWckIcz69Wb9uHcc5t9s592+8hGUI8GAb9228rg74q394Ynuu6Un/XbOb3ILywPFFR40lMzUpihGJiMj/s3ff8VFVeR/HPyeFQEICJKG30IsooShLEVGx08QuuhSFta2ylsf2KOLirm1XZfdRWFBQsODqKuoiIAp2RUBFBCwUAakJSAkhpNznjzNJJjN3SIZMymS+79drXpl777nnnAkwzG/OOb8jIiKlC3WAE0fxyAfGmFpAOvC55wN9oZ2UnN4VjB88P13XxlCcxSzQ2ppQ1wOA4zi/AGuBE4wxqWW5ByicznbMjG+VzXEcnl76c9FxTJRhwsC2x7hDRERERKR6CPUanB1AV6/jgdig51OfcnWBA8fZxlLPz7ONMVHemdSMMYlAfyAbu97nWL7wlOtvjEn0zqRmjInCjsh4t1cWzTw/88tYvjDTm+9molXmrW+389jC9WzdV7z2ZmTP5jSrX6cKeyUiIiIiUjahHsH5EOhsjPkfTxrkP2PXsCz0KdcN2HY8DTiOswGbwjkNuNHn8mTsaMgL3ov2jTGdjTGdfeo5BMzxlH/Ap56bPPUv8kyp866niW+fjDFRno0+GwGfOY6zz+taH89Ilu89Z2A3JQWYe4yXXGne+nY7N7/8dYngBqBTk8Qq6pGIiIiISHBCPYLzEDAC+KvnYYAljuN8VVjAGNMRuyB/WjnauQH4DJhqjDkTWAf0wWYl+xG416f8usLmfc7fAwwCbjXGpAPLsWuDhgO78Q+gzgUeM8Z8BGwAMrFT7U7zvKad+CdPeAQ7bW0ZxUHdSRTv53Of4zifleVFVzTvaWne/r1iG9cM0BQ1EREREan+QhrgOI7zozGmP3ArdjRjOfCYT7EzgW+Bd8rRzgbPhqEPYoOO87HT46Zi9+DZW8Z6Mo0xfYFJ2MDsVGzQMgu433Ec31GmJcC/sNPgumNTSGdhg6o5wFSXtucAF2Kzx50HxAK7gFeBfzqO83EQL71CeScV8PZzgPMiIiIiItVNqEdwcBxnDTDuGNefAZ4JQTtbgbFlLOs7cuN9bS9wi+dRWj1r8B/VKe2eZ4Fng7mnqnRoVJf1Ow/6nW/fqG4V9EZEREREJHghD3Ck+tmUkcVl0z8vtVxegfvWQTec3h6At7/dztwvfgmq7Yt7teCS3i39zl83ZyX7Dh8Nqq7ZY0+hTq3oEuc++SmDf3zwU1D1nNmlERMGtvM7f/d/VrNxT3D7rT5xWbpfAobvt+/nwbfXBlVPz9YNuPPczn7nH1m4nlW/7HO5I7D7h3blhGYls59v/y2bP837Jqh62jZM4K8jT/I7/6+PNvD+ut1B1fXHMzowoEPJ5ILZR/MZM2t5UPU0iK/FtKt7+Z3/94qtvLYyuGV9V/2uNUO7N/M7P/q55RzJLWueEGveH/r6nVuydhczPg4uf8iQk5pydd80v/MTX/maHfuPBFXXM1f1Ijmh5PK/lb/s5dGFPwS4w13/9qncfGYHv/OT3/6etduDyxXzl5En0q5hyS9MNuw5xD3/+S7AHe66Nkti0tAT/M5Pff8nPv05I6i6/ufcTvRqnVzi3N6so1w/d2VQ9TStV5snL+/hd37O55t5Z/WOoOoaf2pbBnf1TzRalvdxb7Vjo3l+3Cl+5/Ve7k/v5WWj9/KyieT38k0Z1WPfegU4ESArJ48vN5Vp1l4JUQaevLwHwzxvHDv3Hwm6nt+1TXE9v2rLPnYfzAmqrnzHPwDLzMoJuk9tG7pn5f7u1/2s+TW4f+Rub54HsoP/fdeNc/+n+OPOg0HXdSA7z+/ckdz8oOvJOupfD9iAOdi6rszy/7POd5yg62mU6L4X07Z92UHXNbiLe6b6FZv3knU0uP8U3ew+GPzfzRObu2/L9e22/UH/p5GbX+B3bl9WbtB9apxU2/X82u0Hgq7rcI7/7/VwTvB/NwP5efehoOval5Xrdy43vyDoetqkur+v/JJ5OOi6hqc3dz0fbD0JPkFEIb2X+9N7ednovbxsIvm9PCvH/e9bZStXgGOM2YjNkjbYcZxNnuOychzH8f/qRaqNFg3ii4IbEREREZFwUN4RnDRsgBPrdVxW7vOhJOQS4mLo0ya59ILAuh0HOHDERt8pdUsOiTapV7vM9RRq0cB9/5yerRoEPa0h2vgvpUpJiAu6T4G+aT2xeT0SagX3T6J2rP83pEl1yv77LtQxQCrujk0SORTktyFJdfxfQ+3Y6KD7FOjb0TapCUHXlZLg/21dtDFB19Mg3i/jOmD/ngVbV5N67t9m9U5LDnpag5tGicH/3WydEu96vnuLegG/8QwkNtp/F4AGCbFB9ynQGryuzZKCqgcgPs7/30t8XPB/NwO13b5R3eD/TiXE+p2LjY4Kup6mAf4+tU6JD7quQH/Wwdbj9v4Eei93o/fystF7edlE8nv5e3ExBPeOUDGM4zJUXOabjWntefqr4zh5Xsdl4jhOcJOAJWjGmJU9e/bsuXJl2eaTn/vkR0WJBgZ3acTM0SdXZPdEREREpIbo1asXq1atWuU4jv8iq0pUrhEc3wBFAUv4y8wqjrvdvqkREREREanO/Me9JGIVFDjs9Qpwkuu6DyGLiIiIiFRXIQ1wjDEtjTG/N8Z0OkaZTp4yLULZtpTfgSO55Hulik5JUIAjIiIiIuEl1CM4NwOzylBuNkFumCkVL+NQyWVhvkkGRERERESqu1AHOGcD3zuOE3D3Ic+1NcA5IW5byinzUMn89lqDIyIiIiLhJtQBTkvg5zKU2wC0CnHbUk7e628Av51zRURERESqu1AHOLUB/22h/eUA7onZpcpk+AQ4qXU1giMiIiIi4SXUAc6vQM8ylOsF7Axx21JOe33W4LhtgCciIiIiUp2FOsBZCrQ1xowJVMAYMxpoB3wQ4ralnDKzitfgJNaOIS7GfRdsEREREZHqKtQBzt+xU9T+ZYx5yBjTtvCCMaaNMeYh4F+eMn8PcdtSTt6bfGp6moiIiIiEo5AGOI7jrAcmeA7vAn4yxuQYY3KwyQfu8rT5B8dxvg9l21J+3lnUlGBARERERMJRqEdwcBznBaAf8DZwGIj1PLI95/o5jjM71O1K+XlnUdMmnyIiIiISjmIqolLHcVYAI4wxUUCq53SG4zgFFdGehEamV5IBbfIpIiIiIuGoQgKcQp6AZndFtiGhkV/gsO+w9wiO1uCIiIiISPipsADHGHMC0BdoCHzvOM5bnvNRQIzjOEePdb9Urt8OH6XAKT7WGhwRERERCUchX4NjjGlljPkAWA1MB6YAI7yK/BHINsacGeq25fjt9dnkU1PURERERCQchTTAMcakAh8Bg4DvgGcA41PsVcABhoeybSmfDJ9NPjVFTURERETCUahHcO4GWgGPAD0cx7nJt4DjODuAdcCAELct5aARHBERERGpCUId4AwFNgH3OI7jHKPcVqBZiNuWcsjMyilxrDTRIiIiIhKOQh3gtARWlRLcABwAGoS4bSmHTJ8pag0U4IiIiIhIGAp1gJMN1C9DudbAbyFuW8rBewSnXp1YYqNDnn9CRERERKTChfpT7BqglzGmXqACxpjmQHdgVYjblnLwXoOj9TciIiIiEq5CHeC8hB3BmW6M8fuU7NkDZyoQB8wNcdtSDt5Z1LT+RkRERETCVagDnJnAp8ClwDpjzFTP+W7GmEew2dMuBD7EBkNSTZQYwVGKaBEREREJUzGhrMxxnDxjzPnADGyQU5gmurfnAfAmMLoMiQikEmUeKl6Dk6wpaiIiIiISpkIa4AA4jnMQuNwYMxk4D2gLRGNTQ7/rOM7XoW5Tyicvv4DfsnOLjlM1RU1EREREwlRIAxxjTBLgOI5z0HGcddgpaVLN7Tuci/d4WrICHBEREREJU6Feg/MbsCTEdUoF815/A5BSV2twRERERCQ8hTrAOQj8FOI6pYJ5r78BZVETERERkfAV6gBnHdAixHVKBcvQCI6IiIiI1BChDnBmAAOMMb1CXK9UoL2+IzjKoiYiIiIiYSqkAY7jOM8CTwPvGWPuNMZ0NMZoOKCay/QawTEGGsQrwBERERGR8BTqLGr5Xod/8TwwxrgVdxzHCXmaagmed4DTIL4W0VGuf14iIiIiItVeqAOMYD4Z61N0NVFik08lGBARERGRMBbSAMdxnFCv6ZFK4J0mWhnURERERCScKSARMg95BThKMCAiIiIiYUwBjpRYg5OSoJwQIiIiIhK+KmSRvzGmFnARMAi7L44DbAeWAa87jpMT8GapVLn5BezPzi061hocEREREQlnIQ9wjDH9gJeAlvgnErgG+KsxZpTjOJ+Eum0J3j6fTT5TNUVNRERERMJYqNNEnwAsBuKBjcDLwGbP5TTgMqA9sNAY08dxnO9D2b4EL+NQyQAnWVPURERERCSMhXoE50FscPNX4D7HcQq8LxpjJnnK3ANMBi4OcfsSpL0+IzhKMiAiIiIi4SzUSQZOA35wHOde3+AGwHGcAsdx/hf4Abs+R6pYZlbJ5VBKEy0iIiIi4SzUAU4dYFUZyq0Caoe4bTkOmYd8R3A0RU1EREREwleoA5wfgKZlKNcU+CnEbctx8B7BiTJQv05sFfZGRERERKR8Qh3gTAMGGmP6ByrguTYQmB7ituU4eK/BSU6oRVSUb+I7EREREZHwEdIkA47j/MsY0xmbJe1p4EVgk+dyGjAKuAF4ynGcaaFsW46PdxY17YEjIiIiIuEu1Gmi870Ob/c83Ew0xkz0Oec4jlMhG49KYN4jOClKES0iIiIiYS7UAUV55jdpblQVyDxUvAYnWSmiRURERCTMhXqKWqjX9EgFy/QawUnVFDURERERCXMKSCJYTl4+B4/kFR0na4qaiIiIiIQ5BTgRbF9WbonjFE1RExEREZEwpwAngmV4rb8BSNEUNREREREJcwpwIph3BjWAlLqaoiYiIiIi4U0BTgTLzPIZwdEUNREREREJcwpwIljmIZ8RHE1RExEREZEwpwAngnmniI6JMiTVjq3C3oiIiIiIlJ8CnAjmvclng4RaREVpr1URERERCW8KcCKYd5IBTU8TERERkZpAAU4Ey/Bag6MEAyIiIiJSEyjAiWAlR3CUIlpEREREwp8CnAjmvQYnWVPURERERKQGUIAToY7k5pN1NL/oOFVT1ERERESkBlCAE6G8U0QDJGuKmoiIiIjUAApwItRe300+NYIjIiIiIjWAApwIlZGVU+JYaaJFREREpCZQgBOh/EdwNEVNRERERMKfApwIlekzgqMsaiIiIiJSE4RtgGOMaWGMec4Ys90Yk2OM2WyMedIY0yDIepI992321LPdU2+LAOUfMca8b4zZaozJNsbsNcZ8bYyZZIxJOUY7/YwxCzzlDxtjVhtjJhpjooN97aHgnWQgNtqQVDumKrohIiIiIhJSYRngGGPaASuBscBy4AlgI3AL8PmxAg2felKAzz33bfDUs9xT70pjTFuX2/4EJADvAU8BLwJ5wAPAamNMS5d2hgMfAQOBN4D/A2p52nulLH0NtUyvKWrJCbUwxlRFN0REREREQipcv7Z/GmgE3Ow4zj8KTxpj/o4NQB4CritDPX8BOgJPOI5zq1c9N2ODl6eBc33uSXIc54hvRcaYh4B7gLuBG7zOJwEzgHxgkOM4Kzzn7wM+AC42xlzuOE6lBjp7vUZwUpQiWkRERERqiLAbwfGMqpwNbMaOhHibBGQBVxtjEkqpJwG42lN+ks/lf3rqP8d3FMctuPF41fOzg8/5i4GGwCuFwY1XPf/rObz+WH2tCJmHitfgKEW0iIiIiNQUYRfgAGd4fi52HKfA+4LjOAeBT4F44Hel1NMXqAN86rnPu54CYLHn8PQy9muo5+fqAP1d6HLPR8BhoJ8xplKHUTJLjOAowBERERGRmiEcp6h18vz8McD1n7AjPB2B98tZD556/BhjbgfqAvWA3sAAbHDzcFnbcRwnzxizCTgBaAusO0Z/j8kY4wRTvuQaHE1RExEREZGaIRwDnHqen/sDXC88X7+C67kdaOx1vBAY4zjOnhC3E3KHj+aRnZtfdKwpaiIiIiJSU4TjFLXSFKYDC2pEI9h6HMdp4jiOAZoAI7EjMF8bY3qGsp2ychzHuD2AVb5lM303+dQUNRERERGpIcIxwCkc8agX4HqST7kKrcdxnF2O47yBnRaXArxQEe2EkncGNYCUupqiJqXY/ytkbqjqXoiIiIiUKhwDnB88P13XxlCcxSzQ2ppQ1wOA4zi/AGuBE4wxqWVpxxgTA7TB7qOzsSzthEJmVk6JY01Rk4De+iP8rTM80RXef7CqeyMiIiJSqnAMcJZ6fp5tjCnRf2NMItAfyAa+KKWeLzzl+nvu864nCjsi491eWTTz/Mz3OveB56fvfjpgN/6MBz5zHCfH5XqFyNAUNSlUkA+71kLGT+7XD+6Egzvs819XupfJOQRHsyqmfyIiIiJBCrsAx3GcDdgUzmnAjT6XJwMJwAuO4xR94jLGdDbGdPap5xAwx1P+AZ96bvLUv8hxnKKRFU89TXz7ZIyJ8mz02QgbrOzzuvwakAFcbozp7XVPbWCK5/CZY7/q0NIUNQHgpcvg4VbwTF/49Cn3Ms17Fz/fvxUO7vIv8/HjMLUHfPUs5OdWTF9FREREyigcs6gB3AB8Bkw1xpyJTa/cB7tnzY/AvT7lC9MvG5/z9wCDgFuNMenAcqALMBzYjX8AdS7wmDHmI2ADkInNpHYaNsnATmC89w2O4xwwxozHBjrLjDGvAHuBYdgU0q8B84J7+eXjvclnrZgoEmpFV2bzUl0c2Q9HD9nngUZn0gZApwugRS8b7NTxSfZ3YDt8MQ3ysuG/t8La+TD6rYrtt4iIiMgxhGWA4zjOBs9oyIPYoON8YAcwFZjsOM7eMtaTaYzpC0wCRgCnYoOWWcD9juNs87llCfAv7DS47tjUzlnYoGoOMNWtbcdx3jTGnIYNvC4CagM/A7d67ilvxregeG/ymZpQC2N84z6JCM17wZbP7fPd6yDnIMQlliyT1t8+AvnsHza4KXTSpaHvp4iIiEgQwjLAAXAcZyswtoxlA36C9wQkt3gepdWzBv9RnTJxHOdTbCBW5Ups8qkEAzWb49ipZfVb+V9rdwYc3ls8OhMbH3z9p98LtevDZ1OhXkvofkX5+ywiIiJSDmEb4Mjx816Dk5Kg9Tc12lczYfF9MPyfcOLFJa+1P9M+yiOuLgy6E3qPg0M7IcpluuOeH+06ndPvgQZp5WtPREREpBRhl2RAys97DY4yqNVgmz+BhXfZKWSvX1OxaZ7rNoQmJ7pfe38yrJ4H/+gN794JeZWWMFBEREQikEZwIozjOCXW4GgPnBps5WwoyCs+rlW38vuwdTmsf8c+L8iFnd9BtM/fucN7Ydlfi49PvBRanuxf17JH4HCGfd40HXqM8i+zbzNkZdi1RLXqQlIz0BozERGRiKIAJ8JkHc0nJ6+g6DhZU9RqrhHToG5j+PyfcMJIGPCnyu+DibbByI5v7PHgyf4BR85BWP6v4uOm6e4Bzup5sHeDfd7tYvcAZ8Us+PRJ+zy6Fty3x7/Mjm/hvfttABSXBKfeBqnt/cv98hnUSrDlEhpC7aTSX6+b3esgc4N9nQDpLuuUftsKX07z9CkRugyFBq39yx3ZD7EJEK23bhERkUD0v2SE2eu7yadGcGqu6Bg45yFo3Q/aDqqakYwWvWD8Ulj7Jmz7yj1wCaXCIAL8M8IVOrADNi4rPj7lWv8yBfkw67zi40F3w6C7/Mu9eyds/NC22+lcuOBv/mWW/wtWPGefx6cGCHC22EC0UJNu7gHO37pAbpZNCNHvj3Zdk4iIiJSgACfCZGSVXP+gNTgRoPMFVdt+VBR0G2kfbkwU1GlQfBwTYFSxdr3icrUS3MuUJcDxLgN2FKfUMgHq+nUV7PFss3XIZRNUKDk1sHDfIV++52u5tFeQb4MbgNzD9vfm5ret9nd1vCNOIiIiYU4BToTxH8HRFLUaIz8XomOruhfBq98S7txcerkJS0svc+qtNltczkGICvD2VjsJWpwCOQcg51D5Ahzvcr73FN3rVX/eEcg7CjE+Xyzk5dhRmdzDgdvzC4ICrKl6907Y9JGdwnfKBEhp515ORESkhlKAE2EyNYJTM2VugDkXwpC/Q/vBVd2bqtOoi30cS8dz7ONYElLh9/NtAJRzEFqe4l6uaXe7ficuCRp1dS/TYxR0PNsGLXFJ7kFo12H2kZ9nAxm34CUqBs75q+1PzgFo3tO/zL7N8MMCwLFrevZthivnHfu11iQF+XDgV0hq7p6yXEREIoICnAjjnUENtAanRjhyAF6+An77BV68BAY/AP1uVvaw8oitY9ctlWbk9NLLJDWzj7KIjoE69d2v1UqAvjcc+/6VzwNO8XGf68rWbrjLOWj3fPr8/yBrD9z5i//vcd8v8J8JnkAzEfreCC16+9e1+RMbYDbq6j/SJiIiYUEBToTJ9JqiVjs2ivha+isQ9j75O2T8YJ87BbD2LehzvT6cRaLT/geS28AX02yK8LaD/MsU5MOHj0L3y23ZmmDOhTaJRSG3EbDDGbD1i+Ljky5zr+vV38PhTIiOs9MdRzwd2r6KiEiF00afEWav9x44ShFdM5x2F3S/0j6v2wQum6vgJlLF1oGev4frP4Ux77iP4v24CD58GKb2sCN/mRsqv5+h1mts8fPYePc02n7rqgKsYSosl58TOOHF5k/g5/ch+7fg+xqI45ReRkREykRf30eYjEPFa3A0Pa2GiK1tv2Vu2t2uy0hqWtU9kqpmDNRt5H7ty2c8Txz4aTEMecK/TPY+u49QoGx1VeXwXohP9j9/0qU2aMvNtsGO4/gHd7EJkHaqDWCOHiqZua9QXg7ke03jbREgrfnHf4MNH9jn7c6Eq/9z7H679Qdg6V9h41KburxBaxuUut37zUt2jVagZBciIlKCApwIU3IERwFOjWEM/C5C1lvI8cs5BFkZxccnXAiJTfzLLZkMK2fZdNONusK4he515R+1gUJFr/fatxk+edJ+0L92CTQ9qeT16Fi4Yp6dchdbx72Olie7BxDeomJg3GLI/Bl+XQGt+vqXKSiAX1cWH7sFXGCnuu3bbIOX9CvgrAf9y2T+BFu/tM8D/Qo3LoP5N9jseD2ugv636EsMEZFSKMCJMN5rcJI1RU0kssTVhes/s2mkv5weOAnBwR3255H99uFm3Vvw5vUQU9smURj9DtRrHvo+790I/+gNTr49/vhvcOnz/uUaB8hiF4yoaGjVxz56jHIvs29Tyd9Jc5dEBQDbv7YbuALs/9W9jHfyiQM7bPAU5TNz/EtPIoujB2H5dH2RISJSBgpwIojjOCVGcFI1RS385OfBWzdB72vsN9IiwTIG2p5mH4Ec8PpAHigD3IHt9mfeERuEuGV/y8qwCQAadoaGHaHrCEjtEFx/k9tC636w+WN7vHY+ZPxs03NXhZR2cNsPsG2FHeUJ9HtMbFYc4BT+rnw16gqt+9vfcWJTOyIWVbv4+pEDJZMndDofGqSF5GWIiNRkCnAiyMGcPI7mFxQdJ2uKWvhZMgm+fRnWvG7XTvS4qqp7JDXRgFth7wb7wbxhgH2FvD+0x9VzX6+zZz3sXG0fAI27uQc46/8L9VraD/oJqf7XB95uA5zGJ8LA26o++1tiE+gyxD4CaXkK1Iq3r6lxN/cy6VfaRyC1k+BPa+C7f3tG3P7gXm75DNi7CU4ZX/W/GxGRakABTgTZe8h3DxxNUQsra9+Cz/9pn+cfhf/eDu3OKPseKyJl1W1k6WVOvNh+mA40OgGw54eSxw07+ZfJy4F5V9kU5w3S4OZv/Nf0tDnNToFLGxA++zud/efQ1FOYGa/H1e7X8/Pg06dg/1b44mlbdtjU0LQtIhKmFOBEkMysnBLHyqIWZtoOgo7nwY/v2uMR/6fgRqpO6372cSzxyTZz2Z4fIOcA1G/tXybzZxvcgF2U/9Ni6HhOyTLGQJtTQ9LtsBUosPthgQ1uAHAgqQLWQYmIhBkFOBEk03cER1PUqp+DO+20nj0/2GkwXYcXX6udBJe/BMv+CgW50O2iquunSFmccKF9gF1PEhXtX8Z3lOfDR6HD2eEzUlPVYuLsNMI962xq795j/csc3AX/HmNH0Bp2gk7naS2PiNRoCnAiSGaWpqhVCwX5doqZWzrbF4bbAAeg/eCSAQ7YDEtn3KtNASX81E5yP9/pPJjwIWT8aPffadlHwU0wOp5jA8JNH8Hude77H+1ZD1s+sw+wiRvcApyf37fnG6S5B6MiImFCAU4EyTzkM0VNIziVJz8X3rjOflud+ZNdND3wDv9yqR2LA5w9PwauTx8ApaaIrQPN0u1Djk9pmfHKshYqNxvmXgQ4EB0Hgx+Avjf4l/vkSTuCDHYj1LaD/Mt8/4adegiQ0BB6jSnLqxARCRkFOBHEewQnoVY0tWP1DV1I5eXAzjX228+ElJLXomPtN6xZu+2x7weOQg07wTrP88OZkHsEYmu7lxURKYu4RGjWw37CWz6FAAAgAElEQVRp4hRAvVb+ZTJ+Ajwjw/k5gTcw/fARyD1sn/f7o3uA891rsN6zqWrjbu4BTvZvdu1Qwy4QrY8iIhJaeleJICU2+VSCgdDZvw1eHW1T4eYfhRHPuKd+bdip9ADnxEvtFJ3UjjZtru+mfyIiwUq/wj4KCiBrj/v7SobPiLHbKE8obXgfXhsHsQl29G7EM9DAJQmFiMhxUIATQbw3+UxJ0PqboGTvg19XQvNeUKdByWsJDYuDG7AbALoFOK36gomyHxyanOjeTsOO9iEiEmpRUZDY2P1ah7NgzH/tly8ZP0JKkBuyBmvbSvszNwu2fOG+/1HOQdjxrR19cttnCexmsju/s2VzDkLn8/3fo3MOwstXQN3Gds1Xh7MDrwkTkRpBAU4EyfBagxNR628cx33NyoEdsHGZ5z/GAzYo8U27nJ8L/+gFv/1ijy+bC12GliwTEwdNTrK7mkPxT19n3FuulyEiUmFq17P7DKUNOHa5u7YUPzcBRpgveZ6i6W6BeL9PNurqHsBs+QJevNi20+gEuP4T9zLzRhUfN/7QP8CJrmU3igVY8xq0OAWufe/Y/atJDu6EHavt+s5Du+Cch6q6RyIVTgFOBCkxghMJU9R++Qzm32SnZNy91f96xg/w5nXFx61+5x/gRMeW3Mhw2wr/AAdsKtym3aFFb7vwVkSkJoqOLUOZMny0GPIEbP3SjuQEmpq2zRMEOQXFSQt8xSWWPM456F8mJs4mTsj3fMnX6Tz3uvKOQkyY/t+Yn2vXbSY28b/21bPw0aPFx4Puhri6ldc3kSqgACdCOI5TIsBJrulT1HZ+By9eCkc9/9kV5PunPa3l+x/jIfe64hIhe699/utK9zL9bjr+voqIRJrGJ9hH73GBy3iP8uRlQ36ef/BUlgAH7BdYW76wQU6XYe5lZp8PJhq6DrNl6rcs/XW4cRw7apLxA7Tq5x40Te0BRw/b/ne/zD2r5k/v2c1v4xLtVOj2Z/qXWfYwrHkd9m6ERl3gOpdRLt/1VBk/QvOeJc/lZsNr19iZDJ3OU5pwCXsKcCLEgew88gqKpwyk1vQRnPcfLA5uwE5B8522UNb/GE+8GGrVtaMzzXuHtp8iIuJuyJOw7Sv7xVL2bzY9tW+Ak9oRxiyw7+dxie4jGACj37JfYv3yKaS297++f5ttC2DrFzawOP+x4Pv8w0L4zwTI2W+Pr//MBnK+Du6y648O7bSvzc3Xc2Htm/Z5ake46Sv/MlkZxQkiMn62iSR8k0h4Bzhx9ew9vr77N/zwX/uo38pOx27a/divVaQaU4ATITKySu6Bk1zT1+Bc9CzMuwo2fWiDE7eNMRukwc1fQ1ySLRMTYFTreP6TExGR8qnf0j66jQxcJq4upPUvW31xde3GqG7WvVPy2G2UJzcbVs62U+cO/Apj3/Vf35nQsDi4AZu0wTfAKci3wU1Rv3y+bCvk/aVboDLewUteNuzf4r+Ja2pH+P1btmzdxv59dhz4cnrx8eF9djNYkTCmACdCeE9PA0ipW8OnqNVOglGvwYcP270afEdvwE4b0Ju4VLKCggL2799PdnY2Rhu2ipSL4zjUqVOHevXqEVWetPotT4ZeY+3+PY4Drfv5lynIhyUPQN4Re3xol/+IUapP9jnf9NsABXnQ9yY7syDnoPsID8BRr2nTgQKcpunQ/iwbvDTsZL+w8xUTF3gTWICjWbYPe36wo2Q9rnJvL+cgxMZr+pqEBQU4ESLzUMkRnIjIohZTC868v6p7IVKkoKCAHTt2UL9+ferXr68AR6ScHMfh8OHD7Nixg6ZNmx5/kNO8l31c8Dc7Pc3tQ/zPS4qDG7BZyXwDnNpJNnhJbAINO9sMm75i4sqWyez384vTXwd6r2h5Mlz1Wul1HUtcXRj5LzjrQVgxC0661L3ckgfs7+CUCZA+CurUL1+7IhVIAU6EyPQbwalhAU7OwcDfcIlUE/v376d+/fokJATY00NEgmKMKfr3tH//fho0cBmtD0ZUNKS0c7+27i37s24Tu19ZVICPUKFKwxxbxz7qNgpNfaVJbAKn3+1+Lfs3+OZlO7Vu0T12St+4d/3Lff+mnfIdl2iz4wVaEyVSwRTgRIjMQyUDnBq1Bufz/4Mvp8FV//GfHiBSjWRnZ1O/vr71FAm1+Pj40AQ4x3Luw3aEx23Kc033zYsl1w31vNq/TEE+/Ht08fGgu2HQXf7l5t9kM53GJUKzdDh7in+ZPT9Axk/FySOadtfUOAmKApwI4b0GJzEuhriYGvJGseY/9tskgGfPgivmQas+VdsnkQCMMZqWJlIBKuXfVmWNpFRHJ4y0ozgrnrPT5bpd5F/GNxNprQB77ez5AXZ8Y58H2ix27Vuw1Cvw+d89/gHOrrXw2jio2xASm9mpcy16le31SI2nACdCZHitwUmuKdPT8vPgw0eKj7P3wZbPFeCIiIiEUlJTOONeOPU2mzjBLeuob4BTnuxwOQeKn0fHue8ldDgT9qyzD3APugD+NQiia0FiU7tR94kX+5fxzbTqFiyXpYxUGwpwIoT3CE6NSTAQHQOj34GXLoXtq+Dka6H/LVXdKxERkZoptjY0dUmcADYF9fWf2+xvOQdskgU3aQOgXgsb6DTq4l6mTEGQT0CV1NS/TH4ubP8G8AQngaaxTxsAu9bY552HwOUv+pd57z747B/2eWw83LvDvS6pFhTgRAjvNTjJCTUoRXTdhjDmHZvDv/8t+kZFRESkKsTUgsZdSy93weOllxl0N/Qea4OYgjz3MvEpNhg5uBMObIek5v5lDu6kKLgBSGpWettSI5QjabyEE+8saqk1ZYpaoVoJcOqtWoAoIiVs3rwZYwxjxowJab3GGAYNGlTm8suWLcMYwwMPPBDSfojUWImNbWKBtAHQdpB7mVZ97EjL+PfhtnUQn+xe7sRLoPUAu+9d/VYV1WNr1/fw2jV2byGpUgpwIsS+w94jOGEa4OQdhSP7Sy8nItVaWRaEp6WlYYxh8+bNldOpKvLVV18xatQoWrduTVxcHElJSbRr146hQ4fy6KOPkpVlPyjdc889GGO48847S61z/PjxGGN48skngeIAy/sRHx9P06ZNGThwIHfccQdff/11hb5OkSpRvyVcNBPG/hdu/hraD3Yv13ucJ+vb3dBtpHuZdmcUlzn1Nv/r+36BOSNhzWvwwgg4vDd0r0OCpilqESC/wCG/oHiINqVuGE5Rcxx464+wczWM+redvysiUgXWrVtHfHx8ueuZO3cuo0ePxnEczjjjDC688EKio6PZtGkTK1as4J133mHkyJG0b9+e8ePH8/DDDzN79mymTJlCbGysa51ZWVnMmzePuLg4rr66ZCrf1q1bF41mHT16lD179rBq1Soef/xxHn/8ca688kqmT59O3boBsl+J1FQnX1N6mXZn2IebggJ4ZRQc2mmPty2HxffBiP8LXR8lKApwIkBeQcnMH2E5Re39B2H1K/b5zLPszs2NT6jaPolIROrcOcDi6SAcPnyYG2+8EWMMixcv5swzz/Qr89lnn5GamgpAmzZtGDx4MO+99x5vv/02I0e6f8v8yiuvcPDgQa688kpSUlJKXEtLS3OdJvfNN9/w+9//npdeeom9e/fy7rsuGziKSGBRUXZt0UuX2pkmDbvA2X+u6l5FNE1RiwB5+QUljsNuitrhvfDNS8XHh3bZBYUiEjH27dtHfHw87dq1w/FN1+oxZMgQjDGsXLnS79r69esZMWIEycnJJCQkMGDAABYvXuxXbvbs2RhjmD17NgsXLmTQoEHUq1evxJS6QGtwdu3axTXXXEPjxo2pU6cO6enpPP/88659XbNmDQcOHKBbt26uwQ1Av379SmwMO2HCBABmzJjhWt77WmHZskhPT2fJkiU0bNiQhQsX8uabb5b5XhHxaPU7GLsQWvaBq/8TeE2QVAoFOBHAdwQnJdyyqMUnw7XvQWonezz0KehwVtX2SaQae+vb7Zz75Ee0u2cB5z75EW99G/5fCDRo0IDLL7+cjRs3smTJEr/r27ZtY+HChfTq1YtevUpu9rdp0yb69u1LZmYmf/jDH7jkkktYuXIl5513HvPmzXNt77XXXmPIkCEkJiZy3XXXcemllx6zf5mZmfTr14/nnnuOjh07MnHiRNLT07nuuut44okn/MoXjq5s3769aJ1NaYYPH06jRo1YvHgxW7Zs8bu+Zs0avvzySzp27Mhpp51WpjoLNWrUiD/84Q8AvPiiS4pcESld464wbpGytVUDmqIWAfJ9A5xwnKJWvxWMWwg/LoT0K6u6NyIhNfnt71m7/UDpBcsg81AOP+8p/sC8fudBbn75a6Yu+TEk6++6Nkti0tDQTA89Vlax3377ze/cDTfcwKxZs5g+fTpnnVXyS46ZM2eSn59f9CHd20cffcTtt9/OY489VnTupptuom/fvlx33XWcd955JCUllbhnwYIFLFiwgHPPPbdMr+Xuu+9m48aNTJw4sURAU9iOr7Zt23LyySfz1Vdf0b9/f8aPH0+/fv044YQTqFXL/T06NjaWMWPG8Oijj/Lcc8/5/f4KR2/Gjx9fpj77GjRoEFOmTGH58uXHdb+IEHi7in2bYce30HV4pXYnUinAiQB5BQ7eCZQbxIdhgAN2JEfBjdRAa7cf4MtNFZtx5+c9WSUCn+pg8uTJQZXv3bs3vXv3Zv78+ezcuZMmTZoAkJ+fz7PPPktiYiJXXHGF33316tXj/vvv96tr1KhRPP/887zxxhuMHj26xPXhw4eXObjJzc3lxRdfJDEx0S/o8G7HmzGG1157jdGjR7Ns2TJuuukmwAYxPXr0YOTIkVx//fV+gdf48eN57LHHmDVrFvfffz9RUXYiRk5ODnPnzqVWrVp+r6Wsmje3+4js2bPnuO4XkQAO7YE5F8LeTXD+Y3DK8X0JIWWnKWoRwHsNTlLtGGrFVPM/9r2blF5RJAI4jhPw0bp1a9d7brjhBvLy8njuueeKzi1YsIBt27Zx1VVXuWYA69mzJ4mJ/ruhF66jcUuRfMopp5T5daxfv57Dhw+Tnp5OvXr1Arbjq1WrVixdupS1a9fy1FNPcfXVV9O2bVuWL1/OXXfdxYknnsimTZtK3NO+fXsGDRrEli1bWLRoUdH5119/nb1793LhhRfSsGHDMvfdW+HaptJSeItIEHIOwYsXwd6NgAMLboevZlZ1r2o8jeBEAO8paqnVPUX0vs0w+wI7WnP1fEhIKfUWkXDXtVlS6YXKaPW2/WTn5vudj68VzYnN/T98ByuUfT0el19+ObfddhszZszgrrvuIioqiunTpwO4Tk8DaNy4sev5whGg/fv999cqvFYWhfeX1k4gXbp0oUuXLkXH69evZ9y4cXz++ef86U9/8lv0P2HCBJYuXcrMmTM577zzADtFD45/ehrY9UDAcQdIIuIitg40722npwHUawWdzq/aPkUABTgRwDvJQLXOoPbbFpg9FA78ah/PD4Xfz4e6+s9WarZQrWkBm2Dg5pf9RyQevugkhnUP/4WvderUYcyYMTzxxBMsXryYbt26sXDhQvr06UP37t1d79m1a5fr+Z077Z4VbqMuwYxiFN5fWjtl1blzZ+bMmUP79u354IMP/K6PHDmS1NRU3n77bXbt2sWhQ4dYtmwZ7dq144wzAuzTUQZLly4FoE+fPsddh4j4iIqGC/4GCQ3tyM3VbygJQSWo5nOVJBTySmzyWY0DnKNZkJddfJx/FJyCwOVFxM+w7s2YekUPOjdJJCbK0LlJIlOv6FEjgptC119/PcYYpk+ffszkAoVWrVrFwYMH/c4vW7YMgB49epSrP507dyY+Pp5vvvnGdTSosJ1gFE6pc0uJXbjOJjc3l+eff56ZM2fiOA7jx48/7ullu3fvLhoJGzVq1HHVISIBGAOn3w03LofU9lXdm4igACcCeK/BSa7OKaIbdYEx/4W6jSG5HYx+GxLdp3yISGDDujdj4cSB/PyX81k4cWCNCm4AOnTowJlnnsk777zDtGnTqF+/PpdddlnA8vv37+fBBx8scW7FihW8+OKL1KtXjwsvvLBc/YmNjWXUqFEcPHjQL8lAYTu+Nm3axNSpU10DIsdxeOihhwAYOHCga5uFU9FmzJjB7NmzizKsHY9vv/2Ws846i4yMDM4//3yGDRt2XPWISCkCTbs/tNsmIpCQ0RS1CFByDU41HsEBaNgJxiywc1aTmlZ1b0SkmrrhhhtYsmQJu3bt4o9//CPx8fEByw4cOJCZM2fy5Zdf0r9/f3bs2MG8efMoKChg+vTpfpnKjsdf/vIX3n//fZ588klWrFjBgAEDito5//zzeeutt0qU379/P7fccgt33HEH/fv3p1u3biQmJrJ7924++OADNm7cSKNGjfjb3/7m2l6nTp0YOHAgH330EQAXXXRRwDVAhTZv3lwUgOXm5pKRkcHKlSuLNkYdNWpU0SiOiFSSIwdg7kWQcwDO+Su0Hwwx1fyzWs5B+HUV/LoCktvCCeX7kqgiKMCJMNV6DU4hDd+KSCmGDRtGamoqGRkZx5yeBtCmTRumTZvGXXfdxbRp08jJyaFnz57cf//9nHPOOSHpT2pqKp9++in33HMPb7/9NitWrKBTp04888wzpKWl+QU4Xbp04Y033mDx4sV88cUXzJs3j7179xIfH0/79u255557mDhx4jEX/E+YMKEowJkwYUKpffzll1+KUnPXrl2b+vXr06FDB26//XZGjRpFenp6OX4DIhK03CPwypWwc7U9fuUKuHVd9V+j80w/u24aoMM51TLAMW7ze6XmMMasrNW4Xc+mY54CqD5z8Q/vhQ8fgcEP2NEakQiwY8cOmjbVyGQobNy4kfbt29O/f38+/vjjqu6OVAP69yVhZ98vNnPs/q32uMUpcO17/uWy94GJhtqVkMVy/692ZObXldDiZOgy1L/Mq6NhrSe7Y3wK3LGhaIPTXr16sWrVqlWO4/Sq+M4GpjU4ESalOozgZO+DOSPgy2nw0qU2uYCISBAef/xxHMcp2iBTRCTsNGgN4xbZURAT5R5MgM2+9lg7eOky+Hou5OeVv+0C/+0EcBx4pi+8+nv49ClYO9/93ha9i59HxUJWRvn7E2KaohZhqjyLmuPAvKuL88Fv+gjm3wiXzK7SbolI9bdlyxZeeuklfvrpJ2bNmkX37t255JJLqrpbIiLHr15zGPWqTTQQHeteZu1bNrPsjwth11pIP45Mh7nZ8O6dkPEj7PkBTr0N+vl8QWQMNO8FGzzp6betcK+r8wVQr6UNdJKaF43eVCcKcCJMla/BMQYG3Q0vroLcLIhPhdPuqto+iUhY2LhxI3fffTfx8fGcddZZPPPMM0RFaSKCiNQAdRu5n9+7qXiNDthRHreAYucayPjBTnvrOhxS2pW8HlMbvn/DJjMAW9ZN896eAMfYe3Kz/ZcSJLe1j2pMAU6ESY6vBlPU0vrbja7+Mx6ueBkada7qHolIGBg0aJDrvjAiIjVW3cZ2lsvat+CnxdA1QBr39x+EnxbZ54lN/AMcYyC1o11fA7DnR/d6ul9uP6c161k5a34qiAKcCFI/PpaY6GrybWerPvDHlYGHY0VEREQiXa14m6XshAtt1rVoly+qjxyAjUuLj/cEGp3pZTdQb9gZmvd0L5PSzj84CkMKcCJIlSQYyMuBqBiIiva/puBGREREpGxia7uf/3mJXaNT6MCv7uXOfzT0faqmFOBEkJSEuMptMC8HXhllUwiOeNo9yBERERGR49d1OExYZkd4UjtCQkpV96jKKcCJIJWaQS3vqM2T/rMnn3tBLlz4L4jWXzkRERGRkImKhmY9qroX1Uo1WZAhlaFSA5xfV9jFcIXWL4A96yuvfRERERGJSApwIkhyZU5Ra90PLpll19/E1IYrX4Em3SqvfRERERGJSJovFEFSK3uTz67D4dI5EFML2g6q3LZFREREJCIpwIkgVbLJZ+fzK79NEREREYlYmqIWQSo9i5qIiIiISCVTgBNBKjXJgIiIiIhIFVCAE0EqfKPPvRth0b2w/RtwnIptS0TCVn5+PjNmzOC0004jOTmZ2NhYGjVqxEknncS1117LW2+9RUFBAa1atcIYw9q1a49ZX3Z2NvXr16dWrVrs3r0bgDFjxmCMKXpER0dTr1492rVrx4gRI/jnP/9JZmZmZbxcERGpZFqDE0E+/imDET2aV1wD370Gn//TPlI6wOi3IalpxbUnImEnPz+fIUOGsHDhQurXr88FF1xAixYt2Lt3Lxs2bOCll15i/fr1DBs2jHHjxjF58mRmzpzJ3//+94B1vvrqq+zfv5+LL76YRo0albg2fPhw0tPTATh48CBbt27l448/Zv78+dx777089dRTjBkzpiJfsoiIVDIFOBFk4rxviIoyDOveLPSVOw589+/i4/wcSGwS+nZEJKy9/PLLLFy4kO7du/Phhx9Sr169EtcPHz7Ml19+CcA111zDlClTmDNnDg8//DC1armPQs+cOROACRMm+F0bMWKEXwCTl5fHc889xy233MLYsWOJi4vjiiuuCMGrExGR6kBT1CLM00t/rpiKs/dBtFcSg24XgzEV05aIhK3PPvsMsFPIfIMbgPj4eE4//XQAWrZsybnnnktGRgZvvPGGa33r16/nk08+oU2bNgwePLhMfYiJiWHChAk8/fTTANx6661kZ2cfz8sREZFqSAFOhPl596GKqTg+Ga7/BG74Ak69HbpfXjHtiEhYS0lJAeDHH38sU/nx48cDxaM0vgrPX3vttZggv1QZPXo0rVu3ZufOnXzwwQdB3SsiItVX2AY4xpgWxpjnjDHbjTE5xpjNxpgnjTENgqwn2XPfZk892z31tnApm2KMudYY84Yx5mdjTLYxZr8x5hNjzDXGGL/fpzEmzRjjHOPxSnl+D8Fq36huxTbQqAuceR807FSx7YjURLMuKH58/aJ7mUX3Fpf5723uZda8XrKuo4f9y+zdVLLM5k/c65p3VXGZEBg5ciSxsbFMmzaNq6++mv/85z/88ssvAcsPGTKEZs2a8f7777Np06YS144ePcoLL7xATEwM48aNC7ovUVFRnHrqqQAsX7486PtFRKR6Css1OMaYdsBnQCNgPrAeOAW4BTjXGNPfcZxS0+MYY1I89XQEPgBeAToDY4ELjDF9HcfZ6HXLJcAzwA5gKbAFaAyMBGYC5xljLnEc1xRi3wJvupxfU/orDp0bTm9fmc2JSDB+8Qoy0ga4l9n1fXG5/KPuZQ5sL1mXk+9fJvdwyTKHA7xlbv0KDu0M3Ocg9ejRg7lz53LLLbcwd+5c5s6dC0BycjIDBw5k3LhxDB06tKh8dHQ048aNY8qUKTz77LNMmTKl6Nr8+fPZs2cPF154IU2aHN+av+bNbeKVPXv2lONViYhIdRKuIzhPY4Obmx3HGeE4zl2O45wBPAF0Ah4qYz1/wQY3TziOc6annhHYQKmRpx1vPwLDgBaO44xyHOdux3HGYYOircBF2GDHzTeO4zzg8ngtiNd9fAx0bpLI1Ct6VEyCARGRIFx66aVs2bKFRYsWcd999zFkyBAKCgp48803GTZsGKNHj8b7e6Jrr72WqKgoZs2aRX5+cbA2Y8YMwD25QFkVthPs9DYREam+wm4ExxjTFjgb2Az8n8/lScAE4GpjzG2O42Qdo54E4Gogy3Oft38CfwLOMca0LRzFcRzHdZK24zg7jTHTsIHVIOD1IF9WherWrB4LJw6s6m6ISGlae43a1G/lXqbxCZCfa5836uxeJqlZybpMtH+Z2PiSZeJT3OtqeTIc3he4z8cpNjaWs88+m7PPPhuw6aNff/11xo0bxwsvvMCFF17IiBEjAGjdujVnnXUWixYtYsGCBQwdOpTNmzezZMkSWrduXVTH8di+fTsADRs2LP+LEhGRaiHsAhzgDM/PxY7jFHhfcBznoDHmU2wA9Dvg/WPU0xeo46nnoE89BcaYxdhg6XRgo8v9vjyfOMgLcL2ZMeYPQAqQCXzuOM7qMtRbvf24GD5+HE68BLqOgLr6kCBy3Mb+t/Qy55RhgLrbRfZxLMltytbeZXNLLxMC0dHRXHrppXz33XdMmTKFDz74oCjAATtKs2jRImbMmMHQoUN59tlncRyHa665hqio45uMUFBQwEcffQRAnz59QvI6RESk6oVjgFO4ej1QCp6fsAFOR44d4JSlHjz1HJMxJgb4vedwYYBiZ3ke3vctA0Y7jrOltDbK0Ae3dT8V77tXYeuX9rHoXrjjZ6idVCVdEZHwl5iYCIDvUsZhw4bRpEkTFixYwNatW5k9e3bR+pzjNXv2bLZs2ULTpk2LUlOLiEj4C8c1OIUbJ+wPcL3wfP1KqgfgYaAbsMBxnEU+1w4DfwZ6AQ08j9OwSQoGAe97psuFn6NZsH5B8XGbUxXciMgxvfzyy7z33nsUFBT4Xdu5c2fRupqBA0tOq42JiWHMmDHk5+czatQotm3bxvnnn1+UJCAYeXl5zJgxgxtvvBFjDE888QS1a9c+vhckIiLVTjiO4JSmcKVoeUc0ylSPMeZm4DZsJrerfa87jrMbuN/n9EfGmLOBT4A+wLXAU+XprOM4ritkjTErgZ7lqTugo4eh+2Xw/Rt2o88TL6mQZkSk5vjyyy956qmnaNKkCQMGDKBNmzYAbNq0if/+979kZ2czfPhwLr74Yr97x48fzyOPPMLHH38MlC25wJtvvsnmzZsByMrKYsuWLXz88cfs2LGDevXqMX36dC677LLQvUAREaly4RjgFI6s+G+BbSX5lKuweowxN2IDk7XAmY7j7C2lzSKO4+QZY2ZiA5yBlDPAqRJ1G8KQJ+DcR2DjUmjdr6p7JCLV3G233UaHDh1YsmQJq1evZtGiRRw5coSUlBQGDRrElVdeyZVXXuma1axt27aceeaZLFmyhBYtWnDeeeeV2t78+fOZP38+UVFRJCQk0LBhQ0455d/eYCwAAB8mSURBVBQGDx7MlVdeSXJyckW8TBERqULhGOD84PkZaG1MB8/P0rbJLlc9xpiJ2LTUa7DBze5S2nNTuPFCeE5RKxRTCzqeU9W9EJEw0LJlS2688UZuvPHG47r/vffeK1O52bNnM3v27ONqQ0REwls4rsFZ6vl5tjGmRP+NMYlAfyAb+KKUer7wlOvvuc+7nihsogLv9ryv34kNbr4BTj/O4AZspjcoW5Y2EREREREpRdgFOI7jbAAWA2mA71eAk7GjIS9474FjjOlsjCmxYYTjOIeAOZ7yD/jUc5On/kWFe+B41XUfNqnASuzITcax+muM6WOMqeVy/gzsXjsAlZOHVURERESkhgvHKWoANwCfAVONMWcC67BrWU7HTim716f8Os9P30nd92Azmd1qjEkHlgNdgOHAbnwCKGPMaOBBIB/4GLjZZZ74ZsdxZnsdPwKc4EkJvc1z7iSK9/O5z3Gcz0p7wdVKzkEwUVArvGfWiYiIiEjNE5YBjuM4G4wxvbHBxrnA+cAOYCowuayL/R3HyTTG9AUmASOAU7GbcM4C7nccZ5vPLW08P6OBiQGq/RCY7XU8B7gQOBk4D4gFdgGvAv90HOfjsvS1WlkxC5Y9DJ0vgBMvhvaDIcplp3QRERERkUoWlgEOgOM4W4GxZSzrmkLZc20vcIvnUVo9D+A/na20e54Fng3mnmpvzWuQm2U3+dz0Idy6rvR7REREREQqQditwZEqtudH2PFt8XG3izR6IyIiIiLVhgIcCU5SMxgxDdqdadfhnOi/GZ+IiIiISFUJ2ylqUkXi6kL6FfZxaA8kpFZ1j0REREREiijAkeNXt2FV90BEREREpARNURMRERERkRpDAY6IiIiIiNQYCnCkbPb8CNtWgONUdU9ERERERALSGhwpm8+mwtdzoEEanHQZDLobTMDthUREREREqoRGcKR0eTmw9i37fN9m2PypghsRqRTGGAYNGlTuegYNGoSJgPet5557DmMMy5cvL3E+VL/HyjB06FDatWvH0aNHq7orIhKmFOBI6bZ8Djn7i4+1942IlIMxJiyCjdKCgsLXEegxe/bsorIPPPBA0fnRo0cHrPPDDz8sKpeWlhZUfw8dOsT//u//MnToUE455ZSg7q1Mhb+LZcuWuV7/85//zKZNm5g6dWrldkxEagxNUZPStR0Ef1wF370Ga9+ErsOrukciEiHWrVtHfHx8VXfjmCZNmuR6Pj093e9cTEwM//73v3nqqaeoX7++3/UZM2YQExNDXl5e0P2YOnUqO3bs4K677gr63uokPT2dc889l4ceeojrr7+ehISEqu6SiIQZBThSNintYNCd9iEiUkk6d+5c1V0o1QMPPFDmskOGDOHNN9/kxRdf5MYbbyxxbd++fbz++usMHTqUN954I6g+5OfnM23aNDp06EC/fv2Curc6Gj16NO+++y4vv/wy1157bVV3R0TCjKaoiYhItRVomtiOHTsYO3YsjRo1ok6dOqSnp/P888+zbNkyjDEBg468vDz+8pe/0KFDB+Li4mjZsiV33nlnifUes2fPLppC5z1l7Fj1ltW5555LixYtmDFjht+1OXPmcOTIEcaPHx90ve+99x5bt27lsssuO2a57du3c/XVVxf93nr16sVLL70UsPyiRYs4//zzSU1NJS4ujnbt2nHHHXfw22+/+ZVdvXo1V1xxBWlpacTFxdGwYUN69uzJxIkTyc3NBSAtLY3JkycDcPrpp5f43XobPnw4tWvX5tlnnw32VyEiohEcEREJL7t376Zfv35s3ryZgQMH0q9fP3bu3MkNN9zA2Weffcx7r7zySj7++GPOO+88kpKSWLBgAY8++ii7d+9m1qxZgJ0iNWnSJCZPnkzr1q0ZM2ZM0f3lXagfHR3NuHHjePDBB1mxYgW9e/cuujZjxgzS0tIYPHhw0PUuWbIEgAEDBgQss2/fPvr160f9+vUZO3Ysv/32G6+++iqjRo3i119/5Y477ihR/sEHH2TSpEkkJyczZMgQGjVqxOrVq3n88cdZsGABn3/+OUlJSYANbvr06YMxhmHDhtGmTRsOHDjAzz//zNNPP82UKVOIjY1l4sSJvPnmm3z44YeMHj064Dqj2rVr06tXLz7//HP2799PvXr1gv6diEjkUoAjIlJNXDb986DK146N5vlx/ovJ3/52O3O/+CWoui7u1YJLerf0O3/dnJXsO2xHN+b9oW9QdVaUu+++m82bN/M///M/PPLII0XnJ06cWOri+g0bNvD999+TnJwMwEMPPUT37t154YUX+Otf/0qTJk1IT08nPT2dyZMnk5aWVuqojdv1tLS0EoGRt2uuuYYpU6YwY8aMogDniy++YM2aNUyZMuW4EjB88sknACUCJl+rV6/mkksu4ZVXXiEqyk7guOuuu+jVqxf33nsvF110EW3btgVg6dKlTJo0ib59+7JgwYIS64Vmz57N2LFjmTRpEk/8f3v3HmZHVSZq/P1yIbQBmiYgCUQwXEQEYSQokCAEcUACQWYEUeQSEFCYI4PKIwfRE+NRT2AAJTCBkbvICbeRIJAERiDhHs8B5DZy8ZzEEAkIIWnInZA1f1R12+ns3d272Z3dXfv9Pc9+Kl21Vu1VK6tq11e1atXPfw7ADTfcwMqVK5k2bRpf/OK6z2kuXry49Tmqs88+myVLljB79mzGjx/fYcD46U9/mkcffZRHH32UsWPHVlwnkuqXAY7K+8/fQvOrsNs/wmbDal0aqfDmzH27ovSDN+pfcv7rzSsrXte+OwwpOf+p+Yv567urKlpXT1q9ejVTp06lsbGRH/zgB+ss23PPPTnxxBO5+uqry+a/4IILWoMbgMGDB/O1r32t9Y7KEUccUXGZWrpctXXggQeWDXC22247DjnkEKZOncoll1zC4MGDueqqq+jfvz8nn3xyxd8PMH/+fAYOHMiQIaX/HyG7e3TBBRe0BjcAI0aM4KyzzmLixInceOONrQMmtIxgdtVVV603GML48eO59NJLuemmm1oDnBYNDQ3rfW9TU1O3tmno0KGt2yZJlTDAUXmP/yu8+gTce342ctqXb6h1iSTVuZdeeokVK1aw9957s+mmm663fP/99+8wwCl1h+MjH8nuXC1evLhbZUopVZzntNNOY+bMmdx8880cc8wx3HLLLRx++OFss8023RpBbdGiRZ0GEttttx0jRoxYb/6YMWOYOHEiTz/9dOu8xx9/nIEDB3Lbbbdx2223rZdn9erVvPnmmyxatIghQ4Zw7LHHcumll3LUUUdx9NFH8/nPf57Ro0ez4447VrwtLVoC0bfeeqvb65BUnwxwVNriP2fBDQAJGrp3BU5S1+0zYovOE7Wx8cDSd3CGNm5c8bqGN61/5R1gr+2aWruo9QbNzdk7ubbeeuuSy8vNb1FqaOYBA7Kfwvfff/8Dlq7rjjzySIYOHcrVV1/Ne++9x7Jly7o1uECLhoYGVq5c2WGacnXTcqekpW4hC5jWrFlT8u5UW0uXLmXIkCF85jOf4eGHH+anP/0pt99+OzfeeCMAu+yyCxMmTOCrX/1qJZsDwIoVK4DSd4UkqSMGOCrt7f8Hg7eCZW9mf3/ymNqWR6oD1XrGZdye2zBuz22qsq4rTxhZlfVUS8tD7W+88UbJ5eXm9zYDBgxg/PjxTJo0iQULFjB8+HAOO+ywbq/vwx/+MK+88grvvfceAwcOLJmmXN28/vrrAOs8yN/Y2MjatWt5++2ud3Xcb7/9uPvuu1m1ahVPPvkkM2fO5LLLLuO4445jq622qnjwhEWLFgHZtklSJRwmWqXt+Dn4zotw/G/gM9+A7XrHw8WS6tvHP/5xGhoaePbZZ3n33XfXW97ysH019OvXr0fv6px66qlEBAsWLOCUU06hf//Sd+S6Yo899gCyLnzlzJ8/n3nz5q03f9asWQB86lOfap237777snjxYl544YWKyzJo0CBGjRrFj3/849Znee68887W5S3b2Vndvvjii0DpF6ZKUkcMcFRe/wGw08Ew9kLoZ1ORVHsbbbQRxx57LM3NzfzkJz9ZZ9kzzzzDr371q6p915AhQ3j11Vertr72dtxxR2bOnMkdd9zBWWed9YHW1TIa2RNPPFE2zfvvv8+5557L2rVrW+fNnTuXyZMnM2DAAI4//vjW+d/+9reB7Fmh1157bb11LVu2bJ3vevjhh9fp4tai5a5RyyhqQOtACJ0NHvDEE0+w5ZZbsvvuu3eYTpLas4uaJKkmyo0yBjBlypR1TorbmjRpEg888AAXXnghc+bMYdSoUSxcuJBbb72VsWPHMm3atHVGCuuugw8+mJtvvplx48YxcuRIBgwYwAEHHMABBxzwgdfdorP39nTVUUcdxdlnn829997LqaeeWjLNHnvswZw5cxg5ciSHHHIIzc3N3HLLLSxZsoQLL7xwnQEBDj74YCZNmsR5553HzjvvzNixYxkxYgRLly7lz3/+M7Nnz2b//fdn5syZAFx88cXcd999jBkzhh122IFNNtmEF154gRkzZtDU1MTpp5/euu6DDjqIfv36cd555/H888+3Do7QdlS8l156ifnz53P66ad3a9hsSfXNAKderFkNK5fAgEGwcYkXpj3yc3jzJVixBIZ+Ej53/oYvo6S6csMN5Udm/MUvflE2wNl666157LHH+P73v8/06dOZM2cOu+yyC1OmTGHw4MFMmzat9VmdD+LSSy8lIrj//vuZPn06a9euZcKECVUNcKpl+PDhjBs3jrvuuovFixeXHFGtqamJGTNm8L3vfY/rrruOd955h0984hOcc845HHfcceulP/fccxk9ejSTJ0/mkUce4c4776SxsZFtt92W008/fZ08Z555Jk1NTcyZM4dHH32UNWvWMHz4cM4880y++93vsv3227em3XXXXbnhhhu46KKLmDJlSuvgCG0DnJa2ccYZZ1StjiTVj+jO8JbqOyLiyb22GbDXk6flJwqH/gz2+6f1E17997Dg99m/RxwAJ9214Qop1YmFCxcybJjvlOpJ559/Pj/72c+YOXMmhx56aK2Ls0E99thjjB49mksuuaS1i1lftGrVKnbYYQd23XVXfve733U5n/uXVHsjR47kqaeeeiqlVNMRanywoh6kv/W3ZsWS0mkaNu88jST1EqWeC3nuueeYPHkyW2yxBQceeGANSlVbo0aN4phjjuGCCy5g+fLltS5Ot11xxRW8/vrrXHzxxbUuiqQ+yi5q9SDajMyzskzwsukw2Gx4Fuhs+bENUy5J6qa9996bnXbaid13353BgwfzyiuvcM8997B27VquvPJKNt5441oXsSYuuugirr32WubOnctuu+1W6+J0y6BBg7jmmmvYc889a10USX2UXdQKLiKe3Otj2+715C/Pyl7WOXQP2G6fWhdLqkt2oameiRMnMm3aNObNm8e7777L5ptvzr777ss555zTOqKY6ov7l1R7vaWLmndw6sEmW8OB36t1KSSpaiZMmMCECRNqXQxJUi/kMziSJEmSCsMAR5IkSVJhGOBIkiRJKgwDHEmSJEmFYYAjSRtISglHrpSqz31LUlsGOJK0gTQ0NPTpFzBKvdXy5ctpaGiodTEk9RIGOJK0gTQ2NrJkyRKWLVvm1WapClJKLFu2jCVLltDY2Fjr4kjqJXwPjiRtIP369WPYsGE0NzfT3NxMRNS6SFKfllKioaGBYcOG0a+f12wlZQxwJGkD6tevH01NTTQ1NdW6KJIkFZKXOyRJkiQVhgGOJEmSpMIwwJEkSZJUGAY4kiRJkgrDAEeSJElSYRjgSJIkSSoMAxxJkiRJhRG+TbvYImJRQ0PDFrvuumutiyJJkqQC++Mf/8iKFSveTikNqWU5DHAKLiLmApsB82pclL5sr3z6VE1LURzWZ3VZn9VlfVaX9Vld1mf1WafVtRdASilqWQgDHKkTEZGg9jtrUVif1WV9Vpf1WV3WZ3VZn9VnnVZXb6lPn8GRJEmSVBgGOJIkSZIKwwBHkiRJUmEY4EiSJEkqDAMcSZIkSYXhKGqSJEmSCsM7OJIkSZIKwwBHkiRJUmEY4EiSJEkqDAMcSZIkSYVhgCNJkiSpMAxwJEmSJBWGAY4kSZKkwjDAkSRJklQYBjiSJEmSCsMAR5IkSVJhGOBIkiRJKgwDHEmSJEmFYYAjSZIkqTAMcCRJkiQVhgGO6lZEDImIUyPijoj4U0SsiIjmiHgkIr4eEf3apf9oRKQOPjfXalt6i4iY10H9vF4mz6iImB4Rb0fE8oh4NiLOjoj+G7r8vUlEjO+kvaWIeL9NetsnEBFHR8RlEfFwRLyTb/uvO8lTcRuMiCMiYlZ+zFgaEXMi4qTqb1FtVVKfEbFzRJwbEQ9ExKsRsToi3oiIOyPioDJ5Omvn3+zZLdywKqzPbu/TEXFSRPw+b5vNeVs9oue2rHYqrNPru3Bcvb9dnrppo1HheVGbfL3uGDqgWiuS+qBjgCuAhcCDwHxga+AfgauBwyLimJRSapfvGWBaifU934Nl7UuagV+UmL+0/YyI+CLw78BK4BbgbWAc8HNgNNn/Ub36AzCxzLLPAp8DZpRYVu/t8wfAnmTtbQHw8Y4Sd6cNRsR/Ay4DFgG/BlYDRwPXR8QnU0rnVGtjeoFK6vN/AscC/wlMJ6vLXYAjgSMj4p9TSpPL5L2TrM2393+7We7eqqL2maton46Ii4Dv5uu/CtgI+ApwV0R8K6V0eTfK3ZtVUqfTgHlllp0A7EDp4yrURxut+Lyo1x5DU0p+/NTlh+wEcRzQr938oflOnYAvtZn/0Xze9bUue2/9kP1wzOti2s2AvwKrgL3bzN8YeCyv66/Uept64wd4PK+fI9vMs31m9XAQsDMQwJi8Tn5dJm3FbTCv55X5D/NH28xvAv6U59mv1vVQo/ocD3yqxPwDyU5gVgHDSuRJwPhab2svrM+K92lgVJ7nT0BTu3UtytvuR2tdD7Wq0w7WsTmwPG+jW7ZbVjdtlMrPi3rtMdQuaqpbKaUHUkp3pZTWtpv/OnBl/ueYDV6w+nE0sBVwc0qp9QpYSmkl2RU5gDNqUbDeLCJ2B/YF/gLcU+Pi9DoppQdTSq+k/BezE91pg6cAg4DLU0rz2uRZDPws/7MwXVYqqc+U0vUppadLzJ8NzCK7kzCq+qXsOypsn93R0vZ+mrfJlu+dB/wrWds9uYe+uyaqVKcnAA3Ab1JKb1WpaH1ON86Leu0x1C5qUmnv5dM1JZZtExHfAIaQXYF4PKX07AYrWe83KCKOB7YDlgHPAg+llN5vl+5z+XRmiXU8RHY1bVREDEopreqx0vY938in15SoU7B9VqI7bbCjPDPapdHfdHRMBfi7iDib7MrvX4AHU0oLNkjJer9K9unO2ucP8zQTql7Kvu20fPrLDtLUexsttQ/32mOoAY7UTkQMAE7M/yy1A/59/mmbZxZwUkppfs+Wrk8YCtzYbt7ciDg5v5LbYpd8+nL7FaSU1kTEXGA3sj7Rf+yRkvYxEdEAHA+sJesPXYrts+u60wY7yrMwIpYBwyPiQyml5T1Q5j4nIrYHDiY72XmoTLJ/bvf3+xFxNXB2fjW4nnVpn46IwcC2wNKU0sIS63kln36sh8rZJ0XEfsAngZdTSg92kLRu22gH50W99hhqFzVpfZOA3YHpKaV728xfTvYQ7UiyvqJNZH3LHyS7ZXt//gNTz64jO5EZCgwm+9H4N7I+tzMiYs82aRvzaXOZdbXM37z6xeyzvkxWHzNSSq+2W2b7rFx32mBX8zSWWV5XImIQcBNZl5Qfte02lZsLfIvspGcwsA1ZO59Hdrfy2g1W2N6n0n3aY2r3nJ5Pryqz3DZa/ryo1x5DDXCkNiLiLLLRZ14k65PbKqX015TS/0gpPZVSWpJ/HgIOAeYAOwGnbvBC9yIppYl5H943UkrLU0rPp5S+CVxC1r/5RxWsLlpWW+1y9mEtP8T/1n6B7bNHdKcN2m5z+RCxN5KNpHQLcFH7NCml2Smly1NKL+fHjIUppdvIHhxfDHy13YWRutGD+3Tdt80WEdFIFqysBq4vlabe22hH50VdyZ5PN/gx1ABHykXEPwGXkg1xelBK6e2u5EspreFv3YUO6KHi9XUtDye2rZ/OrtJs1i5dXYuIT5A9oL2AbAjeLrF9dqg7bbCred75AOXq8/Lg5tdkQ8TeChxfyUPg+R3KlnZuu22jg326s7bZ2ZXzenQ88CG6MbhAPbTRLpwX9dpjqAGOBOQPDl5O9l6Bg/IRQyrxZj61C1Bpf82nbevnpXy6Xn/wvL/vCLKHGf9/zxatz+hscIGO2D5L604b7CjPMLI6XlDPz9/kdTeV7N0r/xs4Lj8pr5Tttrz16ialtIzs4fdN8rbY3s75dL1nH+pYy+AC690V76LCttEunhf12mOoAY7qXkScS/ZCqj+Q7cR/7SRLKfvmU0/GS9svn7atnwfy6RdKpD+A7KraY46gBhGxMVnXgLXANd1Yhe2ztO60wY7yHNYuTd2JiI2A28nu3PwKOKEbAXmLffKp7XZ95fZp22cXRcQ+ZC8IfTmlNKubqylkG63gvKj3HkNTL3ixkB8/tfqQDZmZyN5EvEUnafcBNiox/3NkL61KwKhab1MN63K3UnUIbE82ek8Cvt9m/mZkV7980WfndXtCXh93dZDG9rn+to+h8xd9VtQGya5I1s2LPiusz0Fk72ZKZF2o+nVhnZ8tMS+A8/L1vAlsVuttr1F9VrxPU4cv+qykTtulvSZP+91O0tVVG63wvKjXHkMjX6lUdyLiJLKHCt8HLqN0v+R5KaXr8/SzyE7iZ5E9BwGwB38br/2HKaWf9FiBe7mI+BHw38lG+JkLvAvsCBxOdrCbDvxDSml1mzxHkV3tXQncDLwNHEk2Ws3twJeTByki4mFgf+DIlNJdZdLMwvbZ0qaOyv8cChxKdnX14XzeWymlc9qlr6gNRsS3gMlkP9C3kD2gfDQwHLi47fr7ukrqMyKuI3vr+1vAFEo/JDwrtblaHhGJrMvU/yHrXtVINijB7mSjiP1DSum+qm5UDVVYn7Poxj4dERcD38nz3E72gtVjyd6j862U0uVV3agaq3Sfz/NsBrwGDAS2TR08f1NPbbTS86I8T+88htY6UvTjp1YfshG9UiefWW3Sfx24m2xoyKVkVyzm5zvneld46u1DNnzpVLKRVpaQvRTsTeA/yMbPjzL5RpMFP4uBFcBzwLeB/rXept7wAXbN2+KrHdWJ7bO1Hjrbr+eVyFNxGwTGAbPJAvllZCc/J9V6+2tZn2Qn4p0dU3/Ubv3/ktfja2QnSMvzY8jlwA613v4a12e392ngpLxNLsvb6GzgiFpvf63rtE2eM/JlU7uw/rppo12oy3XOi9rk63XHUO/gSJIkSSoMBxmQJEmSVBgGOJIkSZIKwwBHkiRJUmEY4EiSJEkqDAMcSZIkSYVhgCNJkiSpMAxwJEmSJBWGAY4kSZKkwjDAkSRJklQYBjiSJEmSCsMAR5IkSVJhGOBIklQlETE+IlJEXF/rskhSvTLAkSRJklQYBjiSJEmSCsMAR5IkSVJhGOBIkvqs/HmXlP/76xExJyLeyedvns//RET8OCIei4jXImJ1RLwZEdMj4gtl1tv6LE1EbBoR/xIRcyNiVUT8JSKuiIgtKizrbhExP1/v+R986yVJpRjgSJL6vIi4DPglsAq4G3gSSPni7wA/BDYHngHuAOYBhwEzIuI7Hay6EXgUOAX4A3Af8CHgm8B/RMTALpbvoHw9Q4ETU0o/rWDzJEkVGFDrAkiSVAUnAPullH5fYtmNwE9SSvPazoyIfcgClkkRcWtKaUGJvEcB04FRKaWleb5tgCeAvYAvAzd1VLCI+BpwLbASOCyldH8lGyZJqox3cCRJRXBhmeCGlNLs9sFNPn8OcDkwEPhimfUuBb7eEtzk+V7L8wEc3FGhIuI8sgDrTWB/gxtJ6nnewZEkFcFvOloYEZsChwN/B2wBbJQv2jmffqxM1idTSq+XmP9iPt2mTL7+EXEl8A3gOWBsmTtEkqQqM8CRJBXBn8stiIgvknUR62hQgM3KzJ9fZv47+XTjMsu/QvYbuxD4bEqpuYPvliRVkV3UJEl9XkppRan5ETEcmEoW3EwC9iALZvqnlILsDgtAlFn12m4W6WGygQyGAf8rIsqtX5JUZQY4kqQiOwJoAP49pXReSum5lNK7KaWWwGWnHvre+cBngZeBM4BrI8LfXEnaADzYSpKKrKVb2qvtF0TEIOBLPfXF+TM3BwDPA+OBmyLCruGS1MMMcCRJRdYyGMCXImLrlpkRsRFwGbBDT355SukNYAzZe3m+AtyWf7ckqYcY4EiSiuy3wNPAR4BXIuK3EXErMBc4Dpjc0wVIKS0iG076MbL36twZEQ09/b2SVK8McCRJhZVSWgMcCFxINqLZIWTPxjwEjCQLfjZEOZrz734A+AJwT0QM3hDfLUn1JlJKtS6DJEmSJFWFd3AkSZIkFYYBjiRJkqTCMMCRJEmSVBgGOJIkSZIKwwBHkiRJUmEY4EiSJEkqDAMcSZIkSYVhgCNJkiSpMAxwJEmSJBWGAY4kSZKkwjDAkSRJklQYBjiSJEmSCsMAR5IkSVJhGOBIkiRJKgwDHEmSJEmFYYAjSZIkqTAMcCRJkiQVxn8BiskwxSmp32kAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "image/png": { + "height": 281, + "width": 412 + }, + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "ax = hsvd_rank_scores.sort_index().plot(label='HybridSVD', marker='.')\n", + "svd_scores.groupby('rank').max().plot(ax=ax , label='SVD', ls=':')\n", + "ax.axhline(study.best_value, label='LightFM (best)', ls='--')\n", + "ax.legend()\n", + "ax.set_ylabel('precision')\n", + "ax.set_title('Item Cold Start');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result demonstrates that HybridSVD is very capable and has a high potential to significantly outperform all other models. It seems that the quality of HybridSVD can be further improved with higher rank values. There is also some room for improvement in careful tuning of the scaling parameter and the weighting coefficient $\\alpha$. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Interestingly, you can achieve the LightFM's quality with HybridSVD of rank 10, while the standard SVD-based model is unable to get even close to LightFM at any rank value!
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here is a quick check:" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HybridSVD's precision at rank 10 is 0.0366\n", + "LightFM's best precision score is 0.0344\n" + ] + } + ], + "source": [ + "hsvd.rank = 10 # Polara automatically truncates latent factors in SVD-based models\n", + "print(f'HybridSVD\\'s {target_metric} at rank {hsvd.rank} is '\n", + " f'{find_target_metric(hsvd.evaluate(), target_metric):.4f}\\n'\n", + " f'LightFM\\'s best {target_metric} score is {study.best_value:.4f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is time to finally verify the obtained result on the test data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "# Evaluation of models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preparing data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to disable the Polara's built-in data splitting mechanism, we will employ two handy methods `prepare_training_only` and `set_test_data`. The former will instruct the `data_model` to utilize the whole training data for training without splitting it, and the latter will inject the test data, ensuring its overall consistency. " + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing data...\n", + "Done.\n", + "There are 57830 events in the training and 0 events in the holdout.\n", + "Done. There are 4307 events in the holdout.\n" + ] + } + ], + "source": [ + "data_model_sim.prepare_training_only()\n", + "data_model_sim.set_test_data(holdout=test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that in the data splitting provided by the `fetch_stackexchange` function some of the items belong to both the training and the test set. We will ignore it and **treat all items in the testset as cold items**. Recall that in our evaluation setup we generate a list of candidate users for every cold item and than verify the list against the actual interactions present in the holdout." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preparing the models to compare" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to ensure that all models use the same data to avoid any accidental discrepancies in the testing procedure. After that we train the models once again with the values of hyper-parameters, found during the tuning phase, and report the final results. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### LightFM" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LightFM training time: 4.090s\n" + ] + } + ], + "source": [ + "lfm = create_lightfm_model(data_model_sim, item_tags, num_threads, seed)\n", + "lfm.rank = study.best_params['rank']\n", + "lfm.item_alpha = study.best_params['item_alpha']\n", + "lfm.fit_params['epochs'] = study.best_trial.user_attrs['epochs']\n", + "lfm.build()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SVD" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PureSVD(cs)-s training time: 0.112s\n" + ] + } + ], + "source": [ + "svd = ScaledSVDItemColdStart(data_model_sim)\n", + "svd.col_scaling = svd_best_config['col_scaling']\n", + "svd.rank = svd_best_config['rank']\n", + "svd.build()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### HybridSVD" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updating items relations matrix\n", + "Performing sparse Cholesky decomposition for items similarity\n", + "Cholesky decomposition computation time: 01m:47s\n", + "HybridSVD(cs)-s training time: 52.765s\n", + "Building items projector for HybridSVD(cs)-s\n", + " Solving triangular system: 1.852s\n", + " Applying Cholesky factor: 48.022s\n" + ] + } + ], + "source": [ + "hsvd.rank = hsvd_best_rank\n", + "hsvd.build()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analyzing the results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let us first gather all the scores into a single dataframe, which is convenient to anaylise." + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "from polara.evaluation.evaluation_engine import consolidate_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [], + "source": [ + "all_scores = {}\n", + "all_scores['SVD (best)'] = svd.evaluate()\n", + "all_scores['LightFM (best)'] = evaluate_lightfm(lfm)\n", + "all_scores[f'HybridSVD (rank {hsvd.rank})'] = hsvd.evaluate()" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "all_scores_df = pd.concat([consolidate_metrics(scores, model, False)\n", + " for model, scores in all_scores.items()])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "During the tuning phase we focused on the `precision` metric. It is now time to also see other metrics as well, and particularly the `coverage` score. It is calculated as the ratio of all unique recommendations, generated by an algorithms, to all unique entities of the same type present in the training data. Obviously, the maximum value is 1 (100% coverage) and it can be typically only achieved with randomly generated recommendations. The `coverage` metric characterizes the tendency of an algorithm to generate the same recommendations over and over again and is, therefore, linked to the diversity of recommendations. Higher diversity allows mitigating the famous [\"Harry Potter\" problem](https://www.quora.com/Recommendation-Systems-What-exactly-is-Harry-Potter-Problem), sometimes also called the \"bananas problem\". I personally like the term \"[tyranny of the majority](https://medium.com/rtl-tech/my-takeaways-from-netflixs-personalization-workshop-2018-f564a19437b6)\". More diverse recommendations are likely to improve an overall user experience as long as the relevance of recommendations remains relatively high. It may also [increase overall product sales](https://en.wikipedia.org/wiki/The_Long_Tail_(book)). However, in practice, there is an inverse relationship between diversity and accuracy of recommendations: increasing one of them may [decrease the other](https://dl.acm.org/citation.cfm?id=2043957). The underlying phenomena is the [succeptibility of many recommendation algorithms to popularity biases](https://dl.acm.org/citation.cfm?id=2852083) in the *long tail-distributed* data. We have already tried to partially address that problem by introducing the scaling parameter for SVD. Note that SGD-based algotihms like LightFM also have some control over it via different sampling schemes and customized optimization objectives." + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
precision recall miss_rate nDCG coverage
SVD (best)0.02281180.197490.802510.1307380.016765
LightFM (best)0.03227570.2857040.7142960.205120.190934
HybridSVD (rank 135)0.03747260.3288950.6711050.2379910.104315
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(all_scores_df\n", + " .dropna(axis=1) # skip irrelevant metrics\n", + " .loc[:, :'coverage']\n", + " .style.bar(subset=[target_metric, 'coverage'], color='#5fba7d', align='mid')\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As indicated by the green horizontal bars in the table above, standard SVD-based model performs poorly both in terms of recommendations accuracy and in terms of diversity. Likewise, LightFM presents a trade-off between these two metrics. \n", + "However, we have seen that HybridSVD has not yet reached it's best performance. Note that **lower rank values make SVD-based models insensitive to frequent variations in the observed user behavior**, hence leading to a lower diversity of recommendations in the end. Let us try to increase the rank of the decompostition to verify that." + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HybridSVD(cs)-s training time: 02m:42s\n", + "Building items projector for HybridSVD(cs)-s\n", + " Solving triangular system: 7.070s\n", + " Applying Cholesky factor: 02m:28s\n" + ] + } + ], + "source": [ + "hsvd.rank = 400\n", + "hsvd.build()" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "# add new HybridSVD scores to the evaluation results\n", + "all_scores_df.loc[f'HybridSVD (rank {hsvd.rank})', :] = [\n", + " score for metrics in hsvd.evaluate() for score in metrics\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvwAAAIWCAYAAAArh1lrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3gU1dfA8e+hhdASqqBIgoiASg0KBsQgRelVxaAS0ICCJQhIUwKIBQUVfEX4gRCagI0oSpPeFKQoFlAQIiABqQFpgeS+f8zuutnsbjabQCCcz/Pss8nMvXfuzE7gzN1bxBiDUkoppZRSKnfKk9MVUEoppZRSSl0+GvArpZRSSimVi2nAr5RSSimlVC6mAb9SSimllFK5mAb8SimllFJK5WIa8CullFJKKZWLacCvlFJKKaVULqYBv1JKKaWUUrmYBvxKKaWUUkrlYhrwK6WUUkoplYtpwK+UUkoppVQupgG/UkoppZRSuZgG/EoppZRSSuViGvArpa4JIhIqIkZEjJ/5V9nyR2Vz1S4bERluq3NcTtflchKRONt5Ds/pumTGtVrv3EJEImzXPyGn66LU1S5fTldAKXX5iEghoBvQEqgJlAIM8A+wBYgHPjfGnMuxSip1lRGRYCAGwBgzPGdro/whIu2BWsAqY8yqHK6OUjlOA36lcikRaQP8DyjrtPkMkAqE2l6dgNEi8rgxZsWVruMVtg/4HUjK6Yqoq14wEGv7ebiXdIlY99TRy10hlWntsRo7AFblYD2Uuipolx6lciFbt5V4rGD/d+BxoJQxpogxphhWQNMZ6z/CG4FGOVPTK8cY84QxpqoxZn5O10XlDsaYwbZ76v9yui5KKeWNtvArlcuISA1gItYD/UKgs2uXHWNMEvA58LmIPAzcfMUrqpRSSqkrQlv4lcp9XgMCgL+ByIz65xtjPgHecd0uIgEi8qKIbBSRJBE5JyK/i8g7IlLWTVGISJRtEN0q2++PisgGETklIkdEZL6IVHNKX05E3heRBBE5LyK7RWSQiOTN6CRFpIGIfG0r96yI/Cgiz4qI23/XPA3adR3451TuUds5/2QrVzKoTxsR+VJEDolIsoj8IyILROSBDPJVEZE5tvTnRGSniMSKSEBG1yAjIlJERIaIyA+2z/C8iOwSkfEicrNL2kDb52REpHUG5e60pXveaVteEWksIuNEZIuIHLZdh4O2z/1+P+qf4aBM13vOZV85EXlGRL6xnfdZ2zluE5ERtr76rnlWAXudfjcur+FO+7wO2s2mv6E2IrJSRE6KyL8i8r2IPOrpenjjXF9b3YaKyHYROW3bHuySPtT29/m77dqdtn22A0WksIdjFBWRV2zpTjvdA5tF5G0RudNTnbzU2+cB9/Z7hv+688S6foYu6SuKyIci8oft8zkrIn/ZjjlYREpldEylrgnGGH3pS1+55AXchNVH3wAvZaGc0sBWWzkGOA+ccvr9OFDfTb4o2/5VwGjbzxdd8h4DbgMqA/tt204Bl5zSfOCm7FCn/Z1s5RrghNPPBpgP5HOTf5Vtf5TL9gjb9gRb/S/ZruFJpzIN8J6Ha5UfmOWSNsnl97c85G2ENa7COd8F288bgNdtP8f58RlWs52TveyLwL8un2EDlzwzbPs+9lJuHVuaS8ANTtvvdDnn8y7HM8AQD2XG2fYP9/TZeKmP455zs+8zl+OfAFKcft8NlHfJ8wVwxCnNIZdX/4zqnY1/Q6/Yfk5xcz/G+HFP2Ov7JrDR9nOyU9nBTmk7AuecjnfW6d40wHbnz9+WJwj41SlNiu08na/5m7589pn923XaFm77nOx1/9f1M3S5l50/k2TbPeJ8nR/0999RfenranppC79SuUsEYG+J/ioL5cwAamP95/cwUNhYff/vAn4GigPxXlq/agF9sWY6CbLlrYE1nqAEVsAxCyvgr2XbXwx42Zb/GdeWQBcfAcuAW4wxxbHGJLyEFai3t/2cWaWBScCHQDljTLDtPN+37X9eRO5wk+8toCtWcB0JFDXGBAFFgV5YAcUA11ZZESkOfAoUwgoMa9nyFcFqnawJ9PbjPBCRIKzuXCFYYznqAIHGmCJARWCm7dw+d2nV/dj23lasGZ7csZ/HCmPMYaftybbzaYM1dsR+vBuwAtcUYJSI1PPnnPy0C+ueusNWn+JAQay/kx+ASlifuYMxpiPWfW7/vazLa4yPx87q31BNrIHDrwAlbfdjWayHGIA3RKSEj3Vx1QfrobsLUMRWdijWwycichcwF+thdjTWfVQY616tj/WwUN12js5eAG7HemBqDQQYY0pgXfPbgEHAn37W2SfGmA3GmLLAPNumMa6foVPyMVh/pxuBOsaYArZ7pDDW5/QeOshf5RY5/cShL33pK/tewCj+a00UP8u4Fy+tW1gB3HHb/pEu+6Kc8sZmUPZxnFoUndIst+0f5rI91CnvL1jBhGve4fzXUl7YZd8qvLcSGmCyh2uy3UOdKmMFsiewHj7c5X3YXmeX7fbW26NYA6pd8z3mVK84P++DeE/3AfCNLY1zi3Ve4LBt+6Nu8gjWbEfprqMPdbKf7zQ3++K4DC38GdSnBNb0tAao6Oley6AMT/XOrr+hoW7yFnSq9xOZPOc4p7Kbe0m3zpamr4f9xbG6DBqgrtP2hbZtA/2o03AvaVa5u+e83R8+lnvWlqZeZq6jvvR1Lb60hV+p3KWk7f2EMcb4WUZn2/tmY8xi153GatWdaPv1YQ9lJONmXACwHuthBOBDY8xJN2mW2969tfCPNcZccLP9HVv5xYBmXvJ78oaH7V96qNMTWGOh4o0xezzk/QKrK8QdIlLOabv9Ok82xrib1nE28FfGVXarm+39XS/3wRzbu+M6GWNSsFrp4b+WfGcNsQZ4n8c6r8xYYHtvkMl8l4Ux5jhWtymAe7K5+Oz4GzqP1cLsmvc8sMT2q7e/EW+2G2OWutshIpWwPqNzTnV0rcMJYJHtV+e/s1O293JcG661+irlNw34lVKu6tjeV3pJY5+z/zYPg/cSjDGnXTcaY1L5b87yXzyUbe8mUtzL8Ve522iMOQVss/1ax10aL457Cdr/9lCncNt7Z7EG66Z7AQewukaAbTYkESmA1c0EYLWHczHAmkyeA2INxi1v+/VTL/Ua71wnJ/ZuPQ+66TISaXv/xnatXY8dKCJ9bQMe/xGRi04DJe2fy42ZPaesEJG7RWSqWAON/3UZvNnuMtUpO/6GfjPGnPGQ19P96KvvvOyz39MFgL1e7p8utnTO989C2/vzIjJTRFqISFE/63gl2Os7Q0TeFJH6IpLfaw6lrlE6LadSucsx23txERE/W/lL297/9pLmgO1dsFbvdQ1MEr3kTckgjX2/t/94vdXNvq+0lzTupHtAcWL/VsK1TvaWwSK2V0bs/eJLYHWfATjoJb238/TEubXSl2uQpq++MWaDiOzF6uvfCZgMICL5+K/l+mNc2L69WIXVV9vuDFZ3p1Ss8y2F1T/6ihCR/lhjLOzjWuzdr5JtvwdhdZHJ7jplx9+QP/ejr4542We/f/JidT3KiOP+McbMEJEGQE+sLmmPAakish3rG54PjTHe/m240gYAVbAecgbaXudF5Dusb7rijK5CrnIJbeFXKnfZYXsPwPqPLCuyPC1kDvE6fWY2s/8b+oIxRnx4rcpk+f6ci/O/60E+1CnUTRlzbe+RTtuaYQWmSVj9/129hxXs78F6UChhrIXeyhhroGR9P87Fb7YB1qOxruH/YX2jEmCMKWH+G7xpHwB7ue6Zq/VvKMXLPvv9s83HezrKObMxphdWV6ORWA+AF7AG8b8C7BIRf7raXRbGmGNY3dSaYX3jtQ3rm43GwATgFxEp77kEpa4dGvArlbusxhqEBtDWzzLsrX8hXtLY/xO0Dzq90rx1wbC3UHprxcwu9u5Ht2cyn32qQvDtXDLDeeaczNbLbrbtvZGI2Otn79P/hev4CVsXJXv3mK7GmC9s/byd+dJa7OqS7b2glzRBHrZ3wvo/bokx5jljzG+2MQpZrZMvroW/IU/s909l27c6mWaM+dUYE2uMaYw1g1YbrJmJCgPTXbrNZOUzzjJjWWaMecEYUwfrobYX1t/oLcC7l+vYSl1JGvArlYsYYw7wX7/U50SkmC/5RNIsKrXV9n6fy3Zn9kWU/vDSz/hyus/dRlt/YXv/6a3u0mQze1/oNpnp+2uMScaarxysufjTsV17t/syKHsv/wVtHTOb31bGr1gBWh6gi4gUxJruFNx058EKkuyt2dvc7Ado6kdV7IO6y9geKty5y8N2e0Dttj62fvOevnVIdUrnT+v/tfA35In9ni4CNM9qYcaYZGPM18BDtk3lsGa3srN/xm5b0m2fUzV3+zJg/wwz9fkZY04YY/4HDLFtcvtvjVLXGg34lcp9Xsb6Gr088LEtWPNIRB4GXnTaZO/mcAf/tdo6p78BeNr26ydZrq1/+nkIAGOwWgpPAW5nIclm07ECixuBwd4S2ubdd2afDSfaw3zqXbCmh/RHnO29tzitbOymTmKbs98de2D/KFYLbVGshYvcDUS1L14E1vzsrscpBzyXcbXT+QPrXhZbHVzLvRWrJd8d+/zp6epjMxTrnNxxHpCcbjVeH1wLf0NuGWN2At/bfh3tYUAx4BikHeD0u6eHMrBm/bFz7ur0s+29uYd/q/riX9co+2fo9vMTkTwZfINhr+/V2i1LqUzRgF+pXMYY8yPWwjoGaAVsE5HHnINKEQkSkY4ishJrgZqiTvnXAvapBKeKSGcRyWvLF4YVSBfHakUedyXOyY0KwHwRCbXVq5CIvIi1UBHAaGPM2ctdCWPMDv6bOnGEiHwgIrfY94tIERFpJiIz+S/At/sAaz71UsASEalhy5NfRB7DGizr76I/b2L1pS8MrBaRbiLiGFQsIjeLSDSwBejgoYyPsc2zzn8PM/PcdIvBGPMv/wWJU0Wklu04eUSkCVZXs0y3lNu+CbFPifquiDS0lZlHRJoD35I2kHT2re29lYgMEdtCYiJSWkTetp3TMXcZbdPF2gdTd/ej3tfC35A3z2E9aN0JrBWRpvbg2Hbt7xCRl7EW0XLudrZMRMaLSCMRCbRvtI2niLP9msh/QT5Yg3nPYQ10niEiZWx5gkRkKNbaGv78Hdi/QXtQ0k6Ha1cM2C0iQ0WkutPnY79nX7OlW+Imr1LXnss5yb++9KWvnHthdcGwL6Jkf50m7VLyBmuF2EYueUtjdYWwpznnku84cI+bY0aRwSJItuMZIMLDfrdlkHbhrU7ARdvPJ5x+ti82lc9NuavI5OI9vpwX1mwmE1yu6Sn+m53Gvm2lm7z38d/iPware8N5288bsNYFyPTCW7aybwV+cyo7BSvAPetS125eyljnkvZuL2nruZT9r9Pvx7Baut0uZoWXRZKw+lEfdSr3jO1+tE/1+byXz+Zzp3yptvvW/pl8lMFxR7icS4LtFeNjvS/n39Bwf+4Lb/V1k7aF7X601/eC7XNIdrknQpzy/Ohyvx13+qzsn10TN8d63qXME7b8BhiGfwtvlbLdd/a6JNo/Q9v+YJdjJtvSX3La9idQPrN/e/rS19X40hZ+pXIpY0w8VrDUB6tf/wGsqXjzYf3H9xnWLCxVjDFrXPIewVqMqB+wGSugLgDswmrRvsMY420u78vKGPM51kwa32D9Z34J+AmrZbKjMeaSl+zZXZcUY0xvrNk+ZmEtllUACMRalXY+1kJY7d3kXQ3UxvqW5QhW94EErIDufqwgy9967baV3RurG85xrFbNS1grB7+P9cAx00sxs51+/tMYs8nL8TZi3TPxWAFbfqxvMCZhzdLyk5/nsQfrYWIO1jXKi3Uvv4a1QFS69QCcPAIMwpq96iLWtwzrsR5ynszg0COxpmncbssXYnv51MXnav8byogxZhHWrEujsMYknMc691NYD6PDgGrGGOfF4Z7C+pZtJda9b2/l34k1U9KdxpjluDDGjMf6rL7HekjMg/U5dTDGjPSz/kex/o34Auu+Kc1/nyG282iN9VlssqUpivVQ8gNWl69axhoXpdQ1T4wxGadSSimllFJKXZO0hV8ppZRSSqlcTAN+pZRSSimlcjEN+JVSSimllMrFNOBXSimllFIqF9OAXymllFJKqVxMA36llFJKKaVysWwL+EWkvIhMFZGDInJBRBJE5D03y8l7K2OAiCy05f1XRE6JyM8i8o6IlPeQx3h5fe8uj1JKKaWUUteLbJmHX0QqYS3EUQZrGfSdwN1Yi178DjQwxrhdwtylnN1YKxr+hLVCaH6shWPuw1okI8IYs80lj8Fa6CbOTZEHjDFT/DsrpZRSSimlrn3ZFfAvAZoDzxtj3nfa/g7QF5hkjHnah3IKGmPOu9keDfwPWGSMaemyzwCrjTERWTsLpZRSSimlcp8sB/wicgvwJ9Zy8JWMMalO+4oCiVjLkpcxxpzx8xhBwElgtzGmsss+DfiVUkoppZTyIF82lHG/7X2pc7APYIw5LSLrsVr/6wPL/TxGG9v7dg/7g0WkB1AWSAK2GGMue/99EdkLFMN62FFKKaWUUupyCQVOGWMqZjZjdgT8VWzvf3jYvwsr4L8NHwN+EXkKKA8UAaoDTbH66Q/ykKUm8JFLGT8BjxtjfvblmBnUx+PXIIGBgVSrVq1EVo+hlFJKKaWUJzt27ODcuXN+5c2OgD/I9p7kYb99e3AmynwKqOf0+w9ApDFmt5u07wCfYz1wnAeqAgOBzsAKEalljPk7E8fOlGrVqrFly5bLVbxSSimllFKEhYWxdevWBH/yXol5+MX27vNgAWNMfWOMAKWwvh0A2CIiD7pJ288Ys8EYc9QY868xZrMx5iGsh4BSQP8s1h9jjLh7AVuzWrZSSimllFKXU3YE/PYW/CAP+4u5pPOZMeaYMeZbrKD/HDBDRAJ9zD7R9t4os8dVSimllFIqt8iOgP932/ttHvbbZ9Xx1Mc/Q8aYk8B3QGngDh+zHbG9F/b3uEoppZRSSl3rsiPgX2l7by4iacqzTcvZAKt1Pquz5txke7/kY/r6tvc9WTyuUkoppZRS16wsB/zGmD+BpVhTBfVx2T0Cq4V9hvMc/CJSVUSqOicUkRDbnP7piEgv4C5gP/Cz0/Y6IpKuBV9EagCv2X6dldlzUkoppZRSKrfIjll6AHoDG4DxItIE2IE1y05jrK48Q13S77C9i9O22sAXIrLBlucwUBKrpb468C/WNJspTnmeBzqKyAqsh4ELWLP0PAjkBSYDc7LpHJVSSimllLrmZEvAb4z5U0TqAiOxgu2WWCvsjgdGGGOO+1DMVuBd4F6gFVACa5rNPcBYYJwxZr9LnnisQcE1sBYAKwgcAxYBk40xX2Xx1JRSSimllLqmZVcLP7ZgvLuPacXNtn1Av0weMx4r6FdKKaWUUkq5kW0Bv1JKKXWtSU1NJSkpiXPnziGSri1KKaUuG2MMgYGBBAUFkSfP5V0aSwN+pZRS16XU1FQSExMJDg4mODhYA36l1BVljOHs2bMkJiZSrly5yxr0X4mVdpVSSqmrTlJSEsHBwRQuXFiDfaXUFSciFC5cmODgYJKSMr0+baZowK+UUuq6dO7cOQoVKpTT1VBKXecKFSrEuXPnLusxNOBXSil1XRIRbdlXSuW4K/FvkQb8SimllFJK5WIa8CullFJKKZWLacCvlFJKKaVULqYBv1JKKaWUUrmYzsOvlFJKuRE66JucroJXCW+2yukqXFWGDRvGW2+9xa5du7j55psBSEhIoGLFinTr1o24uLicrWAGjDHUrl2bokWLsnbt2pyujsplNOBXSqkcsqNqtWwvs9rOHdlepsrdUlJSmDp1KrNmzeLnn3/m9OnTFC9enLJly3L33XfTtm1b2rZtS2pqKqGhoezfv59ff/2V22+/3WOZ586do1y5cpw9e5YDBw5QpkwZoqKimD59uiNNnjx5KFKkCKVKlaJ69eo0bdqURx99lJIlS2b6HPbv38+YMWPo2bOnI9i/Gtmvwd69ewkNDU2zT0QYMWIE7du357PPPqNz5845U0mVK2nAr5RSSl2nUlJSaN26NYsXLyY4OJhWrVpRvnx5jh8/zp9//snHH3/Mzp07adu2LXny5KFHjx6MGDGCKVOm8M4773gs95NPPiEpKYnOnTtTpkyZNPvatWtHrVq1ADh9+jT79+9n7dq1fPnllwwdOpRx48YRFRWVqfN49dVXuXDhAgMGDMj0NbiatGvXjmrVqjF06FA6deqk08aqbKMBv1JKKXWdmjNnDosXL6ZmzZqsXr2aoKCgNPvPnj3Lxo0bHb8/+eSTjBo1ipkzZ/Lmm29SoEABt+VOmTIFgJ49e6bb1759+3QB/aVLl5g6dSovvPAC3bt3JyAggEcffdSnc0hKSmL27Nk0adLkqm7d91W3bt0YNGgQy5cvp2nTpjldHZVL6KBdpZRS6jq1YcMGwOpq4hrsg7UCaOPGjR2/33zzzTz44IMcPXqU+fPnuy1z586drFu3jooVK/ocsObLl4+ePXsyYcIEAF588UWfVx6dM2cOZ8+e5ZFHHvGabufOnbRv354SJUpQuHBhGjZsyNKlS72W27hxY4oXL07BggWpVq0ao0aN4sKFC+nSrl27ljZt2lC+fHkCAgIoW7Ys9evXZ8SIEY40IuLo0lSxYkXHYkuuXXu6dOkCwEcffeTT+SvlCw34lVJKqeuUvb/8H3/84XOe6Oho4L9WfFf27U899VSmu6R069aNkJAQDh06xIoVK3zKs2zZMgAaNmzoMc3evXu55557OHbsGL169eKhhx5iy5YttGjRgnnz5qVL/+STTxIZGcnu3bvp2LEjffr0oUSJErzyyis8+OCDXLp0yZF28eLFREREsG7dOpo0aUK/fv1o3749AQEBjgcYgNjYWGrWrAnACy+8QGxsLLGxscTExKQ5dkhICDfddBPLli3DGOPTNVAqI9qlRymllLpOdezYkdGjRzNx4kROnz5Nhw4dCAsLIyQkxGOe1q1bc+ONN7J8+XL27t1LxYoVHfuSk5OZMWMG+fLlo0ePHpmuT548ebj33nv566+/2LRpE61aZTwT0bp16yhatCi33XabxzRr1qyhf//+vP32245tzz77LPfccw9PP/00LVq0oFixYgDExcUxdepUOnTowOzZswkMDHTkGT58OCNGjOCDDz7ghRdeAGDy5MmkpqayatUqR0Bvd/To0TR5ExIS+Omnn4iJiUnXsu/srrvuIj4+nh07dngdHK2Ur7SFXymllLpO1a5dm1mzZnHDDTcwa9YsOnXqRGhoKCVLlqRDhw4sWLAgXZ68efPSo0cPjDHpup18+eWXHDlyhDZt2lC2bFm/6nTTTTcBcOTIkQzTJicnc/jwYcqWLev124SgoCCGDRuWZlvdunXp2rUrJ0+eTNM9ady4ceTLl4+pU6emCfYBXnnlFUqWLMns2bPTHcM1LUCpUqUyPAd37Ndu3759fuVXypW28CullA+qT6+e7WV+ku0lKpV5Dz/8MB06dGDlypWsW7eObdu2sW7dOuLj44mPj+eJJ54gLi4uTUD91FNP8frrrzNt2jRGjBhB3rx5Aau1G9wP1vWVvRuLL92Bjh07BkDx4sW9pqtTpw5FixZNtz0iIoLp06ezbds2unXrxtmzZ/npp58oVaoU7733ntuyAgIC2LHjv+lvu3btyhdffEG9evV45JFHaNy4MQ0aNKB8+fIZ1t+TEiVKAGm/IVAqKzTgV0oppa5z+fPnp3nz5jRv3hywpuv8/PPP6dGjBzNmzKBDhw60b9/ekT4kJIRmzZqxZMkSFi5cSJs2bUhISGDZsmWEhIQ4yvHHwYMHAShdunSGae2t6ufPn/ea7oYbbnC73d6SnpSUBMCJEycwxnDkyJE0A2696dixI19//TVjx45l6tSpTJo0CYCwsDDeeOMNmjVr5lM5zuwDlt19a6CUP7RLj1JKKaXSyJs3Lw8//DB9+/YFcDuA1t6Kb2/V/+ijjzDG8OSTT5Inj3/hRWpqKmvWrAGgXr16GaYPDg6mQIECjpZ+Tw4fPux2+6FDhwAcMxTZ32vXro0xxuvLWatWrVixYgUnTpxg+fLl9O3bl19//ZXWrVvz22+/ZXgeruzn47qGgVL+0oBfKaWUUm7Zu8G4my2mbdu2lC1bloULF7J//37i4uIc/fv9FRcXx759+yhXrlya6UC9qV69OomJiZw6dcpjmq1bt3L69Ol021etWgVYAT5AkSJFuOOOO/j11185fvx4putfuHBh7r//ft555x2GDBlCcnIyixYtcuy3d31KSUnxWs7OnTvJkycP1atnf1dCdX3SgF8ppZS6Ts2ZM4dvv/2W1NTUdPsOHTrkaL1v1KhRuv358uUjKiqKlJQUunbtyoEDB2jZsqVj0G1mXLp0icmTJ9OnTx9EhHfffZeCBQv6lDciIoLU1FQ2bdrkMU1SUhIjR45Ms23z5s3Mnj2boKAgOnTo4Nj+4osvkpycTI8ePTh58mS6sk6cOMHWrVsdvy9fvtztmgH2bxUKFSrk2GafBtXbYNwLFy7w448/Urt2bYKDgz2mUyoztA+/UkopdZ3auHEj48aNo2zZsjRs2NAxxebevXv55ptvOHfuHO3ataNz585u80dHRzN69GjWrl0L+DZYNz4+noSEBADOnDnDvn37WLt2LYmJiQQFBTFp0qQMF9Fy1qlTJ8aOHcuSJUs8LvTVqFEjpkyZwsaNG2nQoAGJiYnMmzeP1NRUJk2a5JiSE6BHjx5s2bKFCRMmUKlSJR544AEqVKjA8ePH2bt3L2vWrKF79+5MnDgRgH79+pGQkEBERAShoaEUKFCALVu2sGLFCkJCQhwLaQE0adKEt99+m+joaDp37kyRIkUIDg7m2WefdaRZtWoVycnJdOrUyedroFRGRBd18J+IbKlTp06dLVu25HRVlFKX2WWZpeeNSxknyqRqO3dknEgBkJiYSLly5XK6Gjlq//79fPXVVyxbtozffvuNxMREzp8/T8mSJalduzaRkZFERkZ67ZPfrFkzli1bRvny5UlISHB0W3EVFRXlWGkWrDn3CxcuTOnSpalevTpNmzYlMjLSMUNNZtSpU4dDhw6xfxg2sSAAACAASURBVP/+NMdPSEigYsWKdOvWjYEDBzJo0CDWrFnDhQsXqF27NsOGDeOBBx5wW+bXX3/NxIkT2bRpEydPnqREiRJUqFCB5s2b89hjj1G1alUAPvnkE+bPn8/mzZtJTEwkT548VKhQgXbt2hETE5Nu8PE777zD5MmT2bNnD8nJyYSEhDgegAAiIyP5/PPP2b9/v/bhv4748u9RWFgYW7du3WqMCcts+RrwZ4EG/EpdPzTgz3004M895syZQ2RkJF988UWa7jnXmn/++YfQ0FAiIyM9rmSscqfLHfBrH36llFJKXdO6dOlCvXr1GD58uNsBxteK119/nbx58/Lqq6/mdFVULqMBv1JKKaWuaSLC//73Pzp06OCYx/9aY4yhXLlyzJw5U795UtlOB+0qpZRS6ppXo0YNatSokdPV8JuIMHDgwJyuhsqltIVfKaWUUkqpXEwDfqWUUkoppXIxDfiVUkoppZTKxTTgV0oppZRSKhfTgF8ppZRSSqlcTAN+pZRSSimlcjEN+JVSSimllMrFNOBXSimllFIqF9OAXymllFJKqVxMA36llFJK+UREiIiIyHI5ERERiEjWK3SVmzp1KiLCpk2b0mzPrut4JbRp04ZKlSqRnJyc01VRWZAvpyuglFJKXZWGB+V0DbwbnpQtxdgDb2NMtpR3uYgI9913H6tWrfK435tp06YRFRUFwPDhwxkxYgQATzzxBNOnT3ebZ/Xq1Y7APCQkhISEBJ/r+++///Lyyy/Tpk0b7r77bp/zXWn2a7Fy5Uq3DyGvvvoqderUYfz48fTv3//KV1BlCw34lVJKKeWTHTt2UKhQoZyuhlexsbFut9eqVSvdtnz58vHpp58ybtw4goOD0+2fPHky+fLl49KlS5mux/jx40lMTGTQoEGZzns1qVWrFg8++CCvvfYazzzzDIULF87pKik/aMCvlFJKKZ9UrVo1p6uQoeHDh/uctnXr1sTHxzN79mz69OmTZt+JEyf4/PPPadOmDfPnz89UHVJSUpg4cSKVK1cmPDw8U3mvRt26dWPRokXMmTOHp556Kqero/ygffiVUkop5RNPfc8TExPp3r07ZcqUITAwkFq1ajF9+nRWrVqFiHgMwi9dusTrr79O5cqVCQgI4Oabb2bgwIFp+ovHxcU5uuusXr0aEXG8MhPcu/Pggw9Svnx5Jk+enG7fzJkzOX/+PNHR0Zku99tvv2X//v088sgjXtMdPHiQxx9/3HHdwsLC+Pjjjz2mX7JkCS1btqRUqVIEBARQqVIlBgwYwMmTJ9Ol3b59O48++iihoaEEBARQunRp6tSpQ0xMDBcvXgQgNDTU0bWpcePGaa6ts3bt2lGwYEE++uijzF4KdZXQFn6llFJK+e2ff/4hPDychIQEGjVqRHh4OIcOHaJ37940b97ca97IyEjWrl1LixYtKFasGAsXLuStt97in3/+Ydq0aYDVpSQ2NpYRI0YQEhLi6IcPZHnga968eenRowcjR45k8+bN1K1b17Fv8uTJhIaG0rRp00yXu2zZMgAaNmzoMc2JEycIDw8nODiY7t27c/LkST755BO6du3K33//zYABA9KkHzlyJLGxsZQoUYLWrVtTpkwZtm/fzpgxY1i4cCHfffcdxYoVA6xgv169eogIbdu2pWLFipw6dYrdu3czYcIERo0aRf78+YmJiSE+Pp7Vq1fTrVs3QkND3da1YMGChIWF8d1335GUlERQ0FU+vkWlowG/Ukoppfw2ePBgEhISeOmllxg9erRje0xMTIaDVf/8809+/fVXSpQoAcBrr71GzZo1mTFjBm+88QZly5alVq1a1KpVixEjRhAaGpphq767/aGhoWkeFJw9+eSTjBo1ismTJzsC/u+//55ffvmFUaNG+TWb0Lp16wDSPEC42r59Ow899BBz584lTx6rw8WgQYMICwtj6NChdOrUiVtuuQWAlStXEhsbyz333MPChQvTjDeIi4uje/fuxMbG8u677wIwffp0zp8/T3x8PO3atUtz3BMnTjjGYcTExHDy5ElWr15NVFSU1weou+66i/Xr17N+/XpatmyZ6WuicpZ26VFKKaWUX5KTk5kzZw5BQUG8/PLLafbVrFmTJ554wmv+0aNHO4J9gMKFC9O1a1dSU1PZvHmzX3UaMWJEuldcXJzH9BUqVKB58+bMmTOHM2fOAFbrft68eenevbtfddi3bx/58+enZMmSHtPkzZuX0aNHO4J9gIoVK/L8889z8eJFZs6c6dg+fvx4R71cBxdHRUVRq1YtZs+ene4YgYGB6bYVL148zTF9VbZsWcA6N3Xt0RZ+pZRSSvnl999/59y5c9StW5eiRYum29+wYUOmTJniMb+7FvCbb74ZsFqi/eHP9KLR0dEsXryYuXPn8tBDDzFv3jxatWrFjTfe6NcMPceOHaN48eJe01SoUIGKFSum2x4REcGIESPYtm2bY9t3331H/vz5+fTTT/n000/T5UlOTubIkSMcO3aMkiVL8sgjjzBu3Djat29P586dadq0KQ0aNKBSpUqZPhc7+4PZ0aNH/S5D5RwN+JVSSinll6Qkay2AG264we1+T9vt3E2FmS+fFZqkpKRksXa+a9u2LWXLlmXKlClcvHiRM2fO+DVY1y4wMJDz5897TePp2thb0u3XFqwHiEuXLjkG2Hry77//UrJkSe6++27Wrl3La6+9xmeffeb4tqBKlSrExsby6KOPZuZ0ADh37hzg/lsDdfXLti49IlJeRKaKyEERuSAiCSLynoh4f8RNW8YAEVloy/uviJwSkZ9F5B0RKe8l3+0i8omI/CMi50XkdxEZISJ6VyqllFKXiX2Q6OHDh93u97T9apMvXz6ioqL4/vvvee211yhfvjwtWrTwu7wyZcpw6tQpx2w47ni6NocOHQJIMzA2KCiI4sWLY4zx+goJCXHkueeee/j66685ceIE69ev55VXXuHw4cNERkY6BhVnxrFjxxznpq492RLwi0glYAvQHdgEvAvsAV4AvhMRz53Y0uoF3AisBiYAHwHHgL7AryJS282x6wE/AO2BZcA44BQwDPhWRAL8PzOllFJKeVK1alUCAwPZvn07p0+fTrffPng1O+TJk+eytvo/9dRTiAgHDhygR48e5M2b1++yatSoAVhdnjzZt2+f25V77SsJ1679X8hTv359Tpw4wa+//prpugQEBBAeHs7IkSMdYwG+/PJLx377eWZ0bXfu3Am4X8BMXf2yq4V/AlAGeN4Y094YM8gYcz9W4F8FeM3Hcu40xtQyxnQzxrxkjOlrjIkAegLFXMsRkbzANKAQ0NkYE2mMGQjUAz4HGmA9LCillFIqmxUoUIBHHnmEpKQkRo0alWbfTz/9xIwZM7LtWCVLlmT//v3ZVp6rSpUqsXjxYubPn8/zzz+fpbLss918//33HtOkpKQwcOBAUlNTHdv27t3L+PHjyZcvH4899phje9++VigTHR3NwYMH05V15syZNMdau3Ztmi5BdvZvFZxXS7YPLM5oMO73339PqVKluPPOO72mU1enLPfhF5FbgOZAAvCBy+5YrGD9cRHpZ4w5460sY4ynDm+fAP8DKrtsvw+oBqwxxnzlVE6qiLwEdAKeFpHRxp9RPEoppdR1wtO0lQATJkxIEyQ6e/PNN1mxYgVvvfUWGzduJDw8nMTERD755BNatmxJfHy8X7PCuGrSpAlz586lTZs2hIWFkS9fPho1akSjRo2yXLZdRusG+Kp9+/bExMSwZMkSjyvT1qhRg40bNxIWFkbz5s1JSkpi3rx5nDx5krfeeivNANsmTZrw5ptvMnjwYCpXrkzLli2pWLEi//77L3/99RerV6+mYcOGLF68GICxY8eydOlSIiIiuOWWWyhSpAi//vorixYtonjx4vTs2dNRduPGjcmTJw+DBw/ml19+cQw2dp516ffff2ffvn307NnTr2lKVc7LjkG799velxpjUp13GGNOi8h6rAeC+sByP4/Rxva+3cOxF7tmMMbsEZE/gNuAW4A//Ty2UkopletNnz7d47733nvPY8B/ww03sGHDBoYMGcLChQvZuHEjVapUYcKECRQuXJj4+HhHX/+sGDduHCLC8uXLWbhwIampqcTGxmZrwJ9dypcvT5s2bViwYAEnTpxwO2NP8eLFWbRoES+99BLTpk3j1KlT3H777fTv35/IyMh06QcOHEiDBg0YP34869at48svvyQoKIibbrqJnj17psnTu3dvihcvzsaNG1m/fj2XLl2ifPny9O7dm379+qXp61+tWjWmT5/OmDFjmDBhgmOwsXPAb783nnnmmWy7RurKkqw2fIvI20B/oL8xZqyb/f8H9AF6G2M+9LHMp4DyQBGgOtAU2A80M8bsckr3KdAZqzvP527K+RpoBbQ0xizK7Lk5lePxItWpU4ctW7b4W7RS6hpRfXr1bC/zkzcyP91fRqrt3JHtZeZWiYmJlCtXLqerkasNHTqU119/ncWLF/PAAw/kdHWuqA0bNtCgQQPeeecdR5eca9GFCxe45ZZbqFatml+DfZVvfPn3KCwsjK1bt241xoRltvzs6MNvH0aevrNY2u3p597y7Cms7kD9sL4d2AI0dQ72L+OxlVJKKZUJ7vqV//zzz4wfP54SJUpw33335UCtclZ4eDgPPfQQo0eP5uzZszldHb99+OGHHDp0iLFj07XpqmvIlZiH397Zy+evEowx9QFss/vUwRqsu0VEHjHGpOu+k53H9lAftx3WRGSLrX5KKaXUdatu3brceuut3HnnnRQuXJhdu3bxzTffkJqaysSJEylYsGBOVzFHjBkzhqlTp7J3717uuOOOnK6OXwICAvjoo4+oWbNmTldFZUF2BPz2VvQgD/uLuaTzmTHmGNbUmj8AO4EZIhJijDl3uY+tlFJKKd/06tWL+Ph45syZw+nTpwkODuaBBx6gf//+jhlrrkcVKlRg+PDhOV2NLNF++7lDdgT89klmb/Ow3z6zzh/+HsAYc1JEvsOaa/8OYPOVOrZSSimlvIuNjSU2Njanq6GU8iA7+vCvtL03F5E05YlIUay58M8Bniej9c1NtnfnUW4rbO8Puia2TRd6G/AX1iJgSimllFJKXXeyHPAbY/4ElgKhWLPxOBsBFAZmOM/BLyJVRaSqc0IRCbEF6emISC/gLqyZen522rUa2AE0EpG2TunzAKNtv07UOfiVUkoppdT1KrsG7fYGNgDjRaQJVhBeD2iM1Z1mqEt6+7xxzoNhawNfiMgGW57DQEms+furA/8CjxtjHGs/G2NSRKQ7Vkv/ZyLyGbAPaALUBdZjrfarlFJKKaXUdSk7uvTYW/nrAnFYgX4/oBIwHrjHNvg2I1uxgvMCWHPn9wcexZphZyxwuzFmtZtjb8Rq/f8SawrPvliDeEdizdt/ISvnppRSSiml1LUs26blNMbsB7r7mDbdNJfGmH1YDwr+HPs34CF/8iqllFJKKZWbZUsLv1JKKaWUUurqpAG/UkoppZRSuZgG/EoppZRSSuViGvArpZRSSimVi2nAr5RSSqnLLiEhAREhKioqW8sVESIiInxOv2rVKkSE4cOHZ2s9rhZ//PEHBQoU4O23387pqqST26/933//TWBgIK+88kpOVyWdbJulRymllMpNqk+vntNV8Ornbj9nnMgHItbEed7WqAwNDeWvv/5i7969hIaGZstxr0Y//PAD7733HuvWrePQoUMEBARQunRpbr/9du6991769OlD4cKFGTJkCG+88QYvvfQSo0eP9lpmdHQ0U6ZM4d133yUmJoZVq1bRuHHjNGkCAwMJCgqicuXK1KtXj8jISGrXru3XObz44ouULFmSPn1c10LNXT766CM2bdrEjz/+yM8//8y5c+cYOnQoo0aNcpv+559/Zty4cWzZsoUDBw5w6tQpypQpQ5UqVejduzcdOnRw/C3YDR8+nBEjRnisw6JFi3jwwQcdv9900008/fTTjB07lp49e3LzzTdnz8lmAw34lVJKKXXN2rFjB4UKFcpyObNmzaJbt24YY7j//vvp0KEDefPmZe/evWzevJmvv/6ajh07cuuttxIdHc2bb75JXFwco0aNIn/+/G7LPHPmDPPmzSMgIIDHH388zb6QkBDHtx3JyckcOXKErVu3MmbMGMaMGUNkZCSTJk2iSJEiPp/Dhg0b+Oabb3jttdey5Zpczfr160dSUhLFixfnxhtv5M8///SafsuWLcTHx1O/fn3Cw8MJCgri0KFDLFiwgE6dOvHYY48xc+ZMt3m7devm9kH31ltvTbdtwIABvP/++7z66qv873//8+vcLgcN+JVSSil1zapatWqWyzh79ix9+vRBRFi6dClNmjRJl2bDhg2UKlUKgIoVK9K0aVO+/fZbFixYQMeOHd2WO3fuXE6fPk1kZCQlS5ZMsy80NNRt15Yff/yRJ554go8//pjjx4+zaNEin8/jgw8+IE+ePOkeLnKjuXPnUq1aNUJCQoiLi6N7d+9LQXXp0sVtd7JTp05Rv359Zs2axXPPPcfdd9+dLk1UVJTP3cZuvPFGmjVrxuzZs3n77bcJCgryKd/lpn34lVK5z/Cg7H8ppRxOnDhBoUKFqFSpkseuQK1bt0ZE2LJlS7p9O3fupH379pQoUYLChQvTsGFDli5dmi5dXFwcIkJcXByLFy8mIiKCoKCgNF0vPPXhP3z4ME8++SQ33HADgYGB1KpVi+nTp7ut6y+//MKpU6e488473Qb7AOHh4QQHBzt+79mzJwCTJ092m955nz2tL2rVqsWyZcsoXbo0ixcvJj4+3qd8p06d4rPPPiM8PNxtV5KIiAhEhOTkZEaOHEmVKlUICAhwBMFJSUm8/fbb3H///ZQvX54CBQpQunRp2rZty/fff+/2mPZrf/ToUXr27Em5cuUICAjgjjvuYNq0aT6f8/nz5+ncuTMiQp8+fUhNTc0wz4MPPkhISIjPxyhYsKDb7cWKFeOBBx4AYNeuXT6X502XLl04e/Ysc+fOzZbysoMG/EoppZTKlOLFi9OlSxf27NnDsmXL0u0/cOAAixcvJiwsjLCwsDT79u7dyz333MOxY8fo1asXDz30EFu2bKFFixbMmzfP7fE+++wzWrduTdGiRXn66ad5+OGHvdbv2LFjhIeHM3XqVG677TZiYmKoVasWTz/9NO+++2669PbW94MHD3LmzBmfrkG7du0oU6YMS5cuZd++fen2//LLL2zcuJHbbruN++67z6cy7cqUKUOvXr0AmD17tk951qxZQ3JyMg0bNvSarlOnTkyYMIHw8HBiYmKoXt0aq7Jjxw6GDh1Knjx5aNWqFS+++CLNmjVjxYoV3HvvvSxevNhteSdPnqRBgwZ89913dO7cmSeeeIKDBw/So0cPjw9Yzk6cOEGzZs344osveOONNxzfUlwpZ8+eZcWKFQCOa+Fq3bp1jB07ltGjRzNv3jyOHj3qtcwGDRoA8O2332ZvZbNAu/QopZRSyuvMKSdPnky3rXfv3kybNo1JkybRrFmzNPumTJlCSkqKI2h1tmbNGvr3759mFplnn32We+65h6effpoWLVpQrFixNHkWLlzIwoUL0wyQ9Gbw4MHs2bOHmJiYNAG+/TiubrnlFu666y5++OEHGjRoQHR0NOHh4dxxxx0UKFDA7THy589PVFQUb731FlOnTk13/eyt+9HR0T7V2VVERASjRo1i06ZNPqVft24dAHXr1vWa7q+//uKXX35xdE+yq1atGgcPHky3/cCBA9x999307dvX7fX/6aefePLJJ5k0aRJ58+YFoG/fvtSoUYPRo0fTrVs3r3Vp0aIFu3fvZsaMGTz22GM+nWtW7N69m1mzZpGSksLhw4f55ptvOHjwIIMHD6ZGjRpu87jOuhMQEMCAAQMYOXJkuoG+YPXtDw4OZs2aNZflHPyhAb9SSimlvM5G4k7dunWpW7cuX375JYcOHaJs2bIApKSk8NFHH1G0aFEeffTRdPmCgoIYNmxYurK6du3K9OnTmT9/frogsV27dj4H+xcvXmT27NkULVo0XRDufBxnIsJnn31Gt27dWLVqFc8++yxgBfW1a9emY8eOPPPMM+keRKKjo3n77beZNm0aw4YNc7RMX7hwgVmzZlGgQAGvAa83N910EwBHjhzxKb39W4Zy5cp5Tffqq6+mC+oBj33Ny5cvT+fOnXn//ffZt28fFSpUSLO/UKFCvPPOO45gH+D222+nQYMGrFmzhtOnT1O0aNF05f7444+0bNmSM2fOsGjRIo9dqbLb7t2709zr9ilM+/Xrly5tzZo1mTp1KhEREZQrV45//vmHpUuX8vLLLzNq1ChSUlJ4/fXX3R6nbNmy7Ny5k/Pnz3vsTnQlaZcepZRSSmGM8fjy1Fe6d+/eXLp0ialTpzq2LVy4kAMHDvDYY4+5nWGmTp06bgNAez/8bdu2pdvnbiClJzt37uTs2bPUqlXLbRDrafBlhQoVWLlyJb/99hvjxo3j8ccf55ZbbmHTpk0MGjSI6tWrs3fv3jR5br31ViIiIti3bx9LlixxbP/88885fvw4HTp0oHTp0j7X3Zl9bIS7FmR3jh07Bljdrbzxdi3Xr1/Pww8/zM0330xAQAAigojw/vvvA9Y8864qV66c7kEIcIwjcPft0Lp162jUqBEiwpo1a65YsA9W339jDMnJyezevZuhQ4cyZMgQ2rZtS3Jycpq0HTp0oHv37lSsWJGCBQtSoUIFnnrqKRYuXEj+/PkZM2aMx+49JUqUAMiw+8+VogG/UkoppfzSpUsXihcvzuTJkx0DLSdNmgTgtjsPwA033OB2u/0bgqSkJI/7fGHPn9FxPKlWrRrPP/88M2bMYOfOnezYsYN77rmHffv20bdv33Tp7QNyp0yZ4thm/9nf7jxgjScAfH5gCAwMBKwBsN54Ov/58+fTqFEjvvnmG8LCwnj22Wd55ZVXiI2NdYxBuHDhQrp8zgOZneXLZ3UiSUlJSbdv27ZtnD59murVq2fLLEv+yJ8/P5UqVWLYsGGMHDmSr7/+mvHjx/uUt06dOtx9991cvHiR7777zm2ac+fOAf99LjlNA36llFJK+SUwMJCoqCgSEhJYunSpY7BuvXr1qFmzpts8hw8fdrv90KFDgPuuJb62cjvnz+g4vqpatapjfnb74E5nHTt2pFSpUixYsIDDhw/z559/smrVKipVqsT999+fqWM5W7lyJQD16tXzKX2ZMmWA/1r6PfF0LV955RUKFCjA5s2biY+PZ+zYsYwcOZLhw4dTpUqVTNQ8Y88++yzPPPMMS5YsoW3bto7gOKe0aNECsFYC9pX9QczTIO9jx46RL18+R0t/TtOAXymllFJ+e+aZZxARJk2a5HWwrt3WrVs5ffp0uu32YMvfFWbtqlatSqFChfjxxx/dfluQmaDOzt4Fyd0UpPZ++hcvXmT69OlMmTIFYwzR0dGZelBx9s8//zi+KenatatPeewDTnfu3OnXMXfv3s3tt99OtWrV0mxPTU11DAjOLiLChAkTiImJYenSpbRq1crn2ZEuB3tXJfu3Ehm5ePEiW7duBawB367OnDnD33//TY0aNfy+B7KbBvxKKaWU8lvlypVp0qQJX3/9NRMnTiQ4OJhHHnnEY/qkpCRGjhyZZtvmzZuZPXs2QUFBdOjQIUv1yZ8/P127duX06dPpBu3aj+Nq7969jB8/3u0DgjGG1157DYBGjRq5Paa9687kyZOJi4tzzODjj59++olmzZpx9OhRWrZsSdu2bX3KZx+b4GnO/IyEhoaya9cuR1cisM59xIgR/Pbbb36VmZF3332XwYMHs3LlSh544AFOnTp1WY4D1riBixcvptt+5MgRBg0aBECrVq0c20+fPs2PP/6YLn1ycjIxMTHs27ePqlWrup0VadOmTaSkpNC4ceNsPIOs0Vl6lFJKKZUlvXv3ZtmyZRw+fJjnnnuOQoUKeUzbqFEjpkyZwsaNG2nQoAGJiYnMmzeP1NRUJk2a5HYAaGa9/vrrLF++nPfee4/NmzfTsGFDx3FatmzJV199lSZ9UlISL7zwAgMGDKBBgwbceeedFC1alH/++YcVK1awZ88eypQpw9ixY90er0qVKjRq1MgxDWOnTp08jiGwS0hIcDyQXLx4kaNHj7JlyxbHQmVdu3Z1tPL74s4776RKlSosX76clJSUNLPm+KJv3748/fTT1K5dm06dOpE/f37Wr1/Pb7/9Rps2bViwYEGmyvPV66+/TsGCBYmNjaVZs2YsXrw4w4HHYI2TsH/zsHv3bgAWLFjAgQMHAOubHnsgD1Y3okOHDtGgQQMqVKhA3rx5SUhIYOHChZw7d4727dvTo0cPR/pjx45Ru3ZtatWqRY0aNShXrhxHjhxh5cqV7N27l1KlSjFnzhy3awbYF5Hr1KmT/xcmm2nAr5RSSqksadu2LaVKleLo0aNeu/MAVKxYkYkTJzJo0CAmTpzIhQsXqFOnDsOGDXOseJpVpUqVYv369QwZMoQFCxawefNmqlSpwocffkhoaGi6gL9atWrMnz+fpUuX8v333zNv3jyOHz9OoUKFuPXWWxkyZAgxMTFeB9D27NnTEfD7srLuX3/95ZgesmDBggQHB1O5cmX69+9P165dqVWrVqbP+5lnnnF0k7H3S/dVr169CAgI4L333mP69OkEBgZy7733Mm3aND7//PPLFvADDBs2jMDAQF566SWaNGnC0qVL3U4d6mzdunXpplfdvn0727dvB+C+++5LE/D369eP+Ph4tm3bxpIlS0hOTqZUqVLcf//9PP744zz88MNput+UKFGC5557jk2bNrFkyRKOHz9OgQIFqFSpEgMHDuTFF190jJtwlpqayqxZs6hZs6bbNR9yinhaEltlTES21KlTp467ZcOVUjlouPv5pLOiesUKGSfKpE/euJTtZVbbuSPby8ytEhMTM5yzXPlm6/XxzQAAIABJREFUz5493HrrrTRo0IC1a9fmdHWuW6dOnaJSpUqEh4fz5Zdf5nR1rksLFiygbdu2zJw5M1MLifny71FYWBhbt27daowJ85rQDe3Dr5RSSqksGTNmDMYYx4JVKmcUK1aMESNG8NVXX7F58+acrs51xxhDbGysY4G3q4l26VFKKaVUpu3bt4+PP/6YXbt2MW3aNGrWrMlDDz2U09W67vXq1YuTJ096nJZUXT6HDh2ibdu2tG/f/qqZncdOA36llFJKZdqePXsYPHgwhQoVolmzZnz44YduBzCqKytv3rwMGTIkp6txXSpXrly6maGuFhrwK6WUUirTIiIi3M5Lr5S6+uijuFJKKaWUUrmYBvxKKaWUUkrlYhrwK6WUUkoplYtpwK+UUkoppVQupgG/UkoppZRSuZgG/EoppZRSSuViGvArpZRSSimVi2nAr5RSSimlVC6mAb9SSimllFK5mAb8SimllLrsEhISEBGioqKytVwRISIiwuf0q1atQkQYPnx4ttbjavHHH39QoEAB3n777ZyuSjq5/dqfPXuWsmXL8vjjj+d0VdLJl9MVUEoppa5GO6pWy+kqeFVt545sKUdEADDGeEwTGhrKX3/9xd69ewkNDc2W416NfvjhB9577z3WrVvHoUOHCAgIoHTp0tx+++3ce++99OnTh8KFCzNkyBDeeOMNXnrpJUaPHu21zOjoaKZMmcK7775LTEwMq1atonHjxmnSBAYGEhQUROXKlalXrx6RkZHUrl3br3N48cUXKVmyJH369PEr/7Xq1VdfZdiwYQB8++23NG3a1G266dOn88EHH/Dbb7+RN29eateuTf/+/WndurXb9CkpKbz//vtMnTqVXbt2ERgYSP369Xn55ZcJDw9Pk7ZQoUIMHjyYvn378txzz3H33Xdn70lmgbbwK6WUUuqatWPHDmbMmJHlcmbNmkX9+vWZM2cOlStX5plnniE6OpqaNWvy008/MXDgQBITEwEriBcR4uLiuHjxoscyz5w5w7x58wgICEjX6hsSEkJsbCyxsbHExMTQunVrzpw5w5gxY6hTpw5du3bl33//zdQ5bNiwgW+++YbnnnuOQoUKZf4iXKO2bt3Kq6++SpEiRbym69+/P1FRUSQmJhIdHc1jjz3Gzz//TJs2bfi///u/dOmNMXTp0oW+ffuSnJzMs88+S4cOHVizZg2NGjXiyy+/TJenV69eBAcH8/LLL2fb+WUHbeFXSiml1DWratWqWS7j7Nmz9OnTBxFh6dKlNGnSJF2aDRs2UKpUKQAqVqxI06ZN+fbbb1mwYAEdO3Z0W+7cuXM5ffo0kZGRlCxZMs2+0NBQt11bfvzxR5544gk+/vhjjh8/zqJFi3w+jw8++IA8efJclV1KLpfz58/z+OOPU7duXW699VZmzpzpNt2GDRsYO3YslSpV4ocffqB48eIADBgwgLCwMEcrv/M3WHPnzuWzzz4jPDyc5cuXU7BgQQCefvppGjZsSHR0NPfffz9FixZ15ClYsCCPPPIIkyZNYteuXVSuXPnynXwmaAu/UkoppTLlxIkTFCpUiEr/396dx1tV1/sff31wQEAEJ5IrGjkg5DwrKCJcDTWHHMqbeZ0ycMipur/MUrG0slTScriaomallxxvSpSAKKTm2DVBUcARJxBUVBz4/v5Ya+Nmn705+3A2cFi8no/HfizOGr7ftc9Z7P1e3/Vd37XxxjW7An35y18mInj00UebLJs8eTIHHXQQa621Fp06dWK33XZj9OjRTdYbMWLEgpb0UaNGMWDAALp06bKgGxLU7sP/+uuvc9xxx/G5z32ODh06sM0223D99ddX3dennnqKd955hy222KJq2Afo27cvXbt2XfDzt771LQCuvvrqquuXLyutW49tttmGv/3tb6y77rqMGjWK22+/va7t3nnnnQXhdIMNNmiyfMCAAUQEH330Eeeddx6bbbYZ7du3X3BPxZw5c/jFL37BwIED6dGjB6uuuirrrrsuBxxwAA8++GDVOku/+7feeotvfetbdO/enfbt27P55ptz3XXX1f2eP/zwQw499FAigpNOOon58+fXve2ZZ57JtGnTGDFiBO3a1Y61V155JQBnnXXWgrAP2YnXSSedxLx585rs8xVXXAHAT37ykwVhH2DHHXfka1/7Gm+++SYjR45sUtfhhx9OSolrr7227vexpBn4JUlSi6y55pocfvjhTJ06lb/97W9Nlr/88suMGjWK7bffnu23336hZdOmTWPXXXdl5syZDBkyhMMOO4xHH32UffbZh5tvvrlqfSNHjuTLX/4ynTt3ZujQoXz1q19d5P7NnDmTvn37cu2119KrVy9OO+00ttlmG4YOHcoll1zSZP1S6/urr77K3Llz6/odHHjggXTr1o3Ro0fz4osvNln+1FNP8dBDD9GrVy/22GOPusos6datG0OGDAHgpptuqmub8ePH89FHH7Hbbrstcr1DDjmEyy+/nL59+3Laaaex5ZZbAlnXqLPOOot27dqx3377ccYZZ7DXXnsxZswYdt99d0aNGlW1vNmzZ9OvXz/+/ve/c+ihh/Kf//mfvPrqqxx77LE1T7DKvf322+y1117ceuut/PSnP11wlaIeY8eO5Ve/+hU//elP6dWr1yLXHTNmDACDBw9usmyfffZZaB2AefPmMXHiRDp27Mjuu+9e1zYlO+20E6ussgp//etf63ofS4NdeiRJ0iJHTpk9e3aTeSeeeCLXXXcdV111FXvttddCy6655ho+/fTTBaG13Pjx4/nud7+70CgyJ598MrvuuitDhw5ln332YY011lhom7vvvpu77767alir5swzz2Tq1KmcdtppCwX8Uj2VNtpoI3bccUf+8Y9/0K9fP44//nj69u3L5ptvzqqrrlq1jlVWWYWjjz6aCy+8kGuvvbbJ76/Uun/88cfXtc+VBgwYwE9+8hMefvjhutZ/4IEHANhhhx0Wud4LL7zAU089taB7UkmfPn149dVXm8x/+eWX2WmnnTj99NOr/v6ffPJJjjvuOK666ipWWmklAE4//XS22morfv7zn3PUUUctcl/22WcfnnvuOW644Qa+8Y1v1PVeIbsicfTRR7P77rtzyimnLHLduXPn8sorr7D66qvTvXv3JstL3W6effbZBfOee+45Pv30UzbaaCNWXrlpXK62TUmHDh3YfPPNefzxx3n33XcX6vKzrBj4JUkSw4YNa9H6O+ywAzvssAN33HEHr732Guuttx6QjWry29/+ls6dO/Mf//EfTbbr0qXLgtFUyss64ogjuP7667ntttuahMQDDzyw7rD/8ccfc9NNN9G5c+cmIby8nnIRwciRIznqqKMYN24cJ598MpCF+m233ZaDDz6YE044ocmJyPHHH88vfvELrrvuOs4+++wFLdPz5s3jd7/7HauuuuoiA++irL/++gC8+eabda1fuspQLdCW+/GPf9wk1EP2d6mmR48eHHrooVx22WW8+OKLbLjhhgst79ixIxdffPGCsA/wxS9+kX79+jF+/PiagfeJJ55g3333Ze7cudxzzz01u1LV8u1vf5uZM2cyduzYhbp4VTNnzpxFvsfS/PIT28XZptx6663HE088wSuvvNKQ+0xayy49kiSJlFLN1+c///mq25x44ol88sknC/VVvvvuu3n55Zf5xje+UXXUlO22265qACz1w3/88cebLGvJ8IaTJ0/m/fffZ5tttqka1mqN2b/hhhsyduxYnn76aX71q19x5JFHstFGG/Hwww/z/e9/ny233JJp06YttM0mm2zCgAEDePHFF/nLX/6yYP6f/vQnZs2axVe+8hXWXXfduve9XOneiObCbMnMmTMBFuqfXs2ifpcTJkzgq1/9KhtssAHt27cnIogILrvsMgBeeeWVJttsuummTU6EgAX3EVQLxA888AD9+/cnIhg/fnyLw/6tt97KjTfeyIUXXshGG23Uom0Xpd7fNTT/91lrrbUAeOutt1q/Yw1g4JckSYvl8MMPZ8011+Tqq69ecKPlVVddBVC1Ow/A5z73uarzS1cISi2r1ZbVo7R9c/XU0qdPH0455RRuuOEGJk+ezKRJk9h111158cUXOf3005usX7oh95prrlkwr/Tvxe3OA9n9BEDdJwwdOnQAshtgF6XW+7/tttvo378/f/7zn9l+++05+eST+dGPfsQ555yz4B6EefPmNdmu/EbmcqVuMJ9++mmTZaWuLltuuWWLW79nzZrFkCFDGDhwICeccEJd25RO/KodW+Xzy08Qm9vmnXfeabJNuQ8++AD47O+yrNmlR5IkLZYOHTpw9NFHc8kllzB69Gi22GILRo0axc4778zWW29ddZvXX3+96vzXXnsNqB6gWtLyWtq+uXrq1bt3b2688UY22WSTqjdoHnzwwayzzjrcddddvP7667z33nuMGzeOjTfemIEDB7aornJjx44FYOedd65r/W7dugGftfTXUut3+aMf/YhVV12VRx55hD59Fn7o3JAhQ7jvvvvq2o96nHzyybz55ptcccUVHHDAAdx+++11B+MXX3yRt956izFjxtS8ubd0T0npYWedOnVi/fXX55VXXmHGjBlNuj1NmTIFYKEbfzfZZBNWWmklpk6dyieffNKkH3+1bcqV/g6lv8uyZuCXJEmL7YQTTmD48OFcddVVbL311jVv1i157LHHqvbrHjduHMBiP2G2pHfv3nTs2JEnnniCOXPmNDmBKNXTEqV9rTYEaamf/kUXXcT111/P22+/TUppwcO5Fscbb7yx4ErJEUccUdc2W221FZB1aar1lNlFee6559h8882bhP358+cvuCG4USKCyy+/nPbt2zN8+HD2228/7rrrLjp16tTstmuvvTbHHXdc1WXjx49nypQp7LPPPvzbv/0bW2yxxYJlAwcO5MYbb2TUqFEcc8wxC21XetZB+Qla+/bt6du3L/fffz/3339/k6cjV9um3DPPPMPaa69Njx49mn1PS4NdeiRJ0mLbdNNNGTRoEP/7v//LlVdeSdeuXfna175Wc/05c+Zw3nnnLTTvkUce4aabbqJLly585StfadX+rLLKKhxxxBG8++67TW7aLdVTadq0aVx66aVVu2+klDj//PMB6N+/f9U6S113rr76akaMGLFgBJ/F8eSTT7LXXnvx1ltvse+++3LAAQfUtV3p3oRaY+Y3p2fPnkyZMmVBVyLI3vuwYcN4+umnF6vM5lxyySWceeaZjB07li996UsLusksygYbbMA111xT9dW3b18AzjjjDK655pqFTnyGDh0KwPnnn8/bb7+9YP706dP5zW9+Q/v27ZucCJS6DP3whz9cqKvUP/7xD26++WbWXXddDjnkkCb7OG3aNF5//fUFzz5oC2zhlyRJrXLiiSfyt7/9jddff51vf/vbdOzYsea6/fv355prruGhhx6iX79+zJgxg5tvvpn58+dz1VVXVb0BtKUuuOAC7r33XoYPH84jjzzCbrvttqCefffdlzvvvHOh9efMmcOpp57K9773Pfr168cWW2xB586deeONNxgzZgxTp06lW7duXHTRRVXr22yzzejfvz/jx48HsrHua91DUDJ9+vQFJyQff/wxb731Fo8++uiCB5UdccQRC1r567HFFluw2Wabce+99/Lpp58uNGpOPU4//XSGDh3KtttuyyGHHMIqq6zChAkTePrpp9l///256667WlRevS644AJWW201zjnnHPbaay9GjRrV7I3Hi6Nv376cccYZXHzxxWy11VYceuihfPTRR9x8883MmjWLyy67bKGn7EJ2j8qtt97KyJEj2Xbbbdl///2ZOXMmN998M59++ilXX3111eO19BC5aicDy0rDWvgjokdEXBsRr0bEvIiYHhHDI6Kuv1pEdIqIIyLi9xExOSLmRsS7EfFIRHwnIqoOhBsRaRGvxTvNlSRJdTvggAMWDPW4qO48AF/4wheYOHEia665JldeeSW33HIL2223HXffffcirwy0xDrrrMOECRM45phjmDx5MsOHD+eJJ57giiuuqHrjbZ8+fbjttts47rjjmD17NjfffDMXXnght9xyC127duUHP/gBTz311CJvMC1/mm49T9Z94YUXGDZsGMOGDePiiy/mzjvvpGPHjnz3u9/l8ccf53e/+11dXVzKnXDCCbz22mtVn1rcnCFDhnDdddfRvXt3rr/+em666SY22GADHnroIbbbbrsWl9cSZ599NhdeeCEPP/wwgwYNWmIj21x00UWMGDGC9dZbj//+7//mhhtuYPPNN+euu+5aMBRruYjgD3/4AxdffDErr7wyl112GbfeeuuCk7sDDzywaj3XX399zdb/ZSVqPRK7RYVEbAxMBLoBdwCTgZ2APYFngH4ppUXeRRIRg4F7gFnAWOA5YC1gf2C9vPxBKaUPK7ZLwAvAiCrFvpxSuqbK/IaIiEe322677ao9NlzSMnRu9VETWmPLL2zY/EotdMtPP2l4mX0mT2p4mUVV7eY9LZ6pU6eyySab0K9fP+6///5lvTsrrHfeeYeNN96Yvn37cscddyzr3Vkh/fOf/2Trrbfmxz/+MT/84Q/r3q6ez6Ptt9+exx577LGU0vaLXLGKRnXpuZws7J+SUrqsNDMiLgZOB84HhjZTxmvAN4D/SSl9VFZGZ2Ac0Bc4Cah2PW16SuncVuy/JElaTL/85S9JKVVtJdXSs8YaazBs2DBOOukkHnnkkWafuqvGO/vss+nRowff+c53lvWuLKTVgT8iNgL2BqYDv6lYfA7wLeDIiPhOSmlurXJSSk8AT1SZ/25EXATcBAygeuCXJElL0Ysvvsjvf/97pkyZwnXXXcfWW2/NYYcdtqx3a4U3ZMgQZs+eXXNYUi0577//Pttuuy2nnXZamxl/v6QRLfyl8YhGp5Tmly/Iw/oEshOCXYB7F7OOj/NprevfXSPiWLKuP3OAR1NK9t+XJGkJmTp1KmeeeSYdO3Zkr7324oorrqg5LrqWnpVWWokf/OAHy3o3VkgdO3bknHPOWda7UVUjAv9m+fTZGsunkAX+Xix+4D82n46qsXxr4LflMyLiSeDIlNL/LWad5WW1/kYHSZIKZMCAAVXHpZfU9jTiVLx0d1z1Zw9/Nr/6s5ebEREnA4PJuvtcW2WVi4F+wLpAZ2BHYCTZScCYiFh/ceqVJEmSimBpXHsrPXGgxc0AEXEwMJzsht5DUkofV66TUvpOSmliSumtlNJ7KaVHUkqHAX8C1gG+24p9L9UR1V7AY60tW5IkSVqSGhH4Sy34tcbBW6NivbpExEHAH4E3gAEppakt3K8r82n1x+JJkiRJK4BGBP5n8mmvGss3zae1+vg3ERGHAf8DvA7skVJ6pplNqnkzn7bsqRWSJElSgTQi8I/Np3tHxELl5WPo9wM+AOoaNScivg78AXiVLOxPWcz92iWftvTKgCRpBZBS8qZTScvc0vgsanXgTyk9D4wGepI9GKvcMLIW9hvKx+CPiN4R0eT51BFxFHAj8CLQv7luPBGxXUQ0acGPiK3IHvYF8Lv6340kaUXRoUMH3n///WW9G5JWcO+///4SH7e/UU/aPRGYCFwaEYOAScDOwJ5kXXnOqli/9Oz30g29RMSeZKPwtCO7anBMRFRsxuyU0vCyn08BDo6IMcBLwDygN9moPisBV5NdLZAkaSFdunRhxowZQDZ+dpXvHElaYlJKvP/++8yePZvu3bsv0boaEvhTSs9HxA7AeWRhe19gBnApMCylNKuOYj7PZ1ccjq2xzgtko/aU3E52U/BWZA8AWw2YCdwDXJ1SurOFb0WStIJo164d3bt3Z86cOcyZM8fAL2mpSinRoUMHunfvvsQfWteoFn5SSi8Bx9S5bpNP1ZTSCGBEC+u8nSz0S5LUYu3atWPNNddkzTXXXNa7IklLjM/AliRJkgqsYS38kiSpOCb17tPwMvtMntT8SpIazhZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQXmTbuSJC3ntrx+y4aXeUvDS5S0rNjCL0mSJBWYgV+SJEkqMAO/JEmSVGAGfkmSJKnADPySJElSgRn4JUmSpAIz8EuSJEkFZuCXJEmSCszAL0mSJBWYgV+SJEkqMAO/JEmSVGAGfkmSJKnADPySJElSgRn4JUmSpAIz8EuSJEkFZuCXJEmSCszAL0mSJBWYgV+SJEkqMAO/JEmSVGAGfkmSJKnADPySJElSgRn4JUmSpAIz8EuSJEkFZuCXJEmSCszAL0mSJBWYgV+SJEkqMAO/JEmSVGAGfkmSJKnADPySJElSgRn4JUmSpAIz8EuSJEkFZuCXJEmSCszAL0mSJBWYgV+SJEkqMAO/JEmSVGAGfkmSJKnADPySJElSgRn4JUmSpAIz8EuSJEkFtvKy3gFJklYo53ZpfJlf2LDxZUoqjIYF/ojoAZwHDAbWBmYAtwPDUkpv17F9J+AgYD9gO2ADYD7wDPAH4LKU0kc1tv0icC4wAFgDeAH4I/CzlNIHrXlfkiRJajsm9e7T8DL7TJ7U8DLbkoYE/ojYGJgIdAPuACYDOwGnAoMjol9KaWYzxewO/A6YBYwlO1lYC9gf+CVwcEQMSil9WFH3zsAYYBVgJPASMBA4GxiUbzOvEe9TkiRJWt40qoX/crKwf0pK6bLSzIi4GDgdOB8Y2kwZrwHfAP6nvCU/IjoD44C+wEnARWXLVgKuAzoCB6aU7szntwNuAQ7J6/9Z696eJEmStHxq9U27EbERsDcwHfhNxeJzgLnAkXmXnZpSSk+klG6q7LaTUnqXz0L+gIrN9gD6AONLYT/fZj7wX/mPQyMi6n5DkiRJUoE0YpSegfl0dB60F8jD+gSyFvhdWlHHx/n0kxp1j6rcIKU0FXgW+DywUSvqliRJkpZbjQj8m+XTZ2ssn5JPe7WijmPzaWWwXxp1ExGp2ovs5mJJkiSpzWpE4C+NLzanxvLS/K6LU3hEnEw28s8TwLVLs25JkiRpebc0HrxV6j+fWrxhxMHAcLIbeg9JKX3czCYNq7tcSimqvYDHWlOuJEmStKQ1IvCXWtFrPUlkjYr16hIRB5GNpf8GMCDvk79U6pYkSZKKohHDcj6TT2v1k980n9bqZ99ERBwG/J6sZX9gSmlKjVUbXrekpa/n9//c0PKmr9bQ4iRJWq41ooV/bD7dOx//foF8DP1+wAfAg/UUFhFfJ3uy7qvAHosI+5A9cAuyPv6V5WxEdiLwAlDt6oAkSZJUeK0O/Cml54HRQE+yB2OVGwZ0Am5IKc0tzYyI3hHRu7KsiDgKuBF4EehfoxtPufuASUD/iDigrJx2wM/zH69MKbWqD78kSZK0vGrUk3ZPBCYCl0bEILIQvjOwJ1l3mrMq1p+UTxc8ECsi9iQbhacd2VWDY6o8L2t2Sml46YeU0qcRcQxZS//IiBhJdrIwCNiB7BkAlzTiDUqSJEnLo4YE/pTS8xGxA3AeWfeafYEZwKXAsJTSrDqK+TyfXXE4tsY6L5CN2lNe90MRsSPZ1YS9gc75eucBP0spzWvh25EkSZIKo1Et/KSUXgKOqXPdJk33KaURwIjFrPtp4LDF2VaSJEkqsqUxDr8kSZKkZcTAL0mSJBVYw7r0SJIkaTl2bq3nmLamTJ992hbYwi9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwBoW+COiR0RcGxGvRsS8iJgeEcMjYs0WlLFXRFwUEfdGxKyISBHxQDPbpEW8Hmz9O5MkSZKWXys3opCI2BiYCHQD7gAmAzsBpwKDI6JfSmlmHUWdBBwIfAg8B9R7svACMKLK/Jfr3F6SJEkqpIYEfuBysrB/SkrpstLMiLgYOB04HxhaRzk/B84iO2HYAJhWZ/3TU0rntmSHJUmSpBVBq7v0RMRGwN7AdOA3FYvPAeYCR0ZEp+bKSin9PaX0r5TSp63dL0mSJEmNaeEfmE9Hp5Tmly9IKb0bERPITgh2Ae5tQH3VdI2IY4H1gDnAoykl++9LkiRphdeIwL9ZPn22xvIpZIG/F0su8G8N/LZ8RkQ8CRyZUvq/1hYeEam1ZUiSJEnLQiNG6emST+fUWF6a37UBdVVzMdAPWBfoDOwIjCQ7CRgTEesvoXolSZKkNm9pjMMf+XSJtJKnlL6TUpqYUnorpfReSumRlNJhwJ+AdYDvNqCOqPYCHmtt2ZIkSdKS1IjAX2rB71Jj+RoV6y0tV+bT/ku5XkmSJKnNaETgfyaf9qqxfNN8WquP/5LyZj5tdnQgSZIkqagaEfjH5tO9I2Kh8iKiM1n/+g+ApT1qzi75dOpSrleSJElqM1od+FNKzwOjgZ5kT8otN4yshf2GlNLc0syI6B0RvVtbd0RsV218/4jYiuxhXwC/a209kiRJ0vKqUU/aPRGYCFwaEYOAScDOwJ5kXXnOqlh/Uj6N8pkRsRvwzfzH1fPpphExorROSunosk1OAQ6OiDHAS8A8oDcwGFgJuBr4QyvelyRJkrRca0jgTyk9HxE7AOeRhe19gRnApcCwlNKsOovaBDiqYl63inlHl/37drKbgrciewDYasBM4B7g6pS2uuXgAAAd3UlEQVTSnS17J5IkSVKxNKqFn5TSS8Axda4bNeaPAEa0oM7byUK/JEmSpCqWxjj8kiRJkpYRA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFZiBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKbOVlvQOSJLVVPb//54aXOX21hhcpSYtkC78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFZiBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFZiBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFZiBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFdjKjSooInoA5wGDgbWBGcDtwLCU0tt1lrFXvv02wLbAmsCElNJuzWz3ReBcYACwBvAC8EfgZymlDxbj7UiSJKmVtrx+y4aXeUvDSyy+hgT+iNgYmAh0A+4AJgM7AacCgyOiX0ppZh1FnQQcCHwIPEcW+Jure2dgDLAKMBJ4CRgInA0MiohBKaV5LX5TkiRJUgE0qkvP5WRh/5SU0kEppe+nlAYClwCbAefXWc7PgS2A1YH9m1s5IlYCrgM6AoemlL6eUvp/wM7An4B+wOktfTOSJElSUbQ68EfERsDewHTgNxWLzwHmAkdGRKfmykop/T2l9K+U0qd1Vr8H0AcYn1K6s6yc+cB/5T8OjYioszxJkiSpUBrRwj8wn47Og/YCKaV3gQlkLfC7NKCuWnWPqlyQUpoKPAt8HthoCdQtSZIktXmNCPyb5dNnayyfkk97NaCuZVJ3RKRqL2C71pQrSZIkLWmNCPxd8umcGstL87s2oK62VLckSZLU5i2NcfhL/efTUqhridSdUopqL+Cx1u+iJEmStOQ0IvCXWtG71Fi+RsV6jbQs65YkSZLavEYE/mfyaa1+8pvm01r97JfXuiVJkqQ2rxGBf2w+3TsiFiovIjqTjYX/AfBgA+qqNCafDq5ckA8X2ovsqbtTl0DdkiRJUpvX6sCfUnoeGA30JHtSbrlhQCfghpTS3NLMiOgdEb1bWzdwHzAJ6B8RB5SV347sIV4AV6aUlsX9A5IkSdIyt3KDyjkRmAhcGhGDyEL4zsCeZN1pzqpYf1I+XeiBWBGxG/DN/MfV8+mmETGitE5K6eiyf38aEceQtfSPjIiRwIvAIGAHsmcAXNLK9yZJkiQttxoS+FNKz0fEDsB5ZN1r9gVmAJcCw1JKs+osahPgqIp53SrmHV1R90MRsSPZ1YS9gc5k3XjOA36WUprXsncjSZIkFUejWvhJKb0EHFPnulFj/ghgxGLU/TRwWEu3kyRJkopuaYzDL0mSJGkZMfBLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwBoW+COiR0RcGxGvRsS8iJgeEcMjYs0WlrNWvt30vJxX83J71Fh/ekSkGq/XGvPuJEmSpOXTyo0oJCI2BiYC3YA7gMnATsCpwOCI6JdSmllHOWvn5fQCxgB/BHoDxwD7RcSuKaWpVTadAwyvMv+9xXg7kiRJUmE0JPADl5OF/VNSSpeVZkbExcDpwPnA0DrKuYAs7F+SUjqjrJxTgF/l9Qyust3slNK5i733kiRJUkG1uktPRGwE7A1MB35TsfgcYC5wZER0aqacTsCR+frnVCz+dV7+l/L6JEmSJNWhEX34B+bT0Sml+eULUkrvAhOAjsAuzZSzK9ABmJBvV17OfGB0/uOeVbZtHxHfiIgfRMSpEbFnRKzU0jciSZIkFU0juvRslk+frbF8CtkVgF7Ava0sh7ycSusBN1bMmxYRx6SU7ltEnXWJiNTaMiRJkqRloREt/F3y6Zway0vzuy6hcq4DBpGF/k7AlsBVQE/gnojYupl6JUmSpMJaGuPwRz5tbSt51XJSSsNSSmNSSq+nlN5PKT2VUhoKXEzWRejcVtZLSimqvYDHWlu2JEmStCQ1IvCXWt671Fi+RsV6S7qckivzaf8615ckSZIKpxGB/5l8Wq1vPcCm+bRW3/xGl1PyRj5d5OhAkiRJUpE1IvCPzad7R8RC5UVEZ6Af8AHwYDPlPJiv1y/frrycdmQ3/pbX15xd82m1B3VJkiRJK4RWB/6U0vNkQ2b2BE6qWDyMrIX9hpTS3NLMiOgdEb0rynmPbKSdTjTtd39yXv5fyp+0GxGbR8RalfsUEZ8nG7sf4HctflOSJElSQTTqSbsnAhOBSyNiEDAJ2JlszPxngbMq1p+UT6Ni/g+AAcAZEbEN8DDQBziQrItO5QnFYcD3I2IsMA14F9gY2A9YDbgb+GUr35skSZK03GpI4E8pPR8ROwDnAYOBfYEZwKXAsJTSrDrLmRkRu5I9afcgYHdgJtnQm2enlF6u2GQs2fj925J14ekEzAYeILtacGNKyTH0JUmStMJqVAs/KaWXgGPqXLeyZb982Szg1PzVXDn3Aa1+sJYkSZJUVEtjHH5JkiRJy4iBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBNWyUHqmtmtS7T8PL7DN5UvMrSZIktQEGfrUpW16/ZcPLvKXhJUqSJC0/7NIjSZIkFZiBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAnNYTi2+c7s0vswvbNj4MiVJklZgtvBLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgps5WW9A1o6en7/zw0vc/pqDS9SkiRJDWYLvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmOPwS5IkLWd8vo5awhZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFZiBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFVjDAn9E9IiIayPi1YiYFxHTI2J4RKzZwnLWyrebnpfzal5ujyVdtyRJklQ0KzeikIjYGJgIdAPuACYDOwGnAoMjol9KaWYd5aydl9MLGAP8EegNHAPsFxG7ppSmLom6JUmSpCJqVAv/5WSB+5SU0kEppe+nlAYClwCbAefXWc4FZGH/kpTSoLycg8jCe7e8niVVtyRJklQ4rQ78EbERsDcwHfhNxeJzgLnAkRHRqZlyOgFH5uufU7H413n5X8rra2jdkiRJUlE1ooV/YD4dnVKaX74gpfQuMAHoCOzSTDm7Ah2ACfl25eXMB0bnP+65BOqWJEmSCqkRffg3y6fP1lg+hawVvhdwbyvLIS+n0XUvUkSkWssmTZrE9ttvv7hFLzUzXpnT8DK3b/dew8t8btXnGl7moa/V/PMtttWWg7/58qTRx6fHphrFz87G8vhsHI/Nxloejs1JkyYB9FycbRsR+Lvk01pHXml+1yVQTqPqXmwffPDBrMcee2z6kiq/LXus8UVuBx82vOinG1lYyWNL4N2rYTw21ZZ5fKqt8ths83oC7yzOhg0ZpacZkU9bezq2OOU0pO6UUjS/llqrdCUlpdT2T7O1QvHYVFvm8am2ymOz7WhEH/5SK3qXGsvXqFivkeU0qm5JkiSpkBoR+J/Jp71qLN80n9bqZ9+achpVtyRJklRIkVLretrkD756jmxozI3LR8uJiM7ADLITi3VTSnMXUc7qwBvAfKB7+Ug9EdEOeJ6s79LGpYdvNaputQ1ll/7sQqU2xWNTbZnHp9oqj822o9Ut/Cml58mGzOwJnFSxeBjQCbihPHBHRO+I6F1RznvAjfn651aUc3Je/l/Kn7S7OHVLkiRJK5JWt/DDgpb2iWRPvL0DmATsTDZm/rNA35TSzLL1q57xRcTaeTm9gDHAw0Af4ECy1v++echf7LrVdtkSoLbKY1Ntmcen2iqPzbajIYEfICI2AM4DBgNrk3WnuR0YllKaVbFuzQMgItYie0ruQUB3YCZwD3B2Sunl1tYtSZIkrUgaFvglSZIktT2NGKVHkiRJUhtl4JckSZIKzMAvSZIkFZiBX5IkSSowA78kSZJUYAZ+SZIkqcAM/JIkSVKBGfglSZKkAjPwS5IkSQVm4JckSZIKzMAvSZIkFZiBX5IkSSowA7/alIgYEREpInou5vbjIiI1dq+kxsqP8XEV887N5w9YNnslSSoqA78kSW1ARAzIT/rKX+9HxIyIGB8Rv4iIbesoZ4OI+FlEPBoRb0fExxHxRkT8LSJOjYguNbZbPyJ+HBF/j4i38u3mRMTjEXG5J6PS8itSsjFUbUdEdAe6AM+nlD5ejO03BDqmlCY3fOekBsmvQt2XUhpQNu9c4Bxgz5TSuGWzZ1qW8kA9FngBGJHPXhVYF9gufwH8HhiSUnqvShnfBH4NtAeeBCYCbwNrA7sBmwMzU0rrVGz3LeBXwGrAZOAB4A2gE7AZsHv+71+mlL7XiPcraelZeVnvgFQupTQDmNGK7V9s4O5I0rIwPaV0buXMiNgGuAH4OrAWsE/F8q8DV5MF/ENSSn+uUkY/4DcV844Crsq3OzyldEeV7dYAhgAbLt5bkrQs2aWn4CKiZ35ZeERE9I6I2yNiVkTMjYgHImLvivWPztc/OiIG533i51T2i8/LGhERL0XEvIh4PSJ+HxGb1diPjhHx/yLikYh4NyLei4hJEXFpRHyubL2qffgj4oCIuDe/tD0vIl6NiPsi4sSK9ar24Y+IdhExNCL+kdc9N//3CRHR5P9BqY91RKwTEf9dVu+/IuKYen73WnoqjvNeEXFz3oVhfqkbQkSsFRE/zY+7D/Lj+t7K/wMV5X4tX2dWRHwYEdMj4g8RsUPZOl0i4nsRMSYiXo6IjyLizYi4MyJ2WQpvX21UxXHZMyL+mHeV+TD/LPxyS8pLKT0B/DvwJjA4Ig4qq6szcFn+4+HVwn5exgRg57LtugDD8x+/Vi3s59u9k1L6BXB6S/ZZS0ZE7JR/zr2SfzfNiIjREfHVivW+Gll3sDn5597/RcSZEdG+bJ3VImJ2/plZtSE4Iq7Mj+X9KubXnQXKvt83iohvR8Q/830aly9fNSJOjoi7I+KFvLxZkXVF26eyvLJyvxQRE/Lv9VmR5ZzesYh7AiNi54gYGRGv5Z/ZL0XEVRHxb8386pdbtvCvOL4A/B14iqwlpzvwNeCeiPh6SunmivUPBQYD9wBXAj1LCyJiMHArsApwF/Ac0AM4GNgvIvZMKT1Wtv6aZJeptwaeAa4FPgI2Bo7Ny3q91o5Hdqn5KuC1vL63gG7AVsAxwOV1vP8byVrFXgKuARLwlXzb3YAjqmzTFZiQ7+tIskvdhwLXRsT8lNL1ddSrpWtj4CHgWeAmoAPwTkR8HhhHdhzfD4wi657wZWBURAxJKV1dKiQiArgOOIrseLuVLGT1APYkO44fyVfvA5wPjAf+TNZKuiFwALBPROyfUhq1xN6xlgefBx4GppJ9Fq1F9vl7R0T8e0ppbL0FpZTeiIirgB+SfW7dni86NC/3wZTS6GbKmFf246Fkn3UTU0p/raP+T+rdVy0ZEXE8cAXwKXAnMIXsO3EH4ETglny9C4AzyT7Dfg+8R3ZV6ALgSxGxV0rp45TShxFxM/CtfPldFfW1B75K9j39l7L5LcoCZX5F1kXsz8Dd+fuA7Pj9FVk3tL+SfeZ2B/YH7o6I41NK11Ts29fy9zYvf98zgL5keefJGr+/Y8iuhM3Lf38vAZsC3wT2j4hdCtlbIKXkq8AvsoCT8tcvKpbtAHxMFlDWyOcdna87Hxhcpbw18/XfAr5YsWxzsg+Uxyrm/z4v8wqgXcWyzkCXsp9H5Ov2LJv3KNl/zG5V9medip/HZYf1QvP+Iy/zMWD1svmdyEJbAr5esU3pd3YNsFLZ/C8CnwBPL+u/ra+ax/kFVZaPy4/pwyvmdwWeAD4APlc2/1t5WQ+XH5/5spWA7mU/d6k8DvP5PYBXgUlVliVgXMW8c/P5A5b179NXY14Vx+U5Fcu+lM+/u2zegGrHRpVyB+XrvVA277f5vJ+0cB+vzbf78bL+ffmq6+/1xfx7exaweZXlPfLprvnf9UVgvbLlK5OF8wT8oGx+af2RVco8LF92Udm8xckCI/JyXgG+UKWe9qX9r5jfhayxchbQoWx+53wf5gFbV2zzs7L/ez3L5vcia8R7Dli/YpuBZCcfty3rv/OSeNmlZ8UxBzivfEZK6RGyVtCuZK3d5e5I1Vsl/zNf/5yU0tMV5f2L7Kx524j4IkBEdCNryZoBfDelNL9im3dTSnPq2P9PyD7kFpJSequObY/Np99PZTe5pZTmAv8v//GbVbZ7HzgjpfRp2TZPk7X698kvoatteR0YVj4jIrYG9gD+lFL6Y/mylNJsshtlVwMOKVv07Xw6pPL4TCl9mrJ7TUo/z6l2HKaUXia7MtQ7spvJteJ6AfhJ+YyU0l/IwthOi1HeK/l03bJ53fPpyy0sa72KMheIiK6RDRe70KuF5auxTiAL7T/Ov3MXkn/uwGffez9JKb1WtvwT4DtkDSDfLJv/d7Iro/tHxFoVxR6VT8uvarcoC1S4MKU0rcq+zyvb//L5c8hOTNcEdixbdGC+DzellCpb838CzK5S9wlkVyROTSktdMynlMaQtfjvX8Tvd7v0rDgeSym9W2X+OLL/zNuy8H/mh2uUs2s+3brGB3+vfNoHeJrsP2c7YHwesBfHTcBFwL/yy473ARNSSm/Wuf12ZB9u46osu4/sjL7aUHdTUkrvVJn/Uj7tClT7nWrZeTIt3F0BPjtmu9Q4ZkuhqQ9ARHQCtgBeTyk9Xk+lkd0IeWpeVzeykVXKrU8W7rRieqK84aDMS3x2fLZE5NPUzLzFLaukK9kJcaVzW1iHGqd0X9A9zaxXGtFpTOWClNKzEfEy8IWI6Jo3fECWAc4HDifvKhvZPXZfAh5PKf2zrJiWZoFytfIFEbE58D2gP9lJ7GoVq6xf9u/S9/YDleWklN6LiCfIrpqVK+33HhGxI011I7uK24usd0FhGPhXHLX6yJfO/CvHZX6tcsXc2vn0+GbqWz2fds2nTVqP6pVSujgi3iLrm3gKcBqQIuI+4Hv5lYpF6QLMSil9VKXsT/Kyu1XZrlrrAGRXGyD7UFDbUu24LR2ze+WvWhbrmI2Ir5C15H9I1u/0eWAu2UnmALKrC+1rba8VwqI+SxbnSnvpxsLyRo9X82mPFpZVulq1fuWClNJ0PjshIA+JTdbTUlXv51PpO73WqHczyO416sJnx+cNwI/JGgFL98YdQZYVK+9Za2kWKFc1X0Q2yMGYvL57yVrb3yH7LN2GrEW//LO09B5r5Ztq80v73dzQstX2e7lm4F9xfK7G/NLl3MpuNbVaiUrrbV1xtl9L6YOkVV8SKaUbgBsioivZDTlfIbtk+ZeI6JNSemMRm88B1oqIVVLF2P75iATrkH2oaPlX7bgtHbOnppQuraOMlh6zPybrE7pDSmlS+YL85so96ixHqtee+fShsnkPkH0mDgJ+1IKyJpANfjAIOLshe6clqfzzaVHPmyl97q1H1ghRqXvFeqSUXo6IMcC/R0TvlD3P5iiy7rS/r1F+vVmgXK188UOygRaaPIskIs4kC/zlSt/btfJNtfml/e5S4wp+YdmHf8WxXY0+aQPyaV1dF4AH8+nuda7/MNnZef+8q0SrpJRmp5TuTikdT3YD0Fp17MvjZMd6/yrL+pO11FcbSUDF0KJjNu969hTwuajjqabAJmQ3cVeG/XZkI0BJDZPfFzUk//GmskUjyW5q3DUi/r2ZMspbSUeShci+ETGokfuqJaL0eVZzmMpc6Tt9QOWCiNiE7ErQtLLuPCUj8ulRkT33YSvgnipdaFuaBeqxCdnV+HFVllVrOCm9xyafsxGxOtlVgUpLYr+XCwb+FUcXKlpvIhtL/AiyM97b6iznOrIvh3MiosnNZpGNdz+g9HP+IfFHstaEX0bFmPcRsXrUeMx72TqDa4wNXOqG834z+3xtPv1pRHQsK7cj2Z38kI1woQLKu3zdDxwcEcdWWycitsyDVEnpSsBVlcdnfox3L5s1Hdi0fPzmfFjPc8hG1JAaIr8B/a9kVyXvTindWVqW36N1Sv7jzRHxpRpl7EI2ZGFpuzlk3SQBbomI/Wts15HsZkctW1eQdQX7UbUbYiOi1KWr9L33w4hYt2z5SsAvyfJfte+9W8lazr9BNmoffHYSUK5FWaBO08muxm9VUdZxZPcRVLqDLL8ckf/fKPdDPuv+VO7XZFcsLomIXpULI3sWQCFPBuzSs+IYD3wzInYmu4RbGoe/HdlIJHVd2kopzYyIQ8lOEB6MiHuBf5G14m9IdkPM2ix8o83JZDdBDgUGRMRfyLpAfIHsP/EBVL+htuSPwIcR8QDZB0KQnZ3vSHZTzd+a2effR8SBZOMI/ysibie7pHhQvg+3pJRuWlQZWu59naxv6G8j4hSyrhCzyVq5tiI7PncFSl3DriFrNfpPYEpE3EHWX/rfyIZuu5bPbly8hOxZFY9HxJ/Ivkz6kYX9u8jGkJZaomfZjZCrkAX87fMXZC37Qyo3SindFBEdyELNqPymxYlkQxeuTXaMb002lGL5dtfnrf6XAndGxCSy74k3yPoy9wD2zv99f+PeploqpfR0ZA+cLH3m3EE2Dv/aZENtv0vWJWZiRFwI/BfwVESMJLu3aB+yz7sHgF9UKf+DiPgf4Diy++Zmko2XX7ne4mSB5gwnywQPRMQtZGF+B7LP4pFkz4wo34d38t/F74CJ+Talcfi3JhuUY498n0rbTM4bfq4lywOjyEYnWiXf793JPut7t2C/lw/LelxQX0v2xWfjQI8gu1v+DrIP//fJPtC/VLH+0fn6R9dR7q/JPmg+JGsRmEz2UJmDqqzfCTgL+Gde97tkd+4Pp2x8faqPwz+U7ENlar7tLLJLef8FdK6oZxwV4/Dn89uRfXg9kpfxPtnJwklUPBsgX7/mWNjV9tFX2znOF7FOZ+AH+d/9PbKx96eRfZl9C+hUZZsjyL405uTH+TSysLVdxXpHk43nP5csTN0GbEmNsfWrHV+11vW1/L6aOy4rP6/4bBz+8tcHZCFmPFlA26aOejcAfk7WVXE22Unom2QPQDyN/LkrVbbrQTZKy0P55+zH+bH/JFnA3GNZ/059Lfhb7Qr8ieyk7COym7ZHAYdWrHc4Wbh/N/8M+xfZd/Fqiyh7t7Lj77Jm9qMndWaBer47yR6G+GC+v7OB0WRdb4+mRjYhO4mZSPa9/jZZzukN/G++Tdcq22yZ788LZOP4z+KzB5MOXNZ/3yXxivyNq6Aie6T0NOD6lNLRy3RnJEmSlrC869JUoH1Kab3m1l8R2IdfkiRJy5384XAdK+YFWR/+DcnuSRD24ZckSdLyaReym9RHk93jt3o+bxuyB9udu8z2rI0x8EuSJGl59AxZX/1+wL5kufZlshvQL0iLfkbPCsU+/JIkSVKB2YdfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQCM/BLkiRJBWbglyRJkgrMwC9JkiQVmIFfkiRJKjADvyRJklRgBn5JkiSpwAz8kiRJUoEZ+CVJkqQC+//4dHoJwCrHMwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "image/png": { + "height": 267, + "width": 382 + }, + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "all_scores_df[['precision', 'recall', 'nDCG', 'coverage']].T.plot.bar(rot=0);\n", + "plt.title('Combined evaluation results');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Remarkably, with HybridSVD we were able to substantially increase the diversity of recommendations without spoiling them.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In fact, we have even slightly *increased the accuracy* both in terms of relevance of recommendations (indicated by precision and recall) and in terms of ranking (indicated by nDCG metric)!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "# Conclusion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This tutorial demonstrates capabilities of the HybridSVD model. Surprisingly, among many datasets I used for the [HybridSVD paper](https://arxiv.org/abs/1802.06398) none of them had such a strong dependence on side information, and HybridSVD was often only slightly better than SVD. Here we observe a different picture where standard SVD almost fails and HybridSVD actually extracts useful information, significantly outperforming its competitors. It would be interesting to know whether there are other datasets like that. If you find this tutorial helpful and will use it on another data (or with other models), let me know about your results! Do you observe the same behavior?\n", + "\n", + "Of course, it does not immediately follow from this tutorial that HybridSVD will always be better than any other hybrid model, including LightFM. However, HybridSVD at least presents an easy-to-tune yet very competitive baseline. It inherits the main advantages of the standard PureSVD approach and extends its functionality. Moreover, its hyper-parameters have a straightforward and intuitive effect on the quality of recommendations.\n", + "\n", + "There are certain technical challenges, related to implementation of the algorithm, which I'm not going to discuss here (the post is already too long). Some of them will be addressed in forthcoming papers. However, if you have any specific question in mind, let me know in the comments section below or post an issue in the Polara's github repository." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 6790f85f9a1b581677db54ec9259a4d498b234f7 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 9 Oct 2019 12:53:39 +0300 Subject: [PATCH 57/82] update LightFM tutorial --- .../Comparing LightFM with HybridSVD.ipynb | 100 +++++++++--------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/examples/Comparing LightFM with HybridSVD.ipynb b/examples/Comparing LightFM with HybridSVD.ipynb index 4be090d..5276801 100644 --- a/examples/Comparing LightFM with HybridSVD.ipynb +++ b/examples/Comparing LightFM with HybridSVD.ipynb @@ -11,9 +11,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "LightFM is a very popular tool, which finds its applications both in small projects and in production systems. Aside from the technical aspects, the general idea of representing users and items in terms of feature combinations, used in the LightFM model, is not new. It was implemented in a number of preceeding works, and I'm not even saying about the obvious link to [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf). Another notable member of this family is [SVDFeature](https://arxiv.org/pdf/1109.2271.pdf) , which came around in 2011 and in some sense is even closer to LightFM than FM (the naming, though, is misleading, as it is not an SVD-based model). In fact, the feature combination approach can be traced back to the Microsoft's [MatchBox system](https://www.microsoft.com/en-us/research/wp-content/uploads/2009/01/www09.pdf) from 2009, where they use the term *trait vector* for a combined representation. Still, even though the idea itself was not new, the LightFM model was made convenient and extreemly easy to use and had a fairly good computational performance, which, I believe, has made this framework so popular among practitioners.\n", + "LightFM is a very popular tool, which finds its applications both in small projects and in production systems. On a high level, the general idea of representing users and items in terms of combinations of their features, used in the LightFM model, is not new. It was implemented in a number of preceeding works, and I'm not even saying about the obvious link to [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf). Another notable member of this family is [SVDFeature](https://arxiv.org/pdf/1109.2271.pdf) , which came around in 2011 and in some sense was even closer to LightFM than FM (the naming, though, is misleading, as it is not an SVD-based model). In fact, the feature combination approach can be traced back to the Microsoft's [MatchBox system](https://www.microsoft.com/en-us/research/wp-content/uploads/2009/01/www09.pdf) from 2009. The authors used the term *trait vector* for a combined representation and employed elaborate probabilistic framework on top of it. Still, even though the idea itself was not new, the LightFM model was made convenient and extremely easy to use, which, probably, has made this framework popular among practitioners. Moreover, it provided several ranking optimization schemes and had a fairly good computational performance.\n", "\n", - "On the other hand, I have also seen some complains from data scientists about actual prediction quality of LightFM. Moreover, I could not find more or less rigorous comparison of LightFM with strong baselines, where all compared models would undergo appropriate tuning. The [examples section](https://lyst.github.io/lightfm/docs/examples.html) of the LightFM's documentation merely provides quick-start demos, not real performance tests. The [paper from the RecSys workshop](http://ceur-ws.org/Vol-1448/paper4.pdf) also does not provide too much evidence on the model's performance. Numerous online tutorials only repeat basic configuration steps, leaving aside the whole tuning aspect. Considering how often practitioners are advised to start with LightFM (according to data science chats and forums I'm aware of), I have decided **to put the LightFM capabilities to the real test**." + "On the other hand, I have also seen some complains from data scientists about actual prediction quality of LightFM. I could not find a more or less rigorous evaluation of LightFM, where it would be compared against strong baselines and, more importantly, all models would undergo appropriate tuning. The [examples section](https://lyst.github.io/lightfm/docs/examples.html) of the LightFM's documentation merely provides quick-start demos, not real performance tests. The [paper from the RecSys workshop](http://ceur-ws.org/Vol-1448/paper4.pdf) also does not provide too much evidence on the model's performance. Numerous online tutorials only repeat basic configuration steps, leaving the whole tuning aspect aside. Considering how often practitioners are advised to start with LightFM (in my experience), I have decided **to put the LightFM capabilities to the real test**." ] }, { @@ -27,10 +27,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The baseline models in this tutorial will be based on my favorite SVD (the real one, not just some matrix factorization). I have summarized the reasons why I promote SVD-based recommendation algorithms as the default choice in my [previous post](//slug:to-svd-or-not-to-svd). However, I have not yet described how to use SVD in the cold start scenario. The theoretical foundation of this can be found in our [recent paper](https://arxiv.org/abs/1802.06398), where we demonstrate how to adopt both PureSVD and HybridSVD models for the cold start scenario. Here, I will provide only the minimum necessary material for understanding what is going on, focusing more on technical aspects instead. For all experiments we will be using [Polara](https://github.com/evfro/polara), as it provides easy-to-use high-level API for both LightFM and SVD-based models, as well as a set of convenient tools for building the entire experimentation pipeline.\n", + "The baseline models in this tutorial will be based on my favorite SVD (the real one, not just some matrix factorization). I have summarized the reasons why I promote SVD-based recommendation algorithms as the default choice in my [previous post](//slug:to-svd-or-not-to-svd). However, I have not yet described how to use SVD in the cold start scenario. The theoretical foundation as well as more experiments can be found in our [recent paper](https://arxiv.org/abs/1802.06398), where we demonstrate how to adopt both [PureSVD](https://www.researchgate.net/publication/221141030_Performance_of_recommender_algorithms_on_top-N_recommendation_tasks) and our HybridSVD model for the cold start scenario. Here, I will provide only the minimum necessary material for understanding what is going on, focusing more on technical aspects instead.\n", "\n", - "Unlike many online materials, we will employ an *advanced optimization procedure for LightFM* to ensure that the obtained model is close to its optimal configuration.\n", - "Conversely, as we will see, *tuning of the SVD-based models is very straightforward*. They depend on fewer hyper-parameters and also have deterministic output. The question remains, whether it is possible to outperform LightFM with these simpler models. Let's find out." + "For all experiments we will employ an *advanced optimization procedure for LightFM* to ensure that the obtained model is close to its optimal configuration.\n", + "Conversely, as we will see, *tuning of the SVD-based models is much more straightforward and less cumbersome*. They depend on fewer hyper-parameters and also have deterministic output. The question remains, whether it is possible to outperform LightFM with these simpler models. Let us find it out." ] }, { @@ -151,7 +151,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We also need to convert item features (tags) into the dataframe with a special structure. This will enable many Polara's built-in functions for data manipulation." + "We also need to convert item features (tags) into a dataframe with a ceartain structure. This will enable many Polara's built-in functions for data manipulation." ] }, { @@ -160,8 +160,8 @@ "metadata": {}, "outputs": [], "source": [ - "# convert sparse matrix into `item - tag list` Series format\n", - "# make use of sparse CSR format\n", + "# convert sparse matrix into `item - tag list` pd.Series object\n", + "# make use of sparse CSR format for efficiency\n", "item_tags = (\n", " pd.Series(\n", " np.array_split(\n", @@ -171,7 +171,7 @@ " # split tags into groups by items\n", " data['item_features'].indptr[1:-1]))\n", " .rename_axis('items')\n", - " .to_frame('tags')\n", + " .to_frame('tags') # Polara expects dataframe\n", ")" ] }, @@ -348,9 +348,9 @@ } ], "source": [ - "print_frames([training_data.head(),\n", - " test_data.head(),\n", - " item_tags.head()])" + "print_frames([training_data.head(), # data for training and validation\n", + " test_data.head(), # data for testing\n", + " item_tags.head()]) # item features data" ] }, { @@ -364,7 +364,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We need to wrap our data into an instance of Polara's recommender data model, which allows to share data among many recommender models and propagate consistent state across all of them. More details on its usage can be found in other [Polara examples](https://github.com/evfro/polara/tree/master/examples). In order to fine-tune models we will additionally split training data into the traininig and validation sets using the Polara's built-in functionality." + "We need to wrap our data into an instance of Polara's `RecommenderData` subclass, designed for automating cold start experiments. This also allows to share data among many recommender models and propagate consistent state across all of them. In order to fine-tune models we will additionally split training data into the traininig and validation sets using the Polara's built-in functionality." ] }, { @@ -421,7 +421,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Let us look at the holdout data:" + "Let us look at the `holdout` data, which will serve as a validation set:" ] }, { @@ -556,7 +556,7 @@ "metadata": {}, "source": [ "**The procedure for both validation and final evaluation will be the following**:\n", - "1. For every unique cold item, we generate a list of candidate users who, according to our model, are most likely to be intersted in this item.\n", + "1. For every unique cold item, we generate a list of candidate users who, according to our models, are most likely to be intersted in this item.\n", "2. We then verify the list of predicted users against the actual users from the holdout.\n", "\n", "We will use standard evaluation metrics to assess the quality of predictions." @@ -583,10 +583,10 @@ "source": [ "As stated in the introduction, LightFM is an efficient and convenient instrument for building hybrid recommender systems. There are, however, several downsides in its implementation (at least at the moment of writing this post).\n", "\n", - "First of all, **there is no off-the-shelf support for the warm start scenario** (e.g., for new users with several known interactions), which would allow estimating strong generalization of the algorithm rather then weak generalization that we test in the standard setup (with only known users).\n", + "First of all, **there is no off-the-shelf support for the warm start scenario** (e.g., for new users with several known interactions), which would allow estimating strong generalization of the algorithm.\n", "Basically, there's no folding-in implementation, even though it seems to be in a high demand (see github issues [here](https://github.com/lyst/lightfm/issues/300), [here](https://github.com/lyst/lightfm/issues/322#issuecomment-401951114) or [here](https://github.com/lyst/lightfm/issues/194#issuecomment-310775451)) and it is not that difficult to implement. Likewise, there is no [session-based recommendations support](https://github.com/lyst/lightfm/issues/362) as well. The [sometimes recommended](https://github.com/lyst/lightfm/issues/347#issuecomment-407383263) `fit_partial` method is not really an answer to these problems. It is going to update both user and item embeddings and sometimes it is not what you may want. For example, fitting newly introduced users would affect latent representation of some know items as well, potentially changing recommendations for already present users without any of their actions. Such an unintended change in recommendations can be confusing for users and lead to a lower satisfaction with a recommendation service.\n", "\n", - "Another aspect, which is critical in certain situations is **the lack of determinism**. LightFM's output is deterministic only in strict conditions: single-threaded training with the fixed input that never changes (even the order of elements). In multithredaded execution the LightFM's optimization process causes racing conditions that are resolved at the hardware level, which you have no control of. This is not unique to LightFM and is shared across all matrix factorization algorithms based on naive SGD optimization. Moreover, even tiny reorderings (e.g., when two items corresponding to a single user switch their places) may lead to [noticably different results](https://github.com/lyst/lightfm/issues/225). Even in this notebook, if you set the number of LightFM threads to 1 (see below), once you restart the notebook kernel, the results may change. The problem seems to be outside of LightFM itself, however, it demonstrates its non-deterministic nature. If your business has specific requirements on, e.g., non-regression testing, this model is not for you. Conversely, **SVD-based models are free of such issues.** This is also among the reasons why matrix factorization models like SVDFeature or FunkSVD should not be mixed with real SVD-based models." + "Another aspect, which can be critical in certain situations is **the lack of determinism**. LightFM's output is deterministic only in strict conditions: single-threaded execution with a fixed input data order. Multithredading would cause racing conditions that are resolved at a low system level beyond user control. This is not unique to LightFM, of course, and is shared across all matrix or tensor factorization algorithms based on naive SGD. Input data reordering and, probably, [some other factors](https://github.com/lyst/lightfm/issues/225) may also lead to noticably different results. Even in this notebook, if you set the number of LightFM threads to 1 (see `num_threads` variable below), once you restart the notebook kernel, the results may change. If your business has specific requirements on, e.g., non-regression testing, this model may not be suitable for you. Conversely, **SVD-based models are free of such issues.** This is also among the reasons why matrix factorization models like SVDFeature or FunkSVD should not be mixed with real SVD-based models." ] }, { @@ -658,13 +658,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As the LightFM algorithm depends on many hyper-parameters and has a stochastic nature, it presents a hard problem for optimization. We will have to give a fair bit of tuning to LightFM. There are many great tools for doing this. Two popular options are [HyperOpt](https://hyperopt.github.io/) and [Optuna](https://optuna.readthedocs.io/); both of them are great. I personally prefer the latter as it typically allows me to write a more concise code. We will be using it in this tutorial as well. If you don't have `optuna` installed you can simply run\n", + "As the LightFM algorithm depends on many hyper-parameters and has a stochastic nature, it presents a hard problem for optimization and requires a fair bit of tuning. There are many great tools that make life a bit easier in this regard, e.g., [HyperOpt](https://hyperopt.github.io/) or [Optuna](https://optuna.readthedocs.io/). I personally prefer the latter as it typically allows writing more concise code. We will be using it in this tutorial as well. If you don't have `optuna` installed you can simply run\n", "```\n", "pip install optuna\n", "```\n", "in your python environment.\n", "\n", - "Instead of performing *random search* for hyper-parameter optimization we will employ a more advanced and flexible techniqe based on *Tree-structured Parzen Estimator* (TPE). It will help to iteratively narrow-down the hyper-parameter search subspace and preemptively disregard unmpromising search regions. A bit more details along with further reading can be found in the [optuna documentation](https://optuna.readthedocs.io/en/latest/reference/samplers.html). There is also a nice comparison of different techniques in this [blog post](http://neupy.com/2016/12/17/hyperparameter_optimization_for_neural_networks.html#tree-structured-parzen-estimators-tpe)." + "Instead of performing *random search* for hyper-parameter optimization we will employ a more advanced and flexible techniqe based on *Tree-structured Parzen Estimator* (TPE). It will help to iteratively narrow-down the hyper-parameter search subspace and preemptively disregard unmpromising search regions. A bit more details along with resources for further reading can be found in the [optuna documentation](https://optuna.readthedocs.io/en/latest/reference/samplers.html). There is also a nice comparison of different techniques in this [blog post](http://neupy.com/2016/12/17/hyperparameter_optimization_for_neural_networks.html#tree-structured-parzen-estimators-tpe)." ] }, { @@ -684,14 +684,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note that tuning for all possible types of hyper-parameters exponentially increases the complexity of the task. The authors of `optuna` even recommend to refrain from tuning unimportant variables. We will therefore **concentrate on 3 main hyper-parameters**:\n", - "1. dimensionality of the latent space (rank of the factor matrices),\n", + "Note that tuning with high number of hyper-parameters exponentially increases the complexity of the task. The authors of `optuna` even recommend to refrain from tuning unimportant variables. We will therefore **concentrate on 3 main hyper-parameters**:\n", + "1. dimensionality of the latent space (rank of the decomposition),\n", "2. importance of item features, controlled by the `item_alpha` regularization parameter,\n", "3. number of epochs.\n", "\n", - "We will leave other hyper-parameters with their default values. My preliminary experiments showed that there was no big difference in the final result after changing them, as long as their values remained within a reasonable range. As always, you are free verify that on your own by adding more hyper-parameters into the search space. You will need to modify the `objective` function defined below.\n", + "We will leave other hyper-parameters with their default values. My preliminary experiments showed that there was no big difference in the final result after changing them, as long as their values remained within a reasonable range. As always, you are free to verify that on your own by adding more hyper-parameters into the search space. You will need to modify the `objective` function defined below.\n", "\n", - "To aid the learning process, we will sample `item_alpha` from a log-uniform distribution and `rank` values from a range of positive integer numbers up to `max_rank`. Note that we also use `set_user_attr` routine to store additional information related to each tuning trial." + "To aid the learning process, we will sample `item_alpha` from a log-uniform distribution and `rank` values from a range of positive integer numbers up to a certain threshold. Note that we also use `set_user_attr` routine to store additional information related to each tuning trial." ] }, { @@ -701,8 +701,8 @@ "outputs": [], "source": [ "def evaluate_lightfm(model):\n", - " 'Convenience function for evaluating lightfm.'\n", - " # disabling user bias terms improves cold start prediction quality\n", + " '''Convenience function for evaluating LightFM.\n", + " It disables user bias terms to improve quality in cold start.'''\n", " model._model.user_biases *= 0.0\n", " return model.evaluate()\n", "\n", @@ -734,8 +734,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "One of the limitations of TPE is that it does not take into account possible dependencies between hyper-parameters. In addition to that, parameters like `rank` or `epochs` significantly influence the training time. In order to avoid spending too much time on tuning yet being able to achieve a reasonably good model, we will use a specific procedure. We take the number of `epochs` out of the main loop.\n", - "Based on that, we want to early exclude bad hyper-parameter values that lead to high enough scores solely due to high number of epochs. Hence, we start from a smaller number of epochs and increase it gradually, leaving out unpromising configurations along the way due to TPE optimization procedure. Hopefully, it won't discard potentially good search directions that will later produce good results with the higher number of epochs. Likewise, we also gradually decrease the trials budget for each subsequent number of epochs, assuming that narrower search space requires less exploration." + "One of the limitations of TPE is that it does not take into account possible dependencies between hyper-parameters. In addition to that, parameters like `rank` or `epochs` significantly influence the training time. In order to avoid spending too much time on getting a reasonably good model, we will use a specific tuning procedure.\n", + "\n", + "We take the number of `epochs` out of the main loop. Based on that, we want to early exclude bad configurations that lead to high enough scores solely due to high number of epochs. Hence, we start from a smaller number of epochs and increase it gradually, letting TPE discard unpromising search directions along the way. Hopefully, it won't discard good search regions that can potentially produce good results with a higher number of epochs. Likewise, we also gradually decrease the trials budget for each subsequent number of epochs, assuming that narrower search space requires less exploration for achieving reasonable quality." ] }, { @@ -1096,19 +1097,19 @@ ] }, { - "cell_type": "code", - "execution_count": 24, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from polara.recommender.coldstart.models import ScaledSVDItemColdStart" + "In the cold start scenario the model depends on side features (needed to construct the matrix $F$). This informaion can be provided via an input argument during instantiation of the model itself. An alternative and more robust way used here is to provide features in the data model constructor. Hence, the required `item_tags` variable will be taken from the `data_model` object." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 24, "metadata": {}, + "outputs": [], "source": [ - "In the cold start scenario the model depends on side features (needed to construct the matrix $F$). This informaion can be provided via an input argument during instantiation of the model itself. An alternative and more robust way used here is to provide features in the data model constructor. Hence, the required `item_tags` variable will be taken from the `data_model` object." + "from polara.recommender.coldstart.models import ScaledSVDItemColdStart" ] }, { @@ -1843,7 +1844,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Hopefully, this structure will be more appropriate for discovering relationships between the real and the latent features, and will improve predictions quality in the cold start. The hybrid functionality of SVD is enabled by the use of additional information about similarities between users and/or items. These similarities are computed beforehand and utilize side features (item tags in our case). The **model still uses SVD for computations**, even though internally it is applied to a special auxiliary matrix instead of the original one. It, therefore, remains the \"real\" SVD-based model **with all its advantages**. Once again, you can look up the details in [our paper](https://arxiv.org/abs/1802.06398). The process of computing the model and generating recommendations remains largely the same, except for the additional handling of similarity information. Polara already provides support for this. We only need to invoke the necessary methods and classes. See the related data pre-processing procedure below. " + "Hopefully, this structure will be more appropriate for discovering relationships between the real and the latent features, and will improve predictions quality in the cold start. The hybrid functionality of SVD is enabled by the use of additional information about similarities between users and/or items. These similarities should be computed beforehand and utilize side features (item tags in our case). The **model still uses SVD for computations**, even though internally it is applied to a special auxiliary matrix instead of the original one. It, therefore, remains the \"real\" SVD-based model **with all its advantages**. Once again, you can look up the details in [our paper](https://arxiv.org/abs/1802.06398). The process of computing the model and generating recommendations remains largely the same, except for the additional handling of similarity information. Polara already provides support for this. We only need to invoke the necessary methods and classes. See the related data pre-processing procedure below. " ] }, { @@ -1894,7 +1895,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This similairty matrix will be feed into the special data model instance, which will take care of providing a consistent on-demand access to similarity information for all dependent recommender models. The input should be in the format of a dictionary, specifiyng types of entities, similarities between them and the corresponding indexing information to decode rows (or columns) of the similarity matrices:" + "This similairty matrix will be fed into the special data model instance, which will take care of providing a consistent on-demand access to similarity information for all dependent recommender models. The input should be in the format of a dictionary, specifiyng types of entities, similarities between them and the corresponding indexing information to decode rows (or columns) of the similarity matrices:" ] }, { @@ -1982,9 +1983,9 @@ "source": [ "In order to generate the latent representation of a cold item the model uses the same linear system as in the standard PureSVD case. The main difference is how the linear transformatin matrix $W$ is computed:\n", "$$\n", - "W=V^\\top SF,\n", + "W=V^\\top SF.\n", "$$\n", - "where $S$ is an item similarity matrix and $V$ is now $S$-orthogonal, e.g., $V^\\top S V = I$. The model also introduces an additional weighting hyper-parameter $\\alpha \\in [0, 1]$:\n", + "$S$ is an item similarity matrix and $V$ is now $S$-orthogonal, e.g., $V^\\top S V = I$. The model also introduces an additional weighting hyper-parameter $\\alpha \\in [0, 1]$:\n", "$$\n", "S=(1-\\alpha)I + \\alpha Z,\n", "$$\n", @@ -2123,7 +2124,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The result is much better than for PureSVD! Let us compare all three models together." + "**The result is much better than for PureSVD!** Let us compare all three models together." ] }, { @@ -2175,7 +2176,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The result demonstrates that HybridSVD is very capable and has a high potential to significantly outperform all other models. It seems that the quality of HybridSVD can be further improved with higher rank values. There is also some room for improvement in careful tuning of the scaling parameter and the weighting coefficient $\\alpha$. " + "We see that HybridSVD has a very good capacity comparing to other models in terms of the prediction quality. It seems that HybridSVD can be further improved with higher rank values. There is also some room for improvement in a more careful tuning of the scaling parameter and the weighting coefficient $\\alpha$. " ] }, { @@ -2381,7 +2382,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Let us first gather all the scores into a single dataframe, which is convenient to anaylise." + "Let us first gather all the scores into a single dataframe, which is more convenient." ] }, { @@ -2399,10 +2400,11 @@ "metadata": {}, "outputs": [], "source": [ - "all_scores = {}\n", - "all_scores['SVD (best)'] = svd.evaluate()\n", - "all_scores['LightFM (best)'] = evaluate_lightfm(lfm)\n", - "all_scores[f'HybridSVD (rank {hsvd.rank})'] = hsvd.evaluate()" + "all_scores = {\n", + " 'SVD (best)': svd.evaluate()\n", + " 'LightFM (best)': evaluate_lightfm(lfm)\n", + " f'HybridSVD (rank {hsvd.rank})': hsvd.evaluate()\n", + "}" ] }, { @@ -2419,7 +2421,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "During the tuning phase we focused on the `precision` metric. It is now time to also see other metrics as well, and particularly the `coverage` score. It is calculated as the ratio of all unique recommendations, generated by an algorithms, to all unique entities of the same type present in the training data. Obviously, the maximum value is 1 (100% coverage) and it can be typically only achieved with randomly generated recommendations. The `coverage` metric characterizes the tendency of an algorithm to generate the same recommendations over and over again and is, therefore, linked to the diversity of recommendations. Higher diversity allows mitigating the famous [\"Harry Potter\" problem](https://www.quora.com/Recommendation-Systems-What-exactly-is-Harry-Potter-Problem), sometimes also called the \"bananas problem\". I personally like the term \"[tyranny of the majority](https://medium.com/rtl-tech/my-takeaways-from-netflixs-personalization-workshop-2018-f564a19437b6)\". More diverse recommendations are likely to improve an overall user experience as long as the relevance of recommendations remains relatively high. It may also [increase overall product sales](https://en.wikipedia.org/wiki/The_Long_Tail_(book)). However, in practice, there is an inverse relationship between diversity and accuracy of recommendations: increasing one of them may [decrease the other](https://dl.acm.org/citation.cfm?id=2043957). The underlying phenomena is the [succeptibility of many recommendation algorithms to popularity biases](https://dl.acm.org/citation.cfm?id=2852083) in the *long tail-distributed* data. We have already tried to partially address that problem by introducing the scaling parameter for SVD. Note that SGD-based algotihms like LightFM also have some control over it via different sampling schemes and customized optimization objectives." + "During the tuning phase we focused on the `precision` metric. It is now time to also see other metrics as well, and particularly the `coverage` score. It is calculated as the ratio of all unique recommendations, generated by an algorithms, to all unique entities of the same type present in the training data. Obviously, the maximum value is 1 (100% coverage) and it can be typically only achieved with randomly generated recommendations. The `coverage` metric characterizes the tendency of an algorithm to generate the same recommendations over and over again and is, therefore, linked to the diversity of recommendations. Higher diversity allows mitigating the famous [\"Harry Potter\" problem](https://www.quora.com/Recommendation-Systems-What-exactly-is-Harry-Potter-Problem), sometimes also called the \"bananas problem\". I personally like the term \"[tyranny of the majority](https://medium.com/rtl-tech/my-takeaways-from-netflixs-personalization-workshop-2018-f564a19437b6)\". More diverse recommendations are likely to improve an overall user experience as long as the relevance of recommendations remains high enough. It may also [improve overall product sales](https://en.wikipedia.org/wiki/The_Long_Tail_(book)). However, in practice, there is an inverse relationship between diversity and accuracy of recommendations: increasing one of them may [decrease the other](https://dl.acm.org/citation.cfm?id=2043957). The underlying phenomena is the [succeptibility of many recommendation algorithms to popularity biases](https://dl.acm.org/citation.cfm?id=2852083) in the *long tail-distributed* data. We have already tried to partially address that problem by introducing the scaling parameter for SVD. Note that SGD-based algotihms like LightFM also have some control over it via different sampling schemes and customized optimization objectives." ] }, { @@ -2503,7 +2505,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As indicated by the green horizontal bars in the table above, standard SVD-based model performs poorly both in terms of recommendations accuracy and in terms of diversity. Likewise, LightFM presents a trade-off between these two metrics. \n", + "As indicated by green horizontal bars in the table above, the standard SVD-based model performs poorly both in terms of recommendations accuracy and in terms of diversity. Likewise, LightFM presents a trade-off between these two metrics. \n", "However, we have seen that HybridSVD has not yet reached it's best performance. Note that **lower rank values make SVD-based models insensitive to frequent variations in the observed user behavior**, hence leading to a lower diversity of recommendations in the end. Let us try to increase the rank of the decompostition to verify that." ] }, @@ -2593,11 +2595,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This tutorial demonstrates capabilities of the HybridSVD model. Surprisingly, among many datasets I used for the [HybridSVD paper](https://arxiv.org/abs/1802.06398) none of them had such a strong dependence on side information, and HybridSVD was often only slightly better than SVD. Here we observe a different picture where standard SVD almost fails and HybridSVD actually extracts useful information, significantly outperforming its competitors. It would be interesting to know whether there are other datasets like that. If you find this tutorial helpful and will use it on another data (or with other models), let me know about your results! Do you observe the same behavior?\n", + "This tutorial demonstrates capabilities of the HybridSVD model. The dataset we used here has a strong dependence on side information, and HybridSVD apparently takes the most out of it, significantly outperforming its competitors. If you find this tutorial helpful and will use it on another data (or with other models), let me know about your results! Do you observe the same behavior?\n", "\n", - "Of course, it does not immediately follow from this tutorial that HybridSVD will always be better than any other hybrid model, including LightFM. However, HybridSVD at least presents an easy-to-tune yet very competitive baseline. It inherits the main advantages of the standard PureSVD approach and extends its functionality. Moreover, its hyper-parameters have a straightforward and intuitive effect on the quality of recommendations.\n", + "Of course, this tutorial does not suggest that HybridSVD will always be better than any other hybrid model, including LightFM. However, HybridSVD at least presents an easy-to-tune yet very competitive baseline. It inherits the main advantages of the standard PureSVD approach and extends its functionality. Moreover, its hyper-parameters have a straightforward and intuitive effect on the quality of recommendations.\n", "\n", - "There are certain technical challenges, related to implementation of the algorithm, which I'm not going to discuss here (the post is already too long). Some of them will be addressed in forthcoming papers. However, if you have any specific question in mind, let me know in the comments section below or post an issue in the Polara's github repository." + "There are certain technical challenges, related to implementation of the algorithm, which I'm not going to discuss here (the post is already too long). Some of them will be addressed in forthcoming papers or in blog posts. However, if you have any specific question in mind, let me know in the comments section below or post an issue in the Polara's github repository." ] } ], From 8f108cc00b5084b54438f0629035963930f12c43 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 9 Oct 2019 13:02:33 +0300 Subject: [PATCH 58/82] add reference to hybridsvd paper --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f1292b4..e4788d2 100644 --- a/README.md +++ b/README.md @@ -149,3 +149,9 @@ See more usage examples in the [Custom evaluation](examples/Custom_evaluation.ip ### Reproducing others work Polara offers even more options to highly customize experimentation pipeline and tailor it to specific needs. See, for example, [Reproducing EIGENREC results](examples/Reproducing_EIGENREC_results.ipynb) notebook to learn how Polara can be used to reproduce experiments from the *"[EIGENREC: generalizing PureSVD for effective and efficient top-N recommendations](https://arxiv.org/abs/1511.06033)"* paper. + +## How to cite +If you find this framework useful for your research, please cite [the following paper](https://dl.acm.org/citation.cfm?id=3347055): +``` +"HybridSVD: when collaborative information is not enough"; Evgeny Frolov and Ivan Oseledets, 2019. In Proceedings of the 13th ACM Conference on Recommender Systems (RecSys '19). ACM, New York, NY, USA, 331-339. +``` From b1279828a2c74d0a799cf4b4d0d2f0afc68d2b21 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Tue, 21 Apr 2020 10:23:15 +0300 Subject: [PATCH 59/82] add MAP and ARHR ranking metrics --- polara/recommender/evaluation.py | 35 +++++++++++++++++++++++++++----- polara/recommender/models.py | 7 ++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/polara/recommender/evaluation.py b/polara/recommender/evaluation.py index ce125af..f5e36b4 100644 --- a/polara/recommender/evaluation.py +++ b/polara/recommender/evaluation.py @@ -1,6 +1,6 @@ from __future__ import division import numpy as np -from scipy.sparse import csr_matrix +from scipy.sparse import csr_matrix, diags from collections import namedtuple @@ -103,11 +103,34 @@ def get_hr_score(hits_rank): hr = hits_rank.getnnz(axis=1).mean() return namedtuple('Relevance', ['hr'])._make([hr]) +def get_rr_scores(hits_rank): + 'Reciprocal Rank scores' + arhr = get_arhr_score(hits_rank) + mrr = get_mrr_score(hits_rank) + return namedtuple('Ranking', ['arhr', 'mrr'])._make([arhr, mrr]) + +def get_arhr_score(hits_rank): + 'Average Reciprocal Hit-Rank score' + return hits_rank.power(-1, 'f8').sum(axis=1).mean() def get_mrr_score(hits_rank): 'Mean Reciprocal Rank score' - mrr = hits_rank.power(-1, 'f8').max(axis=1).mean() - return namedtuple('Ranking', ['mrr'])._make([mrr]) + return hits_rank.power(-1, 'f8').max(axis=1).mean() + +def get_map_score(hits_rank, eval_matrix, topk): + 'Mean Avergage Precision score' + # transform input from (n_users x n_items) to (n_users x topk) + topk_rank = hits_rank._with_data(hits_rank.data, copy=False) + topk_rank.indices = topk_rank.data.astype('i4') - 1 + topk_rank._shape = (hits_rank.shape[0], topk) + + cumsummer = diags([np.ones(topk, dtype='i4')]*topk, offsets=range(topk)) + prec_at_k = (topk_rank>0).dot(cumsummer).multiply(topk_rank.power(-1, 'f8')) + + num_relevant = eval_matrix.getnnz(axis=1) + num_relevant_adjusted = np.where(num_relevant Date: Tue, 21 Apr 2020 10:24:14 +0300 Subject: [PATCH 60/82] correct adagrad and rmsprop adjustment factor --- polara/lib/optimize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polara/lib/optimize.py b/polara/lib/optimize.py index 01d7492..abb7e6b 100644 --- a/polara/lib/optimize.py +++ b/polara/lib/optimize.py @@ -74,7 +74,7 @@ def identity(x, *args): # used to fall back to standard SGD def adagrad(grad, m, cum_sq_grad, smoothing=1e-6): cum_sq_grad_update = cum_sq_grad[m, :] + grad * grad cum_sq_grad[m, :] = cum_sq_grad_update - adjusted_grad = grad / (smoothing + np.sqrt(cum_sq_grad_update)) + adjusted_grad = grad / np.sqrt(smoothing + cum_sq_grad_update) return adjusted_grad @@ -82,7 +82,7 @@ def adagrad(grad, m, cum_sq_grad, smoothing=1e-6): def rmsprop(grad, m, cum_sq_grad, gamma=0.9, smoothing=1e-6): cum_sq_grad_update = gamma * cum_sq_grad[m, :] + (1 - gamma) * (grad * grad) cum_sq_grad[m, :] = cum_sq_grad_update - adjusted_grad = grad / (smoothing + np.sqrt(cum_sq_grad_update)) + adjusted_grad = grad / np.sqrt(smoothing + cum_sq_grad_update) return adjusted_grad From d0b690147f4663f6c06c7bb30ada9d99a399b2d9 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Tue, 21 Apr 2020 10:25:24 +0300 Subject: [PATCH 61/82] allow specifying data model configuration at creation time --- polara/recommender/data.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 875215b..20e6561 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -104,7 +104,7 @@ class RecommenderData(object): '_warm_start', '_holdout_size', '_test_sample', '_permute_tops', '_random_holdout', '_negative_prediction'} - def __init__(self, data, userid, itemid, feedback=None, custom_order=None, seed=None): + def __init__(self, data, userid, itemid, feedback=None, custom_order=None, config=None, seed=None, verbose=True): self.name = None fields = [userid, itemid, feedback] @@ -134,7 +134,10 @@ def __init__(self, data, userid, itemid, feedback=None, custom_order=None, seed= # random_holdout - test_update. Need to implement checks # non-empty set is used to indicate non-initialized state -> # the data will be updated upon the first access of internal data splits + if config is not None: + self.set_configuration(config) self.seed = seed # use with permute_tops, random_choice + self.verify_sessions_length_distribution = True self.ensure_consistency = True # drop test entities if not present in training self.build_index = True # reindex data, avoid gaps in user and item index @@ -147,7 +150,7 @@ def __init__(self, data, userid, itemid, feedback=None, custom_order=None, seed= self._notify = EventNotifier([self.on_change_event, self.on_update_event]) # on_change indicates whether full data has been changed -> rebuild model # on_update indicates whether only test data has been changed -> renew recommendations - self.verbose = True + self.verbose = verbose def __str__(self): name = self.__class__.__name__ @@ -173,10 +176,16 @@ def _set_defaults(self, params=None): def get_configuration(self): # [1:] omits undersacores in properties names, i.e. uses external name - # in that case it prints worning if change is pending + # in that case it prints warning if change is pending config = {attr[1:]: getattr(self, attr[1:]) for attr in self._config} return config + def set_configuration(self, params): + for name, value in params.items(): + if hasattr(self, name): + setattr(self, name, value) + else: + print(f'Property {name} is undefined.') @classmethod def default_configuration(cls): From 6ff92c47edc07181524d82bc64e635be0d6fe5de Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Tue, 21 Apr 2020 10:26:37 +0300 Subject: [PATCH 62/82] comply with pandas DataFrame.take arguments change: - is_copy is deprectated, it's alwasy True --- polara/recommender/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 20e6561..4e55fb7 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -13,7 +13,7 @@ def random_choice(df, num, random_state): n = df.shape[0] if n > num: - return df.take(random_state.choice(n, num, replace=False), is_copy=False) + return df.take(random_state.choice(n, num, replace=False)) else: return df From 694464889d60334f8318a68dec208e0fa219b09b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 23 Apr 2020 15:16:34 +0300 Subject: [PATCH 63/82] add fast inner product computation on matrices with specified indices --- polara/lib/sparse.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/polara/lib/sparse.py b/polara/lib/sparse.py index 352685c..ccb0430 100644 --- a/polara/lib/sparse.py +++ b/polara/lib/sparse.py @@ -55,6 +55,38 @@ def sparse_dot(left_mat, right_mat, dense_output=False, tocsr=False): return result +def inner_product_at(target='parallel', **kwargs): + @guvectorize([ + 'f4[:,:], f4[:,:], i4[:], i4[:], f4[:]', + 'f4[:,:], f4[:,:], i8[:], i8[:], f4[:]', + 'f8[:,:], f8[:,:], i4[:], i4[:], f8[:]', + 'f8[:,:], f8[:,:], i8[:], i8[:], f8[:]'], + '(i,k),(j,k),(),()->()', + target=target, nopython=True, **kwargs) + def inner_product_at_wrapped(u, v, ui, vi, res): + rank = v.shape[1] + tmp = 0 + for f in range(rank): + tmp += u[ui[0], f] * v[vi[0], f] + res[0] = tmp + return inner_product_at_wrapped + +# roughly equivalent to +# @njit(parallel=True) +# def inner_product_at(u, v, uidx, vidx): +# size = len(uidx) +# res = np.empty(size) +# rank = v.shape[1] +# for k in prange(size): +# i = uidx[k] +# j = vidx[k] +# tmp = 0 +# for f in range(rank): +# tmp += u[i, f] * v[j, f] +# res[k] = tmp +# return res + + def rescale_matrix(matrix, scaling, axis, binary=True, return_scaling_values=False): '''Function to scale either rows or columns of the sparse rating matrix''' scaling_values = None From 0c76d2bc5ef709f103182f23cc1983386affaac7 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 23 Apr 2020 15:18:29 +0300 Subject: [PATCH 64/82] add support for unseen items sampling for evaluation a la PureSVD, RecWalk, PersDiff --- polara/recommender/data.py | 58 ++++++++++++++++++++++++++++++++++++ polara/recommender/models.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 4e55fb7..9bfffe6 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -935,6 +935,64 @@ def set_test_data(self, testset=None, holdout=None, warm_start=False, test_users num_events = self.test.holdout.shape[0] print(f'Done. There are {num_events} events in the holdout.') +class RandomSampleEvaluationMixin(): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.unseen_interactions = None + self.unseen_items_num = None + self._holdout_item_prefix = 'x' + + def adapt_holdout(self): + holdout = self.test.holdout + userid = self.fields.userid + itemid = self.fields.itemid + # holdout items are expected to be in the + # first columns of the predicted scores array + # hence, holdout item index is monotonic and + # starts from 0 -> can use cumcount + # example: sequence {user: [ind1 ,ind2]} + # will be converted to {user: [0, 1]} + holdout_item_index = ( + holdout + .groupby(userid, sort=False) + [itemid] + .transform('cumcount') + ) + holdout_item_field = f'{self._holdout_item_prefix}_{itemid}' + holdout.loc[:, holdout_item_field] = holdout_item_index + + def set_unseen_interactions(self, interactions, reindex=True, warm_start=False): + n_unseen_items = len(interactions.iloc[0]) + assert interactions.apply(len).eq(n_unseen_items).all(), 'Number of unseen items is inconsistent' + if reindex: + if warm_start: + # TODO modify `set_test_holdout` to generate internal user index + raise NotImplementedError + else: + userid = self.fields.userid + itemid = self.fields.itemid + get_index = self.get_entity_index + # reindexing users + user_index = get_index(userid, index_id='training') + user_index_map = user_index.set_index('old').new + interactions = interactions.loc[user_index_map.index] + new_user_index = pd.Index( + interactions.index.map(user_index_map), name=userid + ) + if new_user_index.isnull().any(): + raise IndexError('Input is inconsistent with existing data.') + interactions = pd.Series( + index=new_user_index, data=interactions.values, name=itemid + ) + # reindexing items + item_index = get_index(itemid, index_id='training') + item_index_map = item_index.set_index('old').new + interactions = interactions.apply(lambda x: item_index_map.loc[x].values) + + self.unseen_interactions = interactions + self.unseen_items_num = n_unseen_items + self.adapt_holdout() + class LongTailMixin(object): def __init__(self, *args, **kwargs): diff --git a/polara/recommender/models.py b/polara/recommender/models.py index 4616663..728ffae 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -20,6 +20,7 @@ from polara.lib.tensor import hooi from polara.lib.sparse import sparse_dot, inverse_permutation, rescale_matrix +from polara.lib.sparse import inner_product_at from polara.lib.sparse import unfold_tensor_coordinates, tensor_outer_at from polara.tools.timing import track_time @@ -1077,3 +1078,54 @@ def predict_feedback(self): feedback_idx = self.data.index.feedback.set_index('new') predicted_feedback = feedback_idx.loc[predictions, 'old'].values return predicted_feedback + + +class RandomSampleEvaluationSVDMixin(): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # as deifined in RandomSampleEvaluationMixin in data models + prefix = self.data._holdout_item_prefix + self._prediction_target = f'{prefix}_{self.data.fields.itemid}' + + def compute_holdout_scores(self, user_factors, item_factors): + holdout = self.data.test.holdout + userid = self.data.fields.userid + itemid = self.data.fields.itemid + holdout_size = self.data.holdout_size + assert holdout_size >= 1 # only fixed number of holdout items is supported + + # "rebasing" user index (see comments in `get_recommmendations`) + useridx, _ = pd.factorize(holdout[userid], sort=False) # already sorted via data moodel + useridx = useridx.reshape(-1, holdout_size) + itemidx = holdout[itemid].values.reshape(-1, holdout_size) + return inner_product_at()(user_factors, item_factors, useridx, itemidx) + + def compute_random_item_scores(self, user_factors, item_factors): + holdout = self.data.test.holdout + userid = self.data.fields.userid + itemid = self.data.fields.itemid + n_unseen = self.data.unseen_items_num + test_users = holdout[userid].drop_duplicates() # preserve sorted order via data moodel + n_users = len(test_users) + + # "rebasing" user index (see comments in `get_recommmendations`) + useridx = np.broadcast_to(np.arange(n_users)[:, None], (n_users, n_unseen)) + unseen_interactions = self.data.unseen_interactions.loc[test_users] + itemidx = np.concatenate(unseen_interactions.values).reshape(-1, n_unseen) + return inner_product_at()(user_factors, item_factors, useridx, itemidx) + + def get_recommendations(self): + itemid = self.data.fields.itemid + if self._prediction_target == itemid: + return super().get_recommendations() + + item_factors = self.factors[itemid] + test_matrix, _ = self.get_test_matrix() + user_factors = test_matrix.dot(item_factors) + # from now on will need to work with "rebased" user indices + # to properly index contiguous user matrices + holdout_scores = self.compute_holdout_scores(user_factors, item_factors) + unseen_scores = self.compute_random_item_scores(user_factors, item_factors) + # combine all scores and rank selected items + scores = np.concatenate((holdout_scores, unseen_scores), axis=1) + return np.apply_along_axis(self.topsort, 1, scores, self.topk) From 1080f14506d61e337a38d7fd7773c883d6ac8482 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sat, 9 May 2020 20:24:39 +0300 Subject: [PATCH 65/82] allow on-the-fly computation of mf scores on negative samples --- polara/lib/sampler.py | 58 ++++++++++++++++++++++++++++++++++++ polara/recommender/models.py | 43 +++++++++++++++++++++++--- 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 polara/lib/sampler.py diff --git a/polara/lib/sampler.py b/polara/lib/sampler.py new file mode 100644 index 0000000..673b15e --- /dev/null +++ b/polara/lib/sampler.py @@ -0,0 +1,58 @@ +from random import randrange +from random import seed as set_seed +from numba import njit, prange + +# Sampler below is an adapted "jit-friendly" version of the accepted answer at +# https://stackoverflow.com/questions/18921302/how-to-incrementally-sample-without-replacement +@njit +def sample_unseen(n, size, exclude): + ''' + This is a generator to sample a desired number of integers from a range + (starting from zero) excluding black-listed elements. It samples items + one by one, which makes it memory efficient and convenient for "on-the-fly" + calculations with sampled elements. + ''' + # initialize dicts with desired type + state = {n: n} # n will never get sampled, can safely use + track = {n: n} + # reindex excluded items, placing them in the end + for i, item in enumerate(exclude): + pos = n - i - 1 + x = track.get(item, item) + t = state.get(pos, pos) + state[x] = t + track[t] = x + state.pop(pos, n) + track.pop(item, n) + track = None + # ensure excluded items are not sampled + remaining = n - len(exclude) + # gradually sample from the decreased size range + for _ in range(size): + i = randrange(remaining) + yield state[i] if i in state else i # avoid numba bug with dict.get(i,i) + remaining -= 1 + state[i] = state.get(remaining, remaining) + state.pop(remaining, n) + + +@njit(parallel=True) +def mf_random_item_scoring(user_factors, item_factors, indptr, indices, + size, seedseq, res): + ''' + Calculate matrix factorization scores over a sample of random items + excluding the already observed ones. + ''' + num_items, rank = item_factors.shape + for i in prange(len(indptr)-1): + head = indptr[i] + tail = indptr[i+1] + observed = indices[head:tail] + user_coef = user_factors[i, :] + set_seed(seedseq[i]) # randomization control for sampling in a thread + for j, rnd_item in enumerate(sample_unseen(num_items, size, observed)): + item_coef = item_factors[rnd_item, :] + tmp = 0 + for k in range(rank): + tmp += user_coef[k] * item_coef[k] + res[i, j] = tmp diff --git a/polara/recommender/models.py b/polara/recommender/models.py index 728ffae..ebc9438 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -15,10 +15,12 @@ from polara.recommender.evaluation import get_hits, get_relevance_scores, get_ranking_scores, get_experience_scores from polara.recommender.evaluation import get_hr_score, get_rr_scores from polara.recommender.evaluation import assemble_scoring_matrices +from polara.recommender.evaluation import matrix_from_observations from polara.recommender.utils import array_split from polara.lib.optimize import simple_pmf_sgd from polara.lib.tensor import hooi +from polara.lib.sampler import mf_random_item_scoring from polara.lib.sparse import sparse_dot, inverse_permutation, rescale_matrix from polara.lib.sparse import inner_product_at from polara.lib.sparse import unfold_tensor_coordinates, tensor_outer_at @@ -1098,12 +1100,12 @@ def compute_holdout_scores(self, user_factors, item_factors): useridx, _ = pd.factorize(holdout[userid], sort=False) # already sorted via data moodel useridx = useridx.reshape(-1, holdout_size) itemidx = holdout[itemid].values.reshape(-1, holdout_size) - return inner_product_at()(user_factors, item_factors, useridx, itemidx) + inner_product = inner_product_at(target='parallel') + return inner_product(user_factors, item_factors, useridx, itemidx) def compute_random_item_scores(self, user_factors, item_factors): holdout = self.data.test.holdout userid = self.data.fields.userid - itemid = self.data.fields.itemid n_unseen = self.data.unseen_items_num test_users = holdout[userid].drop_duplicates() # preserve sorted order via data moodel n_users = len(test_users) @@ -1112,7 +1114,29 @@ def compute_random_item_scores(self, user_factors, item_factors): useridx = np.broadcast_to(np.arange(n_users)[:, None], (n_users, n_unseen)) unseen_interactions = self.data.unseen_interactions.loc[test_users] itemidx = np.concatenate(unseen_interactions.values).reshape(-1, n_unseen) - return inner_product_at()(user_factors, item_factors, useridx, itemidx) + inner_product = inner_product_at(target='parallel') + return inner_product(user_factors, item_factors, useridx, itemidx) + + def compute_random_item_scores_gen(self, user_factors, item_factors, + profile_matrix, n_unseen): + userid = self.data.fields.userid + itemid = self.data.fields.itemid + holdout = self.data.test.holdout + n_users = profile_matrix.shape[0] + seed = self.data.seed + + holdout_matrix = matrix_from_observations( + holdout, userid, itemid, profile_matrix.shape, feedback=None + ) + all_seen = profile_matrix + holdout_matrix # only need nnz indices + + scores = np.zeros((n_users, n_unseen)) + seedseq = np.random.SeedSequence(seed).generate_state(n_users) + mf_random_item_scoring( + user_factors, item_factors, all_seen.indptr, all_seen.indices, + n_unseen, seedseq, scores + ) + return scores def get_recommendations(self): itemid = self.data.fields.itemid @@ -1125,7 +1149,18 @@ def get_recommendations(self): # from now on will need to work with "rebased" user indices # to properly index contiguous user matrices holdout_scores = self.compute_holdout_scores(user_factors, item_factors) - unseen_scores = self.compute_random_item_scores(user_factors, item_factors) + if self.data.unseen_interactions is None: + n_unseen = self.data.unseen_items_num + if n_unseen is None: + raise ValueError('Number of items to sample is unspecified.') + + unseen_scores = self.compute_random_item_scores_gen( + user_factors, item_factors, test_matrix, n_unseen + ) + else: + unseen_scores = self.compute_random_item_scores( + user_factors, item_factors + ) # combine all scores and rank selected items scores = np.concatenate((holdout_scores, unseen_scores), axis=1) return np.apply_along_axis(self.topsort, 1, scores, self.topk) From 70658eec8451e0d2df254d0836ff9315c6d4aada Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sat, 9 May 2020 20:26:00 +0300 Subject: [PATCH 66/82] add convenience function to convert evaluation score tuples into pandas --- polara/recommender/evaluation.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/polara/recommender/evaluation.py b/polara/recommender/evaluation.py index f5e36b4..c0eeec3 100644 --- a/polara/recommender/evaluation.py +++ b/polara/recommender/evaluation.py @@ -1,5 +1,6 @@ -from __future__ import division +from itertools import chain import numpy as np +import pandas as pd from scipy.sparse import csr_matrix, diags from collections import namedtuple @@ -251,3 +252,12 @@ def get_experience_scores(recommendations, total): cov = len(np.unique(recommendations)) / total scores = namedtuple('Experience', ['coverage'])._make([cov]) return scores + + +def convert_scores_to_series(metrics, name='scores'): + if not isinstance(metrics, list): + metrics = [metrics] + return pd.DataFrame.from_records( + chain(*map(lambda x: x._asdict().items(), metrics)), + columns=['metric', name] + ).set_index('metric')[name] From bfb291d8c4366530272e1863ee07575c5a780ffc Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 06:48:52 +0300 Subject: [PATCH 67/82] fix import of `rescale_matrix` function --- polara/recommender/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index ebc9438..9f9aadf 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -20,8 +20,9 @@ from polara.lib.optimize import simple_pmf_sgd from polara.lib.tensor import hooi +from polara.preprocessing.matrices import rescale_matrix from polara.lib.sampler import mf_random_item_scoring -from polara.lib.sparse import sparse_dot, inverse_permutation, rescale_matrix +from polara.lib.sparse import sparse_dot, inverse_permutation from polara.lib.sparse import inner_product_at from polara.lib.sparse import unfold_tensor_coordinates, tensor_outer_at from polara.tools.timing import track_time From 662b4617dbb90c35cb6ea54f17f48b87366a2b9b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 06:50:47 +0300 Subject: [PATCH 68/82] add convenience function to read .npz files via url --- polara/recommender/utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/polara/recommender/utils.py b/polara/recommender/utils.py index 6646e67..4dcdfc8 100644 --- a/polara/recommender/utils.py +++ b/polara/recommender/utils.py @@ -1,4 +1,5 @@ -from __future__ import division +from io import BytesIO +from urllib.request import urlopen import numpy as np from polara.tools.systools import get_available_memory from polara.recommender import defaults @@ -50,3 +51,10 @@ def array_split(shp, result_width, scores_multiplier, dtypes=None): chunk_size = get_chunk_size(shp, result_width, scores_multiplier, dtypes=dtypes) split = range_division(shp[0], chunk_size) return split + + +def read_npz_form_url(url, allow_pickle=False): + '''Read numpy's .npz file directly from source url.''' + with urlopen(url) as response: + file_handle = BytesIO(response.read()) + return np.load(file_handle, allow_pickle=allow_pickle) From 42e6ee6b1fb225d68486f641cebe0d054e0266fa Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 06:57:24 +0300 Subject: [PATCH 69/82] refactor matrix inner product computation with negative samples --- polara/recommender/models.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index 9f9aadf..deece0e 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -1097,25 +1097,32 @@ def compute_holdout_scores(self, user_factors, item_factors): holdout_size = self.data.holdout_size assert holdout_size >= 1 # only fixed number of holdout items is supported - # "rebasing" user index (see comments in `get_recommmendations`) - useridx, _ = pd.factorize(holdout[userid], sort=False) # already sorted via data moodel - useridx = useridx.reshape(-1, holdout_size) + # "rebase" user index (see comments in `get_recommmendations`) + useridx = pd.factorize( + holdout[userid], sort=False # already sorted via data moodel + )[0].reshape(-1, holdout_size) itemidx = holdout[itemid].values.reshape(-1, holdout_size) inner_product = inner_product_at(target='parallel') + # for general matrix factorization user must take care of submitting + # user_factors of the correct shape, otherwise, if holdout contains + # only a subset of all users, the answer will be incorrect return inner_product(user_factors, item_factors, useridx, itemidx) def compute_random_item_scores(self, user_factors, item_factors): holdout = self.data.test.holdout userid = self.data.fields.userid - n_unseen = self.data.unseen_items_num - test_users = holdout[userid].drop_duplicates() # preserve sorted order via data moodel + test_users = holdout[userid].drop_duplicates().values # preserve sorted + test_items = self.data.unseen_interactions.loc[test_users].values + # "rebase" user index (see comments in `get_recommmendations`) n_users = len(test_users) - - # "rebasing" user index (see comments in `get_recommmendations`) - useridx = np.broadcast_to(np.arange(n_users)[:, None], (n_users, n_unseen)) - unseen_interactions = self.data.unseen_interactions.loc[test_users] - itemidx = np.concatenate(unseen_interactions.values).reshape(-1, n_unseen) + n_items = self.data.unseen_items_num + useridx = np.broadcast_to(np.arange(n_users)[:, None], (n_users, n_items)) + itemidx = np.concatenate(test_items).reshape(n_users, n_items) + # perform vectorized scalar products at bulk inner_product = inner_product_at(target='parallel') + # for general matrix factorization user must take care of submitting + # user_factors of the correct shape, otherwise, if holdout contains + # only a subset of all users, the answer will be incorrect return inner_product(user_factors, item_factors, useridx, itemidx) def compute_random_item_scores_gen(self, user_factors, item_factors, From 6d772bf4796764f28e900e38a0b15a1cca8f8937 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 06:59:25 +0300 Subject: [PATCH 70/82] update EigenRec example --- examples/Reproducing_EIGENREC_results.ipynb | 56 +++++++++++++-------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/examples/Reproducing_EIGENREC_results.ipynb b/examples/Reproducing_EIGENREC_results.ipynb index 11da8d3..e520c9a 100644 --- a/examples/Reproducing_EIGENREC_results.ipynb +++ b/examples/Reproducing_EIGENREC_results.ipynb @@ -133,7 +133,7 @@ ], "source": [ "data = get_movielens_data() # will automatically download it\n", - " # alternatively you can specify a path to the local copy as an argument to the function\n", + "# alternatively you can specify a path to the local copy as an argument to the function\n", "data.head()" ] }, @@ -158,7 +158,7 @@ "metadata": {}, "source": [ "The *EigenRec* paper follows a specific experimentation setup, mainly based on the settings, proposed in my another favorite paper [Performance of recommender algorithms on top-n recommendation tasks](https://dl.acm.org/citation.cfm?id=1864708.1864721), devoted to the *PureSVD* model itself. For evaluation purposes, the authors sample 1.4% of all available ratings and additionally shrink the resulting sample by leaving 5-star ratings only. Quote from the paper (Section 4.2.1): \n", - "
\"...we form a probeset $\\mathcal{P}$ by randomly sampling 1.4% of the ratings of the dataset, and we use each item $v_j$,rated with 5-star by user $u_i$ in $\\mathcal{P}$ to create the test set $\\mathcal{T}$...\"
" + "
\"...we form a probeset $\\mathcal{P}$ by randomly sampling 1.4% of the ratings of the dataset, and we use each item $v_j$, rated with 5-star by user $u_i$ in $\\mathcal{P}$ to create the test set $\\mathcal{T}$...\"
" ] }, { @@ -180,7 +180,7 @@ "output_type": "stream", "text": [ "Preparing data...\n", - "2 unique movieid's within 2 holdout interactions were filtered. Reason: not in the training data.\n", + "2 unique movieid entities within 2 holdout interactions were filtered. Reason: not in the training data.\n", "Done.\n", "There are 986206 events in the training and 14001 events in the holdout.\n" ] @@ -191,7 +191,6 @@ "data_model.holdout_size = 0.014 # sample this fraction of ratings from data\n", "data_model.random_holdout = True # sample ratings randomly (not just 5-star)\n", "data_model.warm_start = False # allow test users to be part of the training (excluding holdout items)\n", - "\n", "data_model.prepare() # perform sampling" ] }, @@ -301,14 +300,23 @@ "cell_type": "code", "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done. There are 3140 events in the holdout.\n" + ] + } + ], "source": [ - "data_model.set_test_data(holdout=data_model.test.holdout.query('rating==5'), # select only 5-star ratings\n", - " warm_start=data_model.warm_start, \n", - " reindex=False, # avoid reindexing users and items second time\n", - " ensure_consistency=False # do not try to filter out unseen entities (they are already excluded)\n", - " # leaving it as True wouldn't change the result but would lead to extra checks\n", - " )" + "data_model.set_test_data(\n", + " holdout=data_model.test.holdout.query('rating==5'), # select only 5-star ratings\n", + " warm_start=data_model.warm_start, \n", + " reindex=False, # avoid reindexing users and items second time\n", + " ensure_consistency=False # do not try to filter out unseen entities (already excluded)\n", + " # leaving it as True wouldn't change the result but would lead to extra checks\n", + ")" ] }, { @@ -575,24 +583,30 @@ " \n", " itemidx = holdout[itemid] # holdout items of the test users\n", " useridx = pd.factorize(holdout[userid])[0] # have to \"rebase\" user index;\n", - " # necessary for indexing rows of the matrix with test user ratings\n", + " # necessary for indexing rows of the matrix with test user ratings\n", " \n", " # prediction scores for holdout items\n", " test_matrix, seen_data = self.get_test_matrix() \n", " item_factors = self.factors[itemid] # right singular vectors, matrix V\n", - " user_factors = test_matrix.dot(item_factors) # according to product V^T r for every test user\n", - " holdout_scores = (user_factors[useridx, :] * item_factors[itemidx.values, :]).sum(axis=1).squeeze()\n", + " user_factors = test_matrix.dot(item_factors) # similarly to PCA\n", + " holdout_scores = (\n", + " user_factors[useridx, :] * item_factors[itemidx.values, :]\n", + " ).sum(axis=1).squeeze()\n", " \n", " # scores for randomly sampled unseen items\n", " all_items = self.data.index.itemid.new # all unique (reindexed) items\n", " rs = np.random.RandomState(self.seed) # fixing random state to control random output\n", - " sampled_scores = sample_scores_flat(useridx, itemidx,\n", - " seen_data, all_items,\n", - " user_factors, item_factors,\n", - " self.n_rnd_items, random_state=rs)\n", + " sampled_scores = sample_scores_flat(\n", + " useridx, itemidx,\n", + " seen_data, all_items,\n", + " user_factors, item_factors,\n", + " self.n_rnd_items, random_state=rs\n", + " )\n", " \n", " # combine all scores and rank selected items \n", - " scores = np.concatenate((holdout_scores[:, None], sampled_scores), axis=1) # stack into array with 1001 columns\n", + " scores = np.concatenate( # stack into array with 1001 columns\n", + " (holdout_scores[:, None], sampled_scores), axis=1\n", + " )\n", " rankings = np.apply_along_axis(np.argsort, 1, -scores)\n", " return rankings" ] @@ -1116,9 +1130,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python [conda env:py3_polara]", "language": "python", - "name": "python3" + "name": "conda-env-py3_polara-py" }, "language_info": { "codemirror_mode": { From 65902f9b9c8821f22b7b6264d7b9cc39ac4c8fe8 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 07:03:02 +0300 Subject: [PATCH 71/82] minor refactoring --- polara/recommender/data.py | 4 ++-- polara/recommender/evaluation.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 9bfffe6..a81f006 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -622,8 +622,8 @@ def _reindex_train_items(self): def get_entity_index(self, entity, index_id='training'): entity_type = self.fields._fields[self.fields.index(entity)] index_data = getattr(self.index, entity_type) - try: # check whether custom index is introduced (as in e.g. coldstart) + # TODO catch index_id='test' for warm_start = True entity_idx = getattr(index_data, index_id) except AttributeError: # fall back to standard case entity_idx = index_data @@ -885,7 +885,7 @@ def get_test_shape(self, tensor_mode=False): def set_test_data(self, testset=None, holdout=None, warm_start=False, test_users=None, - reindex=True, ensure_consistency=True, holdout_size=None, copy=True): + reindex=True, ensure_consistency=True, holdout_size=None, copy=True): '''Should be used only with custom data.''' if warm_start and ((testset is None) and (test_users is None)): raise ValueError('When warm_start is True, information about test users must be present. ' diff --git a/polara/recommender/evaluation.py b/polara/recommender/evaluation.py index c0eeec3..41ccf21 100644 --- a/polara/recommender/evaluation.py +++ b/polara/recommender/evaluation.py @@ -57,7 +57,8 @@ def matrix_from_observations(observations, key, target, shape, feedback=None): # set data and indices manually to avoid index dtype checks # and thus prevent possible unnecesssary copies of indices indices = observations[target].values - indptr = np.r_[0, np.where(np.diff(observations[key].values))[0]+1, n_observations] + keys = observations[key].values + indptr = np.r_[0, np.where(np.diff(keys))[0]+1, n_observations] matrix = no_copy_csr_matrix(data, indices, indptr, shape, dtype) return matrix From 27d8e367c237984db6b2489e6f0aafb11842d404 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 07:08:54 +0300 Subject: [PATCH 72/82] add a collection of convenience functions to effectively preprocess data in dvarious formats --- polara/lib/sampler.py | 149 +++++++++++++++++++---- polara/lib/sparse.py | 24 ---- polara/preprocessing/dataframes.py | 183 +++++++++++++++++++++++++++++ polara/preprocessing/matrices.py | 93 +++++++++++++++ polara/tools/preprocessing.py | 19 +-- polara/tools/random.py | 22 ++++ 6 files changed, 427 insertions(+), 63 deletions(-) create mode 100644 polara/preprocessing/dataframes.py create mode 100644 polara/preprocessing/matrices.py create mode 100644 polara/tools/random.py diff --git a/polara/lib/sampler.py b/polara/lib/sampler.py index 673b15e..d22d475 100644 --- a/polara/lib/sampler.py +++ b/polara/lib/sampler.py @@ -1,58 +1,165 @@ +import heapq from random import randrange from random import seed as set_seed +import numpy as np from numba import njit, prange # Sampler below is an adapted "jit-friendly" version of the accepted answer at # https://stackoverflow.com/questions/18921302/how-to-incrementally-sample-without-replacement -@njit -def sample_unseen(n, size, exclude): - ''' - This is a generator to sample a desired number of integers from a range - (starting from zero) excluding black-listed elements. It samples items - one by one, which makes it memory efficient and convenient for "on-the-fly" - calculations with sampled elements. - ''' - # initialize dicts with desired type - state = {n: n} # n will never get sampled, can safely use + + +@njit(fastmath=True) +def prime_sampler_state(n, exclude): + """ + Initialize state to be used in fast sampler. Helps ensure excluded items are + never sampled by placing them outside of sampling region. + """ + # initialize typed numba dicts + state = {n: n} + state.pop(n) track = {n: n} + track.pop(n) + + n_pos = n - len(state) - 1 # reindex excluded items, placing them in the end for i, item in enumerate(exclude): - pos = n - i - 1 + pos = n_pos - i x = track.get(item, item) t = state.get(pos, pos) state[x] = t track[t] = x state.pop(pos, n) track.pop(item, n) - track = None - # ensure excluded items are not sampled + return state + + +@njit(fastmath=True) +def sample_unseen(n, size, exclude): + """ + This is a generator to sample a desired number of integers from a range + (starting from zero) excluding black-listed elements. It samples items + one by one, which makes it memory efficient and convenient for "on-the-fly" + calculations with sampled elements. + """ + # exclude items by moving them out of samling region + state = prime_sampler_state(n, exclude) remaining = n - len(exclude) # gradually sample from the decreased size range for _ in range(size): i = randrange(remaining) - yield state[i] if i in state else i # avoid numba bug with dict.get(i,i) + yield state[i] if i in state else i # avoid numba bug with dict.get(i,i) remaining -= 1 state[i] = state.get(remaining, remaining) state.pop(remaining, n) +@njit(fastmath=True) +def sample_fill(sample_size, sampler_state, remaining, result): + """ + Sample a desired number of integers from a range (starting from zero) + excluding black-listed elements defined in sample state. Used in + conjunction with `prime_sample_state` method, which initializes state. + Inspired by Fischer-Yates shuffle. + """ + # gradually sample from the decreased size range + for k in range(sample_size): + i = randrange(remaining) + result[k] = sampler_state.get(i, i) + remaining -= 1 + sampler_state[i] = sampler_state.get(remaining, remaining) + sampler_state.pop(remaining, -1) + + @njit(parallel=True) -def mf_random_item_scoring(user_factors, item_factors, indptr, indices, - size, seedseq, res): - ''' +def mf_random_item_scoring( + user_factors, item_factors, indptr, indices, size, seedseq, res +): + """ Calculate matrix factorization scores over a sample of random items excluding the already observed ones. - ''' + """ num_items, rank = item_factors.shape - for i in prange(len(indptr)-1): + for i in prange(len(indptr) - 1): head = indptr[i] - tail = indptr[i+1] + tail = indptr[i + 1] observed = indices[head:tail] user_coef = user_factors[i, :] - set_seed(seedseq[i]) # randomization control for sampling in a thread + set_seed(seedseq[i]) # randomization control for sampling in a thread for j, rnd_item in enumerate(sample_unseen(num_items, size, observed)): item_coef = item_factors[rnd_item, :] tmp = 0 for k in range(rank): tmp += user_coef[k] * item_coef[k] res[i, j] = tmp + + +@njit(parallel=True) +def sample_row_wise(indptr, indices, n_cols, n_samples, seed_seq): + """ + For every row of a CSR matrix, samples indices not present in this row. + """ + n_rows = len(indptr) - 1 + result = np.empty((n_rows, n_samples), dtype=indices.dtype) + for i in prange(n_rows): + head = indptr[i] + tail = indptr[i + 1] + seen_inds = indices[head:tail] + state = prime_sampler_state(n_cols, seen_inds) + remaining = n_cols - len(seen_inds) + set_seed(seed_seq[i]) + sample_fill(n_samples, state, remaining, result[i, :]) + return result + + +@njit(parallel=True) +def sample_element_wise(indptr, indices, n_cols, n_samples, seed_seq): + """ + For every nnz entry of a CSR matrix, samples indices not present + in its corresponding row. + """ + result = np.empty((indptr[-1], n_samples), dtype=indices.dtype) + for i in prange(len(indptr) - 1): + head = indptr[i] + tail = indptr[i + 1] + + seen_inds = indices[head:tail] + state = prime_sampler_state(n_cols, seen_inds) + remaining = n_cols - len(seen_inds) + set_seed(seed_seq[i]) + for j in range(head, tail): + sampler_state = state.copy() + sample_fill(n_samples, sampler_state, remaining, result[j, :]) + return result + + +@njit +def split_top_continuous(tasks, priorities): + """ + Sample a sequence of unique tasks of the highest priority ensuring that + no task will have another instance with the priority level above the lowest priority in the sequence. + Usecases: avoiding issues with "recommendations from future" when splitting test data by timestamp. + """ + priority_queue = [(-priorities[0], 0)] + priority_queue.pop() + + for idx, priority in enumerate(priorities): + heapq.heappush(priority_queue, (-priority, idx)) + + topseq = {} # continuous sequence of top-priority tasks + nonseq_idx = [] # top-priority tasks that interrupt continuous sequence + + unique_tasks = set(tasks) + while unique_tasks: + _, idx = heapq.heappop(priority_queue) + task = tasks[idx] + try: + visited = topseq[task] + except: + unique_tasks.remove(task) + else: + nonseq_idx.append(visited) + topseq[task] = idx + + topseq_idx = list(topseq.values()) + lowseq_idx = [idx for _, idx in priority_queue] # all remaining tasks + return topseq_idx, lowseq_idx, nonseq_idx \ No newline at end of file diff --git a/polara/lib/sparse.py b/polara/lib/sparse.py index ccb0430..dd529a7 100644 --- a/polara/lib/sparse.py +++ b/polara/lib/sparse.py @@ -87,30 +87,6 @@ def inner_product_at_wrapped(u, v, ui, vi, res): # return res -def rescale_matrix(matrix, scaling, axis, binary=True, return_scaling_values=False): - '''Function to scale either rows or columns of the sparse rating matrix''' - scaling_values = None - if scaling == 1: # no scaling (standard SVD case) - result = matrix - - if binary: - norm = np.sqrt(matrix.getnnz(axis=axis)) # compute Euclidean norm as if values are binary - else: - norm = spnorm(matrix, axis=axis, ord=2) # compute Euclidean norm - - scaling_values = power(norm, scaling-1, where=norm != 0) - scaling_matrix = diags(scaling_values) - - if axis == 0: # scale columns - result = matrix.dot(scaling_matrix) - if axis == 1: # scale rows - result = scaling_matrix.dot(matrix) - - if return_scaling_values: - result = (result, scaling_values) - return result - - # matvec implementation is based on # http://stackoverflow.com/questions/18595981/improving-performance-of-multiplication-of-scipy-sparse-matrices @njit(nogil=True) diff --git a/polara/preprocessing/dataframes.py b/polara/preprocessing/dataframes.py new file mode 100644 index 0000000..221df03 --- /dev/null +++ b/polara/preprocessing/dataframes.py @@ -0,0 +1,183 @@ +import heapq +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix +from pandas.api.types import is_numeric_dtype +from polara.lib.sampler import split_top_continuous +from polara.tools.random import check_random_state + + +def reindex(raw_data, index, filter_invalid=True, names=None): + ''' + Factorizes column values based on provided pandas index. Allows resetting + index names. Optionally drops rows with entries not present in the index. + ''' + if isinstance(index, pd.Index): + index = [index] + + if isinstance(names, str): + names = [names] + + if isinstance(names, (list, tuple, pd.Index)): + for i, name in enumerate(names): + index[i].name = name + + new_data = raw_data.assign(**{ + idx.name: idx.get_indexer(raw_data[idx.name]) for idx in index + }) + + if filter_invalid: + # pandas returns -1 if label is not present in the index + # checking if -1 is present anywhere in data + maybe_invalid = new_data.eval( + ' or '.join([f'{idx.name} == -1' for idx in index]) + ) + if maybe_invalid.any(): + print(f'Filtered {maybe_invalid.sum()} invalid observations.') + new_data = new_data.loc[~maybe_invalid] + + return new_data + + +def matrix_from_observations( + data, + userid='userid', + itemid='itemid', + user_index=None, + item_index=None, + feedback=None, + preserve_order=False, + shape=None, + dtype=None + ): + ''' + Encodes pandas dataframe into sparse matrix. If index is not provided, + returns new index mapping, which optionally preserves order of original data. + Automatically removes incosnistent data not present in the provided index. + ''' + if (user_index is None) or (item_index is None): + useridx, user_index = pd.factorize(data[userid], sort=preserve_order) + itemidx, item_index = pd.factorize(data[itemid], sort=preserve_order) + user_index.name = userid + item_index.name = itemid + else: + data = reindex(data, (user_index, item_index), filter_invalid=True) + useridx = data[userid].values + itemidx = data[itemid].values + if shape is None: + shape = (len(user_index), len(item_index)) + + if feedback is None: + values = np.ones_like(itemidx, dtype=dtype) + else: + values = data[feedback].values + + matrix = csr_matrix((values, (useridx, itemidx)), dtype=dtype, shape=shape) + return matrix, user_index, item_index + + +def split_holdout( + data, + userid = 'userid', + feedback = None, + sample_max_rated = False, + random_state = None + ): + ''' + Samples 1 item per every user according to the rule sample_max_rated. + It always shuffles the input data. The reason is that even if sampling + top-rated elements, there could be several items with the same top rating. + ''' + idx_grouper = ( + data + .sample(frac=1, random_state=random_state) # randomly permute data + .groupby(userid, as_index=False, sort=False) + ) + if sample_max_rated: # take single item with the highest score + idx = idx_grouper[feedback].idxmax() + else: # data is already shuffled - simply take the 1st element + idx = idx_grouper.head(1).index # sample random element + + observed = data.drop(idx.values) + holdout = data.loc[idx.values] + return observed, holdout + + +def sample_unseen_items(item_group, item_pool, n, random_state): + 'Helper function to run on pandas dataframe grouper' + seen_items = item_group.values + candidates = np.setdiff1d(item_pool, seen_items, assume_unique=True) + return random_state.choice(candidates, n, replace=False) + + +def sample_unseen_interactions( + data, + item_pool, + n_random = 999, + random_state = None, + userid = 'userid', + itemid = 'itemid' + ): + ''' + Randomized sampling of unseen items per every user in data. Assumes data + was already preprocessed to contiguous index. + ''' + random_state = check_random_state(random_state) + return ( + data + .groupby(userid, sort=False)[itemid] + .apply(sample_unseen_items, item_pool, n_random, random_state) + ) + + +def verify_split(train, test, random_holdout, feedback, userid='userid'): + if random_holdout: + return + hold_gr = test.set_index(userid)[feedback] + useridx = hold_gr.index + train_gr = train.query(f'{userid} in @useridx').groupby(userid)[feedback] + assert train_gr.apply(lambda x: x.le(hold_gr.loc[x.name]).all()).all() + + +def to_numeric_array(series): + if not is_numeric_dtype(series): + if not hasattr(series, 'cat'): + series = series.astype('category') + return series.cat.codes.values + return series.values + + +def split_earliest_last(data, userid='userid', priority='timestamp', copy=False): + ''' + It helps avoiding "recommendations from future", when training set contains events that occur later than some events in the holdout and can therefore provide an oracle hint for the algorithm. + ''' + topseq_idx, lowseq_idx, nonseq_idx = split_top_continuous( + to_numeric_array(data[userid]), data[priority].values + ) + + observed = data.iloc[lowseq_idx] + holdout = data.iloc[topseq_idx] + future = data.iloc[nonseq_idx] + + if copy: + observed = observed.copy() + holdout = holdout.copy() + future = future.copy() + + return observed, holdout, future + + +def filter_sessions_by_length(data, session_label='userid', min_session_length=3): + """Filters users with insufficient number of items""" + if data.duplicated().any(): + raise NotImplementedError + + sz = data[session_label].value_counts(sort=False) + has_valid_session_length = sz >= min_session_length + if not has_valid_session_length.all(): + valid_sessions = sz.index[has_valid_session_length] + new_data = data[data[session_label].isin(valid_sessions)].copy() + print('Sessions are filtered by length') + else: + new_data = data + return new_data \ No newline at end of file diff --git a/polara/preprocessing/matrices.py b/polara/preprocessing/matrices.py new file mode 100644 index 0000000..50f7601 --- /dev/null +++ b/polara/preprocessing/matrices.py @@ -0,0 +1,93 @@ +import numpy as np +from numpy import power +from scipy.sparse import diags +from scipy.sparse.linalg import norm as spnorm +import pandas as pd +from polara.tools.random import check_random_state + + +def split_holdout(matrix, sample_max_rated=True, random_state=None): + ''' + Uses CSR format to efficiently access non-zero elements. + ''' + holdout = [] + indptr = matrix.indptr + indices = matrix.indices + data = matrix.data + + random_state = check_random_state(random_state) + for i in range(len(indptr)-1): # for every user i + head = indptr[i] + tail = indptr[i+1] + candidates = indices[head:tail] + if sample_max_rated: + vals = data[head:tail] # user feedback + pos_max, = np.where(vals == vals.max()) + candidates = candidates[pos_max] + holdout.append(random_state.choice(candidates)) + return np.array(holdout) + + +def mask_holdout(matrix, holdout_items, copy=True): + ''' + Zeroize holdout items in the rating matrix. + Requires exactly 1 holdout item per each row. + ''' + masked = matrix.copy() if copy else matrix + masked[np.arange(len(holdout_items)), holdout_items] = 0 + masked.eliminate_zeros() + return masked + + +def sample_unseen_interactions(observations, holdout_items, size=999, random_state=None): + ''' + Randomly samples unseen items per every user from observations matrix. + Takes into account holdout items. Assumes there's only one holdout item per every user. + ''' + n_users, n_items = observations.shape + indptr = observations.indptr + indices = observations.indices + + sample = np.zeros((n_users, size), dtype=indices.dtype) + random_state = check_random_state(random_state) + for i in range(len(indptr)-1): + head = indptr[i] + tail = indptr[i+1] + seen_items = np.concatenate(([holdout_items[i]], indices[head:tail])) + rand_items = sample_unseen(n_items, size, seen_items, random_state) + sample[i, :] = rand_items + return sample + + +def sample_unseen(pool_size, sample_size, exclude, random_state=None): + '''Efficient sampling from a range with exclusion.''' + assert (pool_size-len(exclude)) >= sample_size + random_state = check_random_state(random_state) + src = random_state.rand(pool_size) + np.put(src, exclude, -1) # will never get to the top + return np.argpartition(src, -sample_size)[-sample_size:] + + +def rescale_matrix(matrix, scaling, axis, binary=True, return_scaling_values=False): + '''Function to scale either rows or columns of the sparse rating matrix''' + result = None + scaling_values = None + if scaling == 1: # no scaling (standard SVD case) + result = matrix + + if binary: + norm = np.sqrt(matrix.getnnz(axis=axis)) # compute Euclidean norm as if values are binary + else: + norm = spnorm(matrix, axis=axis, ord=2) # compute Euclidean norm + + scaling_values = power(norm, scaling-1, where=norm != 0) + scaling_matrix = diags(scaling_values) + + if axis == 0: # scale columns + result = matrix.dot(scaling_matrix) + if axis == 1: # scale rows + result = scaling_matrix.dot(matrix) + + if return_scaling_values: + result = (result, scaling_values) + return result diff --git a/polara/tools/preprocessing.py b/polara/tools/preprocessing.py index 9605da0..6c17236 100644 --- a/polara/tools/preprocessing.py +++ b/polara/tools/preprocessing.py @@ -1,18 +1 @@ -# python 2/3 interoperability -from __future__ import print_function - - -def filter_sessions_by_length(data, session_label='userid', min_session_length=3): - """Filters users with insufficient number of items""" - if data.duplicated().any(): - raise NotImplementedError - - sz = data[session_label].value_counts(sort=False) - has_valid_session_length = sz >= min_session_length - if not has_valid_session_length.all(): - valid_sessions = sz.index[has_valid_session_length] - new_data = data[data[session_label].isin(valid_sessions)].copy() - print('Sessions are filtered by length') - else: - new_data = data - return new_data +from polara.preprocessing.dataframes import filter_sessions_by_length \ No newline at end of file diff --git a/polara/tools/random.py b/polara/tools/random.py new file mode 100644 index 0000000..9799992 --- /dev/null +++ b/polara/tools/random.py @@ -0,0 +1,22 @@ +import numpy as np + +def check_random_state(random_state): + ''' + Handles seed or random state as an input. Provides consistent output. + ''' + if random_state is None: + return np.random + if isinstance(random_state, (np.integer, int)): + return np.random.RandomState(random_state) + return random_state + +def random_seeds(size, entropy=None): + '''Generates a sequence of most likely independent seeds.''' + return np.random.SeedSequence(entropy).generate_state(size) + +def seed_generator(seed): + rs = np.random.RandomState(seed) + while True: + new_seed = yield rs.randint(np.iinfo('i4').max) + if new_seed is not None: + rs = np.random.RandomState(new_seed) From f39bfc5ad6f73df2f81a6be8a79a40f514176013 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 07:14:18 +0300 Subject: [PATCH 73/82] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a7213ec..efb3f7d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ polara.egg-info/ examples/.ipynb_checkpoints/ .ipynb_checkpoints/ +.vscode/ From 272de1dc0c05c045567af152ba686ae46f8ad2c5 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 07:15:06 +0300 Subject: [PATCH 74/82] add mixin for quick access to latent factors --- polara/recommender/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index deece0e..4b4854d 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -788,6 +788,16 @@ def slice_recommendations(self, test_data, shape, start, stop, test_users=None): return scores, slice_data +class EmbeddingsMixin: + @property + def user_embeddings(self): + return self.factors[self.data.fields.userid] + + @property + def item_embeddings(self): + return self.factors[self.data.fields.itemid] + + class SVDModel(RecommenderModel): def __init__(self, *args, **kwargs): From a60a75e6b32925150315acd47cba417445f25c91 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 07:15:40 +0300 Subject: [PATCH 75/82] add wrapper for implicit's implementation of BPR --- .../external/implicit/bprwrapper.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 polara/recommender/external/implicit/bprwrapper.py diff --git a/polara/recommender/external/implicit/bprwrapper.py b/polara/recommender/external/implicit/bprwrapper.py new file mode 100644 index 0000000..775ce4a --- /dev/null +++ b/polara/recommender/external/implicit/bprwrapper.py @@ -0,0 +1,76 @@ +import numpy as np +import implicit +from polara.recommender.models import RecommenderModel +from polara.tools.timing import track_time + + +class ImplicitBPR(RecommenderModel): + + def __init__(self, *args, **kwargs): + super(ImplicitBPR, self).__init__(*args, **kwargs) + self._rank = 10 + self.learning_rate = 0.01 + self.regularization = 0.01 + self.num_epochs = 100 + self.num_threads = 0 + self.random_state = None + self.method = 'BPRMF' + self.show_progress = False + self._model = None + + @property + def rank(self): + return self._rank + + @rank.setter + def rank(self, new_value): + if new_value != self._rank: + self._rank = new_value + self._is_ready = False + self._recommendations = None + + def build(self): + # define iALS model instance + self._model = implicit.bpr.BayesianPersonalizedRanking( + factors=self.rank, + learning_rate=self.learning_rate, + regularization=self.regularization, + iterations=self.num_epochs, + num_threads=self.num_threads, + #random_state = self.random_state # doesn't support yet + ) + self._model.random_state = self.random_state # for future releases + # prepare input matrix for learning the model + matrix = self.get_training_matrix() # user_by_item sparse matrix + with track_time(self.training_time, verbose=self.verbose, model=self.method): + # build the model + # implicit takes item_by_user matrix as input, need to transpose + self._model.fit(matrix.T, show_progress=self.show_progress) + + + def get_recommendations(self): + recalculate = self.data.warm_start # used to force folding-in computation + if recalculate: + if self.filter_seen is False: + raise ValueError('The model always filters seen items from results.') + # prepare test matrix with preferences of unseen users + matrix, _ = self.get_test_matrix() + num_users = matrix.shape[0] + users_idx = range(num_users) + + top_recs = np.empty((num_users, self.topk), dtype=np.intp) + for i, user_row in enumerate(users_idx): + recs = self._model.recommend(user_row, matrix, N=self.topk, recalculate_user=recalculate) + top_recs[i, :] = [item for item, _ in recs] + else: + top_recs = super(ImplicitBPR, self).get_recommendations() + return top_recs + + + def slice_recommendations(self, test_data, shape, start, stop, test_users=None): + slice_data = self._slice_test_data(test_data, start, stop) + + user_factors = self._model.user_factors[test_users[start:stop], :] + item_factors = self._model.item_factors + scores = user_factors.dot(item_factors.T) + return scores, slice_data From c02943b21425bf3971d523b669af18aa4100ac4b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 07:16:31 +0300 Subject: [PATCH 76/82] add function for readig yahoo music data --- polara/datasets/yahoo.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 polara/datasets/yahoo.py diff --git a/polara/datasets/yahoo.py b/polara/datasets/yahoo.py new file mode 100644 index 0000000..0528901 --- /dev/null +++ b/polara/datasets/yahoo.py @@ -0,0 +1,35 @@ +import tarfile +import pandas as pd + +def get_yahoo_music_data(path=None, fileid=0, include_test=True, read_attributes=False, read_genres=False): + res = [] + if path: + data_folder = 'ydata-ymusic-user-song-ratings-meta-v1_0' + col_names = ['userid', 'songid', 'rating'] + with tarfile.open(path, 'r:gz') as tar: + handle = tar.getmember(f'{data_folder}/train_{fileid}.txt') + file = tar.extractfile(handle) + data = pd.read_csv(file, sep='\t', header=None, names=col_names) + res.append(data) + if include_test: + handle = tar.getmember(f'{data_folder}/test_{fileid}.txt') + file = tar.extractfile(handle) + data = pd.read_csv(file, sep='\t', header=None, names=col_names) + res.append(data) + + if read_attributes: + handle = tar.getmember(f'{data_folder}/song-attributes.txt') + file = tar.extractfile(handle) + attr = pd.read_csv(file, sep='\t', header=None, index_col=0, + names=['songid', 'albumid', 'artistid', 'genreid']) + res.append(attr) + + if read_genres: + handle = tar.getmember(f'{data_folder}/song-attributes.txt') + file = tar.extractfile(handle) + genres = pd.read_csv(file, sep='\t', header=None, index_col=0, + names=['genreid', 'parent_genre', 'level', 'genre_name']) + res.append(genres) + if len(res) == 1: + res = res[0] + return res From 69d3bde2d9b766c5e167c5471a466bbe864a6bb5 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 07:19:28 +0300 Subject: [PATCH 77/82] add folder for tests --- tests/conftest.py | 2 ++ tests/dataset_fixtures.py | 23 +++++++++++++++++++++++ tests/preprocessing_test.py | 16 ++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/dataset_fixtures.py create mode 100644 tests/preprocessing_test.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7013f17 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,2 @@ + +pytest_plugins = ['dataset_fixtures'] \ No newline at end of file diff --git a/tests/dataset_fixtures.py b/tests/dataset_fixtures.py new file mode 100644 index 0000000..d2ef43c --- /dev/null +++ b/tests/dataset_fixtures.py @@ -0,0 +1,23 @@ +import pytest +import pandas as pd + + +@pytest.fixture +def ts_data_short(): + # -------- TIMELINE -------->> + # u1 | Matrix . LOTR + # u2 | GF . SW1 + # u3 | Matrix . LOTR . SW4 + return pd.DataFrame([ + ('u1', 'Matrix', 0), + ('u3', 'Matrix', 1), + ('u2', 'GF', 2), + ('u1', 'LOTR', 3), + ('u3', 'LOTR', 4), + ('u2', 'SW1', 5), + ('u3', 'SW4', 6), + ], columns=['userid', 'itemid', 'timestamp']) + +@pytest.fixture +def ts_data_short_earliest_last_split(): + return [0, 1, 2], [3, 4, 5], [6] \ No newline at end of file diff --git a/tests/preprocessing_test.py b/tests/preprocessing_test.py new file mode 100644 index 0000000..680d935 --- /dev/null +++ b/tests/preprocessing_test.py @@ -0,0 +1,16 @@ +from pandas.testing import assert_frame_equal +from polara.preprocessing.dataframes import split_earliest_last + + +def test_earliest_last_split(ts_data_short, ts_data_short_earliest_last_split): + observed, hidden, future = split_earliest_last(ts_data_short) + observed_idx, hidden_idx, future_idx = ts_data_short_earliest_last_split + + assert_frame_equal(observed, ts_data_short.iloc[observed_idx], check_like=True) + assert_frame_equal(hidden, ts_data_short.iloc[hidden_idx], check_like=True) + assert_frame_equal(future, ts_data_short.iloc[future_idx], check_like=True) + + + + + From 22747227954f7dd75713875a6f6b10c703c32c60 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Wed, 6 Jan 2021 08:11:51 +0300 Subject: [PATCH 78/82] fix `safe_divide` TypeError for unsized numpy arrays with a single element --- polara/recommender/evaluation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/polara/recommender/evaluation.py b/polara/recommender/evaluation.py index 41ccf21..813828b 100644 --- a/polara/recommender/evaluation.py +++ b/polara/recommender/evaluation.py @@ -17,9 +17,7 @@ def no_copy_csr_matrix(data, indices, indptr, shape, dtype): def safe_divide(a, b, mask=None, dtype=None): pos = mask if mask is not None else a > 0 - res = np.zeros(len(a), dtype=dtype) - np.divide(a, b, where=pos, out=res) - return res + return np.divide(a, b, where=pos, dtype=dtype) def build_rank_matrix(recommendations, shape): From 67e5aec0be46ecf9af6a9f05770d0b086d95416a Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jan 2021 08:40:29 +0300 Subject: [PATCH 79/82] fix `show_recommendations` call bug when the input list of items consists of strings --- polara/recommender/models.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index 4b4854d..0ec0210 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -297,16 +297,16 @@ def _make_user(self, user_info): # converts external user info into internal representation userid, itemid, feedback = self.data.fields - if isinstance(user_info, dict): - user_info = user_info.items() - try: - items_data, feedback_data = zip(*user_info) - except TypeError: + if isinstance(user_info, dict): # item:feedback dictionary + items_data, feedback_data = zip(*user_info.items()) + elif isinstance(user_info, (list, tuple, set, np.ndarray)): # list of items items_data = user_info feedback_data = {} if feedback is not None: feedback_val = self.data.training[feedback].max() feedback_data = {feedback: [feedback_val]*len(items_data)} + else: + raise ValueError("Unrecognized input for `user_info`.") try: item_index = self.data.index.itemid.training @@ -339,18 +339,17 @@ def show_recommendations(self, user_info, topk=None): self.data._test = namedtuple('TestData', 'testset holdout')._make([testset, holdout]) _topk = self.topk - self.topk = topk or _topk - # takes care of both sparse and dense recommendation lists - top_recs = self.get_topk_elements(scores).squeeze() # remove singleton - self.topk = _topk - + if topk is not None: + self.topk = topk try: - item_index = self.data.index.itemid.training - except AttributeError: - item_index = self.data.index.itemid + # takes care of both sparse and dense recommendation lists + top_recs = self.get_topk_elements(scores).squeeze() # remove singleton + finally: + self.topk = _topk seen_idx = seen_idx[1] # only items idx # covert back to external representation + item_index = self.data.get_entity_index(self.data.fields.itemid) item_idx_map = item_index.set_index('new') top_recs = item_idx_map.loc[top_recs, 'old'].values seen_items = item_idx_map.loc[seen_idx, 'old'].values From d77a4854083393be9ec992d6f3289687984d8763 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jan 2021 08:46:14 +0300 Subject: [PATCH 80/82] allow disabling feature normalizationfor LightFM --- polara/recommender/external/lightfm/lightfmwrapper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/polara/recommender/external/lightfm/lightfmwrapper.py b/polara/recommender/external/lightfm/lightfmwrapper.py index 27a1679..56006d6 100644 --- a/polara/recommender/external/lightfm/lightfmwrapper.py +++ b/polara/recommender/external/lightfm/lightfmwrapper.py @@ -19,12 +19,14 @@ def __init__(self, *args, item_features=None, user_features=None, **kwargs): self.item_alpha = 0.0 self.item_identity = True self._item_features_csr = None + self.normalize_item_features = True self.user_features = user_features self.user_features_labels = None self.user_alpha = 0.0 self.user_identity = True self._user_features_csr = None + self.normalize_user_features = True self.loss = 'warp' self.learning_schedule = 'adagrad' @@ -72,7 +74,7 @@ def build(self): self._item_features_csr, self.item_features_labels = stack_features( item_features, add_identity=self.item_identity, - normalize=True, + normalize=self.normalize_item_features, dtype='f4') if self.user_features is not None: user_features = self.user_features.reindex( @@ -81,7 +83,7 @@ def build(self): self._user_features_csr, self.user_features_labels = stack_features( user_features, add_identity=self.user_identity, - normalize=True, + normalize=self.normalize_user_features, dtype='f4') with track_time(self.training_time, verbose=self.verbose, model=self.method): From e93a6ec5dc2b0c68d190e6ec599f9a44f771aae8 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jan 2021 08:51:44 +0300 Subject: [PATCH 81/82] bump current version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6699306..df83166 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ opts = dict(name="polara", description="Fast and flexible recommender systems framework", keywords = "recommender systems", - version = "0.6.5.dev", + version = "0.7.0.dev", license="MIT", author="Evgeny Frolov", platforms=["any"], From 35c545b98c4666aebf28de343a6b2875cb3c91a7 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jan 2021 09:00:08 +0300 Subject: [PATCH 82/82] update setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df83166..dfcadc5 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ "polara/datasets", "polara/lib", "polara/tools", + "polara/preprocessing", "polara/recommender/coldstart", "polara/recommender/hybrid", "polara/recommender/contextual", @@ -20,7 +21,7 @@ opts = dict(name="polara", description="Fast and flexible recommender systems framework", keywords = "recommender systems", - version = "0.7.0.dev", + version = "0.7.0", license="MIT", author="Evgeny Frolov", platforms=["any"],