Merge pull request #325 Version 0.1.5_1-beta

[Version] 0.1.5_1-beta

README.md

@@ -1,4 +1,4 @@
-# OctoBot [0.1.4_4-beta](https://github.com/Drakkar-Software/OctoBot/tree/dev/docs/CHANGELOG.md)
+# OctoBot [0.1.5_1-beta](https://github.com/Drakkar-Software/OctoBot/tree/dev/docs/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/c83a127c42ba4a389ca86a92fba7c53c)](https://www.codacy.com/app/paul.bouquet/OctoBot?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot&utm_campaign=Badge_Grade) [![Build Status](https://api.travis-ci.org/Drakkar-Software/OctoBot.svg?branch=dev)](https://travis-ci.org/Drakkar-Software/OctoBot) [![Code Factor](https://www.codefactor.io/repository/github/Drakkar-Software/OctoBot/badge)](https://www.codefactor.io/repository/github/Drakkar-Software/OctoBot/overview/dev) [![Build Status](https://semaphoreci.com/api/v1/herklos/octobot/branches/dev/shields_badge.svg)](https://semaphoreci.com/herklos/octobot) [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot/badge.svg?branch=dev)](https://coveralls.io/github/Drakkar-Software/OctoBot?branch=dev) [![Codefresh build status]( https://g.codefresh.io/api/badges/build?repoOwner=Drakkar-Software&repoName=OctoBot&branch=dev&pipelineName=OctoBot&accountName=herklos_marketplace&type=cf-1)](https://g.codefresh.io/repositories/Drakkar-Software/OctoBot/builds?filter=trigger:build;branch:dev;service:5b06a377435197b088b1757a~OctoBot) [![Build status](https://ci.appveyor.com/api/projects/status/jr9o8sghywnued9x?svg=true)](https://ci.appveyor.com/project/Herklos/octobot) 
 <p align="center">
 <img src="../assets/octopus.svg" alt="Octobot Logo" height="400" width="400">

backtesting/backtesting.py

@@ -4,6 +4,7 @@
 
 from backtesting import get_bot
 from config.cst import *
+from tools.pretty_printer import PrettyPrinter
 
 
 class Backtesting:
@@ -14,17 +15,53 @@ def __init__(self, config, exchange_simulator, exit_at_end=True):
         self.force_exit_at_end = exit_at_end
         self.exchange_simulator = exchange_simulator
         self.logger = logging.getLogger(self.__class__.__name__)
+        self.ended_symbols = set()
+        self.symbols_to_test = set()
+        self.init_symbols_to_test()
 
-    def end(self):
-        for symbol in self.exchange_simulator.get_symbols():
-            self.report(symbol)
+    def end(self, symbol):
+        self.ended_symbols.add(symbol)
+        if len(self.ended_symbols) == len(self.symbols_to_test):
+
+            self.logger.info(" **** Backtesting report ****")
+            self.logger.info(" ========= Trades =========")
+            self.print_trades_history()
+
+            self.logger.info(" ========= Symbols price evolution =========")
+            for symbol_to_test in self.symbols_to_test:
+                self.print_symbol_report(symbol_to_test)
+
+            self.logger.info(" ========= Octobot end state =========")
+            self.print_global_report()
+
+            if self.force_exit_at_end:
+                os._exit(0)
+
+    def print_trades_history(self):
+        trader = next(iter(get_bot().get_exchange_trader_simulators().values()))
+        trades_history = trader.get_trades_manager().get_trade_history()
+        trades_history_string = ""
+        for trade in trades_history:
+            trades_history_string += PrettyPrinter.trade_pretty_printer(trade) + "\n"
+        self.logger.info(trades_history_string.strip())
+
+    def print_global_report(self):
+        trader = next(iter(get_bot().get_exchange_trader_simulators().values()))
+        trade_manager = trader.get_trades_manager()
+        _, profitability, _, market_average_profitability = trade_manager.get_profitability(True)
+        reference_market = trade_manager.get_reference()
+        portfolio = trader.get_portfolio()
+
+        self.logger.info(f"End portfolio: "
+                         f"{PrettyPrinter.global_portfolio_pretty_print(portfolio.get_portfolio(),' | ')}")
+
+        self.logger.info(f"Global market profitability (vs {reference_market}) : "
+                         f"{market_average_profitability}% | Octobot : {profitability}%")
 
         backtesting_time = time.time() - self.begin_time
-        self.logger.info("Simulation lasted {0} sec".format(backtesting_time))
-        if self.force_exit_at_end:
-            os._exit(0)
+        self.logger.info(f"Simulation lasted {backtesting_time} sec")
 
-    def report(self, symbol):
+    def print_symbol_report(self, symbol):
         market_data = self.exchange_simulator.get_data()[symbol][self.exchange_simulator.MIN_ENABLED_TIME_FRAME.value]
 
         self.time_delta = self.begin_time - market_data[0][PriceIndexes.IND_PRICE_TIME.value] / 1000
@@ -39,8 +76,13 @@ def report(self, symbol):
         market_delta = self.get_market_delta(market_data)
 
         # log
-        self.logger.info(
-            "Profitability : Market {0}% | OctoBot : {1}%".format(market_delta * 100, total_profitability))
+        self.logger.info(f"{symbol} Profitability : Market {market_delta * 100}%")
+
+    def init_symbols_to_test(self):
+        for crypto_currency_data in self.config[CONFIG_CRYPTO_CURRENCIES].values():
+            for symbol in crypto_currency_data[CONFIG_CRYPTO_PAIRS]:
+                if symbol in self.exchange_simulator.get_symbols():
+                    self.symbols_to_test.add(symbol)
 
     @staticmethod
     def get_market_delta(market_data):
@@ -57,3 +99,9 @@ def get_market_delta(market_data):
     @staticmethod
     def enabled(config):
         return CONFIG_BACKTESTING in config and config[CONFIG_BACKTESTING][CONFIG_ENABLED_OPTION]
+
+
+class BacktestingEndedException(Exception):
+    def __init__(self, symbol=""):
+        self.msg = f"Backtesting finished for {symbol}."
+        super().__init__(self.msg)

backtesting/collector/data_collector.py

@@ -28,7 +28,7 @@ def create_exchange_data_collectors(self):
 
                 exchange_data_collector = ExchangeDataCollector(self.config, exchange_inst)
 
-                if len(exchange_data_collector.get_symbols()) == 0 or len(exchange_data_collector.time_frames) == 0:
+                if not exchange_data_collector.get_symbols() or not exchange_data_collector.time_frames:
                     self.logger.warning("{0} exchange not started (not enough symbols or timeframes)"
                                         .format(exchange_class_string))
                 else:

backtesting/collector/data_parser.py

@@ -37,4 +37,3 @@ def merge_arrays(arrays):
                     data[PriceIndexes.IND_PRICE_VOL.value][i]
 
         return time_frames_data
-

config/cst.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
 SHORT_VERSION = "0.1.5"
-MINOR_VERSION = "0"
+MINOR_VERSION = "1"
 VERSION_DEV_PHASE = "beta"
 VERSION = f"{SHORT_VERSION}-{VERSION_DEV_PHASE}"
 LONG_VERSION = f"{SHORT_VERSION}_{MINOR_VERSION}-{VERSION_DEV_PHASE}"
@@ -49,6 +49,7 @@
 CONFIG_TRADER_RISK_MAX = 1
 ORDER_REFRESHER_TIME = 15
 ORDER_REFRESHER_TIME_WS = 1
+UPDATER_MAX_SLEEPING_TIME = 2
 SIMULATOR_LAST_PRICES_TO_CHECK = 50
 ORDER_CREATION_LAST_TRADES_TO_USE = 10
 CONFIG_TRADER_REFERENCE_MARKET = "reference_market"

docs/CHANGELOG.md

@@ -1,18 +1,42 @@
 *It is strongly advised to perform an update of your tentacles after updating OctoBot.*
 
+Changelog for 0.1.5_1-beta
+====================
+*Released date : July 17 2018*
+
+# Concerned issues :
+    #305 [Refactor] refactor overall code
+    #318 [Candles management] adapt candles timestamp to have second timestamp everywhere
+    #319 [Web interface] trades are displayed for all symbols, display only for the selected one
+    #320 [Backtesting] do not start unecessary services on backtesting mode
+    #322 [Web interface] Create portfolio page
+    #323 [Web interface] Create orders page
+    #324 [Web interface] Create trades page
+
+# New features :
+    - Backtesting multi symbol support improved
+    - Backtesting report at the end of a backtesting
+    - Web Interface : New pages (portfolio, orders, trades)
+
+# Improvements:
+    - Improved readability of web interface
+
+# Bug fixes :
+    - Backtesting trades timestamps were wrong on multi symbol backtesting
+
 Changelog for 0.1.5-beta
 ====================
 *Released date : July 15 2018*
 
 # Concerned issues :
+    #252 [OrderManager] "Timed Out" raised during _update_orders_status
     #265 [Web Interface] Create web evaluator_config.json edition interface
     #266 [Web Interface] Create web tentacles manager interface
-    #302 [Web Interface] setup architecture
     #270 [Web interface] Create advanced web interface
-    #308 [Backtesting] Improve backtesting accuracy 
-    #304 [Trade Manager] Ensure get_average_market_profitability resilience
     #294 [Trader simulator] StopLoss orders triggered when they shouldn't
-    #252 [OrderManager] "Timed Out" raised during _update_orders_status
+    #302 [Web Interface] setup architecture
+    #304 [Trade Manager] Ensure get_average_market_profitability resilience
+    #308 [Backtesting] Improve backtesting accuracy
 
 # New features :
     - New features in web interface : tentacles configuration, trades displayed on Dashboard tob

evaluator/Dispatchers/reddit_dispatcher.py

@@ -9,7 +9,7 @@
 
 class RedditDispatcher(AbstractDispatcher):
 
-    MAX_CONNECTION_ATTEMPTS = 3
+    MAX_CONNECTION_ATTEMPTS = 10
 
     def __init__(self, config):
         super().__init__(config)
@@ -18,6 +18,7 @@ def __init__(self, config):
         self.counter = 0
         self.connect_attempts = 0
         self.social_config = {}
+        self.credentials_ok = False
 
         # check presence of twitter instance
         if RedditService.is_setup_correctly(self.config):
@@ -56,6 +57,7 @@ def _start_listener(self):
         subreddit = self.reddit_service.get_endpoint().subreddit(self.subreddits)
         start_time = time.time()
         for entry in subreddit.stream.submissions():
+            self.credentials_ok = True
             self.connect_attempts = 0
             self.counter += 1
             # check if we are in the 100 history or if it's a new entry (new posts are more valuables)
@@ -93,27 +95,33 @@ def _start_dispatcher(self):
                 time.sleep(self._SLEEPING_TIME_BEFORE_RECONNECT_ATTEMPT_SEC)
             except InvalidToken:
                 # expired, try again
-                self.logger.error("Error when receiving Reddit feed: '{0}'".format(e))
+                self.logger.error(f"Error when receiving Reddit feed: '{e}'")
                 self.logger.exception(e)
-                self.logger.info("Try to continue after some time.")
+                self.logger.info(f"Try to continue after {self._SLEEPING_TIME_BEFORE_RECONNECT_ATTEMPT_SEC} seconds.")
                 time.sleep(self._SLEEPING_TIME_BEFORE_RECONNECT_ATTEMPT_SEC)
             except ServerError as e:
                 # server error, try again
-                self.logger.error("Error when receiving Reddit feed: '{0}'".format(e))
+                self.logger.error("Error when receiving Reddit feed: '{e}'")
                 self.logger.exception(e)
-                self.logger.info("Try to continue after some time.")
+                self.logger.info(f"Try to continue after {self._SLEEPING_TIME_BEFORE_RECONNECT_ATTEMPT_SEC} seconds.")
                 time.sleep(self._SLEEPING_TIME_BEFORE_RECONNECT_ATTEMPT_SEC)
             except OAuthException as e:
-                self.logger.error("Error when receiving Reddit feed: '{0}' this may mean {1}"
-                                  .format(e, "that reddit login info in config.json are wrong."))
+                self.logger.error(f"Error when receiving Reddit feed: '{e}' this may mean that reddit login info "
+                                  f"in config.json are wrong")
                 self.logger.exception(e)
                 self.keep_running = False
             except ResponseException as e:
-                self.logger.error("Error when receiving Reddit feed: '{0}' this may mean {1}"
-                                  .format(e, "that reddit configuration in config.json are wrong."))
+                message_complement = "this may mean that reddit login info in config.json are wrong." \
+                    if not self.credentials_ok else \
+                    f"Try to continue after {self._SLEEPING_TIME_BEFORE_RECONNECT_ATTEMPT_SEC} seconds."
+                self.logger.error(f"Error when receiving Reddit feed: '{e}' this may mean {message_complement}")
                 self.logger.exception(e)
-                self.connect_attempts += 1
+                if not self.credentials_ok:
+                    self.connect_attempts += 1
+                else:
+                    self.connect_attempts += 0.1
+                time.sleep(self._SLEEPING_TIME_BEFORE_RECONNECT_ATTEMPT_SEC)
             except Exception as e:
-                self.logger.error("Error when receiving Reddit feed: '{0}'".format(e))
+                self.logger.error(f"Error when receiving Reddit feed: '{e}'")
                 self.logger.exception(e)
                 self.keep_running = False

evaluator/RealTime/__init__.py

@@ -1,2 +1,2 @@
 from .realtime_evaluator import *
-from tentacles.Evaluator.RealTime import *
+from tentacles.Evaluator.RealTime import *

evaluator/Social/__init__.py

@@ -1,2 +1,2 @@
 from .social_evaluator import *
-from tentacles.Evaluator.Social import *
+from tentacles.Evaluator.Social import *

evaluator/Strategies/__init__.py

@@ -1,2 +1,2 @@
 from evaluator.Strategies.strategies_evaluator import *
-from tentacles.Evaluator.Strategies import *
+from tentacles.Evaluator.Strategies import *

evaluator/Strategies/strategies_evaluator.py

@@ -1,7 +1,6 @@
 from abc import *
 
-from config.cst import CONFIG_EVALUATOR_STRATEGIES, STRATEGIES_REQUIRED_TIME_FRAME, CONFIG_FILE_EXT, \
-    STRATEGIES_REQUIRED_EVALUATORS
+from config.cst import CONFIG_EVALUATOR_STRATEGIES, STRATEGIES_REQUIRED_TIME_FRAME, STRATEGIES_REQUIRED_EVALUATORS
 from evaluator.abstract_evaluator import AbstractEvaluator
 from tools.evaluator_divergence_analyser import EvaluatorDivergenceAnalyser
 from tools.time_frame_manager import TimeFrameManager