Merge pull request #380 version 0.1.7-beta

[Version] New version 0.1.7-beta

.gitignore

@@ -32,7 +32,7 @@ wheels/
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
+# *.spec
 
 # Installer logs
 pip-log.txt
@@ -111,19 +111,14 @@ config/evaluator_config.json
 
 # Data
 backtesting/collector/data/**
-
 logs/matrix_history\.data
-
 logs/OctoBot\.log\.*
 
 # Tentacles
 tentacles
 config\.json
 
 *.csv
-
 *.ods
-
 temp_config\.json
-
 log

.travis.yml

@@ -1,4 +1,6 @@
 sudo: enabled
+os:
+  - linux
 language: python
 python:
   - 3.6
@@ -15,7 +17,10 @@ install:
 script:
   - pip3 install -r dev_requirements.txt
   - pytest --cov=. --cov-config=.coveragerc tests/unit_tests tests/functional_tests
+# if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "beta" ];
+#  - bash ./docs/delivery/travis_delivery.sh
+after_failure:
+  - dmesg | grep -i kill
+  - sleep 4 # to prevent sudden log truncation
 after_success:
   - coveralls
-
-

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3
+FROM python:3.6.6
 
 ARG octobot_branch="beta"
 ARG octobot_install_dir="Octobot"

README.md

@@ -1,4 +1,4 @@
-# OctoBot [0.1.6-beta](https://github.com/Drakkar-Software/OctoBot/tree/dev/docs/CHANGELOG.md)
+# OctoBot [0.1.7-beta](https://github.com/Drakkar-Software/OctoBot/tree/dev/docs/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/c83a127c42ba4a389ca86a92fba7c53c)](https://www.codacy.com/app/paul.bouquet/OctoBot?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot&utm_campaign=Badge_Grade) [![Build Status](https://api.travis-ci.org/Drakkar-Software/OctoBot.svg?branch=dev)](https://travis-ci.org/Drakkar-Software/OctoBot) [![Code Factor](https://www.codefactor.io/repository/github/Drakkar-Software/OctoBot/badge)](https://www.codefactor.io/repository/github/Drakkar-Software/OctoBot/overview/dev) [![Build Status](https://semaphoreci.com/api/v1/herklos/octobot/branches/dev/shields_badge.svg)](https://semaphoreci.com/herklos/octobot) [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot/badge.svg?branch=dev)](https://coveralls.io/github/Drakkar-Software/OctoBot?branch=dev) [![Codefresh build status]( https://g.codefresh.io/api/badges/build?repoOwner=Drakkar-Software&repoName=OctoBot&branch=dev&pipelineName=OctoBot&accountName=herklos_marketplace&type=cf-1)](https://g.codefresh.io/repositories/Drakkar-Software/OctoBot/builds?filter=trigger:build;branch:dev;service:5b06a377435197b088b1757a~OctoBot) [![Build status](https://ci.appveyor.com/api/projects/status/jr9o8sghywnued9x?svg=true)](https://ci.appveyor.com/project/Herklos/octobot) [![wercker status](https://app.wercker.com/status/c247b89f2a53068fde745db1f2c530ae/s/ "wercker status")](https://app.wercker.com/project/byKey/c247b89f2a53068fde745db1f2c530ae)
 <p align="center">
 <img src="../assets/octopus.svg" alt="Octobot Logo" height="400" width="400">

backtesting/__init__.py

@@ -1,10 +0,0 @@
-bot_instance = None
-
-
-def __init__(bot):
-    global bot_instance
-    bot_instance = bot
-
-
-def get_bot():
-    return bot_instance

backtesting/backtester.py

@@ -0,0 +1,51 @@
+from trading.exchanges.exchange_simulator.exchange_simulator import ExchangeSimulator
+from backtesting.backtesting_util import start_backtesting_bot, get_standalone_backtesting_bot
+
+
+class Backtester:
+
+    def __init__(self, config, files=[]):
+        self.octobot, self.ignored_files = get_standalone_backtesting_bot(config, files)
+        self.error = None
+
+    def get_ignored_files(self):
+        return self.ignored_files
+
+    def get_is_computing(self):
+        if self.error is not None:
+            return False
+        simulator = self._get_exchange_simulator()
+        if simulator:
+            return simulator.get_is_initializing() or not simulator.get_backtesting().get_is_finished()
+        return False
+
+    def get_progress(self):
+        simulator = self._get_exchange_simulator()
+        if simulator:
+            return simulator.get_progress()
+        return 0
+
+    def get_report(self):
+        simulator = self._get_exchange_simulator()
+        if simulator:
+            report = simulator.get_backtesting().get_dict_formatted_report()
+            if self.error is not None:
+                report["error"] = str(self.error)
+            return report
+        return {}
+
+    def _get_exchange_simulator(self):
+        for exchange in self.octobot.get_exchanges_list().values():
+            if isinstance(exchange.get_exchange(), ExchangeSimulator):
+                return exchange.get_exchange()
+
+    def start_backtesting(self, in_thread=False):
+        self.error = None
+        return start_backtesting_bot(self.octobot, in_thread=in_thread, watcher=self)
+
+    def get_bot(self):
+        return self.octobot
+
+    def set_error(self, error):
+        self.error = error
+

backtesting/backtesting.py

@@ -2,7 +2,6 @@
 import os
 import time
 
-from backtesting import get_bot
 from config.cst import *
 from tools.pretty_printer import PrettyPrinter
 
@@ -18,9 +17,15 @@ def __init__(self, config, exchange_simulator, exit_at_end=True):
         self.symbols_to_test = set()
         self.init_symbols_to_test()
 
+    def get_is_finished(self, symbol=None):
+        if symbol is None:
+            return len(self.ended_symbols) == len(self.symbols_to_test)
+        else:
+            return symbol in self.ended_symbols
+
     def end(self, symbol):
         self.ended_symbols.add(symbol)
-        if len(self.ended_symbols) == len(self.symbols_to_test):
+        if self.get_is_finished():
             try:
                 self.logger.info(" **** Backtesting report ****")
                 self.logger.info(" ========= Trades =========")
@@ -39,54 +44,90 @@ def end(self, symbol):
                 os._exit(0)
 
     def print_trades_history(self):
-        trader = next(iter(get_bot().get_exchange_trader_simulators().values()))
+        trader = self.get_trader()
         trades_history = trader.get_trades_manager().get_trade_history()
         trades_history_string = ""
         for trade in trades_history:
             trades_history_string += PrettyPrinter.trade_pretty_printer(trade) + "\n"
         self.logger.info(trades_history_string.strip())
 
     def print_global_report(self):
-        trader = next(iter(get_bot().get_exchange_trader_simulators().values()))
-        trade_manager = trader.get_trades_manager()
-        profitability, market_average_profitability = self.get_profitability(get_bot())
+        try:
+            trader = self.get_trader()
+
+            profitability, market_average_profitability = self.get_profitability(trader)
+            reference_market = self.get_reference_market(trader)
+            portfolio = self.get_portfolio(trader)
+            accuracy_info = "" if len(self.symbols_to_test) < 2 else \
+                "\nPlease note that multi symbol backtesting is slightly random due to Octbot's multithreaded " \
+                "architecture used to process all symbols as fast as possible. This randomness is kept for " \
+                "backtesting in order to be as close as possible from reality. Single symbol backtesting is " \
+                "100% determinist."
+
+            self.logger.info(f"End portfolio: "
+                             f"{PrettyPrinter.global_portfolio_pretty_print(portfolio,' | ')}")
+
+            self.logger.info(f"Global market profitability (vs {reference_market}) : "
+                             f"{market_average_profitability}% | Octobot : {profitability}%{accuracy_info}")
+
+            backtesting_time = time.time() - self.begin_time
+            self.logger.info(f"Simulation lasted {backtesting_time} sec")
+        except Exception as e:
+            logging.exception(e)
+
+    def _get_symbol_report(self, symbol, trader):
+        market_data = self.exchange_simulator.get_data()[symbol][self.exchange_simulator.MIN_ENABLED_TIME_FRAME.value]
 
-        reference_market = trade_manager.get_reference()
-        portfolio = trader.get_portfolio()
-        accuracy_info = "" if len(self.symbols_to_test) < 2 else "\nPlease note that multi symbol backtesting is " \
-                                                                 "slightly random due to Octbot's multithreaded " \
-                                                                 "architecture used to process all symbols as fast as" \
-                                                                 " possible. This randomness is kept for backtesting " \
-                                                                 "in order to be as close as possible from reality. " \
-                                                                 "Single symbol backtesting is 100% determinist."
+        # profitability
+        total_profitability = 0
+        _, profitability, _, _ = trader.get_trades_manager().get_profitability()
+        total_profitability += profitability
 
-        self.logger.info(f"End portfolio: "
-                         f"{PrettyPrinter.global_portfolio_pretty_print(portfolio.get_portfolio(),' | ')}")
+        # vs market
+        return self.get_market_delta(market_data)
 
-        self.logger.info(f"Global market profitability (vs {reference_market}) : "
-                         f"{market_average_profitability}% | Octobot : {profitability}%{accuracy_info}")
+    def print_symbol_report(self, symbol):
+        self.logger.info(f"{symbol} Profitability : Market {self._get_symbol_report(symbol, self.get_trader()) * 100}%")
 
-        backtesting_time = time.time() - self.begin_time
-        self.logger.info(f"Simulation lasted {backtesting_time} sec")
+    def get_dict_formatted_report(self):
+        SYMBOL_REPORT = "symbol_report"
+        BOT_REPORT = "bot_report"
+        SYMBOLS_WITH_TF = "symbols_with_time_frames_frames"
+        report = {
+            SYMBOL_REPORT: [],
+            BOT_REPORT: {},
+            SYMBOLS_WITH_TF: {}
+        }
 
-    def print_symbol_report(self, symbol):
-        market_data = self.exchange_simulator.get_data()[symbol][self.exchange_simulator.MIN_ENABLED_TIME_FRAME.value]
+        trader = self.get_trader()
 
-        # profitability
-        total_profitability = 0
-        for trader in get_bot().get_exchange_trader_simulators().values():
-            _, profitability, _, _ = trader.get_trades_manager().get_profitability()
-            total_profitability += profitability
+        profitability, market_average_profitability = self.get_profitability(trader)
 
-        # vs market
-        market_delta = self.get_market_delta(market_data)
+        for symbol in self.symbols_to_test:
+            report[SYMBOL_REPORT].append({symbol: self._get_symbol_report(symbol, trader) * 100})
+            report[SYMBOLS_WITH_TF][symbol] = self.exchange_simulator.get_min_time_frame(symbol)
+
+        report[BOT_REPORT] = {
+            "profitability": profitability,
+            "market_average_profitability": market_average_profitability,
+            "reference_market": self.get_reference_market(trader),
+            "end_portfolio": self.get_portfolio(trader)
+        }
+        return report
 
-        # log
-        self.logger.info(f"{symbol} Profitability : Market {market_delta * 100}%")
+    def get_trader(self):
+        return self.exchange_simulator.get_exchange_manager().get_trader()
+
+    @staticmethod
+    def get_reference_market(trader):
+        return trader.get_trades_manager().get_reference()
+
+    @staticmethod
+    def get_portfolio(trader):
+        return trader.get_portfolio().get_portfolio()
 
     @staticmethod
-    def get_profitability(bot):
-        trader = next(iter(bot.get_exchange_trader_simulators().values()))
+    def get_profitability(trader):
         trade_manager = trader.get_trades_manager()
         _, profitability, _, market_average_profitability = trade_manager.get_profitability(True)
         return profitability, market_average_profitability