Merge pull request #338 new version 0.1.5_3-beta

[Version] new version 0.1.5_3-beta

.gitignore

@@ -125,3 +125,5 @@ config\.json
 *.ods
 
 temp_config\.json
+
+log

README.md

@@ -1,4 +1,4 @@
-# OctoBot [0.1.5_2-beta](https://github.com/Drakkar-Software/OctoBot/tree/dev/docs/CHANGELOG.md)
+# OctoBot [0.1.5_3-beta](https://github.com/Drakkar-Software/OctoBot/tree/dev/docs/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/c83a127c42ba4a389ca86a92fba7c53c)](https://www.codacy.com/app/paul.bouquet/OctoBot?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot&utm_campaign=Badge_Grade) [![Build Status](https://api.travis-ci.org/Drakkar-Software/OctoBot.svg?branch=dev)](https://travis-ci.org/Drakkar-Software/OctoBot) [![Code Factor](https://www.codefactor.io/repository/github/Drakkar-Software/OctoBot/badge)](https://www.codefactor.io/repository/github/Drakkar-Software/OctoBot/overview/dev) [![Build Status](https://semaphoreci.com/api/v1/herklos/octobot/branches/dev/shields_badge.svg)](https://semaphoreci.com/herklos/octobot) [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot/badge.svg?branch=dev)](https://coveralls.io/github/Drakkar-Software/OctoBot?branch=dev) [![Codefresh build status]( https://g.codefresh.io/api/badges/build?repoOwner=Drakkar-Software&repoName=OctoBot&branch=dev&pipelineName=OctoBot&accountName=herklos_marketplace&type=cf-1)](https://g.codefresh.io/repositories/Drakkar-Software/OctoBot/builds?filter=trigger:build;branch:dev;service:5b06a377435197b088b1757a~OctoBot) [![Build status](https://ci.appveyor.com/api/projects/status/jr9o8sghywnued9x?svg=true)](https://ci.appveyor.com/project/Herklos/octobot) 
 <p align="center">
 <img src="../assets/octopus.svg" alt="Octobot Logo" height="400" width="400">

backtesting/collector/data_collector.py

@@ -3,7 +3,7 @@
 import ccxt
 
 from backtesting.collector.exchange_collector import ExchangeDataCollector
-from config.cst import *
+from config.cst import CONFIG_TIME_FRAME, CONFIG_EXCHANGES
 from trading.exchanges.exchange_manager import ExchangeManager
 
 
@@ -15,6 +15,8 @@ def __init__(self, config):
         self.exchange_data_collectors_threads = []
         self.logger.info("Create data collectors...")
 
+        self.config[CONFIG_TIME_FRAME] = []
+
         self.create_exchange_data_collectors()
 
     def create_exchange_data_collectors(self):

backtesting/collector/exchange_collector.py

@@ -42,10 +42,8 @@ def stop(self):
         self.keep_running = False
 
     def _set_file_name(self, symbol):
-        return "{0}_{1}_{2}{3}".format(self.exchange.get_name(),
-                                       symbol.replace("/", "_"),
-                                       time.strftime("%Y%m%d_%H%M%S"),
-                                       self.Exchange_Data_Collector_File_Ext)
+        return f"{self.exchange.get_name()}_{symbol.replace('/', '_')}_" \
+               f"{time.strftime('%Y%m%d_%H%M%S')}{self.Exchange_Data_Collector_File_Ext}"
 
     @staticmethod
     def get_file_name(file_name):
@@ -90,7 +88,8 @@ def _update_file(self, symbol):
 
     def run(self):
         self._prepare()
-        self.logger.info("{0} updating...".format(self.exchange.get_name()))
+        self.logger.info(f"Data Collector will now update this file from {self.exchange.get_name()} "
+                         f"for each new time frame update...")
         while self.keep_running:
             now = time.time()
 
@@ -106,8 +105,7 @@ def run(self):
                         self.file_contents[symbol][time_frame.value].append(result_df)
                         self._data_updated = True
                         self.time_frame_update[symbol][time_frame] = now
-                        self.logger.info(
-                            "{0} ({2}) on {1} updated".format(symbol, self.exchange.get_name(), time_frame))
+                        self.logger.info(f"{symbol} ({self.exchange.get_name()}) on {time_frame} updated")
 
                 if self._data_updated:
                     self._update_file(symbol)

config/cst.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
 SHORT_VERSION = "0.1.5"
-MINOR_VERSION = "2"
+MINOR_VERSION = "3"
 VERSION_DEV_PHASE = "beta"
 VERSION = f"{SHORT_VERSION}-{VERSION_DEV_PHASE}"
 LONG_VERSION = f"{SHORT_VERSION}_{MINOR_VERSION}-{VERSION_DEV_PHASE}"
@@ -40,14 +40,14 @@
 
 # Trading
 CONFIG_EXCHANGES = "exchanges"
-CONFIG_EXCHANGE_WEB_SOCKET = "web_socket"
+CONFIG_EXCHANGE_WEB_SOCKET = "web-socket"
 CONFIG_EXCHANGE_KEY = "api-key"
 CONFIG_EXCHANGE_SECRET = "api-secret"
 CONFIG_TRADER = "trader"
 CONFIG_TRADING = "trading"
 CONFIG_TRADER_MODES = "modes"
-CONFIG_SIMULATOR = "trader_simulator"
-CONFIG_STARTING_PORTFOLIO = "starting_portfolio"
+CONFIG_SIMULATOR = "trader-simulator"
+CONFIG_STARTING_PORTFOLIO = "starting-portfolio"
 CONFIG_TRADER_RISK = "risk"
 CONFIG_TRADER_MODE = "mode"
 CONFIG_TRADER_RISK_MIN = 0.05
@@ -57,7 +57,7 @@
 UPDATER_MAX_SLEEPING_TIME = 2
 SIMULATOR_LAST_PRICES_TO_CHECK = 50
 ORDER_CREATION_LAST_TRADES_TO_USE = 10
-CONFIG_TRADER_REFERENCE_MARKET = "reference_market"
+CONFIG_TRADER_REFERENCE_MARKET = "reference-market"
 DEFAULT_REFERENCE_MARKET = "BTC"
 MARKET_SEPARATOR = "/"
 CURRENCY_DEFAULT_MAX_PRICE_DIGITS = 8
@@ -68,21 +68,21 @@
 CONFIG_PORTFOLIO_TOTAL = "total"
 
 # Notification
-CONFIG_NOTIFICATION_TYPE = "notification_type"
+CONFIG_NOTIFICATION_TYPE = "notification-type"
 CONFIG_NOTIFICATION_INSTANCE = "notifier"
 CONFIG_CATEGORY_NOTIFICATION = "notification"
-CONFIG_NOTIFICATION_GLOBAL_INFO = "global_info"
-CONFIG_NOTIFICATION_PRICE_ALERTS = "price_alerts"
+CONFIG_NOTIFICATION_GLOBAL_INFO = "global-info"
+CONFIG_NOTIFICATION_PRICE_ALERTS = "price-alerts"
 CONFIG_NOTIFICATION_TRADES = "trades"
 NOTIFICATION_STARTING_MESSAGE = f"OctoBot v{LONG_VERSION} starting..."
 NOTIFICATION_STOPPING_MESSAGE = f"OctoBot v{LONG_VERSION} stopping..."
 REAL_TRADER_STR = "[Real Trader] "
 SIMULATOR_TRADER_STR = "[Simulator] "
 
 # DEBUG options
-CONFIG_DEBUG_OPTION_PERF = "performance_analyser"
+CONFIG_DEBUG_OPTION_PERF = "performance-analyser"
 CONFIG_DEBUG_OPTION_PERF_REFRESH_TIME_MIN = 5
-CONFIG_DEBUG_OPTION = "DEV_MODE"
+CONFIG_DEBUG_OPTION = "DEV-MODE"
 
 # SERVICES
 CONFIG_CATEGORY_SERVICES = "services"
@@ -116,6 +116,7 @@
 
 # Evaluator
 CONFIG_EVALUATOR = "evaluator"
+CONFIG_FORCED_EVALUATOR = "forced_evaluator"
 CONFIG_EVALUATOR_SOCIAL = "Social"
 CONFIG_EVALUATOR_REALTIME = "RealTime"
 CONFIG_EVALUATOR_TA = "TA"
@@ -128,8 +129,9 @@
 DEFAULT_REST_REAL_TIME_EVALUATOR_REFRESH_RATE_SECONDS = 60
 CONFIG_REFRESH_RATE = "refresh_rate_seconds"
 CONFIG_TIME_FRAME = "time_frame"
+CONFIG_FORCED_TIME_FRAME = "forced_time_frame"
 CONFIG_FILE_EXT = ".json"
-CONFIG_CRYPTO_CURRENCIES = "crypto_currencies"
+CONFIG_CRYPTO_CURRENCIES = "crypto-currencies"
 CONFIG_CRYPTO_PAIRS = "pairs"
 CONFIG_EVALUATORS_WILDCARD = [CONFIG_WILDCARD]
 
@@ -229,6 +231,7 @@
 UPDATED_CONFIG_SEPARATOR = "_"
 GLOBAL_CONFIG_KEY = "global_config"
 EVALUATOR_CONFIG_KEY = "evaluator_config"
+COIN_MARKET_CAP_CURRENCIES_LIST_URL = "https://api.coinmarketcap.com/v2/listings/"
 
 
 class TentacleManagerActions(Enum):
@@ -283,6 +286,7 @@ class TimeFrames(Enum):
     THIRTY_MINUTES = "30m"
     ONE_HOUR = "1h"
     TWO_HOURS = "2h"
+    THREE_HOURS = "3h"
     FOUR_HOURS = "4h"
     HEIGHT_HOURS = "8h"
     TWELVE_HOURS = "12h"
@@ -302,6 +306,7 @@ class TimeFrames(Enum):
     TimeFrames.THIRTY_MINUTES: 30,
     TimeFrames.ONE_HOUR: 60,
     TimeFrames.TWO_HOURS: 120,
+    TimeFrames.THREE_HOURS: 180,
     TimeFrames.FOUR_HOURS: 240,
     TimeFrames.HEIGHT_HOURS: 480,
     TimeFrames.TWELVE_HOURS: 720,

config/default_config.json

@@ -1,5 +1,5 @@
 {
-  "crypto_currencies":{
+  "crypto-currencies":{
     "Bitcoin": {
       "pairs" : ["BTC/USDT"]
     }
@@ -8,21 +8,21 @@
   },
   "services": {},
   "notification":{
-    "global_info": true,
-    "price_alerts": true,
+    "global-info": true,
+    "price-alerts": true,
     "trades": true,
-    "notification_type": []
+    "notification-type": []
   },
   "trader":{
     "enabled": false,
     "risk": 0.5,
-    "reference_market": "BTC",
+    "reference-market": "BTC",
     "mode": "DailyTradingMode"
   },
-  "trader_simulator":{
+  "trader-simulator":{
     "enabled": true,
     "risk": 0.5,
-    "starting_portfolio": {
+    "starting-portfolio": {
       "BTC": 10,
       "USDT": 1000
     }

dev_requirements.txt

@@ -6,5 +6,3 @@ coveralls
 
 tox
 tox-travis
-
-matplotlib==2.2.2

docs/CHANGELOG.md

@@ -1,5 +1,22 @@
 *It is strongly advised to perform an update of your tentacles after updating OctoBot.*
 
+Changelog for 0.1.5_3-beta
+====================
+*Released date : July 26 2018*
+
+**Warning** :
+- All config keys with "_" changed to "-" (for example crypto_currencies -> crypto-currencies)
+
+# Concerned issues :
+    #312 [Web Interface] Services configuration
+    #311 [Web Interface] Symbols configuration
+    #334 [Strategy configuration] create a strategy configuration otpimizer
+
+# New features :
+    - Web Interface : Services & Symbols configuration
+    - Startegy optimizer
+    - Encrypter command
+
 Changelog for 0.1.5_2-beta
 ====================
 *Released date : July 24 2018*
@@ -12,13 +29,11 @@ Changelog for 0.1.5_2-beta
     #269 [Tool] Implement ConfigManager
     #307 [Tentacle Installation] Add new key in description
     #309 [Web Interface] Exchange configuration
-    #311 [Web Interface] Symbols configuration
-    #312 [Web Interface] Services configuration
     #331 [Security] Encrypt user api key
 
 # New features :
     - Api key encryption
-    - Web Interface : Full configuration (except notifications)
+    - Web Interface : Exchange configuration
 
 Changelog for 0.1.5_1-beta
 ====================

evaluator/Strategies/strategies_evaluator.py

@@ -1,6 +1,7 @@
 from abc import *
 
-from config.cst import CONFIG_EVALUATOR_STRATEGIES, STRATEGIES_REQUIRED_TIME_FRAME, STRATEGIES_REQUIRED_EVALUATORS
+from config.cst import CONFIG_EVALUATOR_STRATEGIES, STRATEGIES_REQUIRED_TIME_FRAME, STRATEGIES_REQUIRED_EVALUATORS, \
+    CONFIG_FORCED_TIME_FRAME, CONFIG_FORCED_EVALUATOR
 from evaluator.abstract_evaluator import AbstractEvaluator
 from tools.evaluator_divergence_analyser import EvaluatorDivergenceAnalyser
 from tools.time_frame_manager import TimeFrameManager
@@ -35,19 +36,23 @@ def get_config_file_name(cls, config_evaluator_type=CONFIG_EVALUATOR_STRATEGIES)
         return super().get_config_file_name(config_evaluator_type)
 
     @classmethod
-    def get_required_time_frames(cls):
-        config = cls.get_evaluator_config()
-        if STRATEGIES_REQUIRED_TIME_FRAME in config:
-            return TimeFrameManager.parse_time_frames(config[STRATEGIES_REQUIRED_TIME_FRAME])
+    def get_required_time_frames(cls, config):
+        if CONFIG_FORCED_TIME_FRAME in config:
+            return TimeFrameManager.parse_time_frames(config[CONFIG_FORCED_TIME_FRAME])
+        strategy_config = cls.get_evaluator_config()
+        if STRATEGIES_REQUIRED_TIME_FRAME in strategy_config:
+            return TimeFrameManager.parse_time_frames(strategy_config[STRATEGIES_REQUIRED_TIME_FRAME])
         else:
             raise Exception("'{0}' is missing in {1}".format(STRATEGIES_REQUIRED_TIME_FRAME,
                                                              cls.get_config_file_name()))
 
     @classmethod
-    def get_required_evaluators(cls):
-        config = cls.get_evaluator_config()
-        if STRATEGIES_REQUIRED_EVALUATORS in config:
-            return config[STRATEGIES_REQUIRED_EVALUATORS]
+    def get_required_evaluators(cls, config):
+        if CONFIG_FORCED_EVALUATOR in config:
+            return config[CONFIG_FORCED_EVALUATOR]
+        strategy_config = cls.get_evaluator_config()
+        if STRATEGIES_REQUIRED_EVALUATORS in strategy_config:
+            return strategy_config[STRATEGIES_REQUIRED_EVALUATORS]
         else:
             raise Exception("'{0}' is missing in {1}".format(STRATEGIES_REQUIRED_EVALUATORS,
                                                              cls.get_config_file_name()))

evaluator/evaluator_creator.py

@@ -124,7 +124,7 @@ def init_time_frames_from_strategies(config):
         time_frame_list = set()
         for strategies_eval_class in AdvancedManager.create_advanced_evaluator_types_list(StrategiesEvaluator, config):
             if strategies_eval_class.is_enabled(config, False):
-                for time_frame in strategies_eval_class.get_required_time_frames():
+                for time_frame in strategies_eval_class.get_required_time_frames(config):
                     time_frame_list.add(time_frame)
         time_frame_list = TimeFrameManager.sort_time_frames(time_frame_list)
         config[CONFIG_TIME_FRAME] = time_frame_list
@@ -134,7 +134,7 @@ def get_relevant_evaluators_from_strategies(config):
         evaluator_list = set()
         for strategies_eval_class in AdvancedManager.create_advanced_evaluator_types_list(StrategiesEvaluator, config):
             if strategies_eval_class.is_enabled(config, False):
-                required_evaluators = strategies_eval_class.get_required_evaluators()
+                required_evaluators = strategies_eval_class.get_required_evaluators(config)
                 if required_evaluators == CONFIG_EVALUATORS_WILDCARD:
                     return CONFIG_EVALUATORS_WILDCARD
                 else:

evaluator/evaluator_threads_manager.py

@@ -1,9 +1,9 @@
 import logging
 
 from config.cst import CONFIG_EVALUATORS_WILDCARD, EvaluatorMatrixTypes, START_PENDING_EVAL_NOTE, \
-    CONFIG_SAVE_EVALUATION, EVALUATION_SAVING_FILE_ENDING, EVALUATION_SAVING_COLUMN_SEPARATOR, \
-    EVALUATION_SAVING_ROW_SEPARATOR
+    CONFIG_SAVE_EVALUATION
 from evaluator.evaluator import Evaluator
+from tools.exporter import MatrixExporter
 
 """
 This class represent the last level of evaluator management by :
@@ -31,13 +31,12 @@ def __init__(self, config,
         self.symbol_evaluator = symbol_evaluator
 
         self.should_save_evaluations = CONFIG_SAVE_EVALUATION in self.config and self.config[CONFIG_SAVE_EVALUATION]
-        self.evaluation_save_file = self._get_evaluation_save_file_name()
-        self.is_evaluation_save_file_initialised = False
 
         # notify symbol evaluator
         self.symbol_evaluator.add_evaluator_thread_manager(self.exchange, self.time_frame, self.trading_mode, self)
 
         self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
+        self.matrix_exporter = MatrixExporter(self.matrix, self.symbol)
 
         self.thread_name = f"TA THREAD MANAGER - {self.symbol} - {self.exchange.get_name()} - {self.time_frame}"
         self.logger = logging.getLogger(self.thread_name)
@@ -120,46 +119,7 @@ def refresh_matrix(self):
 
     def _save_evaluations_if_necessary(self):
         if self.should_save_evaluations and self.symbol_evaluator.are_all_timeframes_initialized(self.exchange):
-            try:
-                if not self.is_evaluation_save_file_initialised:
-                    with open(self.evaluation_save_file, 'w+') as eval_file:
-                        eval_file.write(self._get_init_evaluation_save_file())
-                        eval_file.write(self._get_formatted_matrix())
-                else:
-                    with open(self.evaluation_save_file, 'a') as eval_file:
-                        eval_file.write(self._get_formatted_matrix())
-            except PermissionError as e:
-                self.logger.error(f"Impossible to save evaluation on {self.evaluation_save_file}: {e}")
-
-
-    def _get_formatted_matrix(self):
-        formatted_matrix = ""
-        for eval_type, eval_dict in self.matrix.matrix.items():
-            for eval_name, eval_results in eval_dict.items():
-                if isinstance(eval_results, dict):
-                    for eval_result in eval_results.values():
-                        formatted_matrix = f"{formatted_matrix}{EVALUATION_SAVING_COLUMN_SEPARATOR}{eval_result}"
-                else:
-                    formatted_matrix = f"{formatted_matrix}{EVALUATION_SAVING_COLUMN_SEPARATOR}{eval_results}"
-        return formatted_matrix[len(EVALUATION_SAVING_COLUMN_SEPARATOR):] + EVALUATION_SAVING_ROW_SEPARATOR
-
-    def _get_init_evaluation_save_file(self):
-        # create file 1st line
-        line = ""
-        for eval_type, eval_dict in self.matrix.matrix.items():
-            for eval_name, eval_results in eval_dict.items():
-                if isinstance(eval_results, dict):
-                    for eval_key in eval_results.keys():
-                        line = f"{line}{EVALUATION_SAVING_COLUMN_SEPARATOR}" \
-                               f"{eval_type.value}/{eval_name}/{eval_key.value}"
-                else:
-                    line = f"{line}{EVALUATION_SAVING_COLUMN_SEPARATOR}{eval_type.value}/{eval_name}"
-        self.is_evaluation_save_file_initialised = True
-        # remove first separator
-        return line[len(EVALUATION_SAVING_COLUMN_SEPARATOR):] + EVALUATION_SAVING_ROW_SEPARATOR
-
-    def _get_evaluation_save_file_name(self):
-        return f"{self.symbol.replace('/','-')}_{EVALUATION_SAVING_FILE_ENDING}"
+            self.matrix_exporter.save()
 
     def start_threads(self):
         pass

evaluator/symbol_evaluator.py

@@ -65,7 +65,7 @@ def init_evaluator_instances_by_strategies(self):
     def add_evaluator_instance_to_strategy_instances_list(self, evaluator, exchange):
         exchange_name = exchange.get_exchange().get_name()
         for strategy in self.evaluator_instances_by_strategies[exchange_name].keys():
-            if EvaluatorCreator.is_relevant_evaluator(evaluator, strategy.get_required_evaluators()):
+            if EvaluatorCreator.is_relevant_evaluator(evaluator, strategy.get_required_evaluators(self.config)):
                 evaluator_parents = evaluator.get_parent_evaluator_classes()
                 for evaluator_type in self.evaluator_instances_by_strategies[exchange_name][strategy].keys():
                     if evaluator_type in evaluator_parents: