Modernize Python 2 code to prepair for Python 3

.travis.yml

@@ -7,8 +7,18 @@ addons:
 
 python:
   - "2.7"
+  - "3.6"
+
+matrix:
+  allow_failures:
+    - python: "3.6"
 
 install:
   - pip install .
 
+before_script:
+  - pip install flake8
+  # TODO: Fix the flake8 tests and the remove the --exit-zero
+  - flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
+
 script: gryphon-runtests

gryphon/dashboards/handlers/block_times.py

@@ -1,7 +1,8 @@
-from admin_base import AdminBaseHandler
+from __future__ import absolute_import
+from .admin_base import AdminBaseHandler
 import tornado.web
 
-from mixins.start_and_end_time import StartAndEndTimeMixin
+from .mixins.start_and_end_time import StartAndEndTimeMixin
 import util.tick_times as tick_times
 
 

gryphon/dashboards/handlers/home.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 import tornado.web
 
-from admin_base import AdminBaseHandler
+from .admin_base import AdminBaseHandler
 
 
 class HomeHandler(AdminBaseHandler):

gryphon/dashboards/handlers/ledger.py

@@ -142,7 +142,7 @@ def get_exchange_for_address(self, address):
         """
 
         if address and address in self.address_map:
-            return address_map[address]
+            return self.address_map[address]
         else:
             return 'External transfer'
 
@@ -261,7 +261,7 @@ def table_entries_from_transaction(self, transaction):
             entry['date'] = date
             entry['details'] = ''.join([
                 '%s:%s ' % (k, v)
-                for k, v in transaction.transaction_details.iteritems()
+                for k, v in transaction.transaction_details.items()
                 if k in ['external_transaction_id', 'notes'] and v not in ['xxx']
             ])
 

gryphon/dashboards/handlers/status.py

@@ -140,7 +140,7 @@ def get_system_balances(self, exchanges):
             system_balance += e.exchange_account_db_object(self.trading_db).balance
 
         total_fiat = sum([
-            balance.to("USD") for currency, balance in system_balance.iteritems()
+            balance.to("USD") for currency, balance in system_balance.items()
             if currency not in Money.CRYPTO_CURRENCIES
         ])
 

gryphon/dashboards/handlers/tick_times.py

@@ -1,7 +1,8 @@
-from admin_base import AdminBaseHandler
+from __future__ import absolute_import
+from .admin_base import AdminBaseHandler
 import tornado.web
 
-from mixins.start_and_end_time import StartAndEndTimeMixin
+from .mixins.start_and_end_time import StartAndEndTimeMixin
 from tinkerpy.exchange.exchange_factory import all_exchanges
 import util.tick_times as tick_times
 

gryphon/dashboards/models/base.py

@@ -1,13 +1,14 @@
 # -*- coding: utf-8 -*-
+from six import text_type
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
 metadata = Base.metadata
 
 def unicode_string(self):
-    return unicode(self).encode('utf-8')
+    return text_type(self).encode('utf-8')
 
-Base.__str__ == unicode_string   
+Base.__str__ == unicode_string
 
 
 # How to migrate a database

gryphon/dashboards/settings.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 import logging
 import tornado
 import tornado.template
 import os
 from os.path import dirname, abspath
 from tornado.options import define, options
 from gryphon.lib.logperf import log_request_perf
-import uimodules
+from . import uimodules
 
 # Make filepaths relative to settings.
 path = lambda root,*a: os.path.join(root, *a)

gryphon/dashboards/util/balances.py

@@ -12,7 +12,7 @@
 def get_balance_time_series_from_audits(audits):
     fiat_balances = []
     btc_balances = []
-    
+
     for audit in audits:
         timestamp = int(Delorean(audit.time_created, "UTC").epoch) * 1000
 
@@ -34,7 +34,7 @@ def get_balance_time_series_from_audits(audits):
                 continue
 
             # convert to Money objects
-            for currency, balance_str in balance_data.iteritems():
+            for currency, balance_str in balance_data.items():
                 balance_data[currency] = Money.loads(balance_str)
 
             balance = Balance(balance_data)
@@ -103,7 +103,7 @@ def get_drift_from_audits(audits):
         if 'drift' in audit.data:
             data = json.loads(audit.data)
 
-            for currency, str_amount in data['drift'].iteritems():
+            for currency, str_amount in data['drift'].items():
                 drift_by_currency += Money.loads(str_amount)
 
     return drift_by_currency

gryphon/data_service/auditors/orderbook_auditor.py

@@ -242,15 +242,15 @@ def audit_orderbook(self, orderbook, orderbook_timestamp):
             if not our_orderbooks:
                 log.msg('No orderbooks to audit against')
 
-            for key, value in fundamental_values.iteritems():
+            for key, value in fundamental_values.items():
                 log.msg(
                     '------ Fundamental Value Closeness:%.6f, DBfv:%s, HTTPfv:%s' % (
                     key,
                     value['db_fundamental_value'],
                     value['http_fundamental_value']
                 ))
 
-            for key, value in change_dict.iteritems():
+            for key, value in change_dict.items():
                 log.msg('------ Change Count: %s' % key)
 
         log.msg(

gryphon/data_service/pollers/orderbook/kraken_cad_orderbook_poller.py

@@ -1,4 +1,5 @@
-from kraken_orderbook_poller import KrakenOrderbook
+from __future__ import absolute_import
+from .kraken_orderbook_poller import KrakenOrderbook
 
 
 class KrakenCADOrderbook(KrakenOrderbook):

gryphon/data_service/pollers/orderbook/kraken_usd_orderbook_poller.py

@@ -1,4 +1,5 @@
-from kraken_orderbook_poller import KrakenOrderbook
+from __future__ import absolute_import
+from .kraken_orderbook_poller import KrakenOrderbook
 
 
 class KrakenUSDOrderbook(KrakenOrderbook):

gryphon/data_service/pollers/orderbook/websocket/bitfinex_orderbook_websocket.py

@@ -130,11 +130,11 @@ def get_orderbook_to_publish(self):
         bids = []
         asks = []
 
-        for price, volume in sorted(self.bids_dict.iteritems(), reverse=True):
+        for price, volume in sorted(self.bids_dict.items(), reverse=True):
             if volume > 0:
                 bids.append([str(price), str(volume), ''])
 
-        for price, volume in sorted(self.asks_dict.iteritems()):
+        for price, volume in sorted(self.asks_dict.items()):
             if volume > 0:
                 asks.append([str(price), str(volume), ''])
 

gryphon/data_service/pollers/orderbook/websocket/bitstamp_orderbook_websocket.py

@@ -211,26 +211,26 @@ def apply_change_to_orderbook(self, change):
         # Remove the 0 volumes from the orderbook.
         self.orderbook['bids'].update(bids_changes)
 
-        for k, v in self.orderbook['bids'].iteritems():
+        for k, v in self.orderbook['bids'].items():
             if v == "0":
                 self.orderbook['bids'].pop(k)
 
         # Re-sort the bids.
         self.orderbook['bids'] = OrderedDict(sorted(
-            self.orderbook['bids'].iteritems(),
-            key=lambda (k, v): float(k),
+            self.orderbook['bids'].items(),
+            key=lambda k_v1: float(k_v1[0]),
             reverse=True,
         ))
 
         self.orderbook['asks'].update(asks_changes)
 
-        for k, v in self.orderbook['asks'].iteritems():
+        for k, v in self.orderbook['asks'].items():
             if v == "0":
                 self.orderbook['asks'].pop(k)
 
         # Re-sort the asks.
         self.orderbook['asks'] = OrderedDict(
-            sorted(self.orderbook['asks'].iteritems(), key=lambda (k, v): float(k)),
+            sorted(self.orderbook['asks'].items(), key=lambda k_v: float(k_v[0])),
         )
 
     def parse_orders(self, orders):
@@ -247,7 +247,7 @@ def get_orderbook_to_publish(self):
         price_key_orderbook = self.orderbook
 
         return {
-            'bids': [[k, v, ''] for k, v in price_key_orderbook['bids'].iteritems()],
-            'asks': [[k, v, ''] for k, v in price_key_orderbook['asks'].iteritems()],
+            'bids': [[k, v, ''] for k, v in price_key_orderbook['bids'].items()],
+            'asks': [[k, v, ''] for k, v in price_key_orderbook['asks'].items()],
         }
 

gryphon/data_service/pollers/orderbook/websocket/coinbase_orderbook_websocket.py

@@ -309,13 +309,13 @@ def get_orderbook_to_publish(self):
 
         sorted_bid_keys = sorted(
             fancy_orderbook['bids'].keys(),
-            key=lambda (k): float(k),
+            key=lambda k: float(k),
             reverse=True,
         )
 
         sorted_ask_keys = sorted(
             fancy_orderbook['asks'].keys(),
-            key=lambda (k): float(k),
+            key=lambda k: float(k),
         )
 
         bids = [[k, str(fancy_orderbook['bids'][k]), ''] for k in sorted_bid_keys]

gryphon/data_service/pollers/trades/trades_poller.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import json
 
+from six import text_type
 import termcolor as tc
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
@@ -50,9 +51,9 @@ def parse_response(self, resp_obj):
         for trade in trades:
             if trade['trade_id'] > self.most_recent_trade_id:
                 trade['price_currency'] = trade['price'].currency
-                trade['price'] = unicode(trade['price'].amount)
+                trade['price'] = text_type(trade['price'].amount)
                 trade['volume_currency'] = trade['volume'].currency
-                trade['volume'] = unicode(trade['volume'].amount)
+                trade['volume'] = text_type(trade['volume'].amount)
                 trade['timestamp'] = int(trade['timestamp'])
                 trade_string = json.dumps(trade, ensure_ascii=False)
                 self.producer.publish_message(trade_string)

gryphon/data_service/scripts/autobahn-tester.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 ###############################################################################
 ##
 ##  Copyright (C) 2011-2013 Tavendo GmbH

gryphon/data_service/scripts/benchmark.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 # Simple test script for benchmarking regular python logging vs twisted's logging.
 
 import logging
@@ -25,7 +26,7 @@ def tx_log():
 
 
 def stop():
-    print "Log Counter: %s" % log_counter
+    print("Log Counter: %s" % log_counter)
     reactor.stop()
 
 

gryphon/data_service/scripts/historical_trade_collector.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import csv
 from datetime import timedelta
 import gzip
@@ -161,7 +162,7 @@ def get_our_recorded_ticker_volume_for_period(exchange, start_date, end_date):
 def audit_ticker_volume_individual_days(exchange_list, start_date=test_start_date, end_date=test_end_date):
     d = start_date
     while d < test_end_date:
-        print '\n\nAuditing: %s' % d
+        print('\n\nAuditing: %s' % d)
         day_after = d + timedelta(days=1)
         audit_all_ticker_volume(exchange_list, d, day_after)
         d = day_after
@@ -191,20 +192,20 @@ def audit_ticker_volume(exchange, start_date, end_date):
         end_date,
     )
 
-    print '%s  Our Volume:%s  Ticker Volume:%s, Accuracy: %s' % (
+    print('%s  Our Volume:%s  Ticker Volume:%s, Accuracy: %s' % (
         exchange,
         our_volume,
         ticker_volume,
         our_volume / ticker_volume,
-    )
+    ))
 
 
 def audit_bw_volume(exchange, start_date, end_date):
     bw_exchange = BitcoinWisdom(exchange=exchange)
     bw_volume = bw_exchange.volume_in_period(start_date, end_date)
 
     our_volume = get_our_recorded_exchange_trade_volume_for_period(exchange, start_date, end_date)
-    print '%s  Our Volume:%s  BW Volume:%s, Accuracy: %s' % (exchange, our_volume, bw_volume, our_volume / bw_volume)
+    print('%s  Our Volume:%s  BW Volume:%s, Accuracy: %s' % (exchange, our_volume, bw_volume, our_volume / bw_volume))
 
 
 def compare_all_exchanges():
@@ -222,15 +223,15 @@ def compare_ours_to_history(our_exchange_id, exchange, price_currency, volume_cu
     start = parse('2015-11-27 0:0:0').datetime.replace(tzinfo=None)
     end = parse('2015-11-27 11:59:59').datetime.replace(tzinfo=None)
 
-    print our_exchange_id.upper()
+    print(our_exchange_id.upper())
     hist_in_range = [t for t in hist_trades if t[0] >= start and t[0] <= end]
     ours_in_range = [t for t in our_trades if t[0] >= start and t[0] <= end]
 
     for t in hist_in_range:
         if t not in ours_in_range:
-            print 'Hist Trade not in ours: %s' % t
+            print('Hist Trade not in ours: %s' % t)
 
     for t in ours_in_range:
         if t not in hist_in_range:
-            print'Our trade not in history: %s' % t
-    print'\n\n\n\n\n'
+            print('Our trade not in history: %s' % t)
+    print('\n\n\n\n\n')

gryphon/execution/bots/overwatch.py

@@ -220,7 +220,7 @@ def check_spreads_are_normal(db):
 
     sanity = True
 
-    for exchange_name, fv in native_fvs.iteritems():
+    for exchange_name, fv in native_fvs.items():
         if abs(fv - current_core_fv) > INTER_EXCHANGE_SPREAD_THRESHOLD:
             sanity = False
             break

gryphon/execution/bots/shoebox.py

@@ -301,7 +301,7 @@ def manual_btc_withdrawals(db):
 
     logger.info('Running manual BTC withdrawals')
 
-    for name, target in MANUAL_BTC_EXCHANGES.iteritems():
+    for name, target in MANUAL_BTC_EXCHANGES.items():
         exchange_db = exchange_factory.make_exchange_data_from_key(name, db)
         if exchange_db.balance['BTC'] > target:
             withdrawal_amount = exchange_db.balance['BTC'] - target

gryphon/execution/controllers/balance.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from gryphon.execution.lib.exchange_color import exchange_color
 from gryphon.lib.exchange.exchange_factory import *
 from gryphon.lib.logger import get_logger