fixed etnetera antidos wrapper for other urls; added http_debug optio…

…n; minor fixes

proxytools/cli.py

@@ -218,13 +218,19 @@ def superproxy(config, listen, pool_size, stop_timeout, dozer):
     pool_size = pool_size or conf.pop('pool_size', 500)
     stop_timeout = stop_timeout or conf.pop('stop_timeout', 5)
     dozer = conf.pop('dozer', False)
-    iface, port = listen.split(':')
+    http_debug = conf.pop('http_debug', False)
 
     app = WSGISuperProxy(proxylist, **conf)
+
     if dozer:
         from dozer import Dozer
         app = Dozer(app)
 
+    if http_debug:
+        import http.client
+        http.client.HTTPConnection.debuglevel = 10 if http_debug is True else http_debug
+
+    iface, port = listen.split(':')
     server = WSGIServer((iface, int(port)), app, spawn=Pool(pool_size))
     server.stop_timeout = stop_timeout
 

proxytools/extensions/etnetera_antidos.py

@@ -3,17 +3,18 @@
 # https://www.etnetera.cz/en/what-we-do/ewa-cdn
 # Etnetera Web Accelerator AntiDOS
 EWA_ANTIDOS_REGEXP = re.compile('document.cookie="EWAAD=([\d\w]+);')
-EWA_URLS = [
+EWA_URLS = (
     'https://www.mzv.cz/',
-]
+)
 
 ewaad_cache = {}
 
 
 def proxy_request_wrapper(request):
-    def wrapper(meth, url, data, headers, **kwargs):
+    def wrapper(meth, url, headers=None, **kwargs):
         if not url.startswith(EWA_URLS):
-            return request(meth, url, data, headers, **kwargs)
+            return request(meth, url, headers=headers, **kwargs)
+        headers = headers or {}
         if 'Cookie' in headers:
             # It will be in conflict with cookies parameter,
             # will not raise exception but request would be malformed
@@ -23,15 +24,15 @@ def wrapper(meth, url, data, headers, **kwargs):
         if ewaad:
             # logger.debug('EWAAD from cache %s %s', proxy, ewaad)
             kwargs['cookies'] = {'EWAAD': ewaad}
-        resp = request(meth, url, data, headers, **kwargs)
+        resp = request(meth, url, headers=headers, **kwargs)
         match = EWA_ANTIDOS_REGEXP.search(resp.text)
         if not match:
             return resp
         ewaad = match.groups()[0]
         # logger.debug('EWAAD resolved %s %s', proxy, ewaad)
         ewaad_cache[proxy] = ewaad
         kwargs['cookies'] = {'EWAAD': ewaad}
-        resp = request(meth, url, data, headers, **kwargs)
+        resp = request(meth, url, headers=headers, **kwargs)
         match = EWA_ANTIDOS_REGEXP.search(resp.text)
         if match:
             raise RuntimeError('EWAAD not resolved')

proxytools/proxylist.py

@@ -73,6 +73,8 @@ def __init__(self, fetcher=None, checker=None, min_size=50, max_fail=3, max_simu
             fetcher.proxy = self.proxy
             if self.checker:
                 fetcher.checker = self.checker
+            elif fetcher.checker:
+                fetcher.checker.proxy = self.proxy
             fetcher.blacklist = CompositeContains(self.active_proxies,
                                                   self.blacklist_proxies)
         self.fetcher = fetcher

proxytools/requests.py

@@ -423,7 +423,7 @@ def __init__(self, superproxy_url, proxy_persist=False, adapter={}, **kwargs):
         kwargs['proxies'] = {'http': superproxy_url, 'https': superproxy_url}
         super().__init__(**kwargs)
 
-    def request(self, method, url, headers={}, **kwargs):
+    def request(self, method, url, headers=None, **kwargs):
         # TODO: for now redirects are done without proxy,
         # because it uses self.send method directly, and we
         # can't easily pass proxy_kwargs there
@@ -436,6 +436,8 @@ def request(self, method, url, headers={}, **kwargs):
 
         proxy_kwargs = {k: kwargs.pop(k) for k in tuple(kwargs)
                         if k.startswith('proxy_')}
+
+        headers = headers or {}
         for key, value in proxy_kwargs.items():
             if key in ['proxy_timeout', 'proxy_allow_no_proxy']:
                 key = key[6:]

tests/conftest.py

@@ -1,4 +1,2 @@
-from proxytools.utils import gevent_monkey_patch
-
-
-gevent_monkey_patch()
+from gevent import monkey
+monkey.patch_all()

tests/test_requests.py

@@ -17,7 +17,7 @@ def test_proxylist_session():
     def worker(x):
         started = time.time()
         print('Fetch start', x)
-        resp = session.get('https://httpbin.org/get')
+        resp = session.get('http://httpbin.org/get')
         assert 'origin' in resp.json(), resp.json()
         print('Fetch succeed', x, time.time() - started, resp._proxy.addr, resp._proxy.speed)
         # except Exception as exc: