From aa0a680a00a30e29562be1fb0575f7a64765c5e3 Mon Sep 17 00:00:00 2001 From: Dingding Fan Date: Fri, 13 Dec 2019 14:46:24 +1300 Subject: [PATCH] indenting changes made after running flake8 --- gsimporter/_util.py | 8 +++-- gsimporter/api.py | 74 ++++++++++++++++++++++++-------------------- gsimporter/client.py | 45 ++++++++++++++------------- 3 files changed, 70 insertions(+), 57 deletions(-) diff --git a/gsimporter/_util.py b/gsimporter/_util.py index 942b732..d09e185 100644 --- a/gsimporter/_util.py +++ b/gsimporter/_util.py @@ -2,17 +2,19 @@ import tempfile from os import path -_shp_exts = ["dbf","prj","shx"] +_shp_exts = ["dbf", "prj", "shx"] _shp_exts = _shp_exts + [s.upper() for s in _shp_exts] + def shp_files(fpath): basename, ext = path.splitext(fpath) - paths = [ "%s.%s" % (basename,ext) for ext in _shp_exts ] + paths = ["%s.%s" % (basename, ext) for ext in _shp_exts] paths.append(fpath) return [f for f in paths if path.exists(f)] + def create_zip(fpaths): - _,payload = tempfile.mkstemp(suffix='.zip') + _, payload = tempfile.mkstemp(suffix='.zip') zipf = ZipFile(payload, "w") for fp in fpaths: basename = path.basename(fp) diff --git a/gsimporter/api.py b/gsimporter/api.py index 7316af7..6651b4e 100644 --- a/gsimporter/api.py +++ b/gsimporter/api.py @@ -17,22 +17,27 @@ _logger = logging.getLogger("gsuploader") + class BadRequest(Exception): '''Encapsulate a 400 or other 'invalid' request''' pass + class RequestFailed(Exception): '''Encapsulate a 500 or other 'failed' request''' pass + class NotFound(Exception): '''Encapsulate an HTTP 404 or other 'missing' request''' pass + class BindingFailed(Exception): '''Something in the API has changed''' pass + def parse_response(args, parent=None): '''Parse a top-level concept from the provided args. :param args: a tuple of headers, response from an httplib2 request @@ -43,23 +48,25 @@ def parse_response(args, parent=None): try: resp = json.loads(response) except ValueError as ex: - _logger.warn('invalid JSON response: %s',response) + _logger.warn('invalid JSON response: %s', response) raise ex if "import" in resp: return Session(json=resp['import'], parent=parent) elif "task" in resp: return Task(resp['task'], parent=parent) elif "imports" in resp: - return [ Session(json=j, parent=parent) for j in resp['imports'] ] + return [Session(json=j, parent=parent) for j in resp['imports']] elif "tasks" in resp: # non-recognized file tasks have null source.format - return [ Task(t, parent=parent) for t in resp['tasks'] ] + return [Task(t, parent=parent) for t in resp['tasks']] raise Exception("Unknown response %s" % resp) _Binding = collections.namedtuple('Binding', [ 'name', 'expected', 'ro', 'binding' ]) + + def _binding(name, expected=True, ro=True, binding=None): return _Binding(name, expected, ro, binding) @@ -145,7 +152,7 @@ def _to_json_object(self, deep=True, top_level=True): json[binding.name] = val self._to_json_object_custom(json) if top_level and self._object_name: - json = { self._object_name : json } + json = {self._object_name: json} return json def _to_json_object_custom(self, json): @@ -194,7 +201,7 @@ class Target(_UploadBase): def _bind_json(self, json): if 'href' in json: self.href = json.get('href') - store_type = [ k for k in Target._store_types if k in json] + store_type = [k for k in Target._store_types if k in json] if store_type: if len(store_type) != 1: self.binding_failed('invalid store entry: %s', list(json.keys())) @@ -215,12 +222,12 @@ def change_datastore(self, store_name=None, workspace=None): :param store_name: An optional existing datastore name :param workspace: An optional workspace to use for referencing the store ''' - dataStore = { 'enabled' : True } # workaround for importer bug + dataStore = {'enabled': True} # workaround for importer bug if store_name: dataStore['name'] = store_name if workspace: - dataStore['workspace'] = { 'name' : str(workspace) } - target_rep = { self.store_type : dataStore } + dataStore['workspace'] = {'name': str(workspace)} + target_rep = {self.store_type: dataStore} self._client().put_json(self._url(None), json.dumps(target_rep)) @@ -254,11 +261,11 @@ class Layer(_UploadBase): ) def set_target_layer_name(self, name): - data = { 'layer' : { 'name' : name }} + data = {'layer': {'name': name}} self._client().put_json(self._url(None), json.dumps(data)) def set_target_srs(self, srs): - data = { 'layer' : { 'srs': srs }} + data = {'layer': {'srs': srs}} self._client().put_json(self._url(None), json.dumps(data)) @@ -269,7 +276,7 @@ class Task(_UploadBase): _binding('href'), _binding('state'), _binding('progress'), - _binding('updateMode', expected=False), # workaround for older versions + _binding('updateMode', expected=False), # workaround for older versions _binding('data', binding=Data), # a missing target implies the source must be imported into db _binding('target', binding=Target, expected=False), @@ -295,35 +302,35 @@ def set_target(self, store_name=None, workspace=None, store_type=None): if workspace is None: raise Exception("workspace required if target is not set") if store_type not in Target._store_types: - raise Exception("store_type must be one of %s" % (Target._store_types,)) + raise Exception("store_type must be one of %s" % (Target._store_types)) self.target = Target(None, self) self.target.store_type = store_type self.target.href = self._url(None) + "/target" self.target.change_datastore(store_name, workspace) - def set_update_mode(self,update_mode): - data = { 'task' : { - 'updateMode' : update_mode + def set_update_mode(self, update_mode): + data = {'task': { + 'updateMode': update_mode }} self._client().put_json(self._url(None), json.dumps(data)) - def set_charset(self,charset): - data = { 'task' : { - 'source' : { - 'charset' : charset + def set_charset(self, charset): + data = {'task': { + 'source': { + 'charset': charset } }} self._client().put_json(self._url(None), json.dumps(data)) def set_srs(self, srs): - data = { 'layer' : { 'srs' : srs }} + data = {'layer': {'srs': srs}} self._client().put_json(self._url(None), json.dumps(data)) def delete(self): """Delete the task""" resp, content = self._client().delete(self._url(None)) if resp.status != 204: - raise Exception('expected 204 response code, got %s' % resp.status,content) + raise Exception('expected 204 response code, got %s' % resp.status, content) def set_transforms(self, transforms, save=True): """Set the transforms of this Item. transforms is a list of dicts""" @@ -335,7 +342,7 @@ def save_transforms(self): "type": self.transform_type, "transforms": self.transforms } - data = { 'task' : { 'transformChain' : chain}} + data = {'task': {'transformChain': chain}} self._client().put_json(self._url(None), json.dumps(data)) def add_transforms(self, transforms, save=True): @@ -345,9 +352,9 @@ def add_transforms(self, transforms, save=True): def remove_transforms(self, transforms, by_field=None, save=True): '''remove transforms by equality or list of field values''' if by_field: - self.transforms = [ t for t in self.transforms if t[by_field] not in transforms ] + self.transforms = [t for t in self.transforms if t[by_field] not in transforms] else: - self.transforms = [ t for t in self.transforms if t not in transforms ] + self.transforms = [t for t in self.transforms if t not in transforms] save and self.save_transforms() def get_progress(self): @@ -369,8 +376,8 @@ def get_progress(self): if unicode_error: progress['message'] += ' - it looks like an invalid character' return progress - except ValueError as ex: - _logger.warn('invalid JSON response: %s',response) + except ValueError: + _logger.warn('invalid JSON response: %s', response) raise RequestFailed('invalid JSON') else: raise Exception("Item does not have a progress endpoint") @@ -405,12 +412,13 @@ def upload_task(self, files, use_url=False, initial_opts=None): # @todo getting the task response updates the session tasks, but # neglects to retreive the overall session status field fname = os.path.basename(files[0]) - _,ext = os.path.splitext(fname) + _, ext = os.path.splitext(fname) + def addopts(base): if initial_opts: # pass options in as value:key parameters, this allows multiple # options per key - base = base + '&' + '&'.join(['option=%s:%s' % (v,k) for k,v in initial_opts.items()]) + base = base + '&' + '&'.join(['option=%s:%s' % (v, k) for k, v in initial_opts.items()]) return base if use_url: if ext == '.zip': @@ -426,7 +434,7 @@ def addopts(base): else: url = self._url("imports/%s/tasks?expand=3" % self.id) resp = self._client().post_multipart(addopts(url), files) - tasks = parse_response( resp ) + tasks = parse_response(resp) if not isinstance(tasks, list): tasks = [tasks] for t in tasks: @@ -436,16 +444,16 @@ def addopts(base): def commit(self, sync=False): """Run the session""" #@todo check task status if we don't have it already? - url = self._url("imports/%s",self.id) + url = self._url("imports/%s", self.id) if sync: url = url + "?sync=true&exec=true" resp, content = self._client().post(url) if resp.status != 204: - raise Exception("expected 204 response code, got %s" % resp.status,content) + raise Exception("expected 204 response code, got %s" % resp.status, content) def delete(self): """Delete this import session""" - url = self._url("imports/%s",self.id) + url = self._url("imports/%s", self.id) resp, content = self._client().delete(url) if resp.status != 204: - raise Exception('expected 204 response code, got %s' % resp.status,content) + raise Exception('expected 204 response code, got %s' % resp.status, content) diff --git a/gsimporter/client.py b/gsimporter/client.py index 0c75291..0f7a8eb 100644 --- a/gsimporter/client.py +++ b/gsimporter/client.py @@ -15,12 +15,13 @@ try: from urllib.parse import urlparse, urlencode -except ImportError: #python2 compatible +except ImportError: # python2 compatible from urlparse import urlparse from urllib import urlencode _logger = logging.getLogger(__name__) + class Client(object): def __init__(self, url, username=None, password=None): @@ -46,12 +47,12 @@ def get_sessions(self): ''' return self._call(self.client.get_imports) - def get_session(self,id): + def get_session(self, id): '''Get an existing 'expanded' session by id. :param id: the integer id of the session to get :return: an expanded Session object ''' - return self._call(self.client.get_import,id) + return self._call(self.client.get_import, id) def start_import(self, import_id=None, mosaic=False, name=None, target_store=None, charset_encoding="UTF-8"): """Create a new import session. @@ -79,7 +80,7 @@ def upload(self, fpath, use_url=False, import_id=None, mosaic=False, :param initial_opts: default None, dict of initial import options :returns: a gsimporter.api.Session object """ - files = [ fpath ] + files = [fpath] if fpath.lower().endswith(".shp"): files = _util.shp_files(fpath) @@ -115,16 +116,17 @@ def upload_files(self, files, use_url=False, import_id=None, mosaic=False, return session - # pickle protocol - client object cannot be serialized # this allows api objects to be seamlessly pickled and loaded without restarting # the connection more explicitly but this will have consequences if other state is stored # in the uploader or client objects + def __getstate__(self): cl = self.client - return {'url':cl.service_url,'username':cl.username,'password':cl.password} - def __setstate__(self,state): - self.client = _Client(state['url'],state['username'],state['password']) + return {'url': cl.service_url, 'username': cl.username, 'password': cl.password} + + def __setstate__(self, state): + self.client = _Client(state['url'], state['username'], state['password']) class _Client(object): @@ -150,7 +152,7 @@ def __init__(self, url, username, password): maxsize=4, block=True) - self.headers = urllib3.util.make_headers(basic_auth=':'.join([username,password])) + self.headers = urllib3.util.make_headers(basic_auth=':'.join([username, password])) def url(self, path=None): if path: @@ -165,7 +167,7 @@ def delete(self, url): def put_json(self, url, data): return self._request(url, "PUT", data, { - "Content-type" : "application/json" + "Content-type": "application/json" }) def _parse_errors(self, content): @@ -194,26 +196,26 @@ def _request(self, url, method="GET", data=None, headers={}): if resp.status < 200 or resp.status > 299: if resp.status == 400: raise BadRequest(*self._parse_errors(content)) - raise RequestFailed(resp.status,content) + raise RequestFailed(resp.status, content) return resp, content def post_upload_url(self, url, upload_url): data = urlencode({ - 'url' : upload_url + 'url': upload_url }) return self._request(url, "POST", data, { # importer very picky - 'Content-type' : "application/x-www-form-urlencoded" + 'Content-type': "application/x-www-form-urlencoded" }) - def put_zip(self,url,payload): + def put_zip(self, url, payload): message = open(payload) with message: - return self._request(url,"PUT",message,{ + return self._request(url, "PUT", message, { "Content-type": "application/zip", }) - def get_import(self,i): + def get_import(self, i): return parse_response(self._request(self.url("imports/%s?expand=3" % i))) def get_imports(self): @@ -275,7 +277,7 @@ def post_multipart(self, url, files, fields=[]): BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' CRLF = '\r\n' L = [] - _logger.info("post_multipart %s %s %s",url,files,fields) + _logger.info("post_multipart %s %s %s", url, files, fields) for (key, value) in fields: L.append('--' + BOUNDARY) L.append('Content-Disposition: form-data; name="%s"' % str(key)) @@ -305,7 +307,7 @@ def post_multipart(self, url, files, fields=[]): L.append('') return self._request( url, 'POST', CRLF.join(L), { - 'Content-Type' : 'multipart/form-data; boundary=%s' % BOUNDARY + 'Content-Type': 'multipart/form-data; boundary=%s' % BOUNDARY } ) @@ -313,14 +315,15 @@ def post_multipart(self, url, files, fields=[]): def _get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + def _debug(resp, content): if _logger.isEnabledFor(logging.DEBUG): - _logger.debug("response : %s",pprint.pformat(resp)) + _logger.debug("response: %s", pprint.pformat(resp)) if "content-type" in resp and resp['content-type'] == 'application/json': try: content = json.loads(content) - content = json.dumps(content,indent=2) + content = json.dumps(content, indent=2) except ValueError: pass - _logger.debug("content : %s",content) + _logger.debug("content : %s", content)