diff --git a/bin/query.py b/bin/query.py index 43a0a60..b7c0718 100755 --- a/bin/query.py +++ b/bin/query.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 """ Interact with an Oracle or postgres database from the shell. @@ -60,11 +60,11 @@ def query_to_cur(dbh, qry, args): """ return a cursor to a query """ if args.debug: - print >> sys.stderr, datetime.datetime.strftime(datetime.datetime.now(), "%D %H:%m:%S"), qry + print(datetime.datetime.strftime(datetime.datetime.now(), "%D %H:%m:%S"), qry, file=sys.stderr) t0 = time.time() cur = dbh.cursor() cur.execute(qry) - print "query took", time.time() - t0, "seconds" + print("query took", time.time() - t0, "seconds") return cur def stringify(datum, floatfmt="%8.2f"): @@ -72,7 +72,7 @@ def stringify(datum, floatfmt="%8.2f"): if isinstance(datum, float): return floatfmt % datum - return "{0}".format(datum) + return f"{datum}" def printPrettyFromCursor(cur, args): """ print data returned from a query in nice aligned columns""" @@ -93,7 +93,7 @@ def printPrettyFromCursor(cur, args): fmat = ' '.join(['{:<%d}' % width for width in widths]) for row in rows: row = [stringify(r) for r in row] - print fmat.format(*row) + print(fmat.format(*row)) def printCSVFromCursor(cur, args): """ output the query results as a CSV""" @@ -114,7 +114,7 @@ def do1Query(dbh, qry, args): """ doc """ cur = query_to_cur(dbh, qry, args) if args.log: - open(args.log, "a").write("%s: %s\n" % (time.time(), qry)) + open(args.log, "a").write(f"{time.time()}: {qry}\n") if args.format == "pretty": printPrettyFromCursor(cur, args) elif args.format == "csv": @@ -123,9 +123,8 @@ def do1Query(dbh, qry, args): sys.exit(1) def query(args): - """ doc """ - ## FM import cx_Oracle - dbh = despydb.DesDbi(os.path.join(os.getenv("HOME"), ".desservices.ini"), args.section) + """ Send the query to the database and render the results as requested """ + dbh = despydb.DesDbi(args.service, args.section) if args.query not in "-+": do1Query(dbh, args.query, args) elif args.query == "-": @@ -143,8 +142,10 @@ def query(args): dbh.close() def main(): - """Create command line arguments""" + """ Parse command line arguments and pass them to the query engine + """ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--service', default=os.path.join(os.getenv("HOME"), ".desservices.ini")) parser.add_argument('--section', '-s', default='db-desoper', help='section in the .desservices file w/ DB connection information') parser.add_argument('--debug', '-d', help='print debug info', default=False, action='store_true') diff --git a/python/despydb/desdbi.py b/python/despydb/desdbi.py index 86b9543..1f66992 100755 --- a/python/despydb/desdbi.py +++ b/python/despydb/desdbi.py @@ -26,7 +26,7 @@ import copy import time import socket -from collections import OrderedDict +import collections from despyserviceaccess import serviceaccess # importing of DB specific modules done down inside code @@ -34,7 +34,7 @@ import despydb.errors as errors import despydb.desdbi_defs as defs -class DesDbi(object): +class DesDbi: """ Provide a dialect-neutral interface to a DES database. During Instantiation of this class, service access parameters are found and @@ -160,22 +160,22 @@ def connect(self): done = True except Exception as e: lasterr = str(e).strip() - print lasterr + print(lasterr) timestamp = time.strftime("%x %X", time.localtime()) - print "%s: Could not connect to database, try %i/%i" %(timestamp, trycnt, MAXTRIES) + print(f"{timestamp}: Could not connect to database, try {trycnt}/{MAXTRIES}") if trycnt < MAXTRIES: - print "\tRetrying...\n" + print("\tRetrying...\n") time.sleep(TRY_DELAY) else: - print " Error, could not connect to the database after %i retries: %s" %(MAXTRIES, lasterr) + print(f" Error, could not connect to the database after {MAXTRIES} retries: {lasterr}") if not done: - print "Exechost:", socket.gethostname() - print "Connection information:", str(self) - print "" - raise Exception("Aborting attempt to connect to database. Last error message: %s" % lasterr) - elif trycnt > 1: # only print success message if we've printed failure message - print "Successfully connected to database after retrying." + print("Exechost:", socket.gethostname()) + print("Connection information:", str(self)) + print("") + raise Exception(f"Aborting attempt to connect to database. Last error message: {lasterr}") + if trycnt > 1: # only print success message if we've printed failure message + print("Successfully connected to database after retrying.") def reconnect(self): """ Reconnect to the database (but ony if the connection is no longer live). @@ -183,7 +183,7 @@ def reconnect(self): if not self.ping(): self.connect() else: - print 'Connection still good, not reconnecting' + print('Connection still good, not reconnecting') def autocommit(self, state=None): """ Return and optionally set autocommit mode. @@ -241,12 +241,12 @@ def exec_sql_expression(self, expression): tuple A tuple containing a single result for each column. """ - if hasattr(expression, '__iter__'): + if isinstance(expression, (list, set, dict)): s = ','.join(expression) else: s = expression - stmt = self.get_expr_exec_format() % s + stmt = self.get_expr_exec_format().format(s) cursor = self.cursor() cursor.execute(stmt) res = cursor.fetchone() @@ -268,10 +268,10 @@ def get_expr_exec_format(self): Examples: expression: con.get_expr_exec_format() - oracle result: SELECT %s FROM DUAL - postgres result: SELECT %s + oracle result: SELECT {} FROM DUAL + postgres result: SELECT {} - expression: con.get_expr_exec_format() % 'func1(), func2()' + expression: con.get_expr_exec_format().format('func1(), func2()') oracle result: SELECT func1(), func2() FROM DUAL postgres result: SELECT func1(), func2() """ @@ -293,7 +293,7 @@ def get_column_metadata(self, table_name): dict """ cursor = self.cursor() - sqlstr = 'SELECT * FROM %s WHERE 0=1' % table_name + sqlstr = f'SELECT * FROM {table_name} WHERE 0=1' if self.type == 'oracle': cursor.parse(sqlstr) #elif self.type == 'postgres': @@ -376,7 +376,7 @@ def get_named_bind_string(self, name): Examples: expression: get_named_bind_string('abc') oracle result: :abc - postgres result: %(abc)s + postgres result: {abc} """ return self.con.get_named_bind_string(name) @@ -398,7 +398,7 @@ def get_positional_bind_string(self, pos=1): Examples: expression: get_positional_bind_string() oracle result: :1 - postgres result: %s + postgres result: {} """ return self.con.get_positional_bind_string(pos) @@ -441,12 +441,12 @@ def get_regex_clause(self, target, pattern, case_sensitive=True): expression: get_regex_clause(get_positional_bind_string(), "prefix.*") oracle result: REGEXP_LIKE(:1, 'prefix.*') - postgres result:(%s ~ 'prefix.*') + postgres result:({} ~ 'prefix.*') """ d = {'target' : target, 'pattern': "'" + pattern + "'"} - return self.get_regex_format(case_sensitive) % d + return self.get_regex_format(case_sensitive).format(**d) def get_regex_format(self, case_sensitive=True): """ Return a format string for constructing a regular expression clause. @@ -476,19 +476,19 @@ def get_regex_format(self, case_sensitive=True): Examples: expression: get_regex_format() - oracle result: REGEXP_LIKE(%(target)s, %(pattern)s) - postgres result: %(target)s ~ %(pattern)s + oracle result: REGEXP_LIKE({target}, {pattern}) + postgres result: {target} ~ {pattern} - expression: get_regex_format() % {"target": "col1", - "pattern": "'pre.*suf'"} + expression: get_regex_format().format(**{"target": "col1", + "pattern": "'pre.*suf'"}) oracle result: REGEXP_LIKE(col1, 'pre.*suf', 'c') postgres result:(col1 ~ 'pre.*suf') - expression: get_regex_format() % { + expression: get_regex_format().format(**{ "target": get_positional_bind_string(), - "pattern": get_positional_bind_string()} + "pattern": get_positional_bind_string()}) oracle result: REGEXP_LIKE(:1, :1, 'c') - postgres result:(%s ~ %s) + postgres result:({} ~ {}) """ return self.con.get_regex_format(case_sensitive) @@ -558,7 +558,6 @@ def insert_many(self, table, columns, rows): rows : array like A sequence of rows to insert. """ - if not rows: return if hasattr(rows[0], 'keys'): @@ -569,7 +568,7 @@ def insert_many(self, table, columns, rows): colStr = ','.join(columns) - stmt = 'INSERT INTO %s(%s) VALUES(%s)' %(table, colStr, vals) + stmt = f'INSERT INTO {table}({colStr}) VALUES({vals})' curs = self.cursor() try: @@ -607,7 +606,7 @@ def insert_many_indiv(self, table, columns, rows): colStr = ','.join(columns) - stmt = 'INSERT INTO %s(%s) VALUES(%s)' %(table, colStr, vals) + stmt = f'INSERT INTO {table}({colStr}) VALUES({vals})' curs = self.cursor() curs.prepare(stmt) @@ -615,10 +614,10 @@ def insert_many_indiv(self, table, columns, rows): try: curs.execute(None, row) except Exception as err: - print "\n\nError: ", err - print "sql>", stmt - print "params:", row - print "\n\n" + print("\n\nError: ", err) + print("sql>", stmt) + print("params:", row) + print("\n\n") raise curs.close() @@ -688,7 +687,7 @@ def query_simple(self, from_, cols='*', where=None, orderby=None, raise TypeError('A table name or other from expression is ' 'required.') - if hasattr(cols, '__iter__') and cols: + if isinstance(cols, (list, set, dict)) and cols: colstr = ','.join(cols) elif cols: colstr = cols @@ -696,21 +695,21 @@ def query_simple(self, from_, cols='*', where=None, orderby=None, raise TypeError('A non-empty sequence of column names or ' 'expressions or a string of such is required.') - if hasattr(where, '__iter__') and where: + if isinstance(where, (list, set, dict)) and where: where_str = ' WHERE ' + ' AND '.join(where) elif where: where_str = ' WHERE ' + where else: where_str = '' - if hasattr(orderby, '__iter__') and orderby: + if isinstance(orderby, (list, set, dict)) and orderby: ord_str = ' ORDER BY ' + ','.join(orderby) elif orderby: ord_str = ' ORDER BY ' + orderby else: ord_str = '' - stmt = "SELECT %s FROM %s%s%s" %(colstr, from_, where_str, ord_str) + stmt = f"SELECT {colstr} FROM {from_}{where_str}{ord_str}" curs = self.cursor() try: @@ -771,7 +770,7 @@ def sequence_drop(self, seq_name): def __str__(self): copydict = copy.deepcopy(self.configdict) del copydict['passwd'] - return '%s' %(copydict) + return f'{copydict}' def table_drop(self, table): """ Drop table; do not generate error if it doesn't exist. @@ -825,7 +824,7 @@ def quote(self, value): str The updated string """ - return "'%s'" % str(value).replace("'", "''") + return "'" + str(value).replace("'", "''") + "'" def get_current_timestamp_str(self): """ Return a string for current timestamp @@ -843,7 +842,7 @@ def query_results_dict(self, sql, tkey): curs.execute(sql) desc = [d[0].lower() for d in curs.description] - result = OrderedDict() + result = collections.OrderedDict() for line in curs: d = dict(zip(desc, line)) result[d[tkey.lower()].lower()] = d @@ -866,7 +865,7 @@ def basic_insert_row(self, table, row): """ ctstr = self.get_current_timestamp_str() - cols = row.keys() + cols = list(row.keys()) namedbind = [] params = {} for col in cols: @@ -876,7 +875,7 @@ def basic_insert_row(self, table, row): namedbind.append(self.get_named_bind_string(col)) params[col] = row[col] - sql = "insert into %s(%s) values(%s)" %(table, ','.join(cols), ','.join(namedbind)) + sql = f"insert into {table}({','.join(cols)}) values({','.join(namedbind)})" curs = self.cursor() @@ -884,10 +883,10 @@ def basic_insert_row(self, table, row): curs.execute(sql, params) except: (_type, value, _) = sys.exc_info() - print "******************************" - print "Error:", _type, value - print "sql> %s\n" %(sql) - print "params> %s\n" %(params) + print("******************************") + print("Error:", _type, value) + print(f"sql> {sql}\n") + print(f"params> {params}\n") raise def basic_update_row(self, table, updatevals, wherevals): @@ -911,43 +910,43 @@ def basic_update_row(self, table, updatevals, wherevals): whclause = [] for c, v in wherevals.items(): if v == ctstr: - whclause.append("%s=%s" %(c, v)) + whclause.append(f"{c}={v}") elif v is None: - whclause.append("%s is NULL" %(c)) + whclause.append(f"{c} is NULL") else: - whclause.append("%s=%s" %(c, self.get_named_bind_string('w_'+c))) - params['w_'+c] = v + whclause.append(f"{c}={self.get_named_bind_string('w_' + c)}") + params['w_' + c] = v upclause = [] for c, v in updatevals.items(): if v == ctstr: - upclause.append("%s=%s" %(c, v)) + upclause.append(f"{c}={v}") else: if isinstance(v, str) and 'TO_DATE' in v.upper(): - upclause.append('%s=%s' % (c, v)) + upclause.append(f'{c}={v}') else: - upclause.append("%s=%s" % (c, self.get_named_bind_string('u_'+c))) - params['u_'+c] = v + upclause.append(f"{c}={self.get_named_bind_string('u_' + c)}") + params['u_' + c] = v - sql = "update %s set %s where %s" %(table, ','.join(upclause), ' and '.join(whclause)) + sql = f"update {table} set {','.join(upclause)} where {' and '.join(whclause)}" curs = self.cursor() try: curs.execute(sql, params) except: (_type, value, _) = sys.exc_info() - print "******************************" - print "Error:", _type, value - print "sql> %s\n" %(sql) - print "params> %s\n" % params + print("******************************") + print("Error:", _type, value) + print(f"sql> {sql}\n") + print(f"params> {params}\n") raise if curs.rowcount == 0: - print "******************************" - print "sql> %s\n" % sql - print "params> %s\n" % params - raise Exception("Error: 0 rows updated in table %s" % table) + print("******************************") + print(f"sql> {sql}\n") + print(f"params> {params}\n") + raise Exception(f"Error: 0 rows updated in table {table}") curs.close() diff --git a/python/despydb/errors.py b/python/despydb/errors.py index ea2e58a..8a14ff0 100644 --- a/python/despydb/errors.py +++ b/python/despydb/errors.py @@ -56,7 +56,7 @@ class UnknownDBTypeError(NotImplementedError): def __init__(self, db_type, msg=None): self.db_type = db_type if not msg: - msg = 'database type: "%s"' % self.db_type + msg = f'database type: "{self.db_type}"' NotImplementedError.__init__(self, msg) @@ -76,6 +76,6 @@ class UnknownCaseSensitiveError(NotImplementedError): def __init__(self, value, msg=None): self.value = value if not msg: - msg = 'Unknown case sensitivity value: "%s"' % value + msg = f'Unknown case sensitivity value: "{value}"' NotImplementedError.__init__(self, msg) diff --git a/python/despydb/oracon.py b/python/despydb/oracon.py index 5b158bc..01c7445 100644 --- a/python/despydb/oracon.py +++ b/python/despydb/oracon.py @@ -50,25 +50,25 @@ _TYPE_MAP = {cx_Oracle.BINARY : bytearray, cx_Oracle.BFILE : cx_Oracle.BFILE, cx_Oracle.BLOB : bytearray, - cx_Oracle.CLOB : unicode, + cx_Oracle.CLOB : str, cx_Oracle.CURSOR : cx_Oracle.CURSOR, cx_Oracle.DATETIME : datetime.datetime, cx_Oracle.FIXED_CHAR : str, - cx_Oracle.FIXED_NCHAR : unicode, + cx_Oracle.FIXED_NCHAR : str, #cx_Oracle.FIXED_UNICODE: unicode, cx_Oracle.INTERVAL : datetime.timedelta, cx_Oracle.LOB : bytearray, cx_Oracle.LONG_BINARY : bytearray, cx_Oracle.LONG_STRING : str, cx_Oracle.NATIVE_FLOAT : float, - cx_Oracle.NCLOB : unicode, + cx_Oracle.NCLOB : bytes, cx_Oracle.NUMBER : float, cx_Oracle.OBJECT : cx_Oracle.OBJECT, cx_Oracle.ROWID : bytearray, cx_Oracle.STRING : str, cx_Oracle.TIMESTAMP : datetime.datetime, #cx_Oracle.UNICODE : unicode - cx_Oracle.NCHAR : unicode + cx_Oracle.NCHAR : str } # Define some symbolic names for oracle error codes to make it clearer what @@ -118,15 +118,14 @@ def __init__(self, access_data): if access_data.get('service', None): kwargs['service'] = access_data['service'] if 'sid' in kwargs: - cdt = "(SID=%s)" % kwargs['sid'] + cdt = f"(SID={kwargs['sid']})" else: - cdt = "(SERVICE_NAME=%s)" % kwargs['service_name'] + cdt = f"(SERVICE_NAME={kwargs['service_name']})" if 'service' in kwargs: - cdt += '(SERVER=%s)' % kwargs['service'] + cdt += f"(SERVER={kwargs['service']})" cx_args['cclass'] = 'DESDM' cx_args['purity'] = cx_Oracle.ATTR_PURITY_SELF - dsn = ("(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=%s)(PORT=%s))" - "(CONNECT_DATA=%s))") %(kwargs['host'], kwargs['port'], cdt) + dsn = f"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST={kwargs['host']})(PORT={kwargs['port']}))(CONNECT_DATA={cdt}))" if access_data.get('threaded', None): cx_args['threaded'] = True @@ -169,7 +168,7 @@ def get_column_types(self, table_name): """ curs = self.cursor() - curs.execute('SELECT * FROM %s WHERE 0=1' % table_name) + curs.execute(f'SELECT * FROM {table_name} WHERE 0=1') types = {d[0].lower(): _TYPE_MAP[d[1]] for d in curs.description} @@ -186,7 +185,7 @@ def get_expr_exec_format(self): The format string. """ - return 'SELECT %s FROM DUAL' + return 'SELECT {} FROM DUAL' def get_named_bind_string(self, name): """ Return a named bind(substitution) string for name with cx_Oracle. @@ -218,7 +217,7 @@ def get_positional_bind_string(self, pos=1): The properly formatted binding string. """ - return ":%d" % pos + return f":{pos:d}" def get_regex_format(self, case_sensitive=True): """ Return a format string for constructing a regular expression clause. @@ -277,7 +276,7 @@ def sequence_drop(self, seq_name): """ - stmt = 'DROP SEQUENCE %s' % seq_name + stmt = f'DROP SEQUENCE {seq_name}' curs = self.cursor() try: @@ -297,7 +296,7 @@ def table_drop(self, table): The name of the table to drop. """ - stmt = 'DROP TABLE %s' % table + stmt = f'DROP TABLE {table}' curs = self.cursor() try: diff --git a/setup.py b/setup.py index 49112d4..91d2303 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ # The main call setup(name='despydb', - version ='2.0.0', + version ='3.0.0', license = "GPL", description = "Provide a dialect-neutral interface to DES databases", author = "The National Center for Supercomputing Applications (NCSA)", diff --git a/ups/despydb.table b/ups/despydb.table index 8677a0e..6ac32b2 100644 --- a/ups/despydb.table +++ b/ups/despydb.table @@ -1,6 +1,8 @@ -setupRequired(cxOracle 5.2.1+0) -setupRequired(despyServiceAccess 2.0.2+0) -setupRequired(psycopg2 2.4.6+8) +setupRequired(cxOracle) +setupRequired(despymisc) +setupRequired(oracleclient) +setupRequired(despyServiceAccess) + envPrepend(PYTHONPATH, ${PRODUCT_DIR}/python) envPrepend(PATH, ${PRODUCT_DIR}/bin)