Skip to content

Commit

Permalink
[IMP] util.snippets: add logging of minimal stats in html conversions
Browse files Browse the repository at this point in the history
Store and return the number of matched and converted record's values
that are processed by `convert_html_columns()`.
Add a `verbose=False` optional argument in `convert_html_content()`
that collects these stats and logs them.

Also, do the same in `util.views.records.convert_html_fields()`.
  • Loading branch information
andreabak committed Mar 17, 2023
1 parent b0b9efc commit 9135ff3
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 5 deletions.
30 changes: 28 additions & 2 deletions src/util/snippets.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,13 +278,19 @@ def convert_html_columns(cr, table, columns, converter_callback, where_column="I
update_sql = ", ".join(f'"{column}" = %({column})s' for column in columns)
update_query = f"UPDATE {table} SET {update_sql} WHERE id = %(id)s"

matched_count = 0
converted_count = 0
with ProcessPoolExecutor(max_workers=util.get_max_workers()) as executor:
convert = Convertor(converters, converter_callback)
for query in util.log_progress(split_queries, logger=_logger, qualifier=f"{table} updates"):
cr.execute(query)
for data in executor.map(convert, cr.fetchall()):
matched_count += 1
if "id" in data:
cr.execute(update_query, data)
converted_count += 1

return matched_count, converted_count


def determine_chunk_limit_ids(cr, table, column_arr, where):
Expand All @@ -307,6 +313,7 @@ def convert_html_content(
cr,
converter_callback,
where_column="IS NOT NULL",
verbose=False,
**kwargs,
):
"""
Expand All @@ -319,17 +326,36 @@ def convert_html_content(
:param str where_column: filtering such as
- "like '%abc%xyz%'"
- "~* '\\yabc.*xyz\\y'"
:param bool verbose: print stats about the conversion
:param dict kwargs: extra keyword arguments to pass to :func:`convert_html_column`
"""

convert_html_columns(
if verbose:
_logger.info("Converting html fields data using %s", repr(converter_callback))

matched_count = 0
converted_count = 0

matched, converted = convert_html_columns(
cr,
"ir_ui_view",
["arch_db"],
converter_callback,
where_column=where_column,
**dict(kwargs, extra_where="type = 'qweb'"),
)
matched_count += matched
converted_count += converted

for table, columns in html_fields(cr):
convert_html_columns(cr, table, columns, converter_callback, where_column=where_column, **kwargs)
matched, converted = convert_html_columns(
cr, table, columns, converter_callback, where_column=where_column, **kwargs
)
matched_count += matched
converted_count += converted

if verbose:
if matched_count:
_logger.info("Converted %d/%d matched html fields values", converted_count, matched_count)
else:
_logger.info("Did not match any html fields values to convert")
22 changes: 19 additions & 3 deletions src/util/views/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -1081,15 +1081,20 @@ def convert_qweb_views(cr, converter):
if views_ids:
convert_views(cr, views_ids, converter)

def convert_html_fields(cr, converter):
def convert_html_fields(cr, converter, verbose=False):
"""
Convert all html fields data in the database using the provided converter.
:param psycopg2.cursor cr: the database cursor.
:param EtreeConverter converter: the converter to use.
:param bool verbose: whether to print stats about the conversion.
:rtype: None
"""
_logger.info("Converting html fields data using %s" % (repr(converter),))
if verbose:
_logger.info("Converting html fields data using %s", repr(converter))

matched_count = 0
converted_count = 0

html_fields = list(snippets.html_fields(cr))
for table, columns in misc.log_progress(html_fields, _logger, "tables", log_hundred_percent=True):
Expand All @@ -1098,7 +1103,18 @@ def convert_html_fields(cr, converter):
"(%s)" % converter.build_where_clause(cr, pg.get_value_or_en_translation(cr, table, column))
for column in columns
)
snippets.convert_html_columns(cr, table, columns, converter.convert_callback, extra_where=extra_where)
# TODO abt: adapt to refactor, maybe make snippets compat w/ converter, instead of adapting?
matched, converted = snippets.convert_html_columns(
cr, table, columns, converter.convert_callback, extra_where=extra_where
)
matched_count += matched
converted_count += converted

if verbose:
if matched_count:
_logger.info("Converted %d/%d matched html fields values", converted_count, matched_count)
else:
_logger.info("Did not match any html fields values to convert")

else:

Expand Down

0 comments on commit 9135ff3

Please sign in to comment.