From d0535c286112348637ece90fa6d8db5ef4203021 Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 18:15:09 +0200 Subject: [PATCH 01/26] hierarchical export import. --- docs/conf.py | 13 +- setup.py | 3 +- src/collective/exportimport/config.py | 6 +- src/collective/exportimport/deserializer.py | 1 + src/collective/exportimport/export_content.py | 128 ++++++++----- src/collective/exportimport/export_other.py | 122 ++++++------ .../exportimport/filesystem_exporter.py | 68 +++++++ .../exportimport/filesystem_importer.py | 97 ++++++++++ src/collective/exportimport/fix_html.py | 63 ++++-- src/collective/exportimport/import_content.py | 180 ++++++++++++------ src/collective/exportimport/import_other.py | 114 +++++------ src/collective/exportimport/interfaces.py | 2 +- src/collective/exportimport/serializer.py | 130 ++++++++----- .../exportimport/templates/export_content.pt | 7 +- .../exportimport/templates/import_content.pt | 17 ++ src/collective/exportimport/testing.py | 11 +- .../tests/test_drop_and_include.py | 88 ++++----- .../exportimport/tests/test_export.py | 69 ++++--- .../exportimport/tests/test_fix_html.py | 68 ++++--- .../exportimport/tests/test_import.py | 76 ++++---- .../exportimport/tests/test_setup.py | 2 +- 21 files changed, 814 insertions(+), 451 deletions(-) create mode 100644 src/collective/exportimport/filesystem_exporter.py create mode 100644 src/collective/exportimport/filesystem_importer.py diff --git a/docs/conf.py b/docs/conf.py index 728ae8e5..6f420407 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,8 +9,9 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import sys + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -42,18 +43,18 @@ master_doc = "index" # General information about the project. -project = u"collective.exportimport" -copyright = u"Philip Bauer (pbauer)" -author = u"Philip Bauer (pbauer)" +project = "collective.exportimport" +copyright = "Philip Bauer (pbauer)" +author = "Philip Bauer (pbauer)" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u"3.0" +version = "3.0" # The full version, including alpha/beta/rc tags. -release = u"3.0" +release = "3.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 6980e329..728b85b1 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- """Installer for the collective.exportimport package.""" -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup import sys diff --git a/src/collective/exportimport/config.py b/src/collective/exportimport/config.py index 17dc2b62..14e63b83 100644 --- a/src/collective/exportimport/config.py +++ b/src/collective/exportimport/config.py @@ -12,7 +12,9 @@ os.path.expandvars(os.getenv("COLLECTIVE_EXPORTIMPORT_CENTRAL_DIRECTORY", "")) ) -SITE_ROOT = 'plone_site_root' +TREE_DIRECTORY = "exported_tree" + +SITE_ROOT = "plone_site_root" # Discussion Item has its own export / import views, don't show it in the exportable content type list -SKIPPED_CONTENTTYPE_IDS = ['Discussion Item'] +SKIPPED_CONTENTTYPE_IDS = ["Discussion Item"] diff --git a/src/collective/exportimport/deserializer.py b/src/collective/exportimport/deserializer.py index bd6fa0bc..fc0a87de 100644 --- a/src/collective/exportimport/deserializer.py +++ b/src/collective/exportimport/deserializer.py @@ -15,6 +15,7 @@ class RichTextFieldDeserializerWithoutUnescape(DefaultFieldDeserializer): """Override default RichTextFieldDeserializer without using html_parser.unescape(). Fixes https://github.com/collective/collective.exportimport/issues/99 """ + def __call__(self, value): content_type = self.field.default_mime_type encoding = "utf8" diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index 0c9135b3..179d30b7 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base from App.config import getConfiguration -from collective.exportimport import _ -from collective.exportimport import config -from collective.exportimport.interfaces import IBase64BlobsMarker -from collective.exportimport.interfaces import IMigrationMarker -from collective.exportimport.interfaces import IPathBlobsMarker -from collective.exportimport.interfaces import IRawRichTextMarker +from collective.exportimport import _, config +from collective.exportimport.filesystem_exporter import FileSystemContentExporter +from collective.exportimport.interfaces import ( + IBase64BlobsMarker, + IMigrationMarker, + IPathBlobsMarker, + IRawRichTextMarker, +) from operator import itemgetter from plone import api from plone.app.layout.viewlets.content import ContentHistoryViewlet @@ -15,16 +17,13 @@ from plone.restapi.serializer.converters import json_compatible from plone.uuid.interfaces import IUUID from Products.CMFPlone.interfaces import IPloneSiteRoot -from Products.CMFPlone.interfaces.constrains import ENABLED -from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes +from Products.CMFPlone.interfaces.constrains import ENABLED, ISelectableConstrainTypes from Products.CMFPlone.utils import safe_unicode from Products.Five import BrowserView from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile -from zope.component import getMultiAdapter -from zope.component import getUtility +from zope.component import getMultiAdapter, getUtility from zope.i18n import translate -from zope.interface import alsoProvides -from zope.interface import noLongerProvides +from zope.interface import alsoProvides, noLongerProvides from zope.schema import getFields import json @@ -34,6 +33,7 @@ import six import tempfile + try: pkg_resources.get_distribution("Products.Archetypes") except pkg_resources.DistributionNotFound: @@ -65,8 +65,7 @@ IRelationList = None HAS_RELATIONS = False else: - from z3c.relationfield.interfaces import IRelationChoice - from z3c.relationfield.interfaces import IRelationList + from z3c.relationfield.interfaces import IRelationChoice, IRelationList HAS_RELATIONS = True @@ -94,7 +93,6 @@ class ExportContent(BrowserView): - template = ViewPageTemplateFile("templates/export_content.pt") QUERY = {} @@ -112,7 +110,7 @@ def __call__( download_to_server=False, migration=True, include_revisions=False, - write_errors=False + write_errors=False, ): self.portal_type = portal_type or [] if isinstance(self.portal_type, str): @@ -122,7 +120,7 @@ def __call__( self.depth = int(depth) self.depth_options = ( - ("-1", _(u"unlimited")), + ("-1", _("unlimited")), ("0", "0"), ("1", "1"), ("2", "2"), @@ -137,9 +135,9 @@ def __call__( ) self.include_blobs = int(include_blobs) self.include_blobs_options = ( - ("0", _(u"as download urls")), - ("1", _(u"as base-64 encoded strings")), - ("2", _(u"as blob paths")), + ("0", _("as download urls")), + ("1", _("as base-64 encoded strings")), + ("2", _("as blob paths")), ) self.include_revisions = include_revisions self.write_errors = write_errors or self.request.form.get("write_errors") @@ -150,7 +148,9 @@ def __call__( return self.template() if not self.portal_type: - api.portal.show_message(_(u"Select at least one type to export"), self.request) + api.portal.show_message( + _("Select at least one type to export"), self.request + ) return self.template() if self.include_blobs == 1: @@ -181,7 +181,7 @@ def __call__( content_generator = self.export_content() number = 0 - if download_to_server: + if download_to_server == 1: directory = config.CENTRAL_DIRECTORY if directory: if not os.path.exists(directory): @@ -205,8 +205,12 @@ def __call__( errors = {"unexported_paths": self.errors} json.dump(errors, f, indent=4) f.write("]") - msg = _(u"Exported {} items ({}) as {} to {} with {} errors").format( - number, ", ".join(self.portal_type), filename, filepath, len(self.errors) + msg = _("Exported {} items ({}) as {} to {} with {} errors").format( + number, + ", ".join(self.portal_type), + filename, + filepath, + len(self.errors), ) logger.info(msg) api.portal.show_message(msg, self.request) @@ -218,6 +222,28 @@ def __call__( noLongerProvides(self.request, IPathBlobsMarker) self.finish() self.request.response.redirect(self.request["ACTUAL_URL"]) + elif download_to_server == 2: + # Will generate a directory tree with one json file per item + portal_id = api.portal.get().getId() + directory = config.CENTRAL_DIRECTORY + if not directory: + cfg = getConfiguration() + directory = cfg.clienthome + rootpath = os.path.join(directory, "exported_tree/%s/content" % portal_id) + if not os.path.exists(rootpath): + os.makedirs(rootpath) + logger.info("Created tree export %s", rootpath) + + self.start() + for number, datum in enumerate(content_generator, start=1): + FileSystemContentExporter(rootpath, datum).save() + self.finish() + + msg = _("Exported {} {} with {} errors").format( + number, self.portal_type, len(self.errors) + ) + logger.info(msg) + api.portal.show_message(msg, self.request) else: with tempfile.TemporaryFile(mode="w+") as f: self.start() @@ -228,12 +254,14 @@ def __call__( f.write(",") json.dump(datum, f, sort_keys=True, indent=4) if number: - if self.errors and self.write_errors: + if self.errors and self.write_errors: f.write(",") errors = {"unexported_paths": self.errors} json.dump(errors, f, indent=4) f.write("]") - msg = _(u"Exported {} {} with {} errors").format(number, self.portal_type, len(self.errors)) + msg = _("Exported {} {} with {} errors").format( + number, self.portal_type, len(self.errors) + ) logger.info(msg) api.portal.show_message(msg, self.request) response = self.request.response @@ -281,7 +309,7 @@ def export_content(self): query = self.build_query() catalog = api.portal.get_tool("portal_catalog") brains = catalog.unrestrictedSearchResults(**query) - logger.info(u"Exporting {} {}".format(len(brains), self.portal_type)) + logger.info("Exporting {} {}".format(len(brains), self.portal_type)) # Override richtext serializer to export links using resolveuid/xxx alsoProvides(self.request, IRawRichTextMarker) @@ -299,18 +327,18 @@ def export_content(self): continue if not index % 100: - logger.info(u"Handled {} items...".format(index)) + logger.info("Handled {} items...".format(index)) try: obj = brain.getObject() except Exception: - msg = u"Error getting brain {}".format(brain.getPath()) - self.errors.append({'path':None, 'message': msg}) + msg = "Error getting brain {}".format(brain.getPath()) + self.errors.append({"path": None, "message": msg}) logger.exception(msg, exc_info=True) continue if obj is None: - msg = u"brain.getObject() is None {}".format(brain.getPath()) + msg = "brain.getObject() is None {}".format(brain.getPath()) logger.error(msg) - self.errors.append({'path':None, 'message': msg}) + self.errors.append({"path": None, "message": msg}) continue obj = self.global_obj_hook(obj) if not obj: @@ -328,8 +356,8 @@ def export_content(self): yield item except Exception: - msg = u"Error exporting {}".format(obj.absolute_url()) - self.errors.append({'path':obj.absolute_url(), 'message':msg}) + msg = "Error exporting {}".format(obj.absolute_url()) + self.errors.append({"path": obj.absolute_url(), "message": msg}) logger.exception(msg, exc_info=True) def portal_types(self): @@ -349,7 +377,9 @@ def portal_types(self): "number": number, "value": fti.id, "title": translate( - safe_unicode(fti.title), domain="plone", context=self.request + safe_unicode(fti.title), + domain="plone", + context=self.request, ), } ) @@ -380,12 +410,12 @@ def update_export_data(self, item, obj): item = self.global_dict_hook(item, obj) if not item: - logger.info(u"Skipping %s", obj.absolute_url()) + logger.info("Skipping %s", obj.absolute_url()) return item = self.custom_dict_hook(item, obj) if not item: - logger.info(u"Skipping %s", obj.absolute_url()) + logger.info("Skipping %s", obj.absolute_url()) return return item @@ -534,15 +564,27 @@ def export_revisions(self, item, obj): item_version = self.update_data_for_migration(item_version, obj) item["exportimport.versions"][version_id] = item_version # inject metadata (missing for Archetypes content): - comment = history_metadata.retrieve(version_id)["metadata"]["sys_metadata"]["comment"] - if comment and comment != item["exportimport.versions"][version_id].get("changeNote"): + comment = history_metadata.retrieve(version_id)["metadata"]["sys_metadata"][ + "comment" + ] + if comment and comment != item["exportimport.versions"][version_id].get( + "changeNote" + ): item["exportimport.versions"][version_id]["changeNote"] = comment - principal = history_metadata.retrieve(version_id)["metadata"]["sys_metadata"]["principal"] - if principal and principal != item["exportimport.versions"][version_id].get("changeActor"): + principal = history_metadata.retrieve(version_id)["metadata"][ + "sys_metadata" + ]["principal"] + if principal and principal != item["exportimport.versions"][version_id].get( + "changeActor" + ): item["exportimport.versions"][version_id]["changeActor"] = principal # current changenote - item["changeNote"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"]["comment"] - item["changeActor"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"]["principal"] + item["changeNote"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"][ + "comment" + ] + item["changeActor"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"][ + "principal" + ] return item diff --git a/src/collective/exportimport/export_other.py b/src/collective/exportimport/export_other.py index ded77b94..58666471 100644 --- a/src/collective/exportimport/export_other.py +++ b/src/collective/exportimport/export_other.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base from App.config import getConfiguration -from collective.exportimport import _ -from collective.exportimport import config +from collective.exportimport import _, config from OFS.interfaces import IOrderedContainer from operator import itemgetter from plone import api @@ -11,14 +10,18 @@ from plone.app.redirector.interfaces import IRedirectionStorage from plone.app.textfield.value import RichTextValue from plone.app.uuid.utils import uuidToObject -from plone.portlets.constants import CONTENT_TYPE_CATEGORY -from plone.portlets.constants import CONTEXT_CATEGORY -from plone.portlets.constants import GROUP_CATEGORY -from plone.portlets.constants import USER_CATEGORY -from plone.portlets.interfaces import ILocalPortletAssignmentManager -from plone.portlets.interfaces import IPortletAssignmentMapping -from plone.portlets.interfaces import IPortletAssignmentSettings -from plone.portlets.interfaces import IPortletManager +from plone.portlets.constants import ( + CONTENT_TYPE_CATEGORY, + CONTEXT_CATEGORY, + GROUP_CATEGORY, + USER_CATEGORY, +) +from plone.portlets.interfaces import ( + ILocalPortletAssignmentManager, + IPortletAssignmentMapping, + IPortletAssignmentSettings, + IPortletManager, +) from plone.restapi.interfaces import ISerializeToJson from plone.restapi.serializer.converters import json_compatible from plone.uuid.interfaces import IUUID @@ -26,11 +29,13 @@ from Products.CMFCore.utils import getToolByName from Products.CMFPlone.interfaces import IPloneSiteRoot from Products.Five import BrowserView -from zope.component import getMultiAdapter -from zope.component import getUtilitiesFor -from zope.component import getUtility -from zope.component import queryMultiAdapter -from zope.component import queryUtility +from zope.component import ( + getMultiAdapter, + getUtilitiesFor, + getUtility, + queryMultiAdapter, + queryUtility, +) from zope.interface import providedBy import json @@ -39,6 +44,7 @@ import pkg_resources import six + try: pkg_resources.get_distribution("Products.Archetypes") except pkg_resources.DistributionNotFound: @@ -81,9 +87,9 @@ class BaseExport(BrowserView): def download(self, data): filename = self.request.form.get("filename") if not filename: - filename = u"{}.json".format(self.__name__) + filename = "{}.json".format(self.__name__) if not data: - msg = _(u"No data to export for {}").format(self.__name__) + msg = _("No data to export for {}").format(self.__name__) logger.info(msg) api.portal.show_message(msg, self.request) return self.request.response.redirect(self.request["ACTUAL_URL"]) @@ -100,7 +106,7 @@ def download(self, data): filepath = os.path.join(directory, filename) with open(filepath, "w") as f: json.dump(data, f, sort_keys=True, indent=4) - msg = _(u"Exported to {}").format(filepath) + msg = _("Exported to {}").format(filepath) logger.info(msg) api.portal.show_message(msg, self.request) return self.request.response.redirect(self.request["ACTUAL_URL"]) @@ -124,13 +130,13 @@ class ExportRelations(BaseExport): def __call__( self, download_to_server=False, debug=False, include_linkintegrity=False ): - self.title = _(u"Export relations") + self.title = _("Export relations") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting relations...") + logger.info("Exporting relations...") data = self.get_all_references(debug, include_linkintegrity) - logger.info(u"Exported %s relations", len(data)) + logger.info("Exported %s relations", len(data)) self.download(data) def get_all_references(self, debug=False, include_linkintegrity=False): @@ -232,7 +238,7 @@ class ExportMembers(BaseExport): def __init__(self, context, request): super(ExportMembers, self).__init__(context, request) self.pms = api.portal.get_tool("portal_membership") - self.title = _(u"Export members, groups and roles") + self.title = _("Export members, groups and roles") self.group_roles = {} def __call__(self, download_to_server=False): @@ -241,10 +247,10 @@ def __call__(self, download_to_server=False): return self.index() data = {} - logger.info(u"Exporting groups and users...") + logger.info("Exporting groups and users...") data["groups"] = self.export_groups() data["members"] = [i for i in self.export_members()] - msg = u"Exported {} groups and {} members".format( + msg = "Exported {} groups and {} members".format( len(data["groups"]), len(data["members"]) ) logger.info(msg) @@ -326,18 +332,17 @@ def _getUserData(self, userId): class ExportTranslations(BaseExport): - DROP_PATH = [] def __call__(self, download_to_server=False): - self.title = _(u"Export translations") + self.title = _("Export translations") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting translations...") + logger.info("Exporting translations...") data = self.all_translations() - logger.info(u"Exported %s groups of translations", len(data)) + logger.info("Exported %s groups of translations", len(data)) self.download(data) def all_translations(self): # noqa: C901 @@ -377,7 +382,7 @@ def all_translations(self): # noqa: C901 # Archetypes and Dexterity with plone.app.multilingual portal_catalog = api.portal.get_tool("portal_catalog") if "TranslationGroup" not in portal_catalog.indexes(): - logger.debug(u"No index TranslationGroup (p.a.multilingual not installed)") + logger.debug("No index TranslationGroup (p.a.multilingual not installed)") return results for uid in portal_catalog.uniqueValuesFor("TranslationGroup"): @@ -397,7 +402,7 @@ def all_translations(self): # noqa: C901 skip = True if not skip and brain.Language in item: logger.info( - u"Duplicate language for {}: {}".format( + "Duplicate language for {}: {}".format( uid, [i.getPath() for i in brains] ) ) @@ -412,14 +417,14 @@ class ExportLocalRoles(BaseExport): """Export all local roles""" def __call__(self, download_to_server=False): - self.title = _(u"Export local roles") + self.title = _("Export local roles") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting local roles...") + logger.info("Exporting local roles...") data = self.all_localroles() - logger.info(u"Exported local roles for %s items", len(data)) + logger.info("Exported local roles for %s items", len(data)) self.download(data) def all_localroles(self): @@ -469,14 +474,14 @@ class ExportOrdering(BaseExport): """Export all local roles""" def __call__(self, download_to_server=False): - self.title = _(u"Export ordering") + self.title = _("Export ordering") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting positions in parent...") + logger.info("Exporting positions in parent...") data = self.all_orders() - logger.info(u"Exported %s positions in parent", len(data)) + logger.info("Exported %s positions in parent", len(data)) self.download(data) def all_orders(self): @@ -505,14 +510,14 @@ class ExportDefaultPages(BaseExport): """Export all default_page settings.""" def __call__(self, download_to_server=False): - self.title = _(u"Export default pages") + self.title = _("Export default pages") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting default pages...") + logger.info("Exporting default pages...") data = self.all_default_pages() - logger.info(u"Exported %s default pages", len(data)) + logger.info("Exported %s default pages", len(data)) self.download(data) def all_default_pages(self): @@ -524,10 +529,10 @@ def all_default_pages(self): try: obj = brain.getObject() except Exception: - logger.info(u"Error getting obj for %s", brain.getURL(), exc_info=True) + logger.info("Error getting obj for %s", brain.getURL(), exc_info=True) continue if obj is None: - logger.error(u"brain.getObject() is None %s", brain.getPath()) + logger.error("brain.getObject() is None %s", brain.getPath()) continue if IPloneSiteRoot.providedBy(obj): # Site root is handled below (in Plone 6 it is returned by a catalog search) @@ -537,7 +542,7 @@ def all_default_pages(self): data = self.get_default_page_info(obj) except Exception: logger.info( - u"Error exporting default_page for %s", + "Error exporting default_page for %s", obj.absolute_url(), exc_info=True, ) @@ -554,7 +559,7 @@ def all_default_pages(self): data["uuid"] = config.SITE_ROOT results.append(data) except Exception: - logger.info(u"Error exporting default_page for portal", exc_info=True) + logger.info("Error exporting default_page for portal", exc_info=True) return results @@ -584,14 +589,14 @@ def get_default_page_info(self, obj): class ExportDiscussion(BaseExport): def __call__(self, download_to_server=False): - self.title = _(u"Export comments") + self.title = _("Export comments") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting discussions...") + logger.info("Exporting discussions...") data = self.all_discussions() - logger.info(u"Exported %s discussions", len(data)) + logger.info("Exported %s discussions", len(data)) self.download(data) def all_discussions(self): @@ -604,7 +609,7 @@ def all_discussions(self): try: obj = brain.getObject() if obj is None: - logger.error(u"brain.getObject() is None %s", brain.getPath()) + logger.error("brain.getObject() is None %s", brain.getPath()) continue conversation = IConversation(obj, None) if not conversation: @@ -625,20 +630,22 @@ def all_discussions(self): class ExportPortlets(BaseExport): def __call__(self, download_to_server=False): - self.title = _(u"Export portlets") + self.title = _("Export portlets") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting portlets...") + logger.info("Exporting portlets...") data = self.all_portlets() - logger.info(u"Exported info for %s items with portlets", len(data)) + logger.info("Exported info for %s items with portlets", len(data)) self.download(data) def all_portlets(self): self.results = [] portal = api.portal.get() - portal.ZopeFindAndApply(self.context, search_sub=True, apply_func=self.get_portlets) + portal.ZopeFindAndApply( + self.context, search_sub=True, apply_func=self.get_portlets + ) self.get_root_portlets() return self.results @@ -663,7 +670,7 @@ def _get_portlets(self, obj, uid): obj_results["uuid"] = uid self.results.append(obj_results) return - + def get_root_portlets(self): site = api.portal.get() self._get_portlets(site, PORTAL_PLACEHOLDER) @@ -675,6 +682,7 @@ def local_portlets_hook(self, portlets): def portlets_blacklist_hook(self, blacklist): return blacklist + def export_local_portlets(obj): """Serialize portlets for one content object Code mostly taken from https://github.com/plone/plone.restapi/pull/669 @@ -739,9 +747,9 @@ def export_portlets_blacklist(obj): obj_results = {} status = assignable.getBlacklistStatus(category) if status is True: - obj_results["status"] = u"block" + obj_results["status"] = "block" elif status is False: - obj_results["status"] = u"show" + obj_results["status"] = "show" if obj_results: obj_results["manager"] = manager_name @@ -776,12 +784,12 @@ def export_plone_redirects(): class ExportRedirects(BaseExport): def __call__(self, download_to_server=False): - self.title = _(u"Export redirects") + self.title = _("Export redirects") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info(u"Exporting redirects...") + logger.info("Exporting redirects...") data = export_plone_redirects() - logger.info(u"Exported %s redirects", len(data)) + logger.info("Exported %s redirects", len(data)) self.download(data) diff --git a/src/collective/exportimport/filesystem_exporter.py b/src/collective/exportimport/filesystem_exporter.py new file mode 100644 index 00000000..abd31670 --- /dev/null +++ b/src/collective/exportimport/filesystem_exporter.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +from six.moves.urllib.parse import unquote, urlparse + +import json +import os + + +class FileSystemExporter(object): + """Base FS Exporter""" + + def __init__(self, rootpath, json_item): + self.item = json_item + self.root = rootpath + + def create_dir(self, dirname): + """Creates a directory if does not exist + + Args: + dirname (str): dirname to be created + """ + dirpath = os.path.join(self.root, dirname) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + def get_parents(self, parent): + """Extracts parents of item + + Args: + parent (dict): Parent info dict + + Returns: + (str): relative path + """ + + if not parent: + return "" + + parent_url = unquote(parent["@id"]) + parent_url_parsed = urlparse(parent_url) + + # Get the path part, split it, remove the always empty first element. + parent_path = parent_url_parsed.path.split("/")[1:] + if ( + len(parent_url_parsed.netloc.split(":")) > 1 + or parent_url_parsed.netloc == "nohost" + ): + # For example localhost:8080, or nohost when running tests. + # First element will then be a Plone Site id. + # Get rid of it. + parent_path = parent_path[1:] + + return "/".join(parent_path) + + +class FileSystemContentExporter(FileSystemExporter): + """Deserializes JSON items into a FS tree""" + + def save(self): + """Saves a json file to filesystem tree + Target directory is related as original parent position in site. + """ + parent_path = self.get_parents(self.item.get("parent")) + self.create_dir(parent_path) + + filename = "%s_%s.json" % (self.item.get("@type"), self.item.get("UID")) + filepath = os.path.join(self.root, parent_path, filename) + with open(filepath, "w") as f: + json.dump(self.item, f, sort_keys=True, indent=4) diff --git a/src/collective/exportimport/filesystem_importer.py b/src/collective/exportimport/filesystem_importer.py new file mode 100644 index 00000000..9d0664f9 --- /dev/null +++ b/src/collective/exportimport/filesystem_importer.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +from glob import iglob +from plone import api +from six.moves.urllib.parse import unquote, urlparse + +import json +import logging +import os +import six + + +if six.PY2: + from pathlib2 import Path +else: + from pathlib import Path + + +class FileSystemImporter(object): + """Base FS Importer""" + + logger = logging.getLogger(__name__) + + def __init__(self, server_tree_file): + self.path = server_tree_file + + def get_parents(self, parent): + """Extracts parents of item + + Args: + parent (dict): Parent info dict + + Returns: + (str): relative path + """ + + if not parent: + return "" + + parent_url = unquote(parent["@id"]) + parent_url_parsed = urlparse(parent_url) + + # Get the path part, split it, remove the always empty first element. + parent_path = parent_url_parsed.path.split("/")[1:] + if ( + len(parent_url_parsed.netloc.split(":")) > 1 + or parent_url_parsed.netloc == "nohost" + ): + # For example localhost:8080, or nohost when running tests. + # First element will then be a Plone Site id. + # Get rid of it. + parent_path = parent_path[1:] + + return "/".join(parent_path) + + +class FileSystemContentImporter(FileSystemImporter): + """Deserializes JSON items into a FS tree""" + + def list_files(self): + """Loads all json files from filesystem tree""" + files = iglob(os.path.join(self.path, "**/*.json"), recursive=True) + return files + + def get_hierarchical_files(self): + """Gets all files and folders""" + root = Path(self.path) + portal = api.portal.get() + assert root.is_dir() + json_files = root.glob("**/*.json") + for json_file in json_files: + self.logger.debug("Importing %s", json_file) + item = json.loads(json_file.read_text()) + item["json_file"] = str(json_file) + + # Modify parent data + json_parent = item.get("parent", {}) + + # Find the real parent nodes + prefix = os.path.commonprefix([str(json_file.parent), self.path]) + path = os.path.relpath(str(json_file.parent), prefix) + parents = self.get_parents(json_parent) + + if json_file.parent == Path(os.path.join(self.path, parents)): + yield item + else: + try: + parent_obj = portal.unrestrictedTraverse(path) + except KeyError: + parent_obj = portal + + if parent_obj: + item["@id"] = item.get("@id") + json_parent.update( + {"@id": parent_obj.absolute_url(), "UID": parent_obj.UID()} + ) + item["parent"] = json_parent + yield item diff --git a/src/collective/exportimport/fix_html.py b/src/collective/exportimport/fix_html.py index 2f21d395..d511fdcc 100644 --- a/src/collective/exportimport/fix_html.py +++ b/src/collective/exportimport/fix_html.py @@ -1,8 +1,8 @@ # -*- coding: UTF-8 -*- from Acquisition import aq_parent from bs4 import BeautifulSoup -from collective.exportimport import _ from collections import defaultdict +from collective.exportimport import _ from logging import getLogger from plone import api from plone.api.exc import InvalidParameterError @@ -11,19 +11,18 @@ from plone.app.textfield.interfaces import IRichText from plone.app.textfield.value import IRichTextValue from plone.dexterity.utils import iterSchemataForType -from plone.portlets.interfaces import IPortletAssignmentMapping -from plone.portlets.interfaces import IPortletManager +from plone.portlets.interfaces import IPortletAssignmentMapping, IPortletManager from plone.uuid.interfaces import IUUID from Products.CMFCore.interfaces import IContentish from Products.Five import BrowserView from six.moves.urllib.parse import urlparse -from zope.component import getUtilitiesFor -from zope.component import queryMultiAdapter +from zope.component import getUtilitiesFor, queryMultiAdapter from zope.interface import providedBy import six import transaction + logger = getLogger(__name__) IMAGE_SCALE_MAP = { @@ -40,7 +39,7 @@ class FixHTML(BrowserView): def __call__(self): - self.title = _(u"Fix links to content and images in richtext") + self.title = _("Fix links to content and images in richtext") if not self.request.form.get("form.submitted", False): return self.index() commit = self.request.form.get("form.commit", True) @@ -48,18 +47,18 @@ def __call__(self): msg = [] fix_count = fix_html_in_content_fields(context=self.context, commit=commit) - msg.append(_(u"Fixed HTML for {} fields in content items").format(fix_count)) + msg.append(_("Fixed HTML for {} fields in content items").format(fix_count)) logger.info(msg[-1]) fix_count = fix_html_in_portlets(context=self.context) - msg.append(_(u"Fixed HTML for {} portlets").format(fix_count)) + msg.append(_("Fixed HTML for {} portlets").format(fix_count)) logger.info(msg[-1]) # TODO: Fix html in tiles # tiles = fix_html_in_tiles() # msg = u"Fixed html for {} tiles".format(tiles) - api.portal.show_message(u" ".join(m + u"." for m in msg), self.request) + api.portal.show_message(" ".join(m + "." for m in msg), self.request) return self.index() @@ -235,7 +234,7 @@ def find_object(base, path): obj = api.portal.get() portal_path = obj.absolute_url_path() + "/" if path.startswith(portal_path): - path = path[len(portal_path):] + path = path[len(portal_path) :] else: obj = aq_parent(base) # relative urls start at the parent... @@ -320,17 +319,21 @@ def table_class_fixer(text, obj=None): query["path"] = "/".join(context.getPhysicalPath()) brains = catalog(**query) total = len(brains) - logger.info("There are %s content items in total, starting migration...", len(brains)) + logger.info( + "There are %s content items in total, starting migration...", len(brains) + ) fixed_fields = 0 fixed_items = 0 for index, brain in enumerate(brains, start=1): try: obj = brain.getObject() except Exception: - logger.warning("Could not get object for: %s", brain.getPath(), exc_info=True) + logger.warning( + "Could not get object for: %s", brain.getPath(), exc_info=True + ) continue if obj is None: - logger.error(u"brain.getObject() is None %s", brain.getPath()) + logger.error("brain.getObject() is None %s", brain.getPath()) continue try: changed = False @@ -339,11 +342,19 @@ def table_class_fixer(text, obj=None): if text and IRichTextValue.providedBy(text) and text.raw: clean_text = text.raw for fixer in fixers: - logger.debug("Fixing html for %s with %s", obj.absolute_url(), fixer.__name__) + logger.debug( + "Fixing html for %s with %s", + obj.absolute_url(), + fixer.__name__, + ) try: clean_text = fixer(clean_text, obj) except Exception: - logger.info(u"Error while fixing html of %s for %s", fieldname, obj.absolute_url()) + logger.info( + "Error while fixing html of %s for %s", + fieldname, + obj.absolute_url(), + ) raise if clean_text and clean_text != text.raw: @@ -355,7 +366,11 @@ def table_class_fixer(text, obj=None): ) setattr(obj, fieldname, textvalue) changed = True - logger.debug(u"Fixed html for field %s of %s", fieldname, obj.absolute_url()) + logger.debug( + "Fixed html for field %s of %s", + fieldname, + obj.absolute_url(), + ) fixed_fields += 1 if changed: fixed_items += 1 @@ -366,12 +381,21 @@ def table_class_fixer(text, obj=None): if fixed_items != 0 and not fixed_items % 1000: # Commit every 1000 changed items. logger.info( - u"Fix html for %s (%s) of %s items (changed %s fields in %s items)", - index, round(index / total * 100, 2), total, fixed_fields, fixed_items) + "Fix html for %s (%s) of %s items (changed %s fields in %s items)", + index, + round(index / total * 100, 2), + total, + fixed_fields, + fixed_items, + ) if commit: transaction.commit() - logger.info(u"Finished fixing html in content fields (changed %s fields in %s items)", fixed_fields, fixed_items) + logger.info( + "Finished fixing html in content fields (changed %s fields in %s items)", + fixed_fields, + fixed_items, + ) if commit: # commit remaining items transaction.commit() @@ -380,7 +404,6 @@ def table_class_fixer(text, obj=None): def fix_html_in_portlets(context=None): - portlets_schemata = { iface: name for name, iface in getUtilitiesFor(IPortletTypeInterface) } diff --git a/src/collective/exportimport/import_content.py b/src/collective/exportimport/import_content.py index d2ce0fc4..25c87327 100644 --- a/src/collective/exportimport/import_content.py +++ b/src/collective/exportimport/import_content.py @@ -1,29 +1,25 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base -from collective.exportimport import _ -from collective.exportimport import config +from App.config import getConfiguration +from collective.exportimport import _, config +from collective.exportimport.filesystem_importer import FileSystemContentImporter from collective.exportimport.interfaces import IMigrationMarker -from datetime import datetime +from datetime import datetime, timedelta from DateTime import DateTime -from datetime import timedelta from Persistence import PersistentMapping from plone import api from plone.api.exc import InvalidParameterError from plone.dexterity.interfaces import IDexterityFTI from plone.i18n.normalizer.interfaces import IIDNormalizer -from plone.namedfile.file import NamedBlobFile -from plone.namedfile.file import NamedBlobImage +from plone.namedfile.file import NamedBlobFile, NamedBlobImage from plone.restapi.interfaces import IDeserializeFromJson -from Products.CMFPlone.interfaces.constrains import ENABLED -from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes +from Products.CMFPlone.interfaces.constrains import ENABLED, ISelectableConstrainTypes from Products.CMFPlone.utils import _createObjectByType from Products.Five import BrowserView from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile -from six.moves.urllib.parse import unquote -from six.moves.urllib.parse import urlparse +from six.moves.urllib.parse import unquote, urlparse from zExceptions import NotFound -from zope.component import getMultiAdapter -from zope.component import getUtility +from zope.component import getMultiAdapter, getUtility from zope.interface import alsoProvides from ZPublisher.HTTPRequest import FileUpload @@ -36,6 +32,7 @@ import six import transaction + try: from plone.app.querystring.upgrades import fix_select_all_existing_collections @@ -74,7 +71,6 @@ def get_absolute_blob_path(obj, blob_path): class ImportContent(BrowserView): - template = ViewPageTemplateFile("templates/import_content.pt") # You can specify a default-target container for all items of a type. @@ -112,7 +108,8 @@ def __call__( return_json=False, limit=None, server_file=None, - iterator=None + server_tree_file=None, + iterator=None, ): request = self.request self.limit = limit @@ -136,17 +133,17 @@ def __call__( status = "success" msg = "" - if server_file and jsonfile: + if server_file and jsonfile and server_tree_file: # This is an error. But when you upload 10 GB AND select a server file, # it is a pity when you would have to upload again. api.portal.show_message( - _(u"json file was uploaded, so the selected server file was ignored."), + _("json file was uploaded, so the selected server file was ignored."), request=self.request, type="warn", ) server_file = None status = "error" - if server_file and not jsonfile: + if server_file and not jsonfile and not server_tree_file: if server_file in self.server_files: for path in self.import_paths: full_path = os.path.join(path, server_file) @@ -161,6 +158,7 @@ def __call__( api.portal.show_message(msg, request=self.request, type="warn") server_file = None status = "error" + if jsonfile: self.portal = api.portal.get() try: @@ -176,7 +174,7 @@ def __call__( status = "error" msg = str(e) api.portal.show_message( - _(u"Exception during upload: {}").format(e), + _("Exception during upload: {}").format(e), request=self.request, ) else: @@ -192,11 +190,18 @@ def __call__( msg = self.do_import(iterator) api.portal.show_message(msg, self.request) + if server_tree_file and not server_file and not jsonfile: + msg = self.do_import( + FileSystemContentImporter(server_tree_file).get_hierarchical_files() + ) + api.portal.show_message(msg, self.request) + self.finish() if return_json: msg = {"state": status, "msg": msg} return json.dumps(msg) + return self.template() def start(self): @@ -207,7 +212,7 @@ def finish(self): def commit_hook(self, added, index): """Hook to do something after importing every x items.""" - msg = u"Committing after creating {} of {} handled items...".format( + msg = "Committing after creating {} of {} handled items...".format( len(added), index ) logger.info(msg) @@ -243,16 +248,51 @@ def server_files(self): listing.sort() return listing + @property + def import_tree_parts(self): + """Returns subdirectories in export tree""" + directory = config.CENTRAL_DIRECTORY + if not directory: + cfg = getConfiguration() + directory = cfg.clienthome + base_path = os.path.join(directory, config.TREE_DIRECTORY) + return [os.path.join(base_path, d, "content") for d in os.listdir(base_path)] + + def process_tree_files(self, files): + """Reads and uploads tree files""" + for path in files: + logger.info("Using server tree file %s", path) + # Open the file in binary mode and use it as jsonfile. + with open(path, "rb") as jsonfile: + try: + if isinstance(jsonfile, str): + data = ijson.items(jsonfile, "item") + elif isinstance(jsonfile, FileUpload) or hasattr(jsonfile, "read"): + data = ijson.items(jsonfile, "item") + else: + raise RuntimeError("Data is neither text, file nor upload.") + except Exception as e: + logger.error(str(e)) + msg = str(e) + api.portal.show_message( + _("Exception during upload: {}").format(e), + request=self.request, + ) + else: + self.start() + msg = self.do_import(data) + self.finish() + def do_import(self, data): start = datetime.now() alsoProvides(self.request, IMigrationMarker) added = self.import_new_content(data) end = datetime.now() delta = end - start - msg = u"Imported {} items".format(len(added)) + msg = "Imported {} items".format(len(added)) transaction.get().note(msg) transaction.commit() - msg = u"{} in {} seconds".format(msg, delta.seconds) + msg = "{} in {} seconds".format(msg, delta.seconds) logger.info(msg) return msg @@ -273,7 +313,9 @@ def must_process(self, item_path): if not self.should_include(item_path): return False elif self.should_drop(item_path): - logger.info(u"Skipping %s, even though listed in INCLUDE_PATHS", item_path) + logger.info( + "Skipping %s, even though listed in INCLUDE_PATHS", item_path + ) return False else: if self.should_drop(item_path): @@ -285,9 +327,9 @@ def import_new_content(self, data): # noqa: C901 added = [] if getattr(data, "len", None): - logger.info(u"Importing {} items".format(len(data))) + logger.info("Importing {} items".format(len(data))) else: - logger.info(u"Importing data") + logger.info("Importing data") for index, item in enumerate(data, start=1): if self.limit and len(added) >= self.limit: break @@ -306,7 +348,7 @@ def import_new_content(self, data): # noqa: C901 new_id = unquote(item["@id"]).split("/")[-1] if new_id != item["id"]: logger.info( - u"Conflicting ids in url ({}) and id ({}). Using {}".format( + "Conflicting ids in url ({}) and id ({}). Using {}".format( new_id, item["id"], new_id ) ) @@ -334,7 +376,7 @@ def import_new_content(self, data): # noqa: C901 if not container: logger.warning( - u"No container (parent was {}) found for {} {}".format( + "No container (parent was {}) found for {} {}".format( item["parent"]["@type"], item["@type"], item["@id"] ) ) @@ -342,7 +384,7 @@ def import_new_content(self, data): # noqa: C901 if not getattr(aq_base(container), "isPrincipiaFolderish", False): logger.warning( - u"Container {} for {} is not folderish".format( + "Container {} for {} is not folderish".format( container.absolute_url(), item["@id"] ) ) @@ -356,7 +398,7 @@ def import_new_content(self, data): # noqa: C901 if self.handle_existing_content == 0: # Skip logger.info( - u"{} ({}) already exists. Skipping it.".format( + "{} ({}) already exists. Skipping it.".format( item["id"], item["@id"] ) ) @@ -365,7 +407,7 @@ def import_new_content(self, data): # noqa: C901 elif self.handle_existing_content == 1: # Replace content before creating it new logger.info( - u"{} ({}) already exists. Replacing it.".format( + "{} ({}) already exists. Replacing it.".format( item["id"], item["@id"] ) ) @@ -374,7 +416,7 @@ def import_new_content(self, data): # noqa: C901 elif self.handle_existing_content == 2: # Update existing item logger.info( - u"{} ({}) already exists. Updating it.".format( + "{} ({}) already exists. Updating it.".format( item["id"], item["@id"] ) ) @@ -386,7 +428,7 @@ def import_new_content(self, data): # noqa: C901 duplicate = item["id"] item["id"] = "{}-{}".format(item["id"], random.randint(1000, 9999)) logger.info( - u"{} ({}) already exists. Created as {}".format( + "{} ({}) already exists. Created as {}".format( duplicate, item["@id"], item["id"] ) ) @@ -414,16 +456,17 @@ def import_new_content(self, data): # noqa: C901 if self.commit and not len(added) % self.commit: self.commit_hook(added, index) except Exception as e: - item_id = item['@id'].split('/')[-1] + item_id = item["@id"].split("/")[-1] container.manage_delObjects(item_id) logger.warning(e) - logger.warning("Didn't add %s %s", item["@type"], item["@id"], exc_info=True) + logger.warning( + "Didn't add %s %s", item["@type"], item["@id"], exc_info=True + ) continue return added def handle_new_object(self, item, index, new): - new, item = self.global_obj_hook_before_deserializing(new, item) # import using plone.restapi deserializers @@ -432,13 +475,15 @@ def handle_new_object(self, item, index, new): try: new = deserializer(validate_all=False, data=item) except TypeError as error: - if 'unexpected keyword argument' in str(error): + if "unexpected keyword argument" in str(error): self.request["BODY"] = json.dumps(item) new = deserializer(validate_all=False) else: raise error except Exception: - logger.warning("Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True) + logger.warning( + "Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True + ) raise # Blobs can be exported as only a path in the blob storage. @@ -454,9 +499,7 @@ def handle_new_object(self, item, index, new): # Happens only when we import content that doesn't have a UID # for instance when importing from non Plone systems. logger.info( - "Created new UID for item %s with type %s.", - item["@id"], - item["@type"] + "Created new UID for item %s with type %s.", item["@id"], item["@type"] ) item["UID"] = uuid @@ -482,9 +525,7 @@ def handle_new_object(self, item, index, new): new.creation_date = creation_date new.aq_base.creation_date_migrated = creation_date logger.info( - "Created item #{}: {} {}".format( - index, item["@type"], new.absolute_url() - ) + "Created item #{}: {} {}".format(index, item["@type"], new.absolute_url()) ) return new @@ -546,7 +587,12 @@ def import_versions(self, container, item): try: new = deserializer(validate_all=False, data=version) except Exception: - logger.warning("Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True) + logger.warning( + "Cannot deserialize %s %s", + item["@type"], + item["@id"], + exc_info=True, + ) return self.save_revision(new, version, initial) @@ -557,7 +603,9 @@ def import_versions(self, container, item): try: new = deserializer(validate_all=False, data=item) except Exception: - logger.warning("Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True) + logger.warning( + "Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True + ) return self.import_blob_paths(new, item) @@ -619,7 +667,7 @@ def save_revision(self, obj, item, initial=False): from plone.app.versioningbehavior import _ as PAV if initial: - comment = PAV(u"initial_version_changeNote", default=u"Initial version") + comment = PAV("initial_version_changeNote", default="Initial version") else: comment = item.get("changeNote") sys_metadata = { @@ -726,15 +774,12 @@ def import_constrains(self, obj, item): return constrains.setConstrainTypesMode(ENABLED) - locally_allowed_types = item["exportimport.constrains"][ - "locally_allowed_types" - ] + locally_allowed_types = item["exportimport.constrains"]["locally_allowed_types"] try: constrains.setLocallyAllowedTypes(locally_allowed_types) except ValueError: logger.warning( - "Cannot setLocallyAllowedTypes on %s", item["@id"], - exc_info=True + "Cannot setLocallyAllowedTypes on %s", item["@id"], exc_info=True ) immediately_addable_types = item["exportimport.constrains"][ @@ -744,8 +789,7 @@ def import_constrains(self, obj, item): constrains.setImmediatelyAddableTypes(immediately_addable_types) except ValueError: logger.warning( - "Cannot setImmediatelyAddableTypes on %s", item["@id"], - exc_info=True + "Cannot setImmediatelyAddableTypes on %s", item["@id"], exc_info=True ) def import_review_state(self, obj, item): @@ -832,7 +876,7 @@ def handle_image_container(self, item): container = api.content.get(path=container_path) if not container: raise RuntimeError( - u"Target folder {} for type {} is missing".format( + "Target folder {} for type {} is missing".format( container_path, item["@type"] ) ) @@ -944,7 +988,9 @@ def create_container(self, item): # Handle folderish Documents provided by plone.volto fti = getUtility(IDexterityFTI, name="Document") - parent_type = "Document" if fti.klass.endswith("FolderishDocument") else "Folder" + parent_type = ( + "Document" if fti.klass.endswith("FolderishDocument") else "Folder" + ) # create original structure for imported content for element in parent_path: if element not in folder: @@ -954,7 +1000,11 @@ def create_container(self, item): id=element, title=element, ) - logger.info(u"Created container %s to hold %s", folder.absolute_url(), item["@id"]) + logger.info( + "Created container %s to hold %s", + folder.absolute_url(), + item["@id"], + ) else: folder = folder[element] @@ -989,16 +1039,18 @@ def fix_portal_type(portal_type): class ResetModifiedAndCreatedDate(BrowserView): def __call__(self): - self.title = _(u"Reset creation and modification date") - self.help_text = _("

Creation- and modification-dates are changed during import." \ - "This resets them to the original dates of the imported content.

") + self.title = _("Reset creation and modification date") + self.help_text = _( + "

Creation- and modification-dates are changed during import." + "This resets them to the original dates of the imported content.

" + ) if not self.request.form.get("form.submitted", False): return self.index() portal = api.portal.get() portal.ZopeFindAndApply(portal, search_sub=True, apply_func=reset_dates) - msg = _(u"Finished resetting creation and modification dates.") + msg = _("Finished resetting creation and modification dates.") logger.info(msg) api.portal.show_message(msg, self.request) return self.index() @@ -1020,12 +1072,16 @@ def reset_dates(obj, path): class FixCollectionQueries(BrowserView): def __call__(self): - self.title = _(u"Fix collection queries") - self.help_text = _(u"""

This fixes invalid collection-criteria that were imported from Plone 4 or 5.

""") + self.title = _("Fix collection queries") + self.help_text = _( + """

This fixes invalid collection-criteria that were imported from Plone 4 or 5.

""" + ) if not HAS_COLLECTION_FIX: api.portal.show_message( - _(u"plone.app.querystring.upgrades.fix_select_all_existing_collections is not available"), + _( + "plone.app.querystring.upgrades.fix_select_all_existing_collections is not available" + ), self.request, ) return self.index() diff --git a/src/collective/exportimport/import_other.py b/src/collective/exportimport/import_other.py index 043337a5..d2fd5161 100644 --- a/src/collective/exportimport/import_other.py +++ b/src/collective/exportimport/import_other.py @@ -1,29 +1,28 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base from BTrees.LLBTree import LLSet -from collective.exportimport import _ -from collective.exportimport import config +from collective.exportimport import _, config +from collective.exportimport.export_other import PORTAL_PLACEHOLDER from collective.exportimport.interfaces import IMigrationMarker from datetime import datetime from OFS.interfaces import IOrderedContainer from operator import itemgetter -from collective.exportimport.export_other import PORTAL_PLACEHOLDER from plone import api from plone.app.discussion.comment import Comment from plone.app.discussion.interfaces import IConversation from plone.app.portlets.interfaces import IPortletTypeInterface from plone.app.redirector.interfaces import IRedirectionStorage -from plone.portlets.interfaces import ILocalPortletAssignmentManager -from plone.portlets.interfaces import IPortletAssignmentMapping -from plone.portlets.interfaces import IPortletAssignmentSettings -from plone.portlets.interfaces import IPortletManager +from plone.portlets.interfaces import ( + ILocalPortletAssignmentManager, + IPortletAssignmentMapping, + IPortletAssignmentSettings, + IPortletManager, +) from plone.restapi.interfaces import IFieldDeserializer from Products.Five import BrowserView from Products.ZCatalog.ProgressHandler import ZLogHandler from zope.annotation.interfaces import IAnnotations -from zope.component import getUtility -from zope.component import queryMultiAdapter -from zope.component import queryUtility +from zope.component import getUtility, queryMultiAdapter, queryUtility from zope.component.interfaces import IFactory from zope.container.interfaces import INameChooser from zope.globalrequest import getRequest @@ -36,6 +35,7 @@ import six import transaction + try: from collective.relationhelpers import api as relapi @@ -90,7 +90,7 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" msg = e api.portal.show_message( - _(u"Failure while uploading: {}").format(e), + _("Failure while uploading: {}").format(e), request=self.request, ) else: @@ -135,7 +135,7 @@ def import_translations(self, data): if len(tg_with_obj) < 2: less_than_2.append(translationgroup) - logger.info(u"Only one item: {}".format(translationgroup)) + logger.info("Only one item: {}".format(translationgroup)) continue imported += 1 @@ -146,7 +146,7 @@ def import_translations(self, data): translation = obj link_translations(canonical, translation, lang) logger.info( - u"Imported {} translation-groups. For {} groups we found only one item. {} groups without content dropped".format( + "Imported {} translation-groups. For {} groups we found only one item. {} groups without content dropped".format( imported, len(less_than_2), len(empty) ) ) @@ -167,9 +167,10 @@ def link_translations(obj, translation, language): try: ITranslationManager(obj).register_translation(language, translation) except TypeError as e: - logger.info(u"Item is not translatable: {}".format(e)) + logger.info("Item is not translatable: {}".format(e)) else: + class ImportTranslations(BrowserView): def __call__(self, jsonfile=None, return_json=False): return "This view only works when using plone.app.multilingual >= 2.0.0" @@ -194,13 +195,13 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _(u"Failure while uploading: {}").format(e), + _("Failure while uploading: {}").format(e), request=self.request, ) else: groups = self.import_groups(data["groups"]) members = self.import_members(data["members"]) - msg = _(u"Imported {} groups and {} members").format(groups, members) + msg = _("Imported {} groups and {} members").format(groups, members) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -247,22 +248,18 @@ def import_members(self, data): for item in data: username = item["username"] if api.user.get(username=username) is not None: - logger.error(u"Skipping: User {} already exists!".format(username)) + logger.error("Skipping: User {} already exists!".format(username)) continue password = item.pop("password") roles = item.pop("roles") groups = item.pop("groups") if not item["email"]: - logger.info( - u"Skipping user {} without email: {}".format(username, item) - ) + logger.info("Skipping user {} without email: {}".format(username, item)) continue try: pr.addMember(username, password, roles, [], item) except ValueError: - logger.info( - u"ValueError {} : {}".format(username, item) - ) + logger.info("ValueError {} : {}".format(username, item)) continue for group in groups: if group not in groupsDict.keys(): @@ -274,7 +271,6 @@ def import_members(self, data): class ImportRelations(BrowserView): - # Overwrite to handle scustom relations RELATIONSHIP_FIELD_MAPPING = { # default relations of Plone 4 > 5 @@ -283,10 +279,11 @@ class ImportRelations(BrowserView): } def __call__(self, jsonfile=None, return_json=False): - if not HAS_RELAPI and not HAS_PLONE6: api.portal.show_message( - _("You need either Plone 6 or collective.relationhelpers to import relations"), + _( + "You need either Plone 6 or collective.relationhelpers to import relations" + ), self.request, ) return self.index() @@ -305,7 +302,7 @@ def __call__(self, jsonfile=None, return_json=False): except Exception as e: status = "error" logger.error(e) - msg = _(u"Failure while uploading: {}").format(e) + msg = _("Failure while uploading: {}").format(e) api.portal.show_message(msg, request=self.request) else: msg = self.do_import(data) @@ -385,12 +382,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _(u"Failure while uploading: {}").format(e), + _("Failure while uploading: {}").format(e), request=self.request, ) else: localroles = self.import_localroles(data) - msg = _(u"Imported {} localroles").format(localroles) + msg = _("Imported {} localroles").format(localroles) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -407,25 +404,29 @@ def import_localroles(self, data): if item["uuid"] == PORTAL_PLACEHOLDER: obj = api.portal.get() else: - logger.info("Could not find object to set localroles on. UUID: {}".format(item["uuid"])) + logger.info( + "Could not find object to set localroles on. UUID: {}".format( + item["uuid"] + ) + ) continue if item.get("localroles"): localroles = item["localroles"] for userid in localroles: obj.manage_setLocalRoles(userid=userid, roles=localroles[userid]) logger.debug( - u"Set roles on {}: {}".format(obj.absolute_url(), localroles) + "Set roles on {}: {}".format(obj.absolute_url(), localroles) ) if item.get("block"): obj.__ac_local_roles_block__ = 1 logger.debug( - u"Disable acquisition of local roles on {}".format( + "Disable acquisition of local roles on {}".format( obj.absolute_url() ) ) if not index % 1000: logger.info( - u"Set local roles on {} ({}%) of {} items".format( + "Set local roles on {} ({}%) of {} items".format( index, round(index / total * 100, 2), total ) ) @@ -457,7 +458,7 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _(u"Failure while uploading: {}").format(e), + _("Failure while uploading: {}").format(e), request=self.request, ) else: @@ -465,7 +466,9 @@ def __call__(self, jsonfile=None, return_json=False): orders = self.import_ordering(data) end = datetime.now() delta = end - start - msg = _(u"Imported {} orders in {} seconds").format(orders, delta.seconds) + msg = _("Imported {} orders in {} seconds").format( + orders, delta.seconds + ) logger.info(msg) api.portal.show_message(msg, self.request) if return_json: @@ -487,7 +490,7 @@ def import_ordering(self, data): ordered.moveObjectToPosition(obj.getId(), item["order"]) if not index % 1000: logger.info( - u"Ordered {} ({}%) of {} items".format( + "Ordered {} ({}%) of {} items".format( index, round(index / total * 100, 2), total ) ) @@ -513,12 +516,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - u"Failure while uploading: {}".format(e), + "Failure while uploading: {}".format(e), request=self.request, ) else: defaultpages = self.import_default_pages(data) - msg = _(u"Changed {} default pages").format(defaultpages) + msg = _("Changed {} default pages").format(defaultpages) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -546,7 +549,7 @@ def import_default_pages(self, data): default_page = item["default_page"] if default_page not in obj: logger.info( - u"Default page not a child: %s not in %s", + "Default page not a child: %s not in %s", default_page, obj.absolute_url(), ) @@ -561,7 +564,7 @@ def import_default_pages(self, data): else: obj.setDefaultPage(default_page) logger.debug( - u"Set %s as default page for %s", default_page, obj.absolute_url() + "Set %s as default page for %s", default_page, obj.absolute_url() ) results += 1 return results @@ -586,12 +589,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _(u"Failure while uploading: {}").format(e), + _("Failure while uploading: {}").format(e), request=self.request, ) else: results = self.import_data(data) - msg = _(u"Imported {} comments").format(results) + msg = _("Imported {} comments").format(results) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -609,7 +612,6 @@ def import_data(self, data): conversation = IConversation(obj) for item in conversation_data["conversation"]["items"]: - if isinstance(item["text"], dict) and item["text"].get("data"): item["text"] = item["text"]["data"] @@ -624,9 +626,7 @@ def import_data(self, data): comment.author_username = item["author_username"] comment.creator = item["author_username"] comment.text = unescape( - item["text"] - .replace(u"\r
", u"\r\n") - .replace(u"
", u"\r\n") + item["text"].replace("\r
", "\r\n").replace("
", "\r\n") ) if item["user_notification"]: @@ -682,12 +682,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _(u"Failure while uploading: {}").format(e), + _("Failure while uploading: {}").format(e), request=self.request, ) else: portlets = self.import_portlets(data) - msg = _(u"Created {} portlets").format(portlets) + msg = _("Created {} portlets").format(portlets) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -703,7 +703,11 @@ def import_portlets(self, data): if item["uuid"] == PORTAL_PLACEHOLDER: obj = api.portal.get() else: - logger.info("Could not find object to set portlet on UUID: {}".format(item["uuid"])) + logger.info( + "Could not find object to set portlet on UUID: {}".format( + item["uuid"] + ) + ) continue registered_portlets = register_portlets(obj, item) results += registered_portlets @@ -721,7 +725,7 @@ def register_portlets(obj, item): for manager_name, portlets in item.get("portlets", {}).items(): manager = queryUtility(IPortletManager, manager_name) if not manager: - logger.info(u"No portlet manager {}".format(manager_name)) + logger.info("No portlet manager {}".format(manager_name)) continue mapping = queryMultiAdapter((obj, manager), IPortletAssignmentMapping) namechooser = INameChooser(mapping) @@ -732,7 +736,7 @@ def register_portlets(obj, item): portlet_type = portlet_data["type"] portlet_factory = queryUtility(IFactory, name=portlet_type) if not portlet_factory: - logger.info(u"No factory for portlet {}".format(portlet_type)) + logger.info("No factory for portlet {}".format(portlet_type)) continue assignment = portlet_factory() @@ -809,7 +813,7 @@ def register_portlets(obj, item): value = deserializer(value) except Exception as e: logger.info( - u"Could not import portlet data {} for field {} on {}: {}".format( + "Could not import portlet data {} for field {} on {}: {}".format( value, field, obj.absolute_url(), str(e) ) ) @@ -817,7 +821,7 @@ def register_portlets(obj, item): field.set(assignment, value) logger.info( - u"Added {} '{}' to {} of {}".format( + "Added {} '{}' to {} of {}".format( portlet_type, name, manager_name, obj.absolute_url() ) ) @@ -865,12 +869,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _(u"Failure while uploading: {}").format(e), + _("Failure while uploading: {}").format(e), request=self.request, ) else: import_plone_redirects(data) - msg = _(u"Redirects imported") + msg = _("Redirects imported") api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} diff --git a/src/collective/exportimport/interfaces.py b/src/collective/exportimport/interfaces.py index c2ef14dc..9dc5523e 100644 --- a/src/collective/exportimport/interfaces.py +++ b/src/collective/exportimport/interfaces.py @@ -20,4 +20,4 @@ class IMigrationMarker(Interface): class ITalesField(Interface): - """a marker interface to export TalesField """ + """a marker interface to export TalesField""" diff --git a/src/collective/exportimport/serializer.py b/src/collective/exportimport/serializer.py index d7695a76..23f2e1d3 100644 --- a/src/collective/exportimport/serializer.py +++ b/src/collective/exportimport/serializer.py @@ -1,33 +1,29 @@ # -*- coding: utf-8 -*- -from collective.exportimport.interfaces import IBase64BlobsMarker -from collective.exportimport.interfaces import IMigrationMarker -from collective.exportimport.interfaces import IPathBlobsMarker -from collective.exportimport.interfaces import IRawRichTextMarker -from collective.exportimport.interfaces import ITalesField +from collective.exportimport.interfaces import ( + IBase64BlobsMarker, + IMigrationMarker, + IPathBlobsMarker, + IRawRichTextMarker, + ITalesField, +) from hurry.filesize import size from plone.app.textfield.interfaces import IRichText from plone.dexterity.interfaces import IDexterityContent -from plone.namedfile.interfaces import INamedFileField -from plone.namedfile.interfaces import INamedImageField -from plone.restapi.interfaces import IFieldSerializer -from plone.restapi.interfaces import IJsonCompatible +from plone.namedfile.interfaces import INamedFileField, INamedImageField +from plone.restapi.interfaces import IFieldSerializer, IJsonCompatible from plone.restapi.serializer.converters import json_compatible from plone.restapi.serializer.dxfields import DefaultFieldSerializer from Products.CMFCore.utils import getToolByName -from zope.component import adapter -from zope.component import getUtility -from zope.interface import implementer -from zope.interface import Interface -from zope.schema.interfaces import IChoice -from zope.schema.interfaces import ICollection -from zope.schema.interfaces import IField -from zope.schema.interfaces import IVocabularyTokenized +from zope.component import adapter, getUtility +from zope.interface import implementer, Interface +from zope.schema.interfaces import IChoice, ICollection, IField, IVocabularyTokenized import base64 import logging import pkg_resources import six + try: pkg_resources.get_distribution("Products.Archetypes") except pkg_resources.DistributionNotFound: @@ -75,6 +71,7 @@ def get_blob_path(blob): # Custom Serializers for Dexterity + @adapter(INamedImageField, IDexterityContent, IBase64BlobsMarker) class ImageFieldSerializerWithBlobs(DefaultFieldSerializer): def __call__(self): @@ -130,9 +127,9 @@ def __call__(self): if value: output = value.raw return { - u"data": json_compatible(output), - u"content-type": json_compatible(value.mimeType), - u"encoding": json_compatible(value.encoding), + "data": json_compatible(output), + "content-type": json_compatible(value.mimeType), + "encoding": json_compatible(value.encoding), } @@ -161,7 +158,13 @@ def __call__(self): except LookupError: # TODO: handle defaultFactory? if v not in [self.field.default, self.field.missing_value]: - logger.info("Term lookup error: %r not in vocabulary %r for field %r of %r", v, value_type.vocabularyName, self.field.__name__, self.context) + logger.info( + "Term lookup error: %r not in vocabulary %r for field %r of %r", + v, + value_type.vocabularyName, + self.field.__name__, + self.context, + ) return json_compatible(value) @@ -184,7 +187,13 @@ def __call__(self): except LookupError: # TODO: handle defaultFactory? if value not in [self.field.default, self.field.missing_value]: - logger.info("Term lookup error: %r not in vocabulary %r for field %r of %r", value, self.field.vocabularyName, self.field.__name__, self.context) + logger.info( + "Term lookup error: %r not in vocabulary %r for field %r of %r", + value, + self.field.vocabularyName, + self.field.__name__, + self.context, + ) return json_compatible(value) @@ -193,20 +202,17 @@ def __call__(self): if HAS_AT: from OFS.Image import Pdata - from plone.app.blob.interfaces import IBlobField - from plone.app.blob.interfaces import IBlobImageField + from plone.app.blob.interfaces import IBlobField, IBlobImageField from plone.restapi.serializer.atfields import ( DefaultFieldSerializer as ATDefaultFieldSerializer, ) from Products.Archetypes.atapi import RichWidget from Products.Archetypes.interfaces import IBaseObject - from Products.Archetypes.interfaces.field import IFileField - from Products.Archetypes.interfaces.field import IImageField - from Products.Archetypes.interfaces.field import ITextField + from Products.Archetypes.interfaces.field import IFileField, IImageField, ITextField if HAS_TALES: - from zope.interface import classImplements from Products.TALESField._field import TALESString + from zope.interface import classImplements # Products.TalesField does not implements any interface # we mark the field class to let queryMultiAdapter intercept @@ -230,7 +236,7 @@ def __call__(self): data = image.data.data if isinstance(image.data, Pdata) else image.data if len(data) > IMAGE_SIZE_WARNING: logger.info( - u"Large image for {}: {}".format( + "Large image for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -261,7 +267,7 @@ def __call__(self): ) if len(data) > FILE_SIZE_WARNING: logger.info( - u"Large file for {}: {}".format( + "Large file for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -284,7 +290,7 @@ def __call__(self): data = image.data.data if isinstance(image.data, Pdata) else image.data if len(data) > IMAGE_SIZE_WARNING: logger.info( - u"Large image for {}: {}".format( + "Large image for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -310,7 +316,7 @@ def __call__(self): ) if len(data) > FILE_SIZE_WARNING: logger.info( - u"Large File for {}: {}".format( + "Large File for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -416,28 +422,43 @@ def __call__(self, version=None, include_items=False): registry = reader.parseRegistry() # Inject new selection-operators that were added in Plone 5 - selection = registry["plone"]["app"]["querystring"]["operation"]["selection"] + selection = registry["plone"]["app"]["querystring"]["operation"][ + "selection" + ] new_operators = ["all", "any", "none"] - for operator in new_operators: + for operator in new_operators: if operator not in selection: # just a dummy method to pass validation selection[operator] = {"operation": "collective.exportimport"} # Inject any operator for some fields any_operator = "plone.app.querystring.operation.selection.any" - fields_with_any_operator = ['Creator', 'Subject', 'portal_type', 'review_state'] + fields_with_any_operator = [ + "Creator", + "Subject", + "portal_type", + "review_state", + ] for field in fields_with_any_operator: - operations = registry["plone"]["app"]["querystring"]["field"][field]["operations"] + operations = registry["plone"]["app"]["querystring"]["field"][field][ + "operations" + ] if any_operator not in operations: - registry["plone"]["app"]["querystring"]["field"][field]["operations"].append(any_operator) + registry["plone"]["app"]["querystring"]["field"][field][ + "operations" + ].append(any_operator) # Inject all operator for Subject - all_operator= "plone.app.querystring.operation.selection.all" + all_operator = "plone.app.querystring.operation.selection.all" fields_with_any_operator = ["Subject"] for field in fields_with_any_operator: - operations = registry["plone"]["app"]["querystring"]["field"][field]["operations"] + operations = registry["plone"]["app"]["querystring"]["field"][field][ + "operations" + ] if all_operator not in operations: - registry["plone"]["app"]["querystring"]["field"][field]["operations"].append(all_operator) + registry["plone"]["app"]["querystring"]["field"][field][ + "operations" + ].append(all_operator) # 3. Migrate criteria using the converters from p.a.contenttypes criteria = self.context.listCriteria() @@ -451,14 +472,18 @@ def __call__(self, version=None, include_items=False): converter = CONVERTERS.get(type_) if converter is None: - msg = u"Unsupported criterion {0}".format(type_) + msg = "Unsupported criterion {0}".format(type_) logger.error(msg) raise ValueError(msg) before = len(query) try: converter(query, criterion, registry) except Exception: - logger.info(u"Error converting criterion %s", criterion.__dict__, exc_info=True) + logger.info( + "Error converting criterion %s", + criterion.__dict__, + exc_info=True, + ) pass # Try to manually convert when no criterion was added @@ -468,21 +493,22 @@ def __call__(self, version=None, include_items=False): if fixed: query.append(fixed) else: - logger.info(u"Check maybe broken collection %s", self.context.absolute_url()) + logger.info( + "Check maybe broken collection %s", + self.context.absolute_url(), + ) # 4. So some manual fixes in the migrated query indexes_to_fix = [ - u"portal_type", - u"review_state", - u"Creator", - u"Subject", + "portal_type", + "review_state", + "Creator", + "Subject", ] operator_mapping = { # old -> new - u"plone.app.querystring.operation.selection.is": - u"plone.app.querystring.operation.selection.any", - u"plone.app.querystring.operation.string.is": - u"plone.app.querystring.operation.selection.any", + "plone.app.querystring.operation.selection.is": "plone.app.querystring.operation.selection.any", + "plone.app.querystring.operation.string.is": "plone.app.querystring.operation.selection.any", } fixed_query = [] for crit in query: @@ -493,7 +519,7 @@ def __call__(self, version=None, include_items=False): for old_operator, new_operator in operator_mapping.items(): if crit["o"] == old_operator: crit["o"] = new_operator - if crit["o"] == u"plone.app.querystring.operation.string.currentUser": + if crit["o"] == "plone.app.querystring.operation.string.currentUser": crit["v"] = "" fixed_query.append(crit) query = fixed_query diff --git a/src/collective/exportimport/templates/export_content.pt b/src/collective/exportimport/templates/export_content.pt index a7672239..9b9f2651 100644 --- a/src/collective/exportimport/templates/export_content.pt +++ b/src/collective/exportimport/templates/export_content.pt @@ -162,9 +162,14 @@ Save to file on server +
+ + +
-
diff --git a/src/collective/exportimport/templates/import_content.pt b/src/collective/exportimport/templates/import_content.pt index dab24813..411d4ff9 100644 --- a/src/collective/exportimport/templates/import_content.pt +++ b/src/collective/exportimport/templates/import_content.pt @@ -34,6 +34,23 @@
+ +

Or you can choose from a tree export in the server in one of these paths:

+ +

No files found.

+
+ +
+ +
+ +
diff --git a/src/collective/exportimport/testing.py b/src/collective/exportimport/testing.py index f5191e7e..4dd426e3 100644 --- a/src/collective/exportimport/testing.py +++ b/src/collective/exportimport/testing.py @@ -1,15 +1,16 @@ # -*- coding: utf-8 -*- from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE -from plone.app.testing import applyProfile -from plone.app.testing import FunctionalTesting -from plone.app.testing import IntegrationTesting -from plone.app.testing import PloneSandboxLayer +from plone.app.testing import ( + applyProfile, + FunctionalTesting, + IntegrationTesting, + PloneSandboxLayer, +) import collective.exportimport class CollectiveExportimportLayer(PloneSandboxLayer): - defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,) def setUpZope(self, app, configurationContext): diff --git a/src/collective/exportimport/tests/test_drop_and_include.py b/src/collective/exportimport/tests/test_drop_and_include.py index 26a91e32..f16d6aa7 100644 --- a/src/collective/exportimport/tests/test_drop_and_include.py +++ b/src/collective/exportimport/tests/test_drop_and_include.py @@ -8,70 +8,70 @@ class NoIncludeAndNoDrop(ImportContent): class IncludeAndNoDrop(ImportContent): - INCLUDE_PATHS = ['/Plone/include'] + INCLUDE_PATHS = ["/Plone/include"] class NoIncludeAndDrop(ImportContent): - DROP_PATHS = ['/Plone/drop'] + DROP_PATHS = ["/Plone/drop"] class IncludeAndDrop(ImportContent): - INCLUDE_PATHS = ['/Plone/include'] - DROP_PATHS = ['/Plone/include/drop', '/Plone/drop'] + INCLUDE_PATHS = ["/Plone/include"] + DROP_PATHS = ["/Plone/include/drop", "/Plone/drop"] class TestDropAndInclude(unittest.TestCase): def test_no_include_and_no_drop(self): view = NoIncludeAndNoDrop(None, None) - self.assertFalse(view.should_drop('/Plone/testdocument')) - self.assertTrue(view.must_process('/Plone/testdocument')) + self.assertFalse(view.should_drop("/Plone/testdocument")) + self.assertTrue(view.must_process("/Plone/testdocument")) def test_include_and_no_drop(self): view = IncludeAndNoDrop(None, None) - self.assertFalse(view.should_drop('/Plone/testdocument')) - self.assertFalse(view.should_include('/Plone/testdocument')) - self.assertTrue(view.should_include('/Plone/include')) - self.assertTrue(view.should_include('/Plone/include/testdocument')) - self.assertFalse(view.must_process('/Plone/testdocument')) - self.assertTrue(view.must_process('/Plone/include')) - self.assertTrue(view.must_process('/Plone/include/testdocument')) + self.assertFalse(view.should_drop("/Plone/testdocument")) + self.assertFalse(view.should_include("/Plone/testdocument")) + self.assertTrue(view.should_include("/Plone/include")) + self.assertTrue(view.should_include("/Plone/include/testdocument")) + self.assertFalse(view.must_process("/Plone/testdocument")) + self.assertTrue(view.must_process("/Plone/include")) + self.assertTrue(view.must_process("/Plone/include/testdocument")) def test_no_include_and_drop(self): view = NoIncludeAndDrop(None, None) - self.assertFalse(view.should_drop('/Plone/testdocument')) - self.assertTrue(view.should_drop('/Plone/drop')) - self.assertTrue(view.should_drop('/Plone/drop/testdocument')) + self.assertFalse(view.should_drop("/Plone/testdocument")) + self.assertTrue(view.should_drop("/Plone/drop")) + self.assertTrue(view.should_drop("/Plone/drop/testdocument")) - self.assertFalse(view.should_include('/Plone/drop/testdocument')) - self.assertFalse(view.should_include('/Plone/testdocument')) + self.assertFalse(view.should_include("/Plone/drop/testdocument")) + self.assertFalse(view.should_include("/Plone/testdocument")) - self.assertFalse(view.must_process('/Plone/drop')) - self.assertTrue(view.must_process('/Plone/testdocument')) - self.assertFalse(view.must_process('/Plone/drop/testdocument')) + self.assertFalse(view.must_process("/Plone/drop")) + self.assertTrue(view.must_process("/Plone/testdocument")) + self.assertFalse(view.must_process("/Plone/drop/testdocument")) def test_include_and_drop(self): view = IncludeAndDrop(None, None) - self.assertTrue(view.should_drop('/Plone/drop')) - self.assertFalse(view.should_drop('/Plone/testdocument')) - self.assertTrue(view.should_drop('/Plone/drop/testdocument')) - self.assertFalse(view.should_drop('/Plone/include/testdocument')) - self.assertTrue(view.should_drop('/Plone/include/drop/testdocument')) - self.assertFalse(view.should_drop('/Plone/include')) - self.assertTrue(view.should_drop('/Plone/include/drop')) - - self.assertFalse(view.should_include('/Plone/drop')) - self.assertFalse(view.should_include('/Plone/testdocument')) - self.assertFalse(view.should_include('/Plone/drop/testdocument')) - self.assertTrue(view.should_include('/Plone/include/testdocument')) - self.assertTrue(view.should_include('/Plone/include/drop/testdocument')) - self.assertTrue(view.should_include('/Plone/include')) - self.assertTrue(view.should_include('/Plone/include/drop')) - - self.assertFalse(view.must_process('/Plone/drop')) - self.assertFalse(view.must_process('/Plone/testdocument')) - self.assertFalse(view.must_process('/Plone/drop/testdocument')) - self.assertTrue(view.must_process('/Plone/include/testdocument')) - self.assertFalse(view.must_process('/Plone/include/drop/testdocument')) - self.assertTrue(view.must_process('/Plone/include')) - self.assertFalse(view.must_process('/Plone/include/drop')) + self.assertTrue(view.should_drop("/Plone/drop")) + self.assertFalse(view.should_drop("/Plone/testdocument")) + self.assertTrue(view.should_drop("/Plone/drop/testdocument")) + self.assertFalse(view.should_drop("/Plone/include/testdocument")) + self.assertTrue(view.should_drop("/Plone/include/drop/testdocument")) + self.assertFalse(view.should_drop("/Plone/include")) + self.assertTrue(view.should_drop("/Plone/include/drop")) + + self.assertFalse(view.should_include("/Plone/drop")) + self.assertFalse(view.should_include("/Plone/testdocument")) + self.assertFalse(view.should_include("/Plone/drop/testdocument")) + self.assertTrue(view.should_include("/Plone/include/testdocument")) + self.assertTrue(view.should_include("/Plone/include/drop/testdocument")) + self.assertTrue(view.should_include("/Plone/include")) + self.assertTrue(view.should_include("/Plone/include/drop")) + + self.assertFalse(view.must_process("/Plone/drop")) + self.assertFalse(view.must_process("/Plone/testdocument")) + self.assertFalse(view.must_process("/Plone/drop/testdocument")) + self.assertTrue(view.must_process("/Plone/include/testdocument")) + self.assertFalse(view.must_process("/Plone/include/drop/testdocument")) + self.assertTrue(view.must_process("/Plone/include")) + self.assertFalse(view.must_process("/Plone/include/drop")) diff --git a/src/collective/exportimport/tests/test_export.py b/src/collective/exportimport/tests/test_export.py index 3b9fd5e6..4f88bc0b 100644 --- a/src/collective/exportimport/tests/test_export.py +++ b/src/collective/exportimport/tests/test_export.py @@ -1,19 +1,15 @@ # -*- coding: utf-8 -*- from collective.exportimport import config -from collective.exportimport.testing import ( - COLLECTIVE_EXPORTIMPORT_FUNCTIONAL_TESTING, # noqa: E501, +from collective.exportimport.testing import ( # noqa: E501, + COLLECTIVE_EXPORTIMPORT_FUNCTIONAL_TESTING, ) from OFS.interfaces import IOrderedContainer from plone import api from plone.app.discussion.interfaces import IConversation -from plone.app.testing import login -from plone.app.testing import SITE_OWNER_NAME -from plone.app.testing import SITE_OWNER_PASSWORD -from plone.app.testing import TEST_USER_ID +from plone.app.testing import login, SITE_OWNER_NAME, SITE_OWNER_PASSWORD, TEST_USER_ID from z3c.relationfield import RelationValue from zope.annotation.interfaces import IAnnotations -from zope.component import createObject -from zope.component import getUtility +from zope.component import createObject, getUtility from zope.intid.interfaces import IIntIds from zope.lifecycleevent import modified @@ -25,6 +21,7 @@ import transaction import unittest + try: from plone.testing import zope @@ -437,9 +434,9 @@ def test_export_relations(self): data, [ { - u"to_uuid": doc2.UID(), - u"relationship": u"relatedItems", - u"from_uuid": doc1.UID(), + "to_uuid": doc2.UID(), + "relationship": "relatedItems", + "from_uuid": doc1.UID(), } ], ) @@ -453,7 +450,7 @@ def test_export_discussion(self): ) conversation = IConversation(doc1) comment = createObject("plone.Comment") - comment.text = u"Comment text" + comment.text = "Comment text" conversation.addComment(comment) transaction.commit() @@ -558,8 +555,8 @@ def test_export_redirects(self): self.assertDictEqual( data, { - u"/plone/doc1": u"/plone/doc1-moved", - u"/plone/doc2": u"/plone/doc2-moved", + "/plone/doc1": "/plone/doc1-moved", + "/plone/doc2": "/plone/doc2-moved", }, ) @@ -602,7 +599,7 @@ def test_export_versions(self): # in Plone 4.3 this is somehow not set... IAnnotations(request)[ "plone.app.versioningbehavior-changeNote" - ] = u"initial_version_changeNote" + ] = "initial_version_changeNote" doc1 = api.content.create( container=portal, type="Document", @@ -624,16 +621,16 @@ def test_export_versions(self): description="A Description", ) - doc1.title = u"Document 1 with changed title" + doc1.title = "Document 1 with changed title" modified(doc1) - doc2.title = u"Document 2 with changed title" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"Föö bar" + doc2.title = "Document 2 with changed title" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "Föö bar" modified(doc2) - doc2.description = u"New description in revision 3" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"I am new!" + doc2.description = "New description in revision 3" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "I am new!" modified(doc2) - folder1.title = u"Folder 1 with changed title" + folder1.title = "Folder 1 with changed title" modified(folder1) transaction.commit() @@ -674,19 +671,19 @@ def test_export_versions(self): self.assertEqual(len(versions), 2) # check first version - self.assertEqual(versions["0"]["title"], u"Document 2") - self.assertEqual(versions["0"]["description"], u"A Description") - self.assertEqual(versions["0"]["changeNote"], u"initial_version_changeNote") + self.assertEqual(versions["0"]["title"], "Document 2") + self.assertEqual(versions["0"]["description"], "A Description") + self.assertEqual(versions["0"]["changeNote"], "initial_version_changeNote") # check version 2 - self.assertEqual(versions["1"]["title"], u"Document 2 with changed title") - self.assertEqual(versions["1"]["description"], u"A Description") - self.assertEqual(versions["1"]["changeNote"], u"Föö bar") + self.assertEqual(versions["1"]["title"], "Document 2 with changed title") + self.assertEqual(versions["1"]["description"], "A Description") + self.assertEqual(versions["1"]["changeNote"], "Föö bar") # final/current version is the item itself - self.assertEqual(item["title"], u"Document 2 with changed title") - self.assertEqual(item["description"], u"New description in revision 3") - self.assertEqual(item["changeNote"], u"I am new!") + self.assertEqual(item["title"], "Document 2 with changed title") + self.assertEqual(item["description"], "New description in revision 3") + self.assertEqual(item["changeNote"], "I am new!") def test_export_blob_as_base64(self): # First create some content with blobs. @@ -702,9 +699,9 @@ def test_export_blob_as_base64(self): container=portal, type="File", id="file1", - title=u"File 1", + title="File 1", ) - file1.file = NamedBlobFile(data=file_data, filename=u"file.pdf") + file1.file = NamedBlobFile(data=file_data, filename="file.pdf") transaction.commit() # Now export @@ -753,9 +750,9 @@ def test_export_blob_as_download_urls(self): container=portal, type="File", id="file1", - title=u"File 1", + title="File 1", ) - file1.file = NamedBlobFile(data=file_data, filename=u"file.pdf") + file1.file = NamedBlobFile(data=file_data, filename="file.pdf") transaction.commit() # Now export @@ -787,5 +784,7 @@ def test_export_blob_as_download_urls(self): self.assertEqual(info["title"], file1.Title()) self.assertEqual(info["file"]["content-type"], "application/pdf") self.assertEqual(info["file"]["filename"], "file.pdf") - self.assertEqual(info["file"]["download"], "http://nohost/plone/file1/@@download/file") + self.assertEqual( + info["file"]["download"], "http://nohost/plone/file1/@@download/file" + ) self.assertEqual(info["file"]["size"], 8561) diff --git a/src/collective/exportimport/tests/test_fix_html.py b/src/collective/exportimport/tests/test_fix_html.py index 1621f4cf..99ba667b 100644 --- a/src/collective/exportimport/tests/test_fix_html.py +++ b/src/collective/exportimport/tests/test_fix_html.py @@ -3,14 +3,14 @@ from collective.exportimport.testing import COLLECTIVE_EXPORTIMPORT_INTEGRATION_TESTING from importlib import import_module from plone import api -from plone.app.testing import login -from plone.app.testing import SITE_OWNER_NAME +from plone.app.testing import login, SITE_OWNER_NAME from plone.app.textfield.value import RichTextValue from plone.namedfile.file import NamedImage from Products.CMFPlone.tests import dummy import unittest + HAS_PLONE_6 = getattr( import_module("Products.CMFPlone.factory"), "PLONE60MARKER", False ) @@ -41,26 +41,26 @@ def create_demo_content(self): container=portal, type="Folder", id="about", - title=u"About", + title="About", ) self.team = api.content.create( container=self.about, type="Document", id="team", - title=u"Team", + title="Team", ) self.contact = api.content.create( container=self.about, type="Document", id="contact", - title=u"Contact", + title="Contact", ) self.image = api.content.create( container=portal, type="Image", - title=u"Image", + title="Image", id="image", - image=NamedImage(dummy.Image(), "image/gif", u"test.gif"), + image=NamedImage(dummy.Image(), "image/gif", "test.gif"), ) def test_html_fixer(self): @@ -121,19 +121,25 @@ def test_html_fixer(self): # image with srcset old_text = '' - fixed_html = ''.format(self.image.UID()) + fixed_html = ''.format( + self.image.UID() + ) output = html_fixer(old_text, self.team) self.assertEqual(output, fixed_html) # relative embed of content old_text = '

' - fixed_html = '

'.format(self.team.UID()) + fixed_html = '

'.format( + self.team.UID() + ) output = html_fixer(old_text, self.team) self.assertEqual(output, fixed_html) # relative video/audio embed old_text = '

' - fixed_html = '

'.format(self.team.UID()) + fixed_html = '

'.format( + self.team.UID() + ) output = html_fixer(old_text, self.team) self.assertEqual(output, fixed_html) @@ -171,12 +177,16 @@ def test_fix_html_form(self): form = self.portal.restrictedTraverse("@@fix_html") html = form() self.assertIn("Fix links to content and images in richtext", html) - self.request.form.update({ - "form.submitted": True, - "form.commit": False, - }) + self.request.form.update( + { + "form.submitted": True, + "form.commit": False, + } + ) html = form() - self.assertIn("Fixed HTML for 1 fields in content items. Fixed HTML for 0 portlets.", html) + self.assertIn( + "Fixed HTML for 1 fields in content items. Fixed HTML for 0 portlets.", html + ) fixed_html = """

Links to uuid

Link to view/form

@@ -200,7 +210,9 @@ def test_fix_html_form(self):

-""".format(self.contact.UID(), self.team.UID(), self.image.UID()) +""".format( + self.contact.UID(), self.team.UID(), self.image.UID() + ) self.assertEqual(fixed_html, doc.text.raw) @@ -218,10 +230,12 @@ def test_fix_html_status_message(self): form = self.portal.restrictedTraverse("@@fix_html") html = form() self.assertIn("Fix links to content and images in richtext", html) - self.request.form.update({ - "form.submitted": True, - "form.commit": False, - }) + self.request.form.update( + { + "form.submitted": True, + "form.commit": False, + } + ) html = form() self.assertIn( "Fixed HTML for 1 fields in content items. Fixed HTML for 0 portlets.", @@ -240,14 +254,18 @@ def test_fix_html_does_not_change_normal_links(self): ) form = self.portal.restrictedTraverse("@@fix_html") html = form() - self.request.form.update({ - "form.submitted": True, - "form.commit": False, - }) + self.request.form.update( + { + "form.submitted": True, + "form.commit": False, + } + ) html = form() fixed_html = 'Result for the fight between Rudd-O and Andufo' self.assertEqual(fixed_html, doc.text.raw) - self.assertIn("Fixed HTML for 0 fields in content items. Fixed HTML for 0 portlets.", html) + self.assertIn( + "Fixed HTML for 0 fields in content items. Fixed HTML for 0 portlets.", html + ) def test_html_fixer_commas_in_href(self): self.create_demo_content() diff --git a/src/collective/exportimport/tests/test_import.py b/src/collective/exportimport/tests/test_import.py index 222254a7..645e613d 100644 --- a/src/collective/exportimport/tests/test_import.py +++ b/src/collective/exportimport/tests/test_import.py @@ -6,14 +6,10 @@ from OFS.interfaces import IOrderedContainer from plone import api from plone.app.redirector.interfaces import IRedirectionStorage -from plone.app.testing import login -from plone.app.testing import SITE_OWNER_NAME -from plone.app.testing import SITE_OWNER_PASSWORD +from plone.app.testing import login, SITE_OWNER_NAME, SITE_OWNER_PASSWORD from plone.app.textfield.value import RichTextValue -from plone.namedfile.file import NamedBlobImage -from plone.namedfile.file import NamedImage -from Products.CMFPlone.interfaces.constrains import ENABLED -from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes +from plone.namedfile.file import NamedBlobImage, NamedImage +from Products.CMFPlone.interfaces.constrains import ENABLED, ISelectableConstrainTypes from Products.CMFPlone.tests import dummy from time import sleep from zope.annotation.interfaces import IAnnotations @@ -28,6 +24,7 @@ import transaction import unittest + try: from plone.testing import zope @@ -76,38 +73,38 @@ def create_demo_content(self): container=portal, type="Link", id="blog", - title=u"Blog", + title="Blog", ) self.about = api.content.create( container=portal, type="Folder", id="about", - title=u"About", + title="About", ) self.events = api.content.create( container=portal, type="Folder", id="events", - title=u"Events", + title="Events", ) self.team = api.content.create( container=self.about, type="Document", id="team", - title=u"Team", + title="Team", ) self.contact = api.content.create( container=self.about, type="Document", id="contact", - title=u"Contact", + title="Contact", ) self.image = api.content.create( container=portal, type="Image", - title=u"Image", + title="Image", id="image", - image=NamedImage(dummy.Image(), "image/gif", u"test.gif"), + image=NamedImage(dummy.Image(), "image/gif", "test.gif"), ) def remove_demo_content(self): @@ -567,6 +564,7 @@ def test_import_content_from_server_file_and_return_json(self): def test_import_content_from_central_directory(self): from collective.exportimport import config + import tempfile # First create some content. @@ -680,7 +678,7 @@ def test_import_imports_but_ignores_constrains(self): container=self.about, type="Collection", id="collection", - title=u"Collection", + title="Collection", ) # constrain self.about to only allow documents constrains = ISelectableConstrainTypes(self.about) @@ -694,7 +692,7 @@ def test_import_imports_but_ignores_constrains(self): container=self.about, type="Collection", id="collection2", - title=u"Collection 2", + title="Collection 2", ) transaction.commit() @@ -738,7 +736,7 @@ def test_import_imports_but_ignores_constrains(self): container=portal["about"], type="Collection", id="collection2", - title=u"Collection 2", + title="Collection 2", ) def test_import_workflow_history(self): @@ -806,9 +804,9 @@ def _disabled_test_import_blob_path(self): self.image = api.content.create( container=portal, type="Image", - title=u"Image", + title="Image", id="image", - image=NamedBlobImage(image_data, "image/gif", u"test.gif"), + image=NamedBlobImage(image_data, "image/gif", "test.gif"), ) self.assertIn("image", portal.contentIds()) transaction.commit() @@ -1225,41 +1223,41 @@ def test_import_versions(self): # in Plone 4.3 this is somehow not set... IAnnotations(request)[ "plone.app.versioningbehavior-changeNote" - ] = u"initial_version_changeNote" + ] = "initial_version_changeNote" doc1 = api.content.create( container=portal, type="Document", id="doc1", - title=u"Document 1", - description=u"A Description", + title="Document 1", + description="A Description", ) folder1 = api.content.create( container=portal, type="Folder", id="folder1", - title=u"Folder 1", + title="Folder 1", ) doc2 = api.content.create( container=folder1, type="Document", id="doc2", - title=u"Document 2", - description=u"A Description", + title="Document 2", + description="A Description", ) modified(doc1) modified(folder1) modified(doc2) - doc1.title = u"Document 1 with changed title" + doc1.title = "Document 1 with changed title" modified(doc1) - doc2.title = u"Document 2 with changed title" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"Föö bar" + doc2.title = "Document 2 with changed title" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "Föö bar" modified(doc2) - doc2.description = u"New description in revision 3" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"I am new!" + doc2.description = "New description in revision 3" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "I am new!" modified(doc2) - folder1.title = u"Folder 1 with changed title" + folder1.title = "Folder 1 with changed title" modified(folder1) transaction.commit() @@ -1268,7 +1266,7 @@ def test_import_versions(self): oldest = repo_tool.getHistory(doc2)._retrieve( doc2, 0, preserve=[], countPurged=False ) - self.assertEqual(oldest.object.title, u"Document 2") + self.assertEqual(oldest.object.title, "Document 2") # Now export complete portal. browser = self.open_page("@@export_content") @@ -1320,8 +1318,8 @@ def test_import_versions(self): self.assertIn("doc1", portal.contentIds()) self.assertEqual(portal["folder1"].portal_type, "Folder") doc2 = portal["folder1"]["doc2"] - self.assertEqual(doc2.title, u"Document 2 with changed title") - self.assertEqual(doc2.description, u"New description in revision 3") + self.assertEqual(doc2.title, "Document 2 with changed title") + self.assertEqual(doc2.description, "New description in revision 3") history = repo_tool.getHistoryMetadata(doc2) self.assertEqual(history.getLength(countPurged=True), 4) @@ -1331,19 +1329,17 @@ def test_import_versions(self): return history_meta = history.retrieve(2) - self.assertEqual( - history_meta["metadata"]["sys_metadata"]["comment"], u"Föö bar" - ) + self.assertEqual(history_meta["metadata"]["sys_metadata"]["comment"], "Föö bar") oldest = repo_tool.getHistory(doc2)._retrieve( doc2, 0, preserve=[], countPurged=False ) - self.assertEqual(oldest.object.title, u"Document 2") + self.assertEqual(oldest.object.title, "Document 2") repo_tool.revert(portal["folder1"]["doc2"], 0) doc2 = portal["folder1"]["doc2"] - self.assertEqual(doc2.title, u"Document 2") - self.assertEqual(doc2.description, u"A Description") + self.assertEqual(doc2.title, "Document 2") + self.assertEqual(doc2.description, "A Description") def test_reset_dates(self): """Reset original modification and creation dates""" diff --git a/src/collective/exportimport/tests/test_setup.py b/src/collective/exportimport/tests/test_setup.py index 506ebd16..a0f40f50 100644 --- a/src/collective/exportimport/tests/test_setup.py +++ b/src/collective/exportimport/tests/test_setup.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Setup tests for this package.""" -from plone import api from collective.exportimport.testing import COLLECTIVE_EXPORTIMPORT_INTEGRATION_TESTING +from plone import api import unittest From 341b17ee1ba7713bc35472f3f8b0fb2cc572216c Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 18:31:27 +0200 Subject: [PATCH 02/26] Revert "hierarchical export import." This reverts commit d0535c286112348637ece90fa6d8db5ef4203021. --- docs/conf.py | 13 +- setup.py | 3 +- src/collective/exportimport/config.py | 6 +- src/collective/exportimport/deserializer.py | 1 - src/collective/exportimport/export_content.py | 128 +++++-------- src/collective/exportimport/export_other.py | 122 ++++++------ .../exportimport/filesystem_exporter.py | 68 ------- .../exportimport/filesystem_importer.py | 97 ---------- src/collective/exportimport/fix_html.py | 63 ++---- src/collective/exportimport/import_content.py | 180 ++++++------------ src/collective/exportimport/import_other.py | 114 ++++++----- src/collective/exportimport/interfaces.py | 2 +- src/collective/exportimport/serializer.py | 130 +++++-------- .../exportimport/templates/export_content.pt | 7 +- .../exportimport/templates/import_content.pt | 17 -- src/collective/exportimport/testing.py | 11 +- .../tests/test_drop_and_include.py | 88 ++++----- .../exportimport/tests/test_export.py | 69 +++---- .../exportimport/tests/test_fix_html.py | 68 +++---- .../exportimport/tests/test_import.py | 76 ++++---- .../exportimport/tests/test_setup.py | 2 +- 21 files changed, 451 insertions(+), 814 deletions(-) delete mode 100644 src/collective/exportimport/filesystem_exporter.py delete mode 100644 src/collective/exportimport/filesystem_importer.py diff --git a/docs/conf.py b/docs/conf.py index 6f420407..728ae8e5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,9 +9,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import os import sys - +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -43,18 +42,18 @@ master_doc = "index" # General information about the project. -project = "collective.exportimport" -copyright = "Philip Bauer (pbauer)" -author = "Philip Bauer (pbauer)" +project = u"collective.exportimport" +copyright = u"Philip Bauer (pbauer)" +author = u"Philip Bauer (pbauer)" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = "3.0" +version = u"3.0" # The full version, including alpha/beta/rc tags. -release = "3.0" +release = u"3.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 728b85b1..6980e329 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """Installer for the collective.exportimport package.""" -from setuptools import find_packages, setup +from setuptools import find_packages +from setuptools import setup import sys diff --git a/src/collective/exportimport/config.py b/src/collective/exportimport/config.py index 14e63b83..17dc2b62 100644 --- a/src/collective/exportimport/config.py +++ b/src/collective/exportimport/config.py @@ -12,9 +12,7 @@ os.path.expandvars(os.getenv("COLLECTIVE_EXPORTIMPORT_CENTRAL_DIRECTORY", "")) ) -TREE_DIRECTORY = "exported_tree" - -SITE_ROOT = "plone_site_root" +SITE_ROOT = 'plone_site_root' # Discussion Item has its own export / import views, don't show it in the exportable content type list -SKIPPED_CONTENTTYPE_IDS = ["Discussion Item"] +SKIPPED_CONTENTTYPE_IDS = ['Discussion Item'] diff --git a/src/collective/exportimport/deserializer.py b/src/collective/exportimport/deserializer.py index fc0a87de..bd6fa0bc 100644 --- a/src/collective/exportimport/deserializer.py +++ b/src/collective/exportimport/deserializer.py @@ -15,7 +15,6 @@ class RichTextFieldDeserializerWithoutUnescape(DefaultFieldDeserializer): """Override default RichTextFieldDeserializer without using html_parser.unescape(). Fixes https://github.com/collective/collective.exportimport/issues/99 """ - def __call__(self, value): content_type = self.field.default_mime_type encoding = "utf8" diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index 179d30b7..0c9135b3 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -1,14 +1,12 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base from App.config import getConfiguration -from collective.exportimport import _, config -from collective.exportimport.filesystem_exporter import FileSystemContentExporter -from collective.exportimport.interfaces import ( - IBase64BlobsMarker, - IMigrationMarker, - IPathBlobsMarker, - IRawRichTextMarker, -) +from collective.exportimport import _ +from collective.exportimport import config +from collective.exportimport.interfaces import IBase64BlobsMarker +from collective.exportimport.interfaces import IMigrationMarker +from collective.exportimport.interfaces import IPathBlobsMarker +from collective.exportimport.interfaces import IRawRichTextMarker from operator import itemgetter from plone import api from plone.app.layout.viewlets.content import ContentHistoryViewlet @@ -17,13 +15,16 @@ from plone.restapi.serializer.converters import json_compatible from plone.uuid.interfaces import IUUID from Products.CMFPlone.interfaces import IPloneSiteRoot -from Products.CMFPlone.interfaces.constrains import ENABLED, ISelectableConstrainTypes +from Products.CMFPlone.interfaces.constrains import ENABLED +from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes from Products.CMFPlone.utils import safe_unicode from Products.Five import BrowserView from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile -from zope.component import getMultiAdapter, getUtility +from zope.component import getMultiAdapter +from zope.component import getUtility from zope.i18n import translate -from zope.interface import alsoProvides, noLongerProvides +from zope.interface import alsoProvides +from zope.interface import noLongerProvides from zope.schema import getFields import json @@ -33,7 +34,6 @@ import six import tempfile - try: pkg_resources.get_distribution("Products.Archetypes") except pkg_resources.DistributionNotFound: @@ -65,7 +65,8 @@ IRelationList = None HAS_RELATIONS = False else: - from z3c.relationfield.interfaces import IRelationChoice, IRelationList + from z3c.relationfield.interfaces import IRelationChoice + from z3c.relationfield.interfaces import IRelationList HAS_RELATIONS = True @@ -93,6 +94,7 @@ class ExportContent(BrowserView): + template = ViewPageTemplateFile("templates/export_content.pt") QUERY = {} @@ -110,7 +112,7 @@ def __call__( download_to_server=False, migration=True, include_revisions=False, - write_errors=False, + write_errors=False ): self.portal_type = portal_type or [] if isinstance(self.portal_type, str): @@ -120,7 +122,7 @@ def __call__( self.depth = int(depth) self.depth_options = ( - ("-1", _("unlimited")), + ("-1", _(u"unlimited")), ("0", "0"), ("1", "1"), ("2", "2"), @@ -135,9 +137,9 @@ def __call__( ) self.include_blobs = int(include_blobs) self.include_blobs_options = ( - ("0", _("as download urls")), - ("1", _("as base-64 encoded strings")), - ("2", _("as blob paths")), + ("0", _(u"as download urls")), + ("1", _(u"as base-64 encoded strings")), + ("2", _(u"as blob paths")), ) self.include_revisions = include_revisions self.write_errors = write_errors or self.request.form.get("write_errors") @@ -148,9 +150,7 @@ def __call__( return self.template() if not self.portal_type: - api.portal.show_message( - _("Select at least one type to export"), self.request - ) + api.portal.show_message(_(u"Select at least one type to export"), self.request) return self.template() if self.include_blobs == 1: @@ -181,7 +181,7 @@ def __call__( content_generator = self.export_content() number = 0 - if download_to_server == 1: + if download_to_server: directory = config.CENTRAL_DIRECTORY if directory: if not os.path.exists(directory): @@ -205,12 +205,8 @@ def __call__( errors = {"unexported_paths": self.errors} json.dump(errors, f, indent=4) f.write("]") - msg = _("Exported {} items ({}) as {} to {} with {} errors").format( - number, - ", ".join(self.portal_type), - filename, - filepath, - len(self.errors), + msg = _(u"Exported {} items ({}) as {} to {} with {} errors").format( + number, ", ".join(self.portal_type), filename, filepath, len(self.errors) ) logger.info(msg) api.portal.show_message(msg, self.request) @@ -222,28 +218,6 @@ def __call__( noLongerProvides(self.request, IPathBlobsMarker) self.finish() self.request.response.redirect(self.request["ACTUAL_URL"]) - elif download_to_server == 2: - # Will generate a directory tree with one json file per item - portal_id = api.portal.get().getId() - directory = config.CENTRAL_DIRECTORY - if not directory: - cfg = getConfiguration() - directory = cfg.clienthome - rootpath = os.path.join(directory, "exported_tree/%s/content" % portal_id) - if not os.path.exists(rootpath): - os.makedirs(rootpath) - logger.info("Created tree export %s", rootpath) - - self.start() - for number, datum in enumerate(content_generator, start=1): - FileSystemContentExporter(rootpath, datum).save() - self.finish() - - msg = _("Exported {} {} with {} errors").format( - number, self.portal_type, len(self.errors) - ) - logger.info(msg) - api.portal.show_message(msg, self.request) else: with tempfile.TemporaryFile(mode="w+") as f: self.start() @@ -254,14 +228,12 @@ def __call__( f.write(",") json.dump(datum, f, sort_keys=True, indent=4) if number: - if self.errors and self.write_errors: + if self.errors and self.write_errors: f.write(",") errors = {"unexported_paths": self.errors} json.dump(errors, f, indent=4) f.write("]") - msg = _("Exported {} {} with {} errors").format( - number, self.portal_type, len(self.errors) - ) + msg = _(u"Exported {} {} with {} errors").format(number, self.portal_type, len(self.errors)) logger.info(msg) api.portal.show_message(msg, self.request) response = self.request.response @@ -309,7 +281,7 @@ def export_content(self): query = self.build_query() catalog = api.portal.get_tool("portal_catalog") brains = catalog.unrestrictedSearchResults(**query) - logger.info("Exporting {} {}".format(len(brains), self.portal_type)) + logger.info(u"Exporting {} {}".format(len(brains), self.portal_type)) # Override richtext serializer to export links using resolveuid/xxx alsoProvides(self.request, IRawRichTextMarker) @@ -327,18 +299,18 @@ def export_content(self): continue if not index % 100: - logger.info("Handled {} items...".format(index)) + logger.info(u"Handled {} items...".format(index)) try: obj = brain.getObject() except Exception: - msg = "Error getting brain {}".format(brain.getPath()) - self.errors.append({"path": None, "message": msg}) + msg = u"Error getting brain {}".format(brain.getPath()) + self.errors.append({'path':None, 'message': msg}) logger.exception(msg, exc_info=True) continue if obj is None: - msg = "brain.getObject() is None {}".format(brain.getPath()) + msg = u"brain.getObject() is None {}".format(brain.getPath()) logger.error(msg) - self.errors.append({"path": None, "message": msg}) + self.errors.append({'path':None, 'message': msg}) continue obj = self.global_obj_hook(obj) if not obj: @@ -356,8 +328,8 @@ def export_content(self): yield item except Exception: - msg = "Error exporting {}".format(obj.absolute_url()) - self.errors.append({"path": obj.absolute_url(), "message": msg}) + msg = u"Error exporting {}".format(obj.absolute_url()) + self.errors.append({'path':obj.absolute_url(), 'message':msg}) logger.exception(msg, exc_info=True) def portal_types(self): @@ -377,9 +349,7 @@ def portal_types(self): "number": number, "value": fti.id, "title": translate( - safe_unicode(fti.title), - domain="plone", - context=self.request, + safe_unicode(fti.title), domain="plone", context=self.request ), } ) @@ -410,12 +380,12 @@ def update_export_data(self, item, obj): item = self.global_dict_hook(item, obj) if not item: - logger.info("Skipping %s", obj.absolute_url()) + logger.info(u"Skipping %s", obj.absolute_url()) return item = self.custom_dict_hook(item, obj) if not item: - logger.info("Skipping %s", obj.absolute_url()) + logger.info(u"Skipping %s", obj.absolute_url()) return return item @@ -564,27 +534,15 @@ def export_revisions(self, item, obj): item_version = self.update_data_for_migration(item_version, obj) item["exportimport.versions"][version_id] = item_version # inject metadata (missing for Archetypes content): - comment = history_metadata.retrieve(version_id)["metadata"]["sys_metadata"][ - "comment" - ] - if comment and comment != item["exportimport.versions"][version_id].get( - "changeNote" - ): + comment = history_metadata.retrieve(version_id)["metadata"]["sys_metadata"]["comment"] + if comment and comment != item["exportimport.versions"][version_id].get("changeNote"): item["exportimport.versions"][version_id]["changeNote"] = comment - principal = history_metadata.retrieve(version_id)["metadata"][ - "sys_metadata" - ]["principal"] - if principal and principal != item["exportimport.versions"][version_id].get( - "changeActor" - ): + principal = history_metadata.retrieve(version_id)["metadata"]["sys_metadata"]["principal"] + if principal and principal != item["exportimport.versions"][version_id].get("changeActor"): item["exportimport.versions"][version_id]["changeActor"] = principal # current changenote - item["changeNote"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"][ - "comment" - ] - item["changeActor"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"][ - "principal" - ] + item["changeNote"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"]["comment"] + item["changeActor"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"]["principal"] return item diff --git a/src/collective/exportimport/export_other.py b/src/collective/exportimport/export_other.py index 58666471..ded77b94 100644 --- a/src/collective/exportimport/export_other.py +++ b/src/collective/exportimport/export_other.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base from App.config import getConfiguration -from collective.exportimport import _, config +from collective.exportimport import _ +from collective.exportimport import config from OFS.interfaces import IOrderedContainer from operator import itemgetter from plone import api @@ -10,18 +11,14 @@ from plone.app.redirector.interfaces import IRedirectionStorage from plone.app.textfield.value import RichTextValue from plone.app.uuid.utils import uuidToObject -from plone.portlets.constants import ( - CONTENT_TYPE_CATEGORY, - CONTEXT_CATEGORY, - GROUP_CATEGORY, - USER_CATEGORY, -) -from plone.portlets.interfaces import ( - ILocalPortletAssignmentManager, - IPortletAssignmentMapping, - IPortletAssignmentSettings, - IPortletManager, -) +from plone.portlets.constants import CONTENT_TYPE_CATEGORY +from plone.portlets.constants import CONTEXT_CATEGORY +from plone.portlets.constants import GROUP_CATEGORY +from plone.portlets.constants import USER_CATEGORY +from plone.portlets.interfaces import ILocalPortletAssignmentManager +from plone.portlets.interfaces import IPortletAssignmentMapping +from plone.portlets.interfaces import IPortletAssignmentSettings +from plone.portlets.interfaces import IPortletManager from plone.restapi.interfaces import ISerializeToJson from plone.restapi.serializer.converters import json_compatible from plone.uuid.interfaces import IUUID @@ -29,13 +26,11 @@ from Products.CMFCore.utils import getToolByName from Products.CMFPlone.interfaces import IPloneSiteRoot from Products.Five import BrowserView -from zope.component import ( - getMultiAdapter, - getUtilitiesFor, - getUtility, - queryMultiAdapter, - queryUtility, -) +from zope.component import getMultiAdapter +from zope.component import getUtilitiesFor +from zope.component import getUtility +from zope.component import queryMultiAdapter +from zope.component import queryUtility from zope.interface import providedBy import json @@ -44,7 +39,6 @@ import pkg_resources import six - try: pkg_resources.get_distribution("Products.Archetypes") except pkg_resources.DistributionNotFound: @@ -87,9 +81,9 @@ class BaseExport(BrowserView): def download(self, data): filename = self.request.form.get("filename") if not filename: - filename = "{}.json".format(self.__name__) + filename = u"{}.json".format(self.__name__) if not data: - msg = _("No data to export for {}").format(self.__name__) + msg = _(u"No data to export for {}").format(self.__name__) logger.info(msg) api.portal.show_message(msg, self.request) return self.request.response.redirect(self.request["ACTUAL_URL"]) @@ -106,7 +100,7 @@ def download(self, data): filepath = os.path.join(directory, filename) with open(filepath, "w") as f: json.dump(data, f, sort_keys=True, indent=4) - msg = _("Exported to {}").format(filepath) + msg = _(u"Exported to {}").format(filepath) logger.info(msg) api.portal.show_message(msg, self.request) return self.request.response.redirect(self.request["ACTUAL_URL"]) @@ -130,13 +124,13 @@ class ExportRelations(BaseExport): def __call__( self, download_to_server=False, debug=False, include_linkintegrity=False ): - self.title = _("Export relations") + self.title = _(u"Export relations") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting relations...") + logger.info(u"Exporting relations...") data = self.get_all_references(debug, include_linkintegrity) - logger.info("Exported %s relations", len(data)) + logger.info(u"Exported %s relations", len(data)) self.download(data) def get_all_references(self, debug=False, include_linkintegrity=False): @@ -238,7 +232,7 @@ class ExportMembers(BaseExport): def __init__(self, context, request): super(ExportMembers, self).__init__(context, request) self.pms = api.portal.get_tool("portal_membership") - self.title = _("Export members, groups and roles") + self.title = _(u"Export members, groups and roles") self.group_roles = {} def __call__(self, download_to_server=False): @@ -247,10 +241,10 @@ def __call__(self, download_to_server=False): return self.index() data = {} - logger.info("Exporting groups and users...") + logger.info(u"Exporting groups and users...") data["groups"] = self.export_groups() data["members"] = [i for i in self.export_members()] - msg = "Exported {} groups and {} members".format( + msg = u"Exported {} groups and {} members".format( len(data["groups"]), len(data["members"]) ) logger.info(msg) @@ -332,17 +326,18 @@ def _getUserData(self, userId): class ExportTranslations(BaseExport): + DROP_PATH = [] def __call__(self, download_to_server=False): - self.title = _("Export translations") + self.title = _(u"Export translations") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting translations...") + logger.info(u"Exporting translations...") data = self.all_translations() - logger.info("Exported %s groups of translations", len(data)) + logger.info(u"Exported %s groups of translations", len(data)) self.download(data) def all_translations(self): # noqa: C901 @@ -382,7 +377,7 @@ def all_translations(self): # noqa: C901 # Archetypes and Dexterity with plone.app.multilingual portal_catalog = api.portal.get_tool("portal_catalog") if "TranslationGroup" not in portal_catalog.indexes(): - logger.debug("No index TranslationGroup (p.a.multilingual not installed)") + logger.debug(u"No index TranslationGroup (p.a.multilingual not installed)") return results for uid in portal_catalog.uniqueValuesFor("TranslationGroup"): @@ -402,7 +397,7 @@ def all_translations(self): # noqa: C901 skip = True if not skip and brain.Language in item: logger.info( - "Duplicate language for {}: {}".format( + u"Duplicate language for {}: {}".format( uid, [i.getPath() for i in brains] ) ) @@ -417,14 +412,14 @@ class ExportLocalRoles(BaseExport): """Export all local roles""" def __call__(self, download_to_server=False): - self.title = _("Export local roles") + self.title = _(u"Export local roles") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting local roles...") + logger.info(u"Exporting local roles...") data = self.all_localroles() - logger.info("Exported local roles for %s items", len(data)) + logger.info(u"Exported local roles for %s items", len(data)) self.download(data) def all_localroles(self): @@ -474,14 +469,14 @@ class ExportOrdering(BaseExport): """Export all local roles""" def __call__(self, download_to_server=False): - self.title = _("Export ordering") + self.title = _(u"Export ordering") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting positions in parent...") + logger.info(u"Exporting positions in parent...") data = self.all_orders() - logger.info("Exported %s positions in parent", len(data)) + logger.info(u"Exported %s positions in parent", len(data)) self.download(data) def all_orders(self): @@ -510,14 +505,14 @@ class ExportDefaultPages(BaseExport): """Export all default_page settings.""" def __call__(self, download_to_server=False): - self.title = _("Export default pages") + self.title = _(u"Export default pages") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting default pages...") + logger.info(u"Exporting default pages...") data = self.all_default_pages() - logger.info("Exported %s default pages", len(data)) + logger.info(u"Exported %s default pages", len(data)) self.download(data) def all_default_pages(self): @@ -529,10 +524,10 @@ def all_default_pages(self): try: obj = brain.getObject() except Exception: - logger.info("Error getting obj for %s", brain.getURL(), exc_info=True) + logger.info(u"Error getting obj for %s", brain.getURL(), exc_info=True) continue if obj is None: - logger.error("brain.getObject() is None %s", brain.getPath()) + logger.error(u"brain.getObject() is None %s", brain.getPath()) continue if IPloneSiteRoot.providedBy(obj): # Site root is handled below (in Plone 6 it is returned by a catalog search) @@ -542,7 +537,7 @@ def all_default_pages(self): data = self.get_default_page_info(obj) except Exception: logger.info( - "Error exporting default_page for %s", + u"Error exporting default_page for %s", obj.absolute_url(), exc_info=True, ) @@ -559,7 +554,7 @@ def all_default_pages(self): data["uuid"] = config.SITE_ROOT results.append(data) except Exception: - logger.info("Error exporting default_page for portal", exc_info=True) + logger.info(u"Error exporting default_page for portal", exc_info=True) return results @@ -589,14 +584,14 @@ def get_default_page_info(self, obj): class ExportDiscussion(BaseExport): def __call__(self, download_to_server=False): - self.title = _("Export comments") + self.title = _(u"Export comments") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting discussions...") + logger.info(u"Exporting discussions...") data = self.all_discussions() - logger.info("Exported %s discussions", len(data)) + logger.info(u"Exported %s discussions", len(data)) self.download(data) def all_discussions(self): @@ -609,7 +604,7 @@ def all_discussions(self): try: obj = brain.getObject() if obj is None: - logger.error("brain.getObject() is None %s", brain.getPath()) + logger.error(u"brain.getObject() is None %s", brain.getPath()) continue conversation = IConversation(obj, None) if not conversation: @@ -630,22 +625,20 @@ def all_discussions(self): class ExportPortlets(BaseExport): def __call__(self, download_to_server=False): - self.title = _("Export portlets") + self.title = _(u"Export portlets") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting portlets...") + logger.info(u"Exporting portlets...") data = self.all_portlets() - logger.info("Exported info for %s items with portlets", len(data)) + logger.info(u"Exported info for %s items with portlets", len(data)) self.download(data) def all_portlets(self): self.results = [] portal = api.portal.get() - portal.ZopeFindAndApply( - self.context, search_sub=True, apply_func=self.get_portlets - ) + portal.ZopeFindAndApply(self.context, search_sub=True, apply_func=self.get_portlets) self.get_root_portlets() return self.results @@ -670,7 +663,7 @@ def _get_portlets(self, obj, uid): obj_results["uuid"] = uid self.results.append(obj_results) return - + def get_root_portlets(self): site = api.portal.get() self._get_portlets(site, PORTAL_PLACEHOLDER) @@ -682,7 +675,6 @@ def local_portlets_hook(self, portlets): def portlets_blacklist_hook(self, blacklist): return blacklist - def export_local_portlets(obj): """Serialize portlets for one content object Code mostly taken from https://github.com/plone/plone.restapi/pull/669 @@ -747,9 +739,9 @@ def export_portlets_blacklist(obj): obj_results = {} status = assignable.getBlacklistStatus(category) if status is True: - obj_results["status"] = "block" + obj_results["status"] = u"block" elif status is False: - obj_results["status"] = "show" + obj_results["status"] = u"show" if obj_results: obj_results["manager"] = manager_name @@ -784,12 +776,12 @@ def export_plone_redirects(): class ExportRedirects(BaseExport): def __call__(self, download_to_server=False): - self.title = _("Export redirects") + self.title = _(u"Export redirects") self.download_to_server = download_to_server if not self.request.form.get("form.submitted", False): return self.index() - logger.info("Exporting redirects...") + logger.info(u"Exporting redirects...") data = export_plone_redirects() - logger.info("Exported %s redirects", len(data)) + logger.info(u"Exported %s redirects", len(data)) self.download(data) diff --git a/src/collective/exportimport/filesystem_exporter.py b/src/collective/exportimport/filesystem_exporter.py deleted file mode 100644 index abd31670..00000000 --- a/src/collective/exportimport/filesystem_exporter.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -from six.moves.urllib.parse import unquote, urlparse - -import json -import os - - -class FileSystemExporter(object): - """Base FS Exporter""" - - def __init__(self, rootpath, json_item): - self.item = json_item - self.root = rootpath - - def create_dir(self, dirname): - """Creates a directory if does not exist - - Args: - dirname (str): dirname to be created - """ - dirpath = os.path.join(self.root, dirname) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - def get_parents(self, parent): - """Extracts parents of item - - Args: - parent (dict): Parent info dict - - Returns: - (str): relative path - """ - - if not parent: - return "" - - parent_url = unquote(parent["@id"]) - parent_url_parsed = urlparse(parent_url) - - # Get the path part, split it, remove the always empty first element. - parent_path = parent_url_parsed.path.split("/")[1:] - if ( - len(parent_url_parsed.netloc.split(":")) > 1 - or parent_url_parsed.netloc == "nohost" - ): - # For example localhost:8080, or nohost when running tests. - # First element will then be a Plone Site id. - # Get rid of it. - parent_path = parent_path[1:] - - return "/".join(parent_path) - - -class FileSystemContentExporter(FileSystemExporter): - """Deserializes JSON items into a FS tree""" - - def save(self): - """Saves a json file to filesystem tree - Target directory is related as original parent position in site. - """ - parent_path = self.get_parents(self.item.get("parent")) - self.create_dir(parent_path) - - filename = "%s_%s.json" % (self.item.get("@type"), self.item.get("UID")) - filepath = os.path.join(self.root, parent_path, filename) - with open(filepath, "w") as f: - json.dump(self.item, f, sort_keys=True, indent=4) diff --git a/src/collective/exportimport/filesystem_importer.py b/src/collective/exportimport/filesystem_importer.py deleted file mode 100644 index 9d0664f9..00000000 --- a/src/collective/exportimport/filesystem_importer.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- -from glob import iglob -from plone import api -from six.moves.urllib.parse import unquote, urlparse - -import json -import logging -import os -import six - - -if six.PY2: - from pathlib2 import Path -else: - from pathlib import Path - - -class FileSystemImporter(object): - """Base FS Importer""" - - logger = logging.getLogger(__name__) - - def __init__(self, server_tree_file): - self.path = server_tree_file - - def get_parents(self, parent): - """Extracts parents of item - - Args: - parent (dict): Parent info dict - - Returns: - (str): relative path - """ - - if not parent: - return "" - - parent_url = unquote(parent["@id"]) - parent_url_parsed = urlparse(parent_url) - - # Get the path part, split it, remove the always empty first element. - parent_path = parent_url_parsed.path.split("/")[1:] - if ( - len(parent_url_parsed.netloc.split(":")) > 1 - or parent_url_parsed.netloc == "nohost" - ): - # For example localhost:8080, or nohost when running tests. - # First element will then be a Plone Site id. - # Get rid of it. - parent_path = parent_path[1:] - - return "/".join(parent_path) - - -class FileSystemContentImporter(FileSystemImporter): - """Deserializes JSON items into a FS tree""" - - def list_files(self): - """Loads all json files from filesystem tree""" - files = iglob(os.path.join(self.path, "**/*.json"), recursive=True) - return files - - def get_hierarchical_files(self): - """Gets all files and folders""" - root = Path(self.path) - portal = api.portal.get() - assert root.is_dir() - json_files = root.glob("**/*.json") - for json_file in json_files: - self.logger.debug("Importing %s", json_file) - item = json.loads(json_file.read_text()) - item["json_file"] = str(json_file) - - # Modify parent data - json_parent = item.get("parent", {}) - - # Find the real parent nodes - prefix = os.path.commonprefix([str(json_file.parent), self.path]) - path = os.path.relpath(str(json_file.parent), prefix) - parents = self.get_parents(json_parent) - - if json_file.parent == Path(os.path.join(self.path, parents)): - yield item - else: - try: - parent_obj = portal.unrestrictedTraverse(path) - except KeyError: - parent_obj = portal - - if parent_obj: - item["@id"] = item.get("@id") - json_parent.update( - {"@id": parent_obj.absolute_url(), "UID": parent_obj.UID()} - ) - item["parent"] = json_parent - yield item diff --git a/src/collective/exportimport/fix_html.py b/src/collective/exportimport/fix_html.py index d511fdcc..2f21d395 100644 --- a/src/collective/exportimport/fix_html.py +++ b/src/collective/exportimport/fix_html.py @@ -1,8 +1,8 @@ # -*- coding: UTF-8 -*- from Acquisition import aq_parent from bs4 import BeautifulSoup -from collections import defaultdict from collective.exportimport import _ +from collections import defaultdict from logging import getLogger from plone import api from plone.api.exc import InvalidParameterError @@ -11,18 +11,19 @@ from plone.app.textfield.interfaces import IRichText from plone.app.textfield.value import IRichTextValue from plone.dexterity.utils import iterSchemataForType -from plone.portlets.interfaces import IPortletAssignmentMapping, IPortletManager +from plone.portlets.interfaces import IPortletAssignmentMapping +from plone.portlets.interfaces import IPortletManager from plone.uuid.interfaces import IUUID from Products.CMFCore.interfaces import IContentish from Products.Five import BrowserView from six.moves.urllib.parse import urlparse -from zope.component import getUtilitiesFor, queryMultiAdapter +from zope.component import getUtilitiesFor +from zope.component import queryMultiAdapter from zope.interface import providedBy import six import transaction - logger = getLogger(__name__) IMAGE_SCALE_MAP = { @@ -39,7 +40,7 @@ class FixHTML(BrowserView): def __call__(self): - self.title = _("Fix links to content and images in richtext") + self.title = _(u"Fix links to content and images in richtext") if not self.request.form.get("form.submitted", False): return self.index() commit = self.request.form.get("form.commit", True) @@ -47,18 +48,18 @@ def __call__(self): msg = [] fix_count = fix_html_in_content_fields(context=self.context, commit=commit) - msg.append(_("Fixed HTML for {} fields in content items").format(fix_count)) + msg.append(_(u"Fixed HTML for {} fields in content items").format(fix_count)) logger.info(msg[-1]) fix_count = fix_html_in_portlets(context=self.context) - msg.append(_("Fixed HTML for {} portlets").format(fix_count)) + msg.append(_(u"Fixed HTML for {} portlets").format(fix_count)) logger.info(msg[-1]) # TODO: Fix html in tiles # tiles = fix_html_in_tiles() # msg = u"Fixed html for {} tiles".format(tiles) - api.portal.show_message(" ".join(m + "." for m in msg), self.request) + api.portal.show_message(u" ".join(m + u"." for m in msg), self.request) return self.index() @@ -234,7 +235,7 @@ def find_object(base, path): obj = api.portal.get() portal_path = obj.absolute_url_path() + "/" if path.startswith(portal_path): - path = path[len(portal_path) :] + path = path[len(portal_path):] else: obj = aq_parent(base) # relative urls start at the parent... @@ -319,21 +320,17 @@ def table_class_fixer(text, obj=None): query["path"] = "/".join(context.getPhysicalPath()) brains = catalog(**query) total = len(brains) - logger.info( - "There are %s content items in total, starting migration...", len(brains) - ) + logger.info("There are %s content items in total, starting migration...", len(brains)) fixed_fields = 0 fixed_items = 0 for index, brain in enumerate(brains, start=1): try: obj = brain.getObject() except Exception: - logger.warning( - "Could not get object for: %s", brain.getPath(), exc_info=True - ) + logger.warning("Could not get object for: %s", brain.getPath(), exc_info=True) continue if obj is None: - logger.error("brain.getObject() is None %s", brain.getPath()) + logger.error(u"brain.getObject() is None %s", brain.getPath()) continue try: changed = False @@ -342,19 +339,11 @@ def table_class_fixer(text, obj=None): if text and IRichTextValue.providedBy(text) and text.raw: clean_text = text.raw for fixer in fixers: - logger.debug( - "Fixing html for %s with %s", - obj.absolute_url(), - fixer.__name__, - ) + logger.debug("Fixing html for %s with %s", obj.absolute_url(), fixer.__name__) try: clean_text = fixer(clean_text, obj) except Exception: - logger.info( - "Error while fixing html of %s for %s", - fieldname, - obj.absolute_url(), - ) + logger.info(u"Error while fixing html of %s for %s", fieldname, obj.absolute_url()) raise if clean_text and clean_text != text.raw: @@ -366,11 +355,7 @@ def table_class_fixer(text, obj=None): ) setattr(obj, fieldname, textvalue) changed = True - logger.debug( - "Fixed html for field %s of %s", - fieldname, - obj.absolute_url(), - ) + logger.debug(u"Fixed html for field %s of %s", fieldname, obj.absolute_url()) fixed_fields += 1 if changed: fixed_items += 1 @@ -381,21 +366,12 @@ def table_class_fixer(text, obj=None): if fixed_items != 0 and not fixed_items % 1000: # Commit every 1000 changed items. logger.info( - "Fix html for %s (%s) of %s items (changed %s fields in %s items)", - index, - round(index / total * 100, 2), - total, - fixed_fields, - fixed_items, - ) + u"Fix html for %s (%s) of %s items (changed %s fields in %s items)", + index, round(index / total * 100, 2), total, fixed_fields, fixed_items) if commit: transaction.commit() - logger.info( - "Finished fixing html in content fields (changed %s fields in %s items)", - fixed_fields, - fixed_items, - ) + logger.info(u"Finished fixing html in content fields (changed %s fields in %s items)", fixed_fields, fixed_items) if commit: # commit remaining items transaction.commit() @@ -404,6 +380,7 @@ def table_class_fixer(text, obj=None): def fix_html_in_portlets(context=None): + portlets_schemata = { iface: name for name, iface in getUtilitiesFor(IPortletTypeInterface) } diff --git a/src/collective/exportimport/import_content.py b/src/collective/exportimport/import_content.py index 25c87327..d2ce0fc4 100644 --- a/src/collective/exportimport/import_content.py +++ b/src/collective/exportimport/import_content.py @@ -1,25 +1,29 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base -from App.config import getConfiguration -from collective.exportimport import _, config -from collective.exportimport.filesystem_importer import FileSystemContentImporter +from collective.exportimport import _ +from collective.exportimport import config from collective.exportimport.interfaces import IMigrationMarker -from datetime import datetime, timedelta +from datetime import datetime from DateTime import DateTime +from datetime import timedelta from Persistence import PersistentMapping from plone import api from plone.api.exc import InvalidParameterError from plone.dexterity.interfaces import IDexterityFTI from plone.i18n.normalizer.interfaces import IIDNormalizer -from plone.namedfile.file import NamedBlobFile, NamedBlobImage +from plone.namedfile.file import NamedBlobFile +from plone.namedfile.file import NamedBlobImage from plone.restapi.interfaces import IDeserializeFromJson -from Products.CMFPlone.interfaces.constrains import ENABLED, ISelectableConstrainTypes +from Products.CMFPlone.interfaces.constrains import ENABLED +from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes from Products.CMFPlone.utils import _createObjectByType from Products.Five import BrowserView from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile -from six.moves.urllib.parse import unquote, urlparse +from six.moves.urllib.parse import unquote +from six.moves.urllib.parse import urlparse from zExceptions import NotFound -from zope.component import getMultiAdapter, getUtility +from zope.component import getMultiAdapter +from zope.component import getUtility from zope.interface import alsoProvides from ZPublisher.HTTPRequest import FileUpload @@ -32,7 +36,6 @@ import six import transaction - try: from plone.app.querystring.upgrades import fix_select_all_existing_collections @@ -71,6 +74,7 @@ def get_absolute_blob_path(obj, blob_path): class ImportContent(BrowserView): + template = ViewPageTemplateFile("templates/import_content.pt") # You can specify a default-target container for all items of a type. @@ -108,8 +112,7 @@ def __call__( return_json=False, limit=None, server_file=None, - server_tree_file=None, - iterator=None, + iterator=None ): request = self.request self.limit = limit @@ -133,17 +136,17 @@ def __call__( status = "success" msg = "" - if server_file and jsonfile and server_tree_file: + if server_file and jsonfile: # This is an error. But when you upload 10 GB AND select a server file, # it is a pity when you would have to upload again. api.portal.show_message( - _("json file was uploaded, so the selected server file was ignored."), + _(u"json file was uploaded, so the selected server file was ignored."), request=self.request, type="warn", ) server_file = None status = "error" - if server_file and not jsonfile and not server_tree_file: + if server_file and not jsonfile: if server_file in self.server_files: for path in self.import_paths: full_path = os.path.join(path, server_file) @@ -158,7 +161,6 @@ def __call__( api.portal.show_message(msg, request=self.request, type="warn") server_file = None status = "error" - if jsonfile: self.portal = api.portal.get() try: @@ -174,7 +176,7 @@ def __call__( status = "error" msg = str(e) api.portal.show_message( - _("Exception during upload: {}").format(e), + _(u"Exception during upload: {}").format(e), request=self.request, ) else: @@ -190,18 +192,11 @@ def __call__( msg = self.do_import(iterator) api.portal.show_message(msg, self.request) - if server_tree_file and not server_file and not jsonfile: - msg = self.do_import( - FileSystemContentImporter(server_tree_file).get_hierarchical_files() - ) - api.portal.show_message(msg, self.request) - self.finish() if return_json: msg = {"state": status, "msg": msg} return json.dumps(msg) - return self.template() def start(self): @@ -212,7 +207,7 @@ def finish(self): def commit_hook(self, added, index): """Hook to do something after importing every x items.""" - msg = "Committing after creating {} of {} handled items...".format( + msg = u"Committing after creating {} of {} handled items...".format( len(added), index ) logger.info(msg) @@ -248,51 +243,16 @@ def server_files(self): listing.sort() return listing - @property - def import_tree_parts(self): - """Returns subdirectories in export tree""" - directory = config.CENTRAL_DIRECTORY - if not directory: - cfg = getConfiguration() - directory = cfg.clienthome - base_path = os.path.join(directory, config.TREE_DIRECTORY) - return [os.path.join(base_path, d, "content") for d in os.listdir(base_path)] - - def process_tree_files(self, files): - """Reads and uploads tree files""" - for path in files: - logger.info("Using server tree file %s", path) - # Open the file in binary mode and use it as jsonfile. - with open(path, "rb") as jsonfile: - try: - if isinstance(jsonfile, str): - data = ijson.items(jsonfile, "item") - elif isinstance(jsonfile, FileUpload) or hasattr(jsonfile, "read"): - data = ijson.items(jsonfile, "item") - else: - raise RuntimeError("Data is neither text, file nor upload.") - except Exception as e: - logger.error(str(e)) - msg = str(e) - api.portal.show_message( - _("Exception during upload: {}").format(e), - request=self.request, - ) - else: - self.start() - msg = self.do_import(data) - self.finish() - def do_import(self, data): start = datetime.now() alsoProvides(self.request, IMigrationMarker) added = self.import_new_content(data) end = datetime.now() delta = end - start - msg = "Imported {} items".format(len(added)) + msg = u"Imported {} items".format(len(added)) transaction.get().note(msg) transaction.commit() - msg = "{} in {} seconds".format(msg, delta.seconds) + msg = u"{} in {} seconds".format(msg, delta.seconds) logger.info(msg) return msg @@ -313,9 +273,7 @@ def must_process(self, item_path): if not self.should_include(item_path): return False elif self.should_drop(item_path): - logger.info( - "Skipping %s, even though listed in INCLUDE_PATHS", item_path - ) + logger.info(u"Skipping %s, even though listed in INCLUDE_PATHS", item_path) return False else: if self.should_drop(item_path): @@ -327,9 +285,9 @@ def import_new_content(self, data): # noqa: C901 added = [] if getattr(data, "len", None): - logger.info("Importing {} items".format(len(data))) + logger.info(u"Importing {} items".format(len(data))) else: - logger.info("Importing data") + logger.info(u"Importing data") for index, item in enumerate(data, start=1): if self.limit and len(added) >= self.limit: break @@ -348,7 +306,7 @@ def import_new_content(self, data): # noqa: C901 new_id = unquote(item["@id"]).split("/")[-1] if new_id != item["id"]: logger.info( - "Conflicting ids in url ({}) and id ({}). Using {}".format( + u"Conflicting ids in url ({}) and id ({}). Using {}".format( new_id, item["id"], new_id ) ) @@ -376,7 +334,7 @@ def import_new_content(self, data): # noqa: C901 if not container: logger.warning( - "No container (parent was {}) found for {} {}".format( + u"No container (parent was {}) found for {} {}".format( item["parent"]["@type"], item["@type"], item["@id"] ) ) @@ -384,7 +342,7 @@ def import_new_content(self, data): # noqa: C901 if not getattr(aq_base(container), "isPrincipiaFolderish", False): logger.warning( - "Container {} for {} is not folderish".format( + u"Container {} for {} is not folderish".format( container.absolute_url(), item["@id"] ) ) @@ -398,7 +356,7 @@ def import_new_content(self, data): # noqa: C901 if self.handle_existing_content == 0: # Skip logger.info( - "{} ({}) already exists. Skipping it.".format( + u"{} ({}) already exists. Skipping it.".format( item["id"], item["@id"] ) ) @@ -407,7 +365,7 @@ def import_new_content(self, data): # noqa: C901 elif self.handle_existing_content == 1: # Replace content before creating it new logger.info( - "{} ({}) already exists. Replacing it.".format( + u"{} ({}) already exists. Replacing it.".format( item["id"], item["@id"] ) ) @@ -416,7 +374,7 @@ def import_new_content(self, data): # noqa: C901 elif self.handle_existing_content == 2: # Update existing item logger.info( - "{} ({}) already exists. Updating it.".format( + u"{} ({}) already exists. Updating it.".format( item["id"], item["@id"] ) ) @@ -428,7 +386,7 @@ def import_new_content(self, data): # noqa: C901 duplicate = item["id"] item["id"] = "{}-{}".format(item["id"], random.randint(1000, 9999)) logger.info( - "{} ({}) already exists. Created as {}".format( + u"{} ({}) already exists. Created as {}".format( duplicate, item["@id"], item["id"] ) ) @@ -456,17 +414,16 @@ def import_new_content(self, data): # noqa: C901 if self.commit and not len(added) % self.commit: self.commit_hook(added, index) except Exception as e: - item_id = item["@id"].split("/")[-1] + item_id = item['@id'].split('/')[-1] container.manage_delObjects(item_id) logger.warning(e) - logger.warning( - "Didn't add %s %s", item["@type"], item["@id"], exc_info=True - ) + logger.warning("Didn't add %s %s", item["@type"], item["@id"], exc_info=True) continue return added def handle_new_object(self, item, index, new): + new, item = self.global_obj_hook_before_deserializing(new, item) # import using plone.restapi deserializers @@ -475,15 +432,13 @@ def handle_new_object(self, item, index, new): try: new = deserializer(validate_all=False, data=item) except TypeError as error: - if "unexpected keyword argument" in str(error): + if 'unexpected keyword argument' in str(error): self.request["BODY"] = json.dumps(item) new = deserializer(validate_all=False) else: raise error except Exception: - logger.warning( - "Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True - ) + logger.warning("Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True) raise # Blobs can be exported as only a path in the blob storage. @@ -499,7 +454,9 @@ def handle_new_object(self, item, index, new): # Happens only when we import content that doesn't have a UID # for instance when importing from non Plone systems. logger.info( - "Created new UID for item %s with type %s.", item["@id"], item["@type"] + "Created new UID for item %s with type %s.", + item["@id"], + item["@type"] ) item["UID"] = uuid @@ -525,7 +482,9 @@ def handle_new_object(self, item, index, new): new.creation_date = creation_date new.aq_base.creation_date_migrated = creation_date logger.info( - "Created item #{}: {} {}".format(index, item["@type"], new.absolute_url()) + "Created item #{}: {} {}".format( + index, item["@type"], new.absolute_url() + ) ) return new @@ -587,12 +546,7 @@ def import_versions(self, container, item): try: new = deserializer(validate_all=False, data=version) except Exception: - logger.warning( - "Cannot deserialize %s %s", - item["@type"], - item["@id"], - exc_info=True, - ) + logger.warning("Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True) return self.save_revision(new, version, initial) @@ -603,9 +557,7 @@ def import_versions(self, container, item): try: new = deserializer(validate_all=False, data=item) except Exception: - logger.warning( - "Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True - ) + logger.warning("Cannot deserialize %s %s", item["@type"], item["@id"], exc_info=True) return self.import_blob_paths(new, item) @@ -667,7 +619,7 @@ def save_revision(self, obj, item, initial=False): from plone.app.versioningbehavior import _ as PAV if initial: - comment = PAV("initial_version_changeNote", default="Initial version") + comment = PAV(u"initial_version_changeNote", default=u"Initial version") else: comment = item.get("changeNote") sys_metadata = { @@ -774,12 +726,15 @@ def import_constrains(self, obj, item): return constrains.setConstrainTypesMode(ENABLED) - locally_allowed_types = item["exportimport.constrains"]["locally_allowed_types"] + locally_allowed_types = item["exportimport.constrains"][ + "locally_allowed_types" + ] try: constrains.setLocallyAllowedTypes(locally_allowed_types) except ValueError: logger.warning( - "Cannot setLocallyAllowedTypes on %s", item["@id"], exc_info=True + "Cannot setLocallyAllowedTypes on %s", item["@id"], + exc_info=True ) immediately_addable_types = item["exportimport.constrains"][ @@ -789,7 +744,8 @@ def import_constrains(self, obj, item): constrains.setImmediatelyAddableTypes(immediately_addable_types) except ValueError: logger.warning( - "Cannot setImmediatelyAddableTypes on %s", item["@id"], exc_info=True + "Cannot setImmediatelyAddableTypes on %s", item["@id"], + exc_info=True ) def import_review_state(self, obj, item): @@ -876,7 +832,7 @@ def handle_image_container(self, item): container = api.content.get(path=container_path) if not container: raise RuntimeError( - "Target folder {} for type {} is missing".format( + u"Target folder {} for type {} is missing".format( container_path, item["@type"] ) ) @@ -988,9 +944,7 @@ def create_container(self, item): # Handle folderish Documents provided by plone.volto fti = getUtility(IDexterityFTI, name="Document") - parent_type = ( - "Document" if fti.klass.endswith("FolderishDocument") else "Folder" - ) + parent_type = "Document" if fti.klass.endswith("FolderishDocument") else "Folder" # create original structure for imported content for element in parent_path: if element not in folder: @@ -1000,11 +954,7 @@ def create_container(self, item): id=element, title=element, ) - logger.info( - "Created container %s to hold %s", - folder.absolute_url(), - item["@id"], - ) + logger.info(u"Created container %s to hold %s", folder.absolute_url(), item["@id"]) else: folder = folder[element] @@ -1039,18 +989,16 @@ def fix_portal_type(portal_type): class ResetModifiedAndCreatedDate(BrowserView): def __call__(self): - self.title = _("Reset creation and modification date") - self.help_text = _( - "

Creation- and modification-dates are changed during import." - "This resets them to the original dates of the imported content.

" - ) + self.title = _(u"Reset creation and modification date") + self.help_text = _("

Creation- and modification-dates are changed during import." \ + "This resets them to the original dates of the imported content.

") if not self.request.form.get("form.submitted", False): return self.index() portal = api.portal.get() portal.ZopeFindAndApply(portal, search_sub=True, apply_func=reset_dates) - msg = _("Finished resetting creation and modification dates.") + msg = _(u"Finished resetting creation and modification dates.") logger.info(msg) api.portal.show_message(msg, self.request) return self.index() @@ -1072,16 +1020,12 @@ def reset_dates(obj, path): class FixCollectionQueries(BrowserView): def __call__(self): - self.title = _("Fix collection queries") - self.help_text = _( - """

This fixes invalid collection-criteria that were imported from Plone 4 or 5.

""" - ) + self.title = _(u"Fix collection queries") + self.help_text = _(u"""

This fixes invalid collection-criteria that were imported from Plone 4 or 5.

""") if not HAS_COLLECTION_FIX: api.portal.show_message( - _( - "plone.app.querystring.upgrades.fix_select_all_existing_collections is not available" - ), + _(u"plone.app.querystring.upgrades.fix_select_all_existing_collections is not available"), self.request, ) return self.index() diff --git a/src/collective/exportimport/import_other.py b/src/collective/exportimport/import_other.py index d2fd5161..043337a5 100644 --- a/src/collective/exportimport/import_other.py +++ b/src/collective/exportimport/import_other.py @@ -1,28 +1,29 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base from BTrees.LLBTree import LLSet -from collective.exportimport import _, config -from collective.exportimport.export_other import PORTAL_PLACEHOLDER +from collective.exportimport import _ +from collective.exportimport import config from collective.exportimport.interfaces import IMigrationMarker from datetime import datetime from OFS.interfaces import IOrderedContainer from operator import itemgetter +from collective.exportimport.export_other import PORTAL_PLACEHOLDER from plone import api from plone.app.discussion.comment import Comment from plone.app.discussion.interfaces import IConversation from plone.app.portlets.interfaces import IPortletTypeInterface from plone.app.redirector.interfaces import IRedirectionStorage -from plone.portlets.interfaces import ( - ILocalPortletAssignmentManager, - IPortletAssignmentMapping, - IPortletAssignmentSettings, - IPortletManager, -) +from plone.portlets.interfaces import ILocalPortletAssignmentManager +from plone.portlets.interfaces import IPortletAssignmentMapping +from plone.portlets.interfaces import IPortletAssignmentSettings +from plone.portlets.interfaces import IPortletManager from plone.restapi.interfaces import IFieldDeserializer from Products.Five import BrowserView from Products.ZCatalog.ProgressHandler import ZLogHandler from zope.annotation.interfaces import IAnnotations -from zope.component import getUtility, queryMultiAdapter, queryUtility +from zope.component import getUtility +from zope.component import queryMultiAdapter +from zope.component import queryUtility from zope.component.interfaces import IFactory from zope.container.interfaces import INameChooser from zope.globalrequest import getRequest @@ -35,7 +36,6 @@ import six import transaction - try: from collective.relationhelpers import api as relapi @@ -90,7 +90,7 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" msg = e api.portal.show_message( - _("Failure while uploading: {}").format(e), + _(u"Failure while uploading: {}").format(e), request=self.request, ) else: @@ -135,7 +135,7 @@ def import_translations(self, data): if len(tg_with_obj) < 2: less_than_2.append(translationgroup) - logger.info("Only one item: {}".format(translationgroup)) + logger.info(u"Only one item: {}".format(translationgroup)) continue imported += 1 @@ -146,7 +146,7 @@ def import_translations(self, data): translation = obj link_translations(canonical, translation, lang) logger.info( - "Imported {} translation-groups. For {} groups we found only one item. {} groups without content dropped".format( + u"Imported {} translation-groups. For {} groups we found only one item. {} groups without content dropped".format( imported, len(less_than_2), len(empty) ) ) @@ -167,10 +167,9 @@ def link_translations(obj, translation, language): try: ITranslationManager(obj).register_translation(language, translation) except TypeError as e: - logger.info("Item is not translatable: {}".format(e)) + logger.info(u"Item is not translatable: {}".format(e)) else: - class ImportTranslations(BrowserView): def __call__(self, jsonfile=None, return_json=False): return "This view only works when using plone.app.multilingual >= 2.0.0" @@ -195,13 +194,13 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _("Failure while uploading: {}").format(e), + _(u"Failure while uploading: {}").format(e), request=self.request, ) else: groups = self.import_groups(data["groups"]) members = self.import_members(data["members"]) - msg = _("Imported {} groups and {} members").format(groups, members) + msg = _(u"Imported {} groups and {} members").format(groups, members) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -248,18 +247,22 @@ def import_members(self, data): for item in data: username = item["username"] if api.user.get(username=username) is not None: - logger.error("Skipping: User {} already exists!".format(username)) + logger.error(u"Skipping: User {} already exists!".format(username)) continue password = item.pop("password") roles = item.pop("roles") groups = item.pop("groups") if not item["email"]: - logger.info("Skipping user {} without email: {}".format(username, item)) + logger.info( + u"Skipping user {} without email: {}".format(username, item) + ) continue try: pr.addMember(username, password, roles, [], item) except ValueError: - logger.info("ValueError {} : {}".format(username, item)) + logger.info( + u"ValueError {} : {}".format(username, item) + ) continue for group in groups: if group not in groupsDict.keys(): @@ -271,6 +274,7 @@ def import_members(self, data): class ImportRelations(BrowserView): + # Overwrite to handle scustom relations RELATIONSHIP_FIELD_MAPPING = { # default relations of Plone 4 > 5 @@ -279,11 +283,10 @@ class ImportRelations(BrowserView): } def __call__(self, jsonfile=None, return_json=False): + if not HAS_RELAPI and not HAS_PLONE6: api.portal.show_message( - _( - "You need either Plone 6 or collective.relationhelpers to import relations" - ), + _("You need either Plone 6 or collective.relationhelpers to import relations"), self.request, ) return self.index() @@ -302,7 +305,7 @@ def __call__(self, jsonfile=None, return_json=False): except Exception as e: status = "error" logger.error(e) - msg = _("Failure while uploading: {}").format(e) + msg = _(u"Failure while uploading: {}").format(e) api.portal.show_message(msg, request=self.request) else: msg = self.do_import(data) @@ -382,12 +385,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _("Failure while uploading: {}").format(e), + _(u"Failure while uploading: {}").format(e), request=self.request, ) else: localroles = self.import_localroles(data) - msg = _("Imported {} localroles").format(localroles) + msg = _(u"Imported {} localroles").format(localroles) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -404,29 +407,25 @@ def import_localroles(self, data): if item["uuid"] == PORTAL_PLACEHOLDER: obj = api.portal.get() else: - logger.info( - "Could not find object to set localroles on. UUID: {}".format( - item["uuid"] - ) - ) + logger.info("Could not find object to set localroles on. UUID: {}".format(item["uuid"])) continue if item.get("localroles"): localroles = item["localroles"] for userid in localroles: obj.manage_setLocalRoles(userid=userid, roles=localroles[userid]) logger.debug( - "Set roles on {}: {}".format(obj.absolute_url(), localroles) + u"Set roles on {}: {}".format(obj.absolute_url(), localroles) ) if item.get("block"): obj.__ac_local_roles_block__ = 1 logger.debug( - "Disable acquisition of local roles on {}".format( + u"Disable acquisition of local roles on {}".format( obj.absolute_url() ) ) if not index % 1000: logger.info( - "Set local roles on {} ({}%) of {} items".format( + u"Set local roles on {} ({}%) of {} items".format( index, round(index / total * 100, 2), total ) ) @@ -458,7 +457,7 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _("Failure while uploading: {}").format(e), + _(u"Failure while uploading: {}").format(e), request=self.request, ) else: @@ -466,9 +465,7 @@ def __call__(self, jsonfile=None, return_json=False): orders = self.import_ordering(data) end = datetime.now() delta = end - start - msg = _("Imported {} orders in {} seconds").format( - orders, delta.seconds - ) + msg = _(u"Imported {} orders in {} seconds").format(orders, delta.seconds) logger.info(msg) api.portal.show_message(msg, self.request) if return_json: @@ -490,7 +487,7 @@ def import_ordering(self, data): ordered.moveObjectToPosition(obj.getId(), item["order"]) if not index % 1000: logger.info( - "Ordered {} ({}%) of {} items".format( + u"Ordered {} ({}%) of {} items".format( index, round(index / total * 100, 2), total ) ) @@ -516,12 +513,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - "Failure while uploading: {}".format(e), + u"Failure while uploading: {}".format(e), request=self.request, ) else: defaultpages = self.import_default_pages(data) - msg = _("Changed {} default pages").format(defaultpages) + msg = _(u"Changed {} default pages").format(defaultpages) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -549,7 +546,7 @@ def import_default_pages(self, data): default_page = item["default_page"] if default_page not in obj: logger.info( - "Default page not a child: %s not in %s", + u"Default page not a child: %s not in %s", default_page, obj.absolute_url(), ) @@ -564,7 +561,7 @@ def import_default_pages(self, data): else: obj.setDefaultPage(default_page) logger.debug( - "Set %s as default page for %s", default_page, obj.absolute_url() + u"Set %s as default page for %s", default_page, obj.absolute_url() ) results += 1 return results @@ -589,12 +586,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _("Failure while uploading: {}").format(e), + _(u"Failure while uploading: {}").format(e), request=self.request, ) else: results = self.import_data(data) - msg = _("Imported {} comments").format(results) + msg = _(u"Imported {} comments").format(results) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -612,6 +609,7 @@ def import_data(self, data): conversation = IConversation(obj) for item in conversation_data["conversation"]["items"]: + if isinstance(item["text"], dict) and item["text"].get("data"): item["text"] = item["text"]["data"] @@ -626,7 +624,9 @@ def import_data(self, data): comment.author_username = item["author_username"] comment.creator = item["author_username"] comment.text = unescape( - item["text"].replace("\r
", "\r\n").replace("
", "\r\n") + item["text"] + .replace(u"\r
", u"\r\n") + .replace(u"
", u"\r\n") ) if item["user_notification"]: @@ -682,12 +682,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _("Failure while uploading: {}").format(e), + _(u"Failure while uploading: {}").format(e), request=self.request, ) else: portlets = self.import_portlets(data) - msg = _("Created {} portlets").format(portlets) + msg = _(u"Created {} portlets").format(portlets) api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} @@ -703,11 +703,7 @@ def import_portlets(self, data): if item["uuid"] == PORTAL_PLACEHOLDER: obj = api.portal.get() else: - logger.info( - "Could not find object to set portlet on UUID: {}".format( - item["uuid"] - ) - ) + logger.info("Could not find object to set portlet on UUID: {}".format(item["uuid"])) continue registered_portlets = register_portlets(obj, item) results += registered_portlets @@ -725,7 +721,7 @@ def register_portlets(obj, item): for manager_name, portlets in item.get("portlets", {}).items(): manager = queryUtility(IPortletManager, manager_name) if not manager: - logger.info("No portlet manager {}".format(manager_name)) + logger.info(u"No portlet manager {}".format(manager_name)) continue mapping = queryMultiAdapter((obj, manager), IPortletAssignmentMapping) namechooser = INameChooser(mapping) @@ -736,7 +732,7 @@ def register_portlets(obj, item): portlet_type = portlet_data["type"] portlet_factory = queryUtility(IFactory, name=portlet_type) if not portlet_factory: - logger.info("No factory for portlet {}".format(portlet_type)) + logger.info(u"No factory for portlet {}".format(portlet_type)) continue assignment = portlet_factory() @@ -813,7 +809,7 @@ def register_portlets(obj, item): value = deserializer(value) except Exception as e: logger.info( - "Could not import portlet data {} for field {} on {}: {}".format( + u"Could not import portlet data {} for field {} on {}: {}".format( value, field, obj.absolute_url(), str(e) ) ) @@ -821,7 +817,7 @@ def register_portlets(obj, item): field.set(assignment, value) logger.info( - "Added {} '{}' to {} of {}".format( + u"Added {} '{}' to {} of {}".format( portlet_type, name, manager_name, obj.absolute_url() ) ) @@ -869,12 +865,12 @@ def __call__(self, jsonfile=None, return_json=False): status = "error" logger.error(e) api.portal.show_message( - _("Failure while uploading: {}").format(e), + _(u"Failure while uploading: {}").format(e), request=self.request, ) else: import_plone_redirects(data) - msg = _("Redirects imported") + msg = _(u"Redirects imported") api.portal.show_message(msg, self.request) if return_json: msg = {"state": status, "msg": msg} diff --git a/src/collective/exportimport/interfaces.py b/src/collective/exportimport/interfaces.py index 9dc5523e..c2ef14dc 100644 --- a/src/collective/exportimport/interfaces.py +++ b/src/collective/exportimport/interfaces.py @@ -20,4 +20,4 @@ class IMigrationMarker(Interface): class ITalesField(Interface): - """a marker interface to export TalesField""" + """a marker interface to export TalesField """ diff --git a/src/collective/exportimport/serializer.py b/src/collective/exportimport/serializer.py index 23f2e1d3..d7695a76 100644 --- a/src/collective/exportimport/serializer.py +++ b/src/collective/exportimport/serializer.py @@ -1,29 +1,33 @@ # -*- coding: utf-8 -*- -from collective.exportimport.interfaces import ( - IBase64BlobsMarker, - IMigrationMarker, - IPathBlobsMarker, - IRawRichTextMarker, - ITalesField, -) +from collective.exportimport.interfaces import IBase64BlobsMarker +from collective.exportimport.interfaces import IMigrationMarker +from collective.exportimport.interfaces import IPathBlobsMarker +from collective.exportimport.interfaces import IRawRichTextMarker +from collective.exportimport.interfaces import ITalesField from hurry.filesize import size from plone.app.textfield.interfaces import IRichText from plone.dexterity.interfaces import IDexterityContent -from plone.namedfile.interfaces import INamedFileField, INamedImageField -from plone.restapi.interfaces import IFieldSerializer, IJsonCompatible +from plone.namedfile.interfaces import INamedFileField +from plone.namedfile.interfaces import INamedImageField +from plone.restapi.interfaces import IFieldSerializer +from plone.restapi.interfaces import IJsonCompatible from plone.restapi.serializer.converters import json_compatible from plone.restapi.serializer.dxfields import DefaultFieldSerializer from Products.CMFCore.utils import getToolByName -from zope.component import adapter, getUtility -from zope.interface import implementer, Interface -from zope.schema.interfaces import IChoice, ICollection, IField, IVocabularyTokenized +from zope.component import adapter +from zope.component import getUtility +from zope.interface import implementer +from zope.interface import Interface +from zope.schema.interfaces import IChoice +from zope.schema.interfaces import ICollection +from zope.schema.interfaces import IField +from zope.schema.interfaces import IVocabularyTokenized import base64 import logging import pkg_resources import six - try: pkg_resources.get_distribution("Products.Archetypes") except pkg_resources.DistributionNotFound: @@ -71,7 +75,6 @@ def get_blob_path(blob): # Custom Serializers for Dexterity - @adapter(INamedImageField, IDexterityContent, IBase64BlobsMarker) class ImageFieldSerializerWithBlobs(DefaultFieldSerializer): def __call__(self): @@ -127,9 +130,9 @@ def __call__(self): if value: output = value.raw return { - "data": json_compatible(output), - "content-type": json_compatible(value.mimeType), - "encoding": json_compatible(value.encoding), + u"data": json_compatible(output), + u"content-type": json_compatible(value.mimeType), + u"encoding": json_compatible(value.encoding), } @@ -158,13 +161,7 @@ def __call__(self): except LookupError: # TODO: handle defaultFactory? if v not in [self.field.default, self.field.missing_value]: - logger.info( - "Term lookup error: %r not in vocabulary %r for field %r of %r", - v, - value_type.vocabularyName, - self.field.__name__, - self.context, - ) + logger.info("Term lookup error: %r not in vocabulary %r for field %r of %r", v, value_type.vocabularyName, self.field.__name__, self.context) return json_compatible(value) @@ -187,13 +184,7 @@ def __call__(self): except LookupError: # TODO: handle defaultFactory? if value not in [self.field.default, self.field.missing_value]: - logger.info( - "Term lookup error: %r not in vocabulary %r for field %r of %r", - value, - self.field.vocabularyName, - self.field.__name__, - self.context, - ) + logger.info("Term lookup error: %r not in vocabulary %r for field %r of %r", value, self.field.vocabularyName, self.field.__name__, self.context) return json_compatible(value) @@ -202,17 +193,20 @@ def __call__(self): if HAS_AT: from OFS.Image import Pdata - from plone.app.blob.interfaces import IBlobField, IBlobImageField + from plone.app.blob.interfaces import IBlobField + from plone.app.blob.interfaces import IBlobImageField from plone.restapi.serializer.atfields import ( DefaultFieldSerializer as ATDefaultFieldSerializer, ) from Products.Archetypes.atapi import RichWidget from Products.Archetypes.interfaces import IBaseObject - from Products.Archetypes.interfaces.field import IFileField, IImageField, ITextField + from Products.Archetypes.interfaces.field import IFileField + from Products.Archetypes.interfaces.field import IImageField + from Products.Archetypes.interfaces.field import ITextField if HAS_TALES: - from Products.TALESField._field import TALESString from zope.interface import classImplements + from Products.TALESField._field import TALESString # Products.TalesField does not implements any interface # we mark the field class to let queryMultiAdapter intercept @@ -236,7 +230,7 @@ def __call__(self): data = image.data.data if isinstance(image.data, Pdata) else image.data if len(data) > IMAGE_SIZE_WARNING: logger.info( - "Large image for {}: {}".format( + u"Large image for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -267,7 +261,7 @@ def __call__(self): ) if len(data) > FILE_SIZE_WARNING: logger.info( - "Large file for {}: {}".format( + u"Large file for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -290,7 +284,7 @@ def __call__(self): data = image.data.data if isinstance(image.data, Pdata) else image.data if len(data) > IMAGE_SIZE_WARNING: logger.info( - "Large image for {}: {}".format( + u"Large image for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -316,7 +310,7 @@ def __call__(self): ) if len(data) > FILE_SIZE_WARNING: logger.info( - "Large File for {}: {}".format( + u"Large File for {}: {}".format( self.context.absolute_url(), size(len(data)) ) ) @@ -422,43 +416,28 @@ def __call__(self, version=None, include_items=False): registry = reader.parseRegistry() # Inject new selection-operators that were added in Plone 5 - selection = registry["plone"]["app"]["querystring"]["operation"][ - "selection" - ] + selection = registry["plone"]["app"]["querystring"]["operation"]["selection"] new_operators = ["all", "any", "none"] - for operator in new_operators: + for operator in new_operators: if operator not in selection: # just a dummy method to pass validation selection[operator] = {"operation": "collective.exportimport"} # Inject any operator for some fields any_operator = "plone.app.querystring.operation.selection.any" - fields_with_any_operator = [ - "Creator", - "Subject", - "portal_type", - "review_state", - ] + fields_with_any_operator = ['Creator', 'Subject', 'portal_type', 'review_state'] for field in fields_with_any_operator: - operations = registry["plone"]["app"]["querystring"]["field"][field][ - "operations" - ] + operations = registry["plone"]["app"]["querystring"]["field"][field]["operations"] if any_operator not in operations: - registry["plone"]["app"]["querystring"]["field"][field][ - "operations" - ].append(any_operator) + registry["plone"]["app"]["querystring"]["field"][field]["operations"].append(any_operator) # Inject all operator for Subject - all_operator = "plone.app.querystring.operation.selection.all" + all_operator= "plone.app.querystring.operation.selection.all" fields_with_any_operator = ["Subject"] for field in fields_with_any_operator: - operations = registry["plone"]["app"]["querystring"]["field"][field][ - "operations" - ] + operations = registry["plone"]["app"]["querystring"]["field"][field]["operations"] if all_operator not in operations: - registry["plone"]["app"]["querystring"]["field"][field][ - "operations" - ].append(all_operator) + registry["plone"]["app"]["querystring"]["field"][field]["operations"].append(all_operator) # 3. Migrate criteria using the converters from p.a.contenttypes criteria = self.context.listCriteria() @@ -472,18 +451,14 @@ def __call__(self, version=None, include_items=False): converter = CONVERTERS.get(type_) if converter is None: - msg = "Unsupported criterion {0}".format(type_) + msg = u"Unsupported criterion {0}".format(type_) logger.error(msg) raise ValueError(msg) before = len(query) try: converter(query, criterion, registry) except Exception: - logger.info( - "Error converting criterion %s", - criterion.__dict__, - exc_info=True, - ) + logger.info(u"Error converting criterion %s", criterion.__dict__, exc_info=True) pass # Try to manually convert when no criterion was added @@ -493,22 +468,21 @@ def __call__(self, version=None, include_items=False): if fixed: query.append(fixed) else: - logger.info( - "Check maybe broken collection %s", - self.context.absolute_url(), - ) + logger.info(u"Check maybe broken collection %s", self.context.absolute_url()) # 4. So some manual fixes in the migrated query indexes_to_fix = [ - "portal_type", - "review_state", - "Creator", - "Subject", + u"portal_type", + u"review_state", + u"Creator", + u"Subject", ] operator_mapping = { # old -> new - "plone.app.querystring.operation.selection.is": "plone.app.querystring.operation.selection.any", - "plone.app.querystring.operation.string.is": "plone.app.querystring.operation.selection.any", + u"plone.app.querystring.operation.selection.is": + u"plone.app.querystring.operation.selection.any", + u"plone.app.querystring.operation.string.is": + u"plone.app.querystring.operation.selection.any", } fixed_query = [] for crit in query: @@ -519,7 +493,7 @@ def __call__(self, version=None, include_items=False): for old_operator, new_operator in operator_mapping.items(): if crit["o"] == old_operator: crit["o"] = new_operator - if crit["o"] == "plone.app.querystring.operation.string.currentUser": + if crit["o"] == u"plone.app.querystring.operation.string.currentUser": crit["v"] = "" fixed_query.append(crit) query = fixed_query diff --git a/src/collective/exportimport/templates/export_content.pt b/src/collective/exportimport/templates/export_content.pt index 9b9f2651..a7672239 100644 --- a/src/collective/exportimport/templates/export_content.pt +++ b/src/collective/exportimport/templates/export_content.pt @@ -162,14 +162,9 @@ Save to file on server
-
- - -
+
diff --git a/src/collective/exportimport/templates/import_content.pt b/src/collective/exportimport/templates/import_content.pt index 411d4ff9..dab24813 100644 --- a/src/collective/exportimport/templates/import_content.pt +++ b/src/collective/exportimport/templates/import_content.pt @@ -34,23 +34,6 @@
- -

Or you can choose from a tree export in the server in one of these paths:

-
    -
    • -
-

No files found.

-
- -
- -
- -
diff --git a/src/collective/exportimport/testing.py b/src/collective/exportimport/testing.py index 4dd426e3..f5191e7e 100644 --- a/src/collective/exportimport/testing.py +++ b/src/collective/exportimport/testing.py @@ -1,16 +1,15 @@ # -*- coding: utf-8 -*- from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE -from plone.app.testing import ( - applyProfile, - FunctionalTesting, - IntegrationTesting, - PloneSandboxLayer, -) +from plone.app.testing import applyProfile +from plone.app.testing import FunctionalTesting +from plone.app.testing import IntegrationTesting +from plone.app.testing import PloneSandboxLayer import collective.exportimport class CollectiveExportimportLayer(PloneSandboxLayer): + defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,) def setUpZope(self, app, configurationContext): diff --git a/src/collective/exportimport/tests/test_drop_and_include.py b/src/collective/exportimport/tests/test_drop_and_include.py index f16d6aa7..26a91e32 100644 --- a/src/collective/exportimport/tests/test_drop_and_include.py +++ b/src/collective/exportimport/tests/test_drop_and_include.py @@ -8,70 +8,70 @@ class NoIncludeAndNoDrop(ImportContent): class IncludeAndNoDrop(ImportContent): - INCLUDE_PATHS = ["/Plone/include"] + INCLUDE_PATHS = ['/Plone/include'] class NoIncludeAndDrop(ImportContent): - DROP_PATHS = ["/Plone/drop"] + DROP_PATHS = ['/Plone/drop'] class IncludeAndDrop(ImportContent): - INCLUDE_PATHS = ["/Plone/include"] - DROP_PATHS = ["/Plone/include/drop", "/Plone/drop"] + INCLUDE_PATHS = ['/Plone/include'] + DROP_PATHS = ['/Plone/include/drop', '/Plone/drop'] class TestDropAndInclude(unittest.TestCase): def test_no_include_and_no_drop(self): view = NoIncludeAndNoDrop(None, None) - self.assertFalse(view.should_drop("/Plone/testdocument")) - self.assertTrue(view.must_process("/Plone/testdocument")) + self.assertFalse(view.should_drop('/Plone/testdocument')) + self.assertTrue(view.must_process('/Plone/testdocument')) def test_include_and_no_drop(self): view = IncludeAndNoDrop(None, None) - self.assertFalse(view.should_drop("/Plone/testdocument")) - self.assertFalse(view.should_include("/Plone/testdocument")) - self.assertTrue(view.should_include("/Plone/include")) - self.assertTrue(view.should_include("/Plone/include/testdocument")) - self.assertFalse(view.must_process("/Plone/testdocument")) - self.assertTrue(view.must_process("/Plone/include")) - self.assertTrue(view.must_process("/Plone/include/testdocument")) + self.assertFalse(view.should_drop('/Plone/testdocument')) + self.assertFalse(view.should_include('/Plone/testdocument')) + self.assertTrue(view.should_include('/Plone/include')) + self.assertTrue(view.should_include('/Plone/include/testdocument')) + self.assertFalse(view.must_process('/Plone/testdocument')) + self.assertTrue(view.must_process('/Plone/include')) + self.assertTrue(view.must_process('/Plone/include/testdocument')) def test_no_include_and_drop(self): view = NoIncludeAndDrop(None, None) - self.assertFalse(view.should_drop("/Plone/testdocument")) - self.assertTrue(view.should_drop("/Plone/drop")) - self.assertTrue(view.should_drop("/Plone/drop/testdocument")) + self.assertFalse(view.should_drop('/Plone/testdocument')) + self.assertTrue(view.should_drop('/Plone/drop')) + self.assertTrue(view.should_drop('/Plone/drop/testdocument')) - self.assertFalse(view.should_include("/Plone/drop/testdocument")) - self.assertFalse(view.should_include("/Plone/testdocument")) + self.assertFalse(view.should_include('/Plone/drop/testdocument')) + self.assertFalse(view.should_include('/Plone/testdocument')) - self.assertFalse(view.must_process("/Plone/drop")) - self.assertTrue(view.must_process("/Plone/testdocument")) - self.assertFalse(view.must_process("/Plone/drop/testdocument")) + self.assertFalse(view.must_process('/Plone/drop')) + self.assertTrue(view.must_process('/Plone/testdocument')) + self.assertFalse(view.must_process('/Plone/drop/testdocument')) def test_include_and_drop(self): view = IncludeAndDrop(None, None) - self.assertTrue(view.should_drop("/Plone/drop")) - self.assertFalse(view.should_drop("/Plone/testdocument")) - self.assertTrue(view.should_drop("/Plone/drop/testdocument")) - self.assertFalse(view.should_drop("/Plone/include/testdocument")) - self.assertTrue(view.should_drop("/Plone/include/drop/testdocument")) - self.assertFalse(view.should_drop("/Plone/include")) - self.assertTrue(view.should_drop("/Plone/include/drop")) - - self.assertFalse(view.should_include("/Plone/drop")) - self.assertFalse(view.should_include("/Plone/testdocument")) - self.assertFalse(view.should_include("/Plone/drop/testdocument")) - self.assertTrue(view.should_include("/Plone/include/testdocument")) - self.assertTrue(view.should_include("/Plone/include/drop/testdocument")) - self.assertTrue(view.should_include("/Plone/include")) - self.assertTrue(view.should_include("/Plone/include/drop")) - - self.assertFalse(view.must_process("/Plone/drop")) - self.assertFalse(view.must_process("/Plone/testdocument")) - self.assertFalse(view.must_process("/Plone/drop/testdocument")) - self.assertTrue(view.must_process("/Plone/include/testdocument")) - self.assertFalse(view.must_process("/Plone/include/drop/testdocument")) - self.assertTrue(view.must_process("/Plone/include")) - self.assertFalse(view.must_process("/Plone/include/drop")) + self.assertTrue(view.should_drop('/Plone/drop')) + self.assertFalse(view.should_drop('/Plone/testdocument')) + self.assertTrue(view.should_drop('/Plone/drop/testdocument')) + self.assertFalse(view.should_drop('/Plone/include/testdocument')) + self.assertTrue(view.should_drop('/Plone/include/drop/testdocument')) + self.assertFalse(view.should_drop('/Plone/include')) + self.assertTrue(view.should_drop('/Plone/include/drop')) + + self.assertFalse(view.should_include('/Plone/drop')) + self.assertFalse(view.should_include('/Plone/testdocument')) + self.assertFalse(view.should_include('/Plone/drop/testdocument')) + self.assertTrue(view.should_include('/Plone/include/testdocument')) + self.assertTrue(view.should_include('/Plone/include/drop/testdocument')) + self.assertTrue(view.should_include('/Plone/include')) + self.assertTrue(view.should_include('/Plone/include/drop')) + + self.assertFalse(view.must_process('/Plone/drop')) + self.assertFalse(view.must_process('/Plone/testdocument')) + self.assertFalse(view.must_process('/Plone/drop/testdocument')) + self.assertTrue(view.must_process('/Plone/include/testdocument')) + self.assertFalse(view.must_process('/Plone/include/drop/testdocument')) + self.assertTrue(view.must_process('/Plone/include')) + self.assertFalse(view.must_process('/Plone/include/drop')) diff --git a/src/collective/exportimport/tests/test_export.py b/src/collective/exportimport/tests/test_export.py index 4f88bc0b..3b9fd5e6 100644 --- a/src/collective/exportimport/tests/test_export.py +++ b/src/collective/exportimport/tests/test_export.py @@ -1,15 +1,19 @@ # -*- coding: utf-8 -*- from collective.exportimport import config -from collective.exportimport.testing import ( # noqa: E501, - COLLECTIVE_EXPORTIMPORT_FUNCTIONAL_TESTING, +from collective.exportimport.testing import ( + COLLECTIVE_EXPORTIMPORT_FUNCTIONAL_TESTING, # noqa: E501, ) from OFS.interfaces import IOrderedContainer from plone import api from plone.app.discussion.interfaces import IConversation -from plone.app.testing import login, SITE_OWNER_NAME, SITE_OWNER_PASSWORD, TEST_USER_ID +from plone.app.testing import login +from plone.app.testing import SITE_OWNER_NAME +from plone.app.testing import SITE_OWNER_PASSWORD +from plone.app.testing import TEST_USER_ID from z3c.relationfield import RelationValue from zope.annotation.interfaces import IAnnotations -from zope.component import createObject, getUtility +from zope.component import createObject +from zope.component import getUtility from zope.intid.interfaces import IIntIds from zope.lifecycleevent import modified @@ -21,7 +25,6 @@ import transaction import unittest - try: from plone.testing import zope @@ -434,9 +437,9 @@ def test_export_relations(self): data, [ { - "to_uuid": doc2.UID(), - "relationship": "relatedItems", - "from_uuid": doc1.UID(), + u"to_uuid": doc2.UID(), + u"relationship": u"relatedItems", + u"from_uuid": doc1.UID(), } ], ) @@ -450,7 +453,7 @@ def test_export_discussion(self): ) conversation = IConversation(doc1) comment = createObject("plone.Comment") - comment.text = "Comment text" + comment.text = u"Comment text" conversation.addComment(comment) transaction.commit() @@ -555,8 +558,8 @@ def test_export_redirects(self): self.assertDictEqual( data, { - "/plone/doc1": "/plone/doc1-moved", - "/plone/doc2": "/plone/doc2-moved", + u"/plone/doc1": u"/plone/doc1-moved", + u"/plone/doc2": u"/plone/doc2-moved", }, ) @@ -599,7 +602,7 @@ def test_export_versions(self): # in Plone 4.3 this is somehow not set... IAnnotations(request)[ "plone.app.versioningbehavior-changeNote" - ] = "initial_version_changeNote" + ] = u"initial_version_changeNote" doc1 = api.content.create( container=portal, type="Document", @@ -621,16 +624,16 @@ def test_export_versions(self): description="A Description", ) - doc1.title = "Document 1 with changed title" + doc1.title = u"Document 1 with changed title" modified(doc1) - doc2.title = "Document 2 with changed title" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "Föö bar" + doc2.title = u"Document 2 with changed title" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"Föö bar" modified(doc2) - doc2.description = "New description in revision 3" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "I am new!" + doc2.description = u"New description in revision 3" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"I am new!" modified(doc2) - folder1.title = "Folder 1 with changed title" + folder1.title = u"Folder 1 with changed title" modified(folder1) transaction.commit() @@ -671,19 +674,19 @@ def test_export_versions(self): self.assertEqual(len(versions), 2) # check first version - self.assertEqual(versions["0"]["title"], "Document 2") - self.assertEqual(versions["0"]["description"], "A Description") - self.assertEqual(versions["0"]["changeNote"], "initial_version_changeNote") + self.assertEqual(versions["0"]["title"], u"Document 2") + self.assertEqual(versions["0"]["description"], u"A Description") + self.assertEqual(versions["0"]["changeNote"], u"initial_version_changeNote") # check version 2 - self.assertEqual(versions["1"]["title"], "Document 2 with changed title") - self.assertEqual(versions["1"]["description"], "A Description") - self.assertEqual(versions["1"]["changeNote"], "Föö bar") + self.assertEqual(versions["1"]["title"], u"Document 2 with changed title") + self.assertEqual(versions["1"]["description"], u"A Description") + self.assertEqual(versions["1"]["changeNote"], u"Föö bar") # final/current version is the item itself - self.assertEqual(item["title"], "Document 2 with changed title") - self.assertEqual(item["description"], "New description in revision 3") - self.assertEqual(item["changeNote"], "I am new!") + self.assertEqual(item["title"], u"Document 2 with changed title") + self.assertEqual(item["description"], u"New description in revision 3") + self.assertEqual(item["changeNote"], u"I am new!") def test_export_blob_as_base64(self): # First create some content with blobs. @@ -699,9 +702,9 @@ def test_export_blob_as_base64(self): container=portal, type="File", id="file1", - title="File 1", + title=u"File 1", ) - file1.file = NamedBlobFile(data=file_data, filename="file.pdf") + file1.file = NamedBlobFile(data=file_data, filename=u"file.pdf") transaction.commit() # Now export @@ -750,9 +753,9 @@ def test_export_blob_as_download_urls(self): container=portal, type="File", id="file1", - title="File 1", + title=u"File 1", ) - file1.file = NamedBlobFile(data=file_data, filename="file.pdf") + file1.file = NamedBlobFile(data=file_data, filename=u"file.pdf") transaction.commit() # Now export @@ -784,7 +787,5 @@ def test_export_blob_as_download_urls(self): self.assertEqual(info["title"], file1.Title()) self.assertEqual(info["file"]["content-type"], "application/pdf") self.assertEqual(info["file"]["filename"], "file.pdf") - self.assertEqual( - info["file"]["download"], "http://nohost/plone/file1/@@download/file" - ) + self.assertEqual(info["file"]["download"], "http://nohost/plone/file1/@@download/file") self.assertEqual(info["file"]["size"], 8561) diff --git a/src/collective/exportimport/tests/test_fix_html.py b/src/collective/exportimport/tests/test_fix_html.py index 99ba667b..1621f4cf 100644 --- a/src/collective/exportimport/tests/test_fix_html.py +++ b/src/collective/exportimport/tests/test_fix_html.py @@ -3,14 +3,14 @@ from collective.exportimport.testing import COLLECTIVE_EXPORTIMPORT_INTEGRATION_TESTING from importlib import import_module from plone import api -from plone.app.testing import login, SITE_OWNER_NAME +from plone.app.testing import login +from plone.app.testing import SITE_OWNER_NAME from plone.app.textfield.value import RichTextValue from plone.namedfile.file import NamedImage from Products.CMFPlone.tests import dummy import unittest - HAS_PLONE_6 = getattr( import_module("Products.CMFPlone.factory"), "PLONE60MARKER", False ) @@ -41,26 +41,26 @@ def create_demo_content(self): container=portal, type="Folder", id="about", - title="About", + title=u"About", ) self.team = api.content.create( container=self.about, type="Document", id="team", - title="Team", + title=u"Team", ) self.contact = api.content.create( container=self.about, type="Document", id="contact", - title="Contact", + title=u"Contact", ) self.image = api.content.create( container=portal, type="Image", - title="Image", + title=u"Image", id="image", - image=NamedImage(dummy.Image(), "image/gif", "test.gif"), + image=NamedImage(dummy.Image(), "image/gif", u"test.gif"), ) def test_html_fixer(self): @@ -121,25 +121,19 @@ def test_html_fixer(self): # image with srcset old_text = '' - fixed_html = ''.format( - self.image.UID() - ) + fixed_html = ''.format(self.image.UID()) output = html_fixer(old_text, self.team) self.assertEqual(output, fixed_html) # relative embed of content old_text = '

' - fixed_html = '

'.format( - self.team.UID() - ) + fixed_html = '

'.format(self.team.UID()) output = html_fixer(old_text, self.team) self.assertEqual(output, fixed_html) # relative video/audio embed old_text = '

' - fixed_html = '

'.format( - self.team.UID() - ) + fixed_html = '

'.format(self.team.UID()) output = html_fixer(old_text, self.team) self.assertEqual(output, fixed_html) @@ -177,16 +171,12 @@ def test_fix_html_form(self): form = self.portal.restrictedTraverse("@@fix_html") html = form() self.assertIn("Fix links to content and images in richtext", html) - self.request.form.update( - { - "form.submitted": True, - "form.commit": False, - } - ) + self.request.form.update({ + "form.submitted": True, + "form.commit": False, + }) html = form() - self.assertIn( - "Fixed HTML for 1 fields in content items. Fixed HTML for 0 portlets.", html - ) + self.assertIn("Fixed HTML for 1 fields in content items. Fixed HTML for 0 portlets.", html) fixed_html = """

Links to uuid

Link to view/form

@@ -210,9 +200,7 @@ def test_fix_html_form(self):

-""".format( - self.contact.UID(), self.team.UID(), self.image.UID() - ) +""".format(self.contact.UID(), self.team.UID(), self.image.UID()) self.assertEqual(fixed_html, doc.text.raw) @@ -230,12 +218,10 @@ def test_fix_html_status_message(self): form = self.portal.restrictedTraverse("@@fix_html") html = form() self.assertIn("Fix links to content and images in richtext", html) - self.request.form.update( - { - "form.submitted": True, - "form.commit": False, - } - ) + self.request.form.update({ + "form.submitted": True, + "form.commit": False, + }) html = form() self.assertIn( "Fixed HTML for 1 fields in content items. Fixed HTML for 0 portlets.", @@ -254,18 +240,14 @@ def test_fix_html_does_not_change_normal_links(self): ) form = self.portal.restrictedTraverse("@@fix_html") html = form() - self.request.form.update( - { - "form.submitted": True, - "form.commit": False, - } - ) + self.request.form.update({ + "form.submitted": True, + "form.commit": False, + }) html = form() fixed_html = 'Result for the fight between Rudd-O and Andufo' self.assertEqual(fixed_html, doc.text.raw) - self.assertIn( - "Fixed HTML for 0 fields in content items. Fixed HTML for 0 portlets.", html - ) + self.assertIn("Fixed HTML for 0 fields in content items. Fixed HTML for 0 portlets.", html) def test_html_fixer_commas_in_href(self): self.create_demo_content() diff --git a/src/collective/exportimport/tests/test_import.py b/src/collective/exportimport/tests/test_import.py index 645e613d..222254a7 100644 --- a/src/collective/exportimport/tests/test_import.py +++ b/src/collective/exportimport/tests/test_import.py @@ -6,10 +6,14 @@ from OFS.interfaces import IOrderedContainer from plone import api from plone.app.redirector.interfaces import IRedirectionStorage -from plone.app.testing import login, SITE_OWNER_NAME, SITE_OWNER_PASSWORD +from plone.app.testing import login +from plone.app.testing import SITE_OWNER_NAME +from plone.app.testing import SITE_OWNER_PASSWORD from plone.app.textfield.value import RichTextValue -from plone.namedfile.file import NamedBlobImage, NamedImage -from Products.CMFPlone.interfaces.constrains import ENABLED, ISelectableConstrainTypes +from plone.namedfile.file import NamedBlobImage +from plone.namedfile.file import NamedImage +from Products.CMFPlone.interfaces.constrains import ENABLED +from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes from Products.CMFPlone.tests import dummy from time import sleep from zope.annotation.interfaces import IAnnotations @@ -24,7 +28,6 @@ import transaction import unittest - try: from plone.testing import zope @@ -73,38 +76,38 @@ def create_demo_content(self): container=portal, type="Link", id="blog", - title="Blog", + title=u"Blog", ) self.about = api.content.create( container=portal, type="Folder", id="about", - title="About", + title=u"About", ) self.events = api.content.create( container=portal, type="Folder", id="events", - title="Events", + title=u"Events", ) self.team = api.content.create( container=self.about, type="Document", id="team", - title="Team", + title=u"Team", ) self.contact = api.content.create( container=self.about, type="Document", id="contact", - title="Contact", + title=u"Contact", ) self.image = api.content.create( container=portal, type="Image", - title="Image", + title=u"Image", id="image", - image=NamedImage(dummy.Image(), "image/gif", "test.gif"), + image=NamedImage(dummy.Image(), "image/gif", u"test.gif"), ) def remove_demo_content(self): @@ -564,7 +567,6 @@ def test_import_content_from_server_file_and_return_json(self): def test_import_content_from_central_directory(self): from collective.exportimport import config - import tempfile # First create some content. @@ -678,7 +680,7 @@ def test_import_imports_but_ignores_constrains(self): container=self.about, type="Collection", id="collection", - title="Collection", + title=u"Collection", ) # constrain self.about to only allow documents constrains = ISelectableConstrainTypes(self.about) @@ -692,7 +694,7 @@ def test_import_imports_but_ignores_constrains(self): container=self.about, type="Collection", id="collection2", - title="Collection 2", + title=u"Collection 2", ) transaction.commit() @@ -736,7 +738,7 @@ def test_import_imports_but_ignores_constrains(self): container=portal["about"], type="Collection", id="collection2", - title="Collection 2", + title=u"Collection 2", ) def test_import_workflow_history(self): @@ -804,9 +806,9 @@ def _disabled_test_import_blob_path(self): self.image = api.content.create( container=portal, type="Image", - title="Image", + title=u"Image", id="image", - image=NamedBlobImage(image_data, "image/gif", "test.gif"), + image=NamedBlobImage(image_data, "image/gif", u"test.gif"), ) self.assertIn("image", portal.contentIds()) transaction.commit() @@ -1223,41 +1225,41 @@ def test_import_versions(self): # in Plone 4.3 this is somehow not set... IAnnotations(request)[ "plone.app.versioningbehavior-changeNote" - ] = "initial_version_changeNote" + ] = u"initial_version_changeNote" doc1 = api.content.create( container=portal, type="Document", id="doc1", - title="Document 1", - description="A Description", + title=u"Document 1", + description=u"A Description", ) folder1 = api.content.create( container=portal, type="Folder", id="folder1", - title="Folder 1", + title=u"Folder 1", ) doc2 = api.content.create( container=folder1, type="Document", id="doc2", - title="Document 2", - description="A Description", + title=u"Document 2", + description=u"A Description", ) modified(doc1) modified(folder1) modified(doc2) - doc1.title = "Document 1 with changed title" + doc1.title = u"Document 1 with changed title" modified(doc1) - doc2.title = "Document 2 with changed title" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "Föö bar" + doc2.title = u"Document 2 with changed title" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"Föö bar" modified(doc2) - doc2.description = "New description in revision 3" - IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = "I am new!" + doc2.description = u"New description in revision 3" + IAnnotations(request)["plone.app.versioningbehavior-changeNote"] = u"I am new!" modified(doc2) - folder1.title = "Folder 1 with changed title" + folder1.title = u"Folder 1 with changed title" modified(folder1) transaction.commit() @@ -1266,7 +1268,7 @@ def test_import_versions(self): oldest = repo_tool.getHistory(doc2)._retrieve( doc2, 0, preserve=[], countPurged=False ) - self.assertEqual(oldest.object.title, "Document 2") + self.assertEqual(oldest.object.title, u"Document 2") # Now export complete portal. browser = self.open_page("@@export_content") @@ -1318,8 +1320,8 @@ def test_import_versions(self): self.assertIn("doc1", portal.contentIds()) self.assertEqual(portal["folder1"].portal_type, "Folder") doc2 = portal["folder1"]["doc2"] - self.assertEqual(doc2.title, "Document 2 with changed title") - self.assertEqual(doc2.description, "New description in revision 3") + self.assertEqual(doc2.title, u"Document 2 with changed title") + self.assertEqual(doc2.description, u"New description in revision 3") history = repo_tool.getHistoryMetadata(doc2) self.assertEqual(history.getLength(countPurged=True), 4) @@ -1329,17 +1331,19 @@ def test_import_versions(self): return history_meta = history.retrieve(2) - self.assertEqual(history_meta["metadata"]["sys_metadata"]["comment"], "Föö bar") + self.assertEqual( + history_meta["metadata"]["sys_metadata"]["comment"], u"Föö bar" + ) oldest = repo_tool.getHistory(doc2)._retrieve( doc2, 0, preserve=[], countPurged=False ) - self.assertEqual(oldest.object.title, "Document 2") + self.assertEqual(oldest.object.title, u"Document 2") repo_tool.revert(portal["folder1"]["doc2"], 0) doc2 = portal["folder1"]["doc2"] - self.assertEqual(doc2.title, "Document 2") - self.assertEqual(doc2.description, "A Description") + self.assertEqual(doc2.title, u"Document 2") + self.assertEqual(doc2.description, u"A Description") def test_reset_dates(self): """Reset original modification and creation dates""" diff --git a/src/collective/exportimport/tests/test_setup.py b/src/collective/exportimport/tests/test_setup.py index a0f40f50..506ebd16 100644 --- a/src/collective/exportimport/tests/test_setup.py +++ b/src/collective/exportimport/tests/test_setup.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Setup tests for this package.""" -from collective.exportimport.testing import COLLECTIVE_EXPORTIMPORT_INTEGRATION_TESTING from plone import api +from collective.exportimport.testing import COLLECTIVE_EXPORTIMPORT_INTEGRATION_TESTING import unittest From 82c794d9f7c9f0567a0785b25f7317c9002738bd Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 18:52:34 +0200 Subject: [PATCH 03/26] hierarchical export. keeps py2 compatibility --- src/collective/exportimport/config.py | 2 + src/collective/exportimport/export_content.py | 24 +++++ .../exportimport/filesystem_exporter.py | 68 +++++++++++++ .../exportimport/filesystem_importer.py | 97 +++++++++++++++++++ src/collective/exportimport/import_content.py | 12 ++- 5 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 src/collective/exportimport/filesystem_exporter.py create mode 100644 src/collective/exportimport/filesystem_importer.py diff --git a/src/collective/exportimport/config.py b/src/collective/exportimport/config.py index 17dc2b62..c99c4b38 100644 --- a/src/collective/exportimport/config.py +++ b/src/collective/exportimport/config.py @@ -16,3 +16,5 @@ # Discussion Item has its own export / import views, don't show it in the exportable content type list SKIPPED_CONTENTTYPE_IDS = ['Discussion Item'] + +TREE_DIRECTORY = "exported_tree" diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index 0c9135b3..282c6de8 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -3,6 +3,7 @@ from App.config import getConfiguration from collective.exportimport import _ from collective.exportimport import config +from collective.exportimport.filesystem_exporter import FileSystemContentExporter from collective.exportimport.interfaces import IBase64BlobsMarker from collective.exportimport.interfaces import IMigrationMarker from collective.exportimport.interfaces import IPathBlobsMarker @@ -218,6 +219,29 @@ def __call__( noLongerProvides(self.request, IPathBlobsMarker) self.finish() self.request.response.redirect(self.request["ACTUAL_URL"]) + elif download_to_server == 2: + # Will generate a directory tree with one json file per item + portal_id = api.portal.get().getId() + directory = config.CENTRAL_DIRECTORY + if not directory: + cfg = getConfiguration() + directory = cfg.clienthome + rootpath = os.path.join(directory, "exported_tree/%s/content" % portal_id) + if not os.path.exists(rootpath): + os.makedirs(rootpath) + logger.info("Created tree export %s", rootpath) + + self.start() + for number, datum in enumerate(content_generator, start=1): + FileSystemContentExporter(rootpath, datum).save() + self.finish() + + msg = _(u"Exported {} {} with {} errors").format( + number, self.portal_type, len(self.errors) + ) + logger.info(msg) + api.portal.show_message(msg, self.request) + self.request.response.redirect(self.request["ACTUAL_URL"]) else: with tempfile.TemporaryFile(mode="w+") as f: self.start() diff --git a/src/collective/exportimport/filesystem_exporter.py b/src/collective/exportimport/filesystem_exporter.py new file mode 100644 index 00000000..abd31670 --- /dev/null +++ b/src/collective/exportimport/filesystem_exporter.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +from six.moves.urllib.parse import unquote, urlparse + +import json +import os + + +class FileSystemExporter(object): + """Base FS Exporter""" + + def __init__(self, rootpath, json_item): + self.item = json_item + self.root = rootpath + + def create_dir(self, dirname): + """Creates a directory if does not exist + + Args: + dirname (str): dirname to be created + """ + dirpath = os.path.join(self.root, dirname) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + def get_parents(self, parent): + """Extracts parents of item + + Args: + parent (dict): Parent info dict + + Returns: + (str): relative path + """ + + if not parent: + return "" + + parent_url = unquote(parent["@id"]) + parent_url_parsed = urlparse(parent_url) + + # Get the path part, split it, remove the always empty first element. + parent_path = parent_url_parsed.path.split("/")[1:] + if ( + len(parent_url_parsed.netloc.split(":")) > 1 + or parent_url_parsed.netloc == "nohost" + ): + # For example localhost:8080, or nohost when running tests. + # First element will then be a Plone Site id. + # Get rid of it. + parent_path = parent_path[1:] + + return "/".join(parent_path) + + +class FileSystemContentExporter(FileSystemExporter): + """Deserializes JSON items into a FS tree""" + + def save(self): + """Saves a json file to filesystem tree + Target directory is related as original parent position in site. + """ + parent_path = self.get_parents(self.item.get("parent")) + self.create_dir(parent_path) + + filename = "%s_%s.json" % (self.item.get("@type"), self.item.get("UID")) + filepath = os.path.join(self.root, parent_path, filename) + with open(filepath, "w") as f: + json.dump(self.item, f, sort_keys=True, indent=4) diff --git a/src/collective/exportimport/filesystem_importer.py b/src/collective/exportimport/filesystem_importer.py new file mode 100644 index 00000000..9d0664f9 --- /dev/null +++ b/src/collective/exportimport/filesystem_importer.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +from glob import iglob +from plone import api +from six.moves.urllib.parse import unquote, urlparse + +import json +import logging +import os +import six + + +if six.PY2: + from pathlib2 import Path +else: + from pathlib import Path + + +class FileSystemImporter(object): + """Base FS Importer""" + + logger = logging.getLogger(__name__) + + def __init__(self, server_tree_file): + self.path = server_tree_file + + def get_parents(self, parent): + """Extracts parents of item + + Args: + parent (dict): Parent info dict + + Returns: + (str): relative path + """ + + if not parent: + return "" + + parent_url = unquote(parent["@id"]) + parent_url_parsed = urlparse(parent_url) + + # Get the path part, split it, remove the always empty first element. + parent_path = parent_url_parsed.path.split("/")[1:] + if ( + len(parent_url_parsed.netloc.split(":")) > 1 + or parent_url_parsed.netloc == "nohost" + ): + # For example localhost:8080, or nohost when running tests. + # First element will then be a Plone Site id. + # Get rid of it. + parent_path = parent_path[1:] + + return "/".join(parent_path) + + +class FileSystemContentImporter(FileSystemImporter): + """Deserializes JSON items into a FS tree""" + + def list_files(self): + """Loads all json files from filesystem tree""" + files = iglob(os.path.join(self.path, "**/*.json"), recursive=True) + return files + + def get_hierarchical_files(self): + """Gets all files and folders""" + root = Path(self.path) + portal = api.portal.get() + assert root.is_dir() + json_files = root.glob("**/*.json") + for json_file in json_files: + self.logger.debug("Importing %s", json_file) + item = json.loads(json_file.read_text()) + item["json_file"] = str(json_file) + + # Modify parent data + json_parent = item.get("parent", {}) + + # Find the real parent nodes + prefix = os.path.commonprefix([str(json_file.parent), self.path]) + path = os.path.relpath(str(json_file.parent), prefix) + parents = self.get_parents(json_parent) + + if json_file.parent == Path(os.path.join(self.path, parents)): + yield item + else: + try: + parent_obj = portal.unrestrictedTraverse(path) + except KeyError: + parent_obj = portal + + if parent_obj: + item["@id"] = item.get("@id") + json_parent.update( + {"@id": parent_obj.absolute_url(), "UID": parent_obj.UID()} + ) + item["parent"] = json_parent + yield item diff --git a/src/collective/exportimport/import_content.py b/src/collective/exportimport/import_content.py index d2ce0fc4..9831fadd 100644 --- a/src/collective/exportimport/import_content.py +++ b/src/collective/exportimport/import_content.py @@ -2,6 +2,7 @@ from Acquisition import aq_base from collective.exportimport import _ from collective.exportimport import config +from collective.exportimport.filesystem_importer import FileSystemContentImporter from collective.exportimport.interfaces import IMigrationMarker from datetime import datetime from DateTime import DateTime @@ -112,6 +113,7 @@ def __call__( return_json=False, limit=None, server_file=None, + server_tree_file=None, iterator=None ): request = self.request @@ -136,7 +138,7 @@ def __call__( status = "success" msg = "" - if server_file and jsonfile: + if server_file and jsonfile and server_tree_file: # This is an error. But when you upload 10 GB AND select a server file, # it is a pity when you would have to upload again. api.portal.show_message( @@ -146,7 +148,7 @@ def __call__( ) server_file = None status = "error" - if server_file and not jsonfile: + if server_file and not jsonfile and not server_tree_file: if server_file in self.server_files: for path in self.import_paths: full_path = os.path.join(path, server_file) @@ -192,6 +194,12 @@ def __call__( msg = self.do_import(iterator) api.portal.show_message(msg, self.request) + if server_tree_file and not server_file and not jsonfile: + msg = self.do_import( + FileSystemContentImporter(server_tree_file).get_hierarchical_files() + ) + api.portal.show_message(msg, self.request) + self.finish() if return_json: From 27142c160ac168d397611cf4314d7f6770d1917f Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 18:59:19 +0200 Subject: [PATCH 04/26] adds form controls for hierarchical export --- .../exportimport/templates/export_content.pt | 6 ++++++ .../exportimport/templates/import_content.pt | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/collective/exportimport/templates/export_content.pt b/src/collective/exportimport/templates/export_content.pt index a7672239..96555636 100644 --- a/src/collective/exportimport/templates/export_content.pt +++ b/src/collective/exportimport/templates/export_content.pt @@ -162,6 +162,12 @@ Save to file on server
+
+ + +
diff --git a/src/collective/exportimport/templates/import_content.pt b/src/collective/exportimport/templates/import_content.pt index dab24813..411d4ff9 100644 --- a/src/collective/exportimport/templates/import_content.pt +++ b/src/collective/exportimport/templates/import_content.pt @@ -34,6 +34,23 @@ + +

Or you can choose from a tree export in the server in one of these paths:

+
    +
    • +
+

No files found.

+
+ +
+ +
+ +
From df8388c69f487d00297d61c376c6227c631a68a1 Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 19:06:38 +0200 Subject: [PATCH 05/26] checks if export_dir exists --- src/collective/exportimport/import_content.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/collective/exportimport/import_content.py b/src/collective/exportimport/import_content.py index 9831fadd..0d21a97c 100644 --- a/src/collective/exportimport/import_content.py +++ b/src/collective/exportimport/import_content.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from Acquisition import aq_base +from App.config import getConfiguration from collective.exportimport import _ from collective.exportimport import config from collective.exportimport.filesystem_importer import FileSystemContentImporter @@ -251,6 +252,18 @@ def server_files(self): listing.sort() return listing + @property + def import_tree_parts(self): + """Returns subdirectories in export tree""" + directory = config.CENTRAL_DIRECTORY + if not directory: + cfg = getConfiguration() + directory = cfg.clienthome + base_path = os.path.join(directory, config.TREE_DIRECTORY) + if os.path.isdir(base_path): + return [os.path.join(base_path, d, "content") for d in os.listdir(base_path)] + return [] + def do_import(self, data): start = datetime.now() alsoProvides(self.request, IMigrationMarker) From 332cd2a1187597e52fa03b137d78181439cc7fbc Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 19:15:48 +0200 Subject: [PATCH 06/26] added translations to spanish --- .../locales/collective.exportimport.pot | 268 ++++++++-------- .../en/LC_MESSAGES/collective.exportimport.po | 268 ++++++++-------- .../es/LC_MESSAGES/collective.exportimport.po | 293 +++++++++--------- 3 files changed, 428 insertions(+), 401 deletions(-) diff --git a/src/collective/exportimport/locales/collective.exportimport.pot b/src/collective/exportimport/locales/collective.exportimport.pot index 28cd1181..da7a28e2 100644 --- a/src/collective/exportimport/locales/collective.exportimport.pot +++ b/src/collective/exportimport/locales/collective.exportimport.pot @@ -4,7 +4,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" -"POT-Creation-Date: 2023-02-17 02:40+0000\n" +"POT-Creation-Date: 2023-10-10 17:09+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI +ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -17,435 +17,447 @@ msgstr "" "Preferred-Encodings: utf-8 latin1\n" "Domain: collective.exportimport\n" -#: collective/exportimport/import_content.py:961 +#: ../import_content.py:1014 msgid "

Creation- and modification-dates are changed during import.This resets them to the original dates of the imported content.

" msgstr "" -#: collective/exportimport/import_content.py:992 +#: ../import_content.py:1045 msgid "

This fixes invalid collection-criteria that were imported from Plone 4 or 5.

" msgstr "" -#: collective/exportimport/templates/import_redirects.pt:32 +#: ../templates/import_redirects.pt:32 msgid "Beware that this import would work only if you keep the same Plone site id and location in the site !" msgstr "" -#: collective/exportimport/import_other.py:517 +#: ../import_other.py:521 msgid "Changed {} default pages" msgstr "" -#: collective/exportimport/templates/import_content.pt:30 +#: ../templates/export_content.pt:146 +msgid "Checking this box puts a list of object paths at the end of the export file that failed to export." +msgstr "" + +#: ../templates/import_content.pt:30 msgid "Choose one" msgstr "" -#: collective/exportimport/templates/export_content.pt:19 +#: ../templates/export_content.pt:19 msgid "Content Types to export" msgstr "" -#: collective/exportimport/import_other.py:687 +#: ../import_other.py:690 msgid "Created {} portlets" msgstr "" -#: collective/exportimport/templates/export_content.pt:71 +#: ../templates/export_content.pt:71 msgid "Depth" msgstr "" -#: collective/exportimport/templates/import_content.pt:56 +#: ../templates/import_content.pt:73 msgid "Do a commit after each number of items" msgstr "" -#: collective/exportimport/templates/export_content.pt:139 -#: collective/exportimport/templates/export_other.pt:19 +#: ../templates/export_content.pt:155 +#: ../templates/export_other.pt:19 msgid "Download to local machine" msgstr "" -#: collective/exportimport/import_content.py:179 +#: ../import_content.py:182 msgid "Exception during upload: {}" msgstr "" -#: collective/exportimport/templates/export_content.pt:155 -#: collective/exportimport/templates/export_other.pt:32 +#: ../templates/export_content.pt:177 +#: ../templates/export_other.pt:32 msgid "Export" msgstr "" -#: collective/exportimport/export_other.py:585 -#: collective/exportimport/templates/links.pt:38 +#: ../export_other.py:587 +#: ../templates/links.pt:38 msgid "Export comments" msgstr "" -#: collective/exportimport/templates/export_content.pt:11 -#: collective/exportimport/templates/links.pt:10 +#: ../templates/export_content.pt:11 +#: ../templates/links.pt:10 msgid "Export content" msgstr "" -#: collective/exportimport/export_other.py:506 -#: collective/exportimport/templates/links.pt:30 +#: ../export_other.py:508 +#: ../templates/links.pt:30 msgid "Export default pages" msgstr "" -#: collective/exportimport/export_other.py:413 -#: collective/exportimport/templates/links.pt:26 +#: ../export_other.py:415 +#: ../templates/links.pt:26 msgid "Export local roles" msgstr "" -#: collective/exportimport/templates/links.pt:22 +#: ../templates/links.pt:22 msgid "Export members" msgstr "" -#: collective/exportimport/export_other.py:233 +#: ../export_other.py:235 msgid "Export members, groups and roles" msgstr "" -#: collective/exportimport/templates/links.pt:34 +#: ../templates/links.pt:34 msgid "Export object positions in parent" msgstr "" -#: collective/exportimport/export_other.py:470 +#: ../export_other.py:472 msgid "Export ordering" msgstr "" -#: collective/exportimport/export_other.py:624 -#: collective/exportimport/templates/links.pt:42 +#: ../export_other.py:628 +#: ../templates/links.pt:42 msgid "Export portlets" msgstr "" -#: collective/exportimport/export_other.py:759 -#: collective/exportimport/templates/links.pt:46 +#: ../export_other.py:779 +#: ../templates/links.pt:46 msgid "Export redirects" msgstr "" -#: collective/exportimport/export_other.py:125 -#: collective/exportimport/templates/links.pt:14 +#: ../export_other.py:127 +#: ../templates/links.pt:14 msgid "Export relations" msgstr "" -#: collective/exportimport/export_other.py:331 -#: collective/exportimport/templates/links.pt:18 +#: ../export_other.py:333 +#: ../templates/links.pt:18 msgid "Export translations" msgstr "" -#: collective/exportimport/export_other.py:101 +#: ../export_other.py:103 msgid "Exported to {}" msgstr "" -#: collective/exportimport/export_content.py:198 -msgid "Exported {} items ({}) as {} to {}" +#: ../export_content.py:209 +msgid "Exported {} items ({}) as {} to {} with {} errors" msgstr "" -#: collective/exportimport/export_content.py:222 -msgid "Exported {} {}" +#: ../export_content.py:239 +msgid "Exported {} {} with {} errors" msgstr "" -#: collective/exportimport/templates/links.pt:8 +#: ../templates/links.pt:8 msgid "Exports" msgstr "" -#: collective/exportimport/import_other.py:91 +#: ../import_other.py:93 msgid "Failure while uploading: {}" msgstr "" -#: collective/exportimport/import_content.py:160 +#: ../import_content.py:163 msgid "File '{}' not found on server." msgstr "" -#: collective/exportimport/templates/import_content.pt:27 +#: ../templates/import_content.pt:27 msgid "File on server to import:" msgstr "" -#: collective/exportimport/import_content.py:1006 +#: ../import_content.py:1059 msgid "Finished fixing collection queries." msgstr "" -#: collective/exportimport/import_content.py:969 +#: ../import_content.py:1022 msgid "Finished resetting creation and modification dates." msgstr "" -#: collective/exportimport/import_content.py:991 -#: collective/exportimport/templates/links.pt:97 +#: ../import_content.py:1044 +#: ../templates/links.pt:97 msgid "Fix collection queries" msgstr "" -#: collective/exportimport/fix_html.py:43 -#: collective/exportimport/templates/links.pt:101 +#: ../fix_html.py:43 +#: ../templates/links.pt:101 msgid "Fix links to content and images in richtext" msgstr "" -#: collective/exportimport/fix_html.py:51 +#: ../fix_html.py:51 msgid "Fixed HTML for {} fields in content items" msgstr "" -#: collective/exportimport/fix_html.py:55 +#: ../fix_html.py:55 msgid "Fixed HTML for {} portlets" msgstr "" -#: collective/exportimport/templates/import_content.pt:38 +#: ../templates/export_content.pt:167 +msgid "Generate a file for each item (as filesytem tree)" +msgstr "" + +#: ../templates/import_content.pt:55 msgid "Handle existing content" msgstr "" -#: collective/exportimport/templates/export_other.pt:44 -#: collective/exportimport/templates/import_defaultpages.pt:31 -#: collective/exportimport/templates/import_discussion.pt:31 +#: ../templates/export_other.pt:44 +#: ../templates/import_defaultpages.pt:31 +#: ../templates/import_discussion.pt:31 msgid "Help" msgstr "" -#: collective/exportimport/templates/import_defaultpages.pt:32 -#: collective/exportimport/templates/import_discussion.pt:32 -#: collective/exportimport/templates/import_localroles.pt:32 +#: ../templates/import_defaultpages.pt:32 +#: ../templates/import_discussion.pt:32 +#: ../templates/import_localroles.pt:32 msgid "Here is a example for the expected format. This is the format created by collective.exportimport when used for export." msgstr "" -#: collective/exportimport/templates/import_redirects.pt:34 +#: ../templates/import_redirects.pt:34 msgid "Here is an example for the expected format. This is the format created by collective.exportimport when used for export." msgstr "" -#: collective/exportimport/templates/import_content.pt:13 -#: collective/exportimport/templates/import_defaultpages.pt:13 -#: collective/exportimport/templates/import_discussion.pt:13 +#: ../templates/import_content.pt:13 +#: ../templates/import_defaultpages.pt:13 +#: ../templates/import_discussion.pt:13 msgid "Here you can upload a json-file." msgstr "" -#: collective/exportimport/templates/import_content.pt:39 +#: ../templates/import_content.pt:56 msgid "How should content be handled that exists with the same id/path?" msgstr "" -#: collective/exportimport/templates/export_content.pt:88 +#: ../templates/export_content.pt:88 msgid "How should data from image- and file-fields be included?" msgstr "" -#: collective/exportimport/import_content.py:127 +#: ../import_content.py:130 msgid "Ignore: Create with a new id" msgstr "" -#: collective/exportimport/templates/import_content.pt:92 -#: collective/exportimport/templates/import_defaultpages.pt:20 -#: collective/exportimport/templates/import_discussion.pt:20 +#: ../templates/import_content.pt:109 +#: ../templates/import_defaultpages.pt:20 +#: ../templates/import_discussion.pt:20 msgid "Import" msgstr "" -#: collective/exportimport/templates/import_content.pt:11 +#: ../templates/import_content.pt:11 msgid "Import Content" msgstr "" -#: collective/exportimport/templates/import_defaultpages.pt:11 +#: ../templates/import_defaultpages.pt:11 msgid "Import Default Pages" msgstr "" -#: collective/exportimport/templates/import_discussion.pt:11 +#: ../templates/import_discussion.pt:11 msgid "Import Discussion" msgstr "" -#: collective/exportimport/templates/import_localroles.pt:11 +#: ../templates/import_localroles.pt:11 msgid "Import Localroles" msgstr "" -#: collective/exportimport/templates/import_members.pt:11 +#: ../templates/import_members.pt:11 msgid "Import Members, Groups and their Roles" msgstr "" -#: collective/exportimport/templates/import_ordering.pt:11 +#: ../templates/import_ordering.pt:11 msgid "Import Object Positions in Parent" msgstr "" -#: collective/exportimport/templates/import_redirects.pt:11 +#: ../templates/import_redirects.pt:11 msgid "Import Redirects" msgstr "" -#: collective/exportimport/templates/import_relations.pt:11 +#: ../templates/import_relations.pt:11 msgid "Import Relations" msgstr "" -#: collective/exportimport/templates/import_content.pt:71 +#: ../templates/import_content.pt:88 msgid "Import all items into the current folder" msgstr "" -#: collective/exportimport/templates/import_content.pt:83 +#: ../templates/import_content.pt:100 msgid "Import all old revisions" msgstr "" -#: collective/exportimport/templates/links.pt:81 +#: ../templates/links.pt:81 msgid "Import comments" msgstr "" -#: collective/exportimport/templates/links.pt:53 +#: ../templates/links.pt:53 msgid "Import content" msgstr "" -#: collective/exportimport/templates/links.pt:73 +#: ../templates/links.pt:73 msgid "Import default pages" msgstr "" -#: collective/exportimport/templates/links.pt:69 +#: ../templates/links.pt:69 msgid "Import local roles" msgstr "" -#: collective/exportimport/templates/links.pt:65 +#: ../templates/links.pt:65 msgid "Import members" msgstr "" -#: collective/exportimport/templates/links.pt:77 +#: ../templates/links.pt:77 msgid "Import object positions in parent" msgstr "" -#: collective/exportimport/templates/import_portlets.pt:11 -#: collective/exportimport/templates/links.pt:85 +#: ../templates/import_portlets.pt:11 +#: ../templates/links.pt:85 msgid "Import portlets" msgstr "" -#: collective/exportimport/templates/links.pt:89 +#: ../templates/links.pt:89 msgid "Import redirects" msgstr "" -#: collective/exportimport/templates/links.pt:57 +#: ../templates/links.pt:57 msgid "Import relations" msgstr "" -#: collective/exportimport/templates/import_translations.pt:11 -#: collective/exportimport/templates/links.pt:61 +#: ../templates/import_translations.pt:11 +#: ../templates/links.pt:61 msgid "Import translations" msgstr "" -#: collective/exportimport/import_other.py:590 +#: ../import_other.py:594 msgid "Imported {} comments" msgstr "" -#: collective/exportimport/import_other.py:201 +#: ../import_other.py:203 msgid "Imported {} groups and {} members" msgstr "" -#: collective/exportimport/import_other.py:392 +#: ../import_other.py:393 msgid "Imported {} localroles" msgstr "" -#: collective/exportimport/import_other.py:464 +#: ../import_other.py:468 msgid "Imported {} orders in {} seconds" msgstr "" -#: collective/exportimport/templates/links.pt:51 +#: ../templates/links.pt:51 msgid "Imports" msgstr "" -#: collective/exportimport/templates/export_content.pt:87 +#: ../templates/export_content.pt:87 msgid "Include blobs" msgstr "" -#: collective/exportimport/templates/export_content.pt:129 +#: ../templates/export_content.pt:129 msgid "Include revisions." msgstr "" -#: collective/exportimport/templates/export_content.pt:113 +#: ../templates/export_content.pt:113 msgid "Modify exported data for migrations." msgstr "" -#: collective/exportimport/templates/import_redirects.pt:33 +#: ../templates/import_redirects.pt:33 msgid "More code is needed if you have another use case." msgstr "" -#: collective/exportimport/export_other.py:84 +#: ../export_other.py:86 msgid "No data to export for {}" msgstr "" -#: collective/exportimport/templates/import_content.pt:25 +#: ../templates/import_content.pt:25 msgid "No files found." msgstr "" -#: collective/exportimport/templates/export_content.pt:63 +#: ../templates/export_content.pt:63 msgid "Path" msgstr "" -#: collective/exportimport/import_other.py:822 +#: ../import_other.py:873 msgid "Redirects imported" msgstr "" -#: collective/exportimport/import_content.py:125 +#: ../import_content.py:128 msgid "Replace: Delete item and create new" msgstr "" -#: collective/exportimport/templates/links.pt:93 +#: ../templates/links.pt:93 msgid "Reset created and modified dates" msgstr "" -#: collective/exportimport/import_content.py:960 +#: ../import_content.py:1013 msgid "Reset creation and modification date" msgstr "" -#: collective/exportimport/templates/export_content.pt:145 -#: collective/exportimport/templates/export_other.pt:25 +#: ../templates/export_content.pt:161 +#: ../templates/export_other.pt:25 msgid "Save to file on server" msgstr "" -#: collective/exportimport/templates/export_content.pt:24 +#: ../templates/export_content.pt:24 msgid "Select all/none" msgstr "" -#: collective/exportimport/export_content.py:150 +#: ../export_content.py:154 msgid "Select at least one type to export" msgstr "" -#: collective/exportimport/templates/export_content.pt:13 +#: ../templates/export_content.pt:13 msgid "Select which content to export as a json-file." msgstr "" -#: collective/exportimport/import_content.py:124 +#: ../import_content.py:127 msgid "Skip: Don't import at all" msgstr "" -#: collective/exportimport/templates/export_content.pt:130 +#: ../templates/export_content.pt:130 msgid "This exports the content-history (versioning) of each exported item. Warning: This can significantly slow down the export!" msgstr "" -#: collective/exportimport/templates/import_content.pt:84 +#: ../templates/import_content.pt:101 msgid "This will import the content-history (versioning) for each item that has revisions. Warning: This can significantly slow down the import!" msgstr "" -#: collective/exportimport/templates/export_content.pt:72 +#: ../templates/export_content.pt:72 msgid "Unlimited: this item and all children, 0: this object only, 1: only direct children of this object, 2-x: children of this object up to the specified level" msgstr "" -#: collective/exportimport/import_content.py:126 +#: ../import_content.py:129 msgid "Update: Reuse and only overwrite imported data" msgstr "" -#: collective/exportimport/templates/export_content.pt:114 +#: ../templates/export_content.pt:114 msgid "Use this if you want to import the data in a newer version of Plone or migrate from Archetypes to Dexterity. Read the documentation to learn which changes are made by this option." msgstr "" -#: collective/exportimport/import_other.py:288 +#: ../templates/export_content.pt:145 +msgid "Write out Errors to file." +msgstr "" + +#: ../import_other.py:289 msgid "You need either Plone 6 or collective.relationhelpers to import relations" msgstr "" -#: collective/exportimport/export_content.py:139 +#: ../export_content.py:142 msgid "as base-64 encoded strings" msgstr "" -#: collective/exportimport/export_content.py:140 +#: ../export_content.py:143 msgid "as blob paths" msgstr "" -#: collective/exportimport/export_content.py:138 +#: ../export_content.py:141 msgid "as download urls" msgstr "" -#: collective/exportimport/templates/export_content.pt:155 +#: ../templates/export_content.pt:177 msgid "export" msgstr "" -#: collective/exportimport/import_content.py:143 +#: ../import_content.py:146 msgid "json file was uploaded, so the selected server file was ignored." msgstr "" #. Default: "Toggle all" -#: collective/exportimport/templates/export_content.pt:22 +#: ../templates/export_content.pt:22 msgid "label_toggle" msgstr "" -#: collective/exportimport/import_content.py:996 +#: ../import_content.py:1049 msgid "plone.app.querystring.upgrades.fix_select_all_existing_collections is not available" msgstr "" #. Default: "Or you can choose a file that is already uploaded on the server in one of these paths:" -#: collective/exportimport/templates/import_content.pt:21 +#: ../templates/import_content.pt:21 msgid "server_paths_list" msgstr "" -#: collective/exportimport/export_content.py:123 +#: ../export_content.py:126 msgid "unlimited" msgstr "" diff --git a/src/collective/exportimport/locales/en/LC_MESSAGES/collective.exportimport.po b/src/collective/exportimport/locales/en/LC_MESSAGES/collective.exportimport.po index 30b2ece5..6ac6bf76 100644 --- a/src/collective/exportimport/locales/en/LC_MESSAGES/collective.exportimport.po +++ b/src/collective/exportimport/locales/en/LC_MESSAGES/collective.exportimport.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" -"POT-Creation-Date: 2023-02-17 02:40+0000\n" +"POT-Creation-Date: 2023-10-10 17:09+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI +ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -14,435 +14,447 @@ msgstr "" "Preferred-Encodings: utf-8 latin1\n" "Domain: DOMAIN\n" -#: collective/exportimport/import_content.py:961 +#: ../import_content.py:1014 msgid "

Creation- and modification-dates are changed during import.This resets them to the original dates of the imported content.

" msgstr "" -#: collective/exportimport/import_content.py:992 +#: ../import_content.py:1045 msgid "

This fixes invalid collection-criteria that were imported from Plone 4 or 5.

" msgstr "" -#: collective/exportimport/templates/import_redirects.pt:32 +#: ../templates/import_redirects.pt:32 msgid "Beware that this import would work only if you keep the same Plone site id and location in the site !" msgstr "" -#: collective/exportimport/import_other.py:517 +#: ../import_other.py:521 msgid "Changed {} default pages" msgstr "" -#: collective/exportimport/templates/import_content.pt:30 +#: ../templates/export_content.pt:146 +msgid "Checking this box puts a list of object paths at the end of the export file that failed to export." +msgstr "" + +#: ../templates/import_content.pt:30 msgid "Choose one" msgstr "" -#: collective/exportimport/templates/export_content.pt:19 +#: ../templates/export_content.pt:19 msgid "Content Types to export" msgstr "" -#: collective/exportimport/import_other.py:687 +#: ../import_other.py:690 msgid "Created {} portlets" msgstr "" -#: collective/exportimport/templates/export_content.pt:71 +#: ../templates/export_content.pt:71 msgid "Depth" msgstr "" -#: collective/exportimport/templates/import_content.pt:56 +#: ../templates/import_content.pt:73 msgid "Do a commit after each number of items" msgstr "" -#: collective/exportimport/templates/export_content.pt:139 -#: collective/exportimport/templates/export_other.pt:19 +#: ../templates/export_content.pt:155 +#: ../templates/export_other.pt:19 msgid "Download to local machine" msgstr "" -#: collective/exportimport/import_content.py:179 +#: ../import_content.py:182 msgid "Exception during upload: {}" msgstr "" -#: collective/exportimport/templates/export_content.pt:155 -#: collective/exportimport/templates/export_other.pt:32 +#: ../templates/export_content.pt:177 +#: ../templates/export_other.pt:32 msgid "Export" msgstr "" -#: collective/exportimport/export_other.py:585 -#: collective/exportimport/templates/links.pt:38 +#: ../export_other.py:587 +#: ../templates/links.pt:38 msgid "Export comments" msgstr "" -#: collective/exportimport/templates/export_content.pt:11 -#: collective/exportimport/templates/links.pt:10 +#: ../templates/export_content.pt:11 +#: ../templates/links.pt:10 msgid "Export content" msgstr "" -#: collective/exportimport/export_other.py:506 -#: collective/exportimport/templates/links.pt:30 +#: ../export_other.py:508 +#: ../templates/links.pt:30 msgid "Export default pages" msgstr "" -#: collective/exportimport/export_other.py:413 -#: collective/exportimport/templates/links.pt:26 +#: ../export_other.py:415 +#: ../templates/links.pt:26 msgid "Export local roles" msgstr "" -#: collective/exportimport/templates/links.pt:22 +#: ../templates/links.pt:22 msgid "Export members" msgstr "" -#: collective/exportimport/export_other.py:233 +#: ../export_other.py:235 msgid "Export members, groups and roles" msgstr "" -#: collective/exportimport/templates/links.pt:34 +#: ../templates/links.pt:34 msgid "Export object positions in parent" msgstr "" -#: collective/exportimport/export_other.py:470 +#: ../export_other.py:472 msgid "Export ordering" msgstr "" -#: collective/exportimport/export_other.py:624 -#: collective/exportimport/templates/links.pt:42 +#: ../export_other.py:628 +#: ../templates/links.pt:42 msgid "Export portlets" msgstr "" -#: collective/exportimport/export_other.py:759 -#: collective/exportimport/templates/links.pt:46 +#: ../export_other.py:779 +#: ../templates/links.pt:46 msgid "Export redirects" msgstr "" -#: collective/exportimport/export_other.py:125 -#: collective/exportimport/templates/links.pt:14 +#: ../export_other.py:127 +#: ../templates/links.pt:14 msgid "Export relations" msgstr "" -#: collective/exportimport/export_other.py:331 -#: collective/exportimport/templates/links.pt:18 +#: ../export_other.py:333 +#: ../templates/links.pt:18 msgid "Export translations" msgstr "" -#: collective/exportimport/export_other.py:101 +#: ../export_other.py:103 msgid "Exported to {}" msgstr "" -#: collective/exportimport/export_content.py:198 -msgid "Exported {} items ({}) as {} to {}" +#: ../export_content.py:209 +msgid "Exported {} items ({}) as {} to {} with {} errors" msgstr "" -#: collective/exportimport/export_content.py:222 -msgid "Exported {} {}" +#: ../export_content.py:239 +msgid "Exported {} {} with {} errors" msgstr "" -#: collective/exportimport/templates/links.pt:8 +#: ../templates/links.pt:8 msgid "Exports" msgstr "" -#: collective/exportimport/import_other.py:91 +#: ../import_other.py:93 msgid "Failure while uploading: {}" msgstr "" -#: collective/exportimport/import_content.py:160 +#: ../import_content.py:163 msgid "File '{}' not found on server." msgstr "" -#: collective/exportimport/templates/import_content.pt:27 +#: ../templates/import_content.pt:27 msgid "File on server to import:" msgstr "" -#: collective/exportimport/import_content.py:1006 +#: ../import_content.py:1059 msgid "Finished fixing collection queries." msgstr "" -#: collective/exportimport/import_content.py:969 +#: ../import_content.py:1022 msgid "Finished resetting creation and modification dates." msgstr "" -#: collective/exportimport/import_content.py:991 -#: collective/exportimport/templates/links.pt:97 +#: ../import_content.py:1044 +#: ../templates/links.pt:97 msgid "Fix collection queries" msgstr "" -#: collective/exportimport/fix_html.py:43 -#: collective/exportimport/templates/links.pt:101 +#: ../fix_html.py:43 +#: ../templates/links.pt:101 msgid "Fix links to content and images in richtext" msgstr "" -#: collective/exportimport/fix_html.py:51 +#: ../fix_html.py:51 msgid "Fixed HTML for {} fields in content items" msgstr "" -#: collective/exportimport/fix_html.py:55 +#: ../fix_html.py:55 msgid "Fixed HTML for {} portlets" msgstr "" -#: collective/exportimport/templates/import_content.pt:38 +#: ../templates/export_content.pt:167 +msgid "Generate a file for each item (as filesytem tree)" +msgstr "" + +#: ../templates/import_content.pt:55 msgid "Handle existing content" msgstr "" -#: collective/exportimport/templates/export_other.pt:44 -#: collective/exportimport/templates/import_defaultpages.pt:31 -#: collective/exportimport/templates/import_discussion.pt:31 +#: ../templates/export_other.pt:44 +#: ../templates/import_defaultpages.pt:31 +#: ../templates/import_discussion.pt:31 msgid "Help" msgstr "" -#: collective/exportimport/templates/import_defaultpages.pt:32 -#: collective/exportimport/templates/import_discussion.pt:32 -#: collective/exportimport/templates/import_localroles.pt:32 +#: ../templates/import_defaultpages.pt:32 +#: ../templates/import_discussion.pt:32 +#: ../templates/import_localroles.pt:32 msgid "Here is a example for the expected format. This is the format created by collective.exportimport when used for export." msgstr "" -#: collective/exportimport/templates/import_redirects.pt:34 +#: ../templates/import_redirects.pt:34 msgid "Here is an example for the expected format. This is the format created by collective.exportimport when used for export." msgstr "" -#: collective/exportimport/templates/import_content.pt:13 -#: collective/exportimport/templates/import_defaultpages.pt:13 -#: collective/exportimport/templates/import_discussion.pt:13 +#: ../templates/import_content.pt:13 +#: ../templates/import_defaultpages.pt:13 +#: ../templates/import_discussion.pt:13 msgid "Here you can upload a json-file." msgstr "" -#: collective/exportimport/templates/import_content.pt:39 +#: ../templates/import_content.pt:56 msgid "How should content be handled that exists with the same id/path?" msgstr "" -#: collective/exportimport/templates/export_content.pt:88 +#: ../templates/export_content.pt:88 msgid "How should data from image- and file-fields be included?" msgstr "" -#: collective/exportimport/import_content.py:127 +#: ../import_content.py:130 msgid "Ignore: Create with a new id" msgstr "" -#: collective/exportimport/templates/import_content.pt:92 -#: collective/exportimport/templates/import_defaultpages.pt:20 -#: collective/exportimport/templates/import_discussion.pt:20 +#: ../templates/import_content.pt:109 +#: ../templates/import_defaultpages.pt:20 +#: ../templates/import_discussion.pt:20 msgid "Import" msgstr "" -#: collective/exportimport/templates/import_content.pt:11 +#: ../templates/import_content.pt:11 msgid "Import Content" msgstr "" -#: collective/exportimport/templates/import_defaultpages.pt:11 +#: ../templates/import_defaultpages.pt:11 msgid "Import Default Pages" msgstr "" -#: collective/exportimport/templates/import_discussion.pt:11 +#: ../templates/import_discussion.pt:11 msgid "Import Discussion" msgstr "" -#: collective/exportimport/templates/import_localroles.pt:11 +#: ../templates/import_localroles.pt:11 msgid "Import Localroles" msgstr "" -#: collective/exportimport/templates/import_members.pt:11 +#: ../templates/import_members.pt:11 msgid "Import Members, Groups and their Roles" msgstr "" -#: collective/exportimport/templates/import_ordering.pt:11 +#: ../templates/import_ordering.pt:11 msgid "Import Object Positions in Parent" msgstr "" -#: collective/exportimport/templates/import_redirects.pt:11 +#: ../templates/import_redirects.pt:11 msgid "Import Redirects" msgstr "" -#: collective/exportimport/templates/import_relations.pt:11 +#: ../templates/import_relations.pt:11 msgid "Import Relations" msgstr "" -#: collective/exportimport/templates/import_content.pt:71 +#: ../templates/import_content.pt:88 msgid "Import all items into the current folder" msgstr "" -#: collective/exportimport/templates/import_content.pt:83 +#: ../templates/import_content.pt:100 msgid "Import all old revisions" msgstr "" -#: collective/exportimport/templates/links.pt:81 +#: ../templates/links.pt:81 msgid "Import comments" msgstr "" -#: collective/exportimport/templates/links.pt:53 +#: ../templates/links.pt:53 msgid "Import content" msgstr "" -#: collective/exportimport/templates/links.pt:73 +#: ../templates/links.pt:73 msgid "Import default pages" msgstr "" -#: collective/exportimport/templates/links.pt:69 +#: ../templates/links.pt:69 msgid "Import local roles" msgstr "" -#: collective/exportimport/templates/links.pt:65 +#: ../templates/links.pt:65 msgid "Import members" msgstr "" -#: collective/exportimport/templates/links.pt:77 +#: ../templates/links.pt:77 msgid "Import object positions in parent" msgstr "" -#: collective/exportimport/templates/import_portlets.pt:11 -#: collective/exportimport/templates/links.pt:85 +#: ../templates/import_portlets.pt:11 +#: ../templates/links.pt:85 msgid "Import portlets" msgstr "" -#: collective/exportimport/templates/links.pt:89 +#: ../templates/links.pt:89 msgid "Import redirects" msgstr "" -#: collective/exportimport/templates/links.pt:57 +#: ../templates/links.pt:57 msgid "Import relations" msgstr "" -#: collective/exportimport/templates/import_translations.pt:11 -#: collective/exportimport/templates/links.pt:61 +#: ../templates/import_translations.pt:11 +#: ../templates/links.pt:61 msgid "Import translations" msgstr "" -#: collective/exportimport/import_other.py:590 +#: ../import_other.py:594 msgid "Imported {} comments" msgstr "" -#: collective/exportimport/import_other.py:201 +#: ../import_other.py:203 msgid "Imported {} groups and {} members" msgstr "" -#: collective/exportimport/import_other.py:392 +#: ../import_other.py:393 msgid "Imported {} localroles" msgstr "" -#: collective/exportimport/import_other.py:464 +#: ../import_other.py:468 msgid "Imported {} orders in {} seconds" msgstr "" -#: collective/exportimport/templates/links.pt:51 +#: ../templates/links.pt:51 msgid "Imports" msgstr "" -#: collective/exportimport/templates/export_content.pt:87 +#: ../templates/export_content.pt:87 msgid "Include blobs" msgstr "" -#: collective/exportimport/templates/export_content.pt:129 +#: ../templates/export_content.pt:129 msgid "Include revisions." msgstr "" -#: collective/exportimport/templates/export_content.pt:113 +#: ../templates/export_content.pt:113 msgid "Modify exported data for migrations." msgstr "" -#: collective/exportimport/templates/import_redirects.pt:33 +#: ../templates/import_redirects.pt:33 msgid "More code is needed if you have another use case." msgstr "" -#: collective/exportimport/export_other.py:84 +#: ../export_other.py:86 msgid "No data to export for {}" msgstr "" -#: collective/exportimport/templates/import_content.pt:25 +#: ../templates/import_content.pt:25 msgid "No files found." msgstr "" -#: collective/exportimport/templates/export_content.pt:63 +#: ../templates/export_content.pt:63 msgid "Path" msgstr "" -#: collective/exportimport/import_other.py:822 +#: ../import_other.py:873 msgid "Redirects imported" msgstr "" -#: collective/exportimport/import_content.py:125 +#: ../import_content.py:128 msgid "Replace: Delete item and create new" msgstr "" -#: collective/exportimport/templates/links.pt:93 +#: ../templates/links.pt:93 msgid "Reset created and modified dates" msgstr "" -#: collective/exportimport/import_content.py:960 +#: ../import_content.py:1013 msgid "Reset creation and modification date" msgstr "" -#: collective/exportimport/templates/export_content.pt:145 -#: collective/exportimport/templates/export_other.pt:25 +#: ../templates/export_content.pt:161 +#: ../templates/export_other.pt:25 msgid "Save to file on server" msgstr "" -#: collective/exportimport/templates/export_content.pt:24 +#: ../templates/export_content.pt:24 msgid "Select all/none" msgstr "" -#: collective/exportimport/export_content.py:150 +#: ../export_content.py:154 msgid "Select at least one type to export" msgstr "" -#: collective/exportimport/templates/export_content.pt:13 +#: ../templates/export_content.pt:13 msgid "Select which content to export as a json-file." msgstr "" -#: collective/exportimport/import_content.py:124 +#: ../import_content.py:127 msgid "Skip: Don't import at all" msgstr "" -#: collective/exportimport/templates/export_content.pt:130 +#: ../templates/export_content.pt:130 msgid "This exports the content-history (versioning) of each exported item. Warning: This can significantly slow down the export!" msgstr "" -#: collective/exportimport/templates/import_content.pt:84 +#: ../templates/import_content.pt:101 msgid "This will import the content-history (versioning) for each item that has revisions. Warning: This can significantly slow down the import!" msgstr "" -#: collective/exportimport/templates/export_content.pt:72 +#: ../templates/export_content.pt:72 msgid "Unlimited: this item and all children, 0: this object only, 1: only direct children of this object, 2-x: children of this object up to the specified level" msgstr "" -#: collective/exportimport/import_content.py:126 +#: ../import_content.py:129 msgid "Update: Reuse and only overwrite imported data" msgstr "" -#: collective/exportimport/templates/export_content.pt:114 +#: ../templates/export_content.pt:114 msgid "Use this if you want to import the data in a newer version of Plone or migrate from Archetypes to Dexterity. Read the documentation to learn which changes are made by this option." msgstr "" -#: collective/exportimport/import_other.py:288 +#: ../templates/export_content.pt:145 +msgid "Write out Errors to file." +msgstr "" + +#: ../import_other.py:289 msgid "You need either Plone 6 or collective.relationhelpers to import relations" msgstr "" -#: collective/exportimport/export_content.py:139 +#: ../export_content.py:142 msgid "as base-64 encoded strings" msgstr "" -#: collective/exportimport/export_content.py:140 +#: ../export_content.py:143 msgid "as blob paths" msgstr "" -#: collective/exportimport/export_content.py:138 +#: ../export_content.py:141 msgid "as download urls" msgstr "" -#: collective/exportimport/templates/export_content.pt:155 +#: ../templates/export_content.pt:177 msgid "export" msgstr "" -#: collective/exportimport/import_content.py:143 +#: ../import_content.py:146 msgid "json file was uploaded, so the selected server file was ignored." msgstr "" #. Default: "Toggle all" -#: collective/exportimport/templates/export_content.pt:22 +#: ../templates/export_content.pt:22 msgid "label_toggle" msgstr "" -#: collective/exportimport/import_content.py:996 +#: ../import_content.py:1049 msgid "plone.app.querystring.upgrades.fix_select_all_existing_collections is not available" msgstr "" #. Default: "Or you can choose a file that is already uploaded on the server in one of these paths:" -#: collective/exportimport/templates/import_content.pt:21 +#: ../templates/import_content.pt:21 msgid "server_paths_list" msgstr "" -#: collective/exportimport/export_content.py:123 +#: ../export_content.py:126 msgid "unlimited" msgstr "" diff --git a/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po b/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po index e640578c..d18e559a 100644 --- a/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po +++ b/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po @@ -2,460 +2,463 @@ msgid "" msgstr "" "Project-Id-Version: collective.exportimport\n" -"POT-Creation-Date: 2023-02-17 02:33+0000\n" +"POT-Creation-Date: 2023-10-10 17:09+0000\n" "PO-Revision-Date: 2023-02-16 22:41-0400\n" "Last-Translator: Leonardo J. Caballero G. \n" "Language-Team: ES \n" -"Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" -"X-Generator: Virtaal 0.7.1\n" "Language-Code: es\n" "Language-Name: Español\n" "Preferred-Encodings: utf-8 latin1\n" "Domain: collective.exportimport\n" +"Language: es\n" +"X-Generator: Virtaal 0.7.1\n" "X-Is-Fallback-For: es-ar es-bo es-cl es-co es-cr es-do es-ec es-es es-sv es-gt es-hn es-mx es-ni es-pa es-py es-pe es-pr es-us es-uy es-ve\n" -#: collective/exportimport/import_content.py:961 +#: ../import_content.py:1014 msgid "

Creation- and modification-dates are changed during import.This resets them to the original dates of the imported content.

" -msgstr "" -"

Las fechas de creación y modificación se cambian durante la importación. " -"Esto las restablece a las fechas originales del contenido importado.

" +msgstr "

Las fechas de creación y modificación se cambian durante la importación. Esto las restablece a las fechas originales del contenido importado.

" -#: collective/exportimport/import_content.py:992 +#: ../import_content.py:1045 msgid "

This fixes invalid collection-criteria that were imported from Plone 4 or 5.

" msgstr "

Esto corrige los criterios de recopilación no válidos que se importaron de Plone 4 o 5.

" -#: collective/exportimport/templates/import_redirects.pt:32 +#: ../templates/import_redirects.pt:32 msgid "Beware that this import would work only if you keep the same Plone site id and location in the site !" -msgstr "" -"¡Tenga en cuenta que esta importación solo funcionaría si mantiene la misma " -"identificación y ubicación del sitio de Plone en el sitio!" +msgstr "¡Tenga en cuenta que esta importación solo funcionaría si mantiene la misma identificación y ubicación del sitio de Plone en el sitio!" -#: collective/exportimport/import_other.py:517 +#: ../import_other.py:521 msgid "Changed {} default pages" msgstr "Se cambiaron {} páginas predeterminadas" -#: collective/exportimport/templates/import_content.pt:30 +#: ../templates/export_content.pt:146 +msgid "Checking this box puts a list of object paths at the end of the export file that failed to export." +msgstr "Seleccionar esta opción pone una lista de rutas, de objetos cuya exportación falló, al final del archivo de exportación" + +#: ../templates/import_content.pt:30 msgid "Choose one" msgstr "Elige uno" -#: collective/exportimport/templates/export_content.pt:19 +#: ../templates/export_content.pt:19 msgid "Content Types to export" msgstr "Tipos de contenido para exportar" -#: collective/exportimport/import_other.py:687 +#: ../import_other.py:690 msgid "Created {} portlets" msgstr "Creados {} portlets" -#: collective/exportimport/templates/export_content.pt:71 +#: ../templates/export_content.pt:71 msgid "Depth" msgstr "Profundidad" -#: collective/exportimport/templates/import_content.pt:56 +#: ../templates/import_content.pt:73 msgid "Do a commit after each number of items" msgstr "Haz una confirmación después de cada número de elementos" -#: collective/exportimport/templates/export_content.pt:139 -#: collective/exportimport/templates/export_other.pt:19 +#: ../templates/export_content.pt:155 +#: ../templates/export_other.pt:19 msgid "Download to local machine" msgstr "Descargar a la máquina local" -#: collective/exportimport/import_content.py:179 +#: ../import_content.py:182 msgid "Exception during upload: {}" msgstr "Excepción durante la carga: {}" -#: collective/exportimport/templates/export_content.pt:155 -#: collective/exportimport/templates/export_other.pt:32 +#: ../templates/export_content.pt:177 +#: ../templates/export_other.pt:32 msgid "Export" msgstr "Exportar" -#: collective/exportimport/export_other.py:585 -#: collective/exportimport/templates/links.pt:38 +#: ../export_other.py:587 +#: ../templates/links.pt:38 msgid "Export comments" msgstr "Exportar comentarios" -#: collective/exportimport/templates/export_content.pt:11 -#: collective/exportimport/templates/links.pt:10 +#: ../templates/export_content.pt:11 +#: ../templates/links.pt:10 msgid "Export content" msgstr "Exportar contenido" -#: collective/exportimport/export_other.py:506 -#: collective/exportimport/templates/links.pt:30 +#: ../export_other.py:508 +#: ../templates/links.pt:30 msgid "Export default pages" msgstr "Exportar páginas predeterminadas" -#: collective/exportimport/export_other.py:413 -#: collective/exportimport/templates/links.pt:26 +#: ../export_other.py:415 +#: ../templates/links.pt:26 msgid "Export local roles" msgstr "Exportar roles locales" -#: collective/exportimport/templates/links.pt:22 +#: ../templates/links.pt:22 msgid "Export members" msgstr "Exportar miembros" -#: collective/exportimport/export_other.py:233 +#: ../export_other.py:235 msgid "Export members, groups and roles" msgstr "Exportar miembros, grupos y roles" -#: collective/exportimport/templates/links.pt:34 +#: ../templates/links.pt:34 msgid "Export object positions in parent" msgstr "Exportar posiciones de objetos en el padre" -#: collective/exportimport/export_other.py:470 +#: ../export_other.py:472 msgid "Export ordering" msgstr "Exportar ordenamiento" -#: collective/exportimport/export_other.py:624 -#: collective/exportimport/templates/links.pt:42 +#: ../export_other.py:628 +#: ../templates/links.pt:42 msgid "Export portlets" msgstr "Exportar portlets" -#: collective/exportimport/export_other.py:759 -#: collective/exportimport/templates/links.pt:46 +#: ../export_other.py:779 +#: ../templates/links.pt:46 msgid "Export redirects" msgstr "Exportar redireccionamientos" -#: collective/exportimport/export_other.py:125 -#: collective/exportimport/templates/links.pt:14 +#: ../export_other.py:127 +#: ../templates/links.pt:14 msgid "Export relations" msgstr "Exportar relaciones" -#: collective/exportimport/export_other.py:331 -#: collective/exportimport/templates/links.pt:18 +#: ../export_other.py:333 +#: ../templates/links.pt:18 msgid "Export translations" msgstr "Exportar traducciones" -#: collective/exportimport/export_other.py:101 +#: ../export_other.py:103 msgid "Exported to {}" msgstr "Exportado a {}" -#: collective/exportimport/export_content.py:198 -msgid "Exported {} items ({}) as {} to {}" -msgstr "Exportado {} elementos ({}) como {} a {}" +#: ../export_content.py:209 +msgid "Exported {} items ({}) as {} to {} with {} errors" +msgstr "Exportados {} elementos ({}) como {} a {}" -#: collective/exportimport/export_content.py:222 -msgid "Exported {} {}" -msgstr "Exportado {} {}" +#: ../export_content.py:239 +msgid "Exported {} {} with {} errors" +msgstr "Exportado {} {} con {} errores" -#: collective/exportimport/templates/links.pt:8 +#: ../templates/links.pt:8 msgid "Exports" msgstr "Exportaciones" -#: collective/exportimport/import_other.py:91 +#: ../import_other.py:93 msgid "Failure while uploading: {}" msgstr "Error al cargar: {}" -#: collective/exportimport/import_content.py:160 +#: ../import_content.py:163 msgid "File '{}' not found on server." msgstr "Archivo '{}' no encontrado en el servidor." -#: collective/exportimport/templates/import_content.pt:27 +#: ../templates/import_content.pt:27 msgid "File on server to import:" msgstr "Archivo en el servidor para importar:" -#: collective/exportimport/import_content.py:1006 +#: ../import_content.py:1059 msgid "Finished fixing collection queries." msgstr "Se terminaron de arreglar las consultas de colección." -#: collective/exportimport/import_content.py:969 +#: ../import_content.py:1022 msgid "Finished resetting creation and modification dates." msgstr "Finalizó el restablecimiento de las fechas de creación y modificación." -#: collective/exportimport/import_content.py:991 -#: collective/exportimport/templates/links.pt:97 +#: ../import_content.py:1044 +#: ../templates/links.pt:97 msgid "Fix collection queries" msgstr "Arreglar consultas de colección" -#: collective/exportimport/fix_html.py:43 -#: collective/exportimport/templates/links.pt:101 +#: ../fix_html.py:43 +#: ../templates/links.pt:101 msgid "Fix links to content and images in richtext" msgstr "Corregir enlaces a contenido e imágenes en texto enriquecido" -#: collective/exportimport/fix_html.py:51 +#: ../fix_html.py:51 msgid "Fixed HTML for {} fields in content items" msgstr "HTML corregido para {} campos en elementos de contenido" -#: collective/exportimport/fix_html.py:55 +#: ../fix_html.py:55 msgid "Fixed HTML for {} portlets" msgstr "HTML corregido para {} portlets" -#: collective/exportimport/templates/import_content.pt:38 +#: ../templates/export_content.pt:167 +msgid "Generate a file for each item (as filesytem tree)" +msgstr "Generar un archivo por cada elemento (exportación jerárquica en el sistema de archivos)" + +#: ../templates/import_content.pt:55 msgid "Handle existing content" msgstr "Manejar el contenido existente" -#: collective/exportimport/templates/export_other.pt:44 -#: collective/exportimport/templates/import_defaultpages.pt:31 -#: collective/exportimport/templates/import_discussion.pt:31 +#: ../templates/export_other.pt:44 +#: ../templates/import_defaultpages.pt:31 +#: ../templates/import_discussion.pt:31 msgid "Help" msgstr "Ayuda" -#: collective/exportimport/templates/import_defaultpages.pt:32 -#: collective/exportimport/templates/import_discussion.pt:32 -#: collective/exportimport/templates/import_localroles.pt:32 +#: ../templates/import_defaultpages.pt:32 +#: ../templates/import_discussion.pt:32 +#: ../templates/import_localroles.pt:32 msgid "Here is a example for the expected format. This is the format created by collective.exportimport when used for export." msgstr "Aquí hay un ejemplo para el formato esperado. Este es el formato creado por el collective.exportimport cuando se utiliza para la exportación." -#: collective/exportimport/templates/import_redirects.pt:34 +#: ../templates/import_redirects.pt:34 msgid "Here is an example for the expected format. This is the format created by collective.exportimport when used for export." msgstr "Aquí hay un ejemplo para el formato esperado. Este es el formato creado por el collective.exportimport cuando se utiliza para la exportación." -#: collective/exportimport/templates/import_content.pt:13 -#: collective/exportimport/templates/import_defaultpages.pt:13 -#: collective/exportimport/templates/import_discussion.pt:13 +#: ../templates/import_content.pt:13 +#: ../templates/import_defaultpages.pt:13 +#: ../templates/import_discussion.pt:13 msgid "Here you can upload a json-file." msgstr "Aquí puede cargar un archivo json." -#: collective/exportimport/templates/import_content.pt:39 +#: ../templates/import_content.pt:56 msgid "How should content be handled that exists with the same id/path?" msgstr "¿Cómo se debe manejar el contenido que existe con el mismo id/path?" -#: collective/exportimport/templates/export_content.pt:88 +#: ../templates/export_content.pt:88 msgid "How should data from image- and file-fields be included?" msgstr "¿Cómo se deben incluir los datos de los campos de imagen y archivo?" -#: collective/exportimport/import_content.py:127 +#: ../import_content.py:130 msgid "Ignore: Create with a new id" msgstr "Ignorar: Crear con un nuevo id" -#: collective/exportimport/templates/import_content.pt:92 -#: collective/exportimport/templates/import_defaultpages.pt:20 -#: collective/exportimport/templates/import_discussion.pt:20 +#: ../templates/import_content.pt:109 +#: ../templates/import_defaultpages.pt:20 +#: ../templates/import_discussion.pt:20 msgid "Import" msgstr "Importar" -#: collective/exportimport/templates/import_content.pt:11 +#: ../templates/import_content.pt:11 msgid "Import Content" msgstr "Importar Contenido" -#: collective/exportimport/templates/import_defaultpages.pt:11 +#: ../templates/import_defaultpages.pt:11 msgid "Import Default Pages" msgstr "Importar páginas predeterminadas" -#: collective/exportimport/templates/import_discussion.pt:11 +#: ../templates/import_discussion.pt:11 msgid "Import Discussion" msgstr "Importar discusión" -#: collective/exportimport/templates/import_localroles.pt:11 +#: ../templates/import_localroles.pt:11 msgid "Import Localroles" msgstr "Importar roles locales" -#: collective/exportimport/templates/import_members.pt:11 +#: ../templates/import_members.pt:11 msgid "Import Members, Groups and their Roles" msgstr "Importar miembros, grupos y sus roles" -#: collective/exportimport/templates/import_ordering.pt:11 +#: ../templates/import_ordering.pt:11 msgid "Import Object Positions in Parent" msgstr "Importar posiciones de objetos en el padre" -#: collective/exportimport/templates/import_redirects.pt:11 +#: ../templates/import_redirects.pt:11 msgid "Import Redirects" msgstr "Importar redireccionamientos" -#: collective/exportimport/templates/import_relations.pt:11 +#: ../templates/import_relations.pt:11 msgid "Import Relations" msgstr "Importar relaciones" -#: collective/exportimport/templates/import_content.pt:71 +#: ../templates/import_content.pt:88 msgid "Import all items into the current folder" msgstr "Importar todos los elementos a la carpeta actual" -#: collective/exportimport/templates/import_content.pt:83 +#: ../templates/import_content.pt:100 msgid "Import all old revisions" msgstr "Importar todas las revisiones antiguas" -#: collective/exportimport/templates/links.pt:81 +#: ../templates/links.pt:81 msgid "Import comments" msgstr "Importar comentarios" -#: collective/exportimport/templates/links.pt:53 +#: ../templates/links.pt:53 msgid "Import content" msgstr "Importar contenido" -#: collective/exportimport/templates/links.pt:73 +#: ../templates/links.pt:73 msgid "Import default pages" msgstr "Importar páginas predeterminadas" -#: collective/exportimport/templates/links.pt:69 +#: ../templates/links.pt:69 msgid "Import local roles" msgstr "Importar roles locales" -#: collective/exportimport/templates/links.pt:65 +#: ../templates/links.pt:65 msgid "Import members" msgstr "Importar miembros" -#: collective/exportimport/templates/links.pt:77 +#: ../templates/links.pt:77 msgid "Import object positions in parent" msgstr "Importar posiciones de objetos en el padre" -#: collective/exportimport/templates/import_portlets.pt:11 -#: collective/exportimport/templates/links.pt:85 +#: ../templates/import_portlets.pt:11 +#: ../templates/links.pt:85 msgid "Import portlets" msgstr "Importar portlets" -#: collective/exportimport/templates/links.pt:89 +#: ../templates/links.pt:89 msgid "Import redirects" msgstr "Importar redireccionamientos" -#: collective/exportimport/templates/links.pt:57 +#: ../templates/links.pt:57 msgid "Import relations" msgstr "Importar relaciones" -#: collective/exportimport/templates/import_translations.pt:11 -#: collective/exportimport/templates/links.pt:61 +#: ../templates/import_translations.pt:11 +#: ../templates/links.pt:61 msgid "Import translations" msgstr "Importar traducciones" -#: collective/exportimport/import_other.py:590 +#: ../import_other.py:594 msgid "Imported {} comments" msgstr "Importados {} comentarios" -#: collective/exportimport/import_other.py:201 +#: ../import_other.py:203 msgid "Imported {} groups and {} members" msgstr "Importados {} grupos y {} miembros" -#: collective/exportimport/import_other.py:392 +#: ../import_other.py:393 msgid "Imported {} localroles" msgstr "Importados {} roles locales" -#: collective/exportimport/import_other.py:464 +#: ../import_other.py:468 msgid "Imported {} orders in {} seconds" msgstr "Importados {} ordenes en {} segundos" -#: collective/exportimport/templates/links.pt:51 +#: ../templates/links.pt:51 msgid "Imports" msgstr "Importaciones" -#: collective/exportimport/templates/export_content.pt:87 +#: ../templates/export_content.pt:87 msgid "Include blobs" msgstr "Incluir blobs" -#: collective/exportimport/templates/export_content.pt:129 +#: ../templates/export_content.pt:129 msgid "Include revisions." msgstr "Incluir revisiones." -#: collective/exportimport/templates/export_content.pt:113 +#: ../templates/export_content.pt:113 msgid "Modify exported data for migrations." msgstr "Modificar datos exportados para migraciones." -#: collective/exportimport/templates/import_redirects.pt:33 +#: ../templates/import_redirects.pt:33 msgid "More code is needed if you have another use case." msgstr "Se necesita más código si tiene otro caso de uso." -#: collective/exportimport/export_other.py:84 +#: ../export_other.py:86 msgid "No data to export for {}" msgstr "No hay datos para exportar {}" -#: collective/exportimport/templates/import_content.pt:25 +#: ../templates/import_content.pt:25 msgid "No files found." msgstr "No se encontraron archivos." -#: collective/exportimport/templates/export_content.pt:63 +#: ../templates/export_content.pt:63 msgid "Path" msgstr "Ruta" -#: collective/exportimport/import_other.py:822 +#: ../import_other.py:873 msgid "Redirects imported" msgstr "Redirecciones importadas" -#: collective/exportimport/import_content.py:125 +#: ../import_content.py:128 msgid "Replace: Delete item and create new" msgstr "Reemplazar: Eliminar elemento y crear nuevo" -#: collective/exportimport/templates/links.pt:93 +#: ../templates/links.pt:93 msgid "Reset created and modified dates" msgstr "Restablecer fechas de creación y modificación" -#: collective/exportimport/import_content.py:960 +#: ../import_content.py:1013 msgid "Reset creation and modification date" msgstr "Restablecer fecha de creación y modificación" -#: collective/exportimport/templates/export_content.pt:145 -#: collective/exportimport/templates/export_other.pt:25 +#: ../templates/export_content.pt:161 +#: ../templates/export_other.pt:25 msgid "Save to file on server" msgstr "Guardar en un archivo en el servidor" -#: collective/exportimport/templates/export_content.pt:24 +#: ../templates/export_content.pt:24 msgid "Select all/none" msgstr "Seleccionar todo/ninguno" -#: collective/exportimport/export_content.py:150 +#: ../export_content.py:154 msgid "Select at least one type to export" msgstr "Seleccione al menos un tipo para exportar" -#: collective/exportimport/templates/export_content.pt:13 +#: ../templates/export_content.pt:13 msgid "Select which content to export as a json-file." msgstr "Seleccione qué contenido exportar como un archivo json." -#: collective/exportimport/import_content.py:124 +#: ../import_content.py:127 msgid "Skip: Don't import at all" msgstr "Omitir: no importar en absoluto" -#: collective/exportimport/templates/export_content.pt:130 +#: ../templates/export_content.pt:130 msgid "This exports the content-history (versioning) of each exported item. Warning: This can significantly slow down the export!" msgstr "Esto exporta el historial de contenido (versiones) de cada elemento exportado. Advertencia: ¡Esto puede ralentizar significativamente la exportación!" -#: collective/exportimport/templates/import_content.pt:84 +#: ../templates/import_content.pt:101 msgid "This will import the content-history (versioning) for each item that has revisions. Warning: This can significantly slow down the import!" msgstr "Esto importará el historial de contenido (versiones) para cada elemento que tenga revisiones. Advertencia: ¡Esto puede ralentizar significativamente la importación!" -#: collective/exportimport/templates/export_content.pt:72 +#: ../templates/export_content.pt:72 msgid "Unlimited: this item and all children, 0: this object only, 1: only direct children of this object, 2-x: children of this object up to the specified level" -msgstr "" -"Ilimitado: este elemento y todos los elementos secundarios, 0: solo este " -"objeto, 1: solo elementos secundarios directos de este objeto, 2-x: " -"elementos secundarios de este objeto hasta el nivel especificado" +msgstr "Ilimitado: este elemento y todos los elementos secundarios, 0: solo este objeto, 1: solo elementos secundarios directos de este objeto, 2-x: elementos secundarios de este objeto hasta el nivel especificado" -#: collective/exportimport/import_content.py:126 +#: ../import_content.py:129 msgid "Update: Reuse and only overwrite imported data" msgstr "Actualizar: reutilizar y solo sobrescribir datos importados" -#: collective/exportimport/templates/export_content.pt:114 +#: ../templates/export_content.pt:114 msgid "Use this if you want to import the data in a newer version of Plone or migrate from Archetypes to Dexterity. Read the documentation to learn which changes are made by this option." msgstr "Use esto si desea importar los datos en una versión más nueva de Plone o migrar de Archetypes a Dexterity. Lea la documentación para saber qué cambios se realizan con esta opción." -#: collective/exportimport/import_other.py:288 +#: ../templates/export_content.pt:145 +msgid "Write out Errors to file." +msgstr "Se han escrito errores en el fichero." + +#: ../import_other.py:289 msgid "You need either Plone 6 or collective.relationhelpers to import relations" msgstr "Necesitas Plone 6 o collective.relationhelpers para importar relaciones" -#: collective/exportimport/export_content.py:139 +#: ../export_content.py:142 msgid "as base-64 encoded strings" msgstr "como cadenas codificadas en base 64" -#: collective/exportimport/export_content.py:140 +#: ../export_content.py:143 msgid "as blob paths" msgstr "como rutas de blob" -#: collective/exportimport/export_content.py:138 +#: ../export_content.py:141 msgid "as download urls" msgstr "como urls de descarga" -#: collective/exportimport/templates/export_content.pt:155 +#: ../templates/export_content.pt:177 msgid "export" msgstr "exportar" -#: collective/exportimport/import_content.py:143 +#: ../import_content.py:146 msgid "json file was uploaded, so the selected server file was ignored." msgstr "archivo json se cargó, por lo que se ignoró el archivo del servidor seleccionado." #. Default: "Toggle all" -#: collective/exportimport/templates/export_content.pt:22 +#: ../templates/export_content.pt:22 msgid "label_toggle" msgstr "Alternar todo" -#: collective/exportimport/import_content.py:996 +#: ../import_content.py:1049 msgid "plone.app.querystring.upgrades.fix_select_all_existing_collections is not available" msgstr "plone.app.querystring.upgrades.fix_select_all_existing_collections no está disponible" #. Default: "Or you can choose a file that is already uploaded on the server in one of these paths:" -#: collective/exportimport/templates/import_content.pt:21 +#: ../templates/import_content.pt:21 msgid "server_paths_list" -msgstr "" -"O puede elegir un archivo que ya está cargado en el servidor en una de estas " -"rutas:" +msgstr "O puede elegir un archivo que ya está cargado en el servidor en una de estas rutas:" -#: collective/exportimport/export_content.py:123 +#: ../export_content.py:126 msgid "unlimited" msgstr "ilimitado" From 388972fac2637ef19a8e9aca81b1e5de5127b615 Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 19:21:24 +0200 Subject: [PATCH 07/26] Fix missing comparison value --- src/collective/exportimport/export_content.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index 282c6de8..efeebda4 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -182,7 +182,7 @@ def __call__( content_generator = self.export_content() number = 0 - if download_to_server: + if download_to_server == 1: directory = config.CENTRAL_DIRECTORY if directory: if not os.path.exists(directory): From 44241d01d2676206ea438988faea13ae62645511 Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 19:29:03 +0200 Subject: [PATCH 08/26] fix translation --- .../locales/es/LC_MESSAGES/collective.exportimport.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po b/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po index d18e559a..c8506662 100644 --- a/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po +++ b/src/collective/exportimport/locales/es/LC_MESSAGES/collective.exportimport.po @@ -134,7 +134,7 @@ msgstr "Exportado a {}" #: ../export_content.py:209 msgid "Exported {} items ({}) as {} to {} with {} errors" -msgstr "Exportados {} elementos ({}) como {} a {}" +msgstr "Exportados {} elementos ({}) como {} a {} con {} errores" #: ../export_content.py:239 msgid "Exported {} {} with {} errors" From b913a6ba02491d9b192b33921acb876ade2681e2 Mon Sep 17 00:00:00 2001 From: rber474 Date: Tue, 10 Oct 2023 19:29:22 +0200 Subject: [PATCH 09/26] changes successful message --- src/collective/exportimport/export_content.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index efeebda4..0cc18ea7 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -236,8 +236,9 @@ def __call__( FileSystemContentExporter(rootpath, datum).save() self.finish() - msg = _(u"Exported {} {} with {} errors").format( - number, self.portal_type, len(self.errors) + msg = _( + u"Exported {} items ({}) as {} to {} with {} errors").format( + number, ", ".join(self.portal_type), filename, rootpath, len(self.errors) ) logger.info(msg) api.portal.show_message(msg, self.request) @@ -252,7 +253,7 @@ def __call__( f.write(",") json.dump(datum, f, sort_keys=True, indent=4) if number: - if self.errors and self.write_errors: + if self.errors and self.write_errors: f.write(",") errors = {"unexported_paths": self.errors} json.dump(errors, f, indent=4) From 84e4f482180210bf25fc897344facbcbfad37106 Mon Sep 17 00:00:00 2001 From: rber474 Date: Wed, 11 Oct 2023 19:35:37 +0200 Subject: [PATCH 10/26] use boostrap classes. convert checkbox to switch --- .../exportimport/templates/export_content.pt | 147 +++++++++--------- 1 file changed, 77 insertions(+), 70 deletions(-) diff --git a/src/collective/exportimport/templates/export_content.pt b/src/collective/exportimport/templates/export_content.pt index 96555636..90194b77 100644 --- a/src/collective/exportimport/templates/export_content.pt +++ b/src/collective/exportimport/templates/export_content.pt @@ -19,59 +19,64 @@ Content Types to export
- -
- +
+ +
+ +
+ - - -
+
+ + +
+
- +
-
- - Unlimited: this item and all children, 0: this object only, 1: only direct children of this object, 2-x: children of this object up to the specified level +
-
- - - How should data from image- and file-fields be included? - +
-
-
-