From 0eddb1eff23a0ea7b6c14a2748a409a2159d1408 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 11 Dec 2024 18:32:47 +0000 Subject: [PATCH] HPCC-33150 Avoid padding logical files with empty parts When targeting a cluster/plane that was wider than the Thor instance, Thor used to ensure the target file width matched the width of the target cluster (or plane), by in part, creating empty physical files. This behaviour becomes more common in a containerized world, where the target plane is normally the default data plane with a width that doesn't relate to a Thor instance. Turn off this behaviour by default, allow it to be re-enabled in configuration or on a per output basis as a fail-safe. Signed-off-by: Jake Smith --- thorlcr/mfilemanager/thmfilemanager.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/thorlcr/mfilemanager/thmfilemanager.cpp b/thorlcr/mfilemanager/thmfilemanager.cpp index f16549d4769..979958caaa4 100644 --- a/thorlcr/mfilemanager/thmfilemanager.cpp +++ b/thorlcr/mfilemanager/thmfilemanager.cpp @@ -119,6 +119,9 @@ class CFileManager : public CSimpleInterface, implements IThorFileManager } else { + // NB: it is very unlikely this code has ever been used. + // The whole of fixTotal should be deleted at some point (see HPCC-33150) + // Check options to shrink 'large' tgtfile to this cluster size const char *wideDestOptStr = globals->queryProp("@wideDestOpt"); if (wideDestOptStr) @@ -489,12 +492,16 @@ class CFileManager : public CSimpleInterface, implements IThorFileManager break; }; #endif - unsigned offset = 0; unsigned total; if (restrictedWidth) total = restrictedWidth; - else + else if (job.getOptBool("legacyLFNTargetWidth")) + { + unsigned offset; // set by fixTotal (always 0), but never used. total = fixTotal(job, groups, offset); + } + else + total = job.querySlaves(); if (nonLocalIndex) ++total; StringBuffer dir;