Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Gordon Smith <GordonJSmith@gmail.com>

# Conflicts:
#	helm/hpcc/Chart.yaml
#	helm/hpcc/templates/_helpers.tpl
#	version.cmake

common/dllserver/thorplugin.cpp

@@ -559,6 +559,8 @@ const StringArray &HelperDll::queryManifestFiles(const char *type, const char *w
                 OwnedIFileIO o = f->open(IFOcreate);
                 assertex(o.get() != nullptr);
                 o->write(0, len, data);
+                o->close();
+
                 list->append(extractName);
                 if (doTrace(traceJava) && streq(type, "jar"))
                     DBGLOG("Extracted jar resource %u size %u to %s in %u ms", id, len, extractName.str(), msTick() - start);

dali/base/dacoven.cpp

@@ -563,12 +563,14 @@ class CCovenServer: public CCovenBase
         Owned<IFile> f = createIFile(storename.get());
         Owned<IFileIO> io = f->open(IFOcreate);
         io->write(0, xml.length(), xml.str());
+        io->close();
         io.clear();
         if (!backupname.isEmpty()) {
             try {
                 f.setown(createIFile(backupname.get()));
                 io.setown(f->open(IFOcreate));
                 io->write(0, xml.length(), xml.str());
+                io->close();
                 io.clear();
             }
             catch (IException *e) {

dali/base/dasds.cpp

@@ -1022,6 +1022,7 @@ void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned re
             sprintf(strNum, "%016" I64F "X", fLen);
             memcpy(headerPtr + deltaHeaderSizeOff, strNum, 16);
             iFileIO->write(0, strlen(deltaHeader), headerPtr);
+            iFileIO->close();
         }
         catch (IException *e)
         {
@@ -1212,6 +1213,7 @@ class CDeltaWriter : implements IThreaded
                 Owned<IFile> iFile = createIFile(rL.str());
                 Owned<IFileIO> fileIO = iFile->open(IFOcreate);
                 fileIO->write(0, length, data);
+                fileIO->close();
             }
             catch (IException *e)
             {
@@ -5286,6 +5288,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             iFileIO->write(0, sizeof(unsigned), crc);
         if (storeInfo)
             storeInfo->cache.set(filename.str());
+        iFileIO->close();
     }
 
     void updateStoreInfo(const char *base, const char *location, unsigned edition, unsigned *crc, CStoreInfo *storeInfo=NULL)
@@ -5397,6 +5400,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
 
             OwnedIFileIO detachIPIO = detachIPIFile->open(IFOcreate);
             detachIPIO->write(0, sizeof(storeHelper.mySessId), &storeHelper.mySessId);
+            detachIPIO->close();
             detachIPIO.clear();
             detachIPIFile->rename(activeDetachIPStr.str());
             // check often do not wait any longer than necessary

dali/datest/datest.cpp

@@ -2927,7 +2927,7 @@ NULL
 #else
     out = fileno(stdout);
 #endif
-    Owned<IFileIO> stdOutFileIO = createIFileIO(out,IFOwrite);
+    Owned<IFileIO> stdOutFileIO = createIFileIO(nullptr,out,IFOwrite);
     if (testParams.ordinality())
     {
         newFileName = testParams.item(0);

dali/dfu/dfurun.cpp

@@ -540,23 +540,31 @@ class CDFUengine: public CInterface, implements IDFUengine
         return result;
     }
 
-    void ensureFilePermissions(const char * planeName, const char * fileName, SecAccessFlags perm, bool write)
+    void ensureFilePermissions(const char * planeName, const char * fileName, SecAccessFlags perm, IUserDescriptor *user, bool write)
     {
         if ((write && !HASWRITEPERMISSION(perm)) || (!write && !HASREADPERMISSION(perm)))
         {
+            StringBuffer context;
+            StringBuffer username;
+            if (user)
+                user->getUserName(username);
+            else
+                username.append("Null user");
+            context.appendf("user: '%s', assigned access %s (%d)", username.str(), getSecAccessFlagName(perm), perm);
+
             if (!isEmptyString(planeName))
             {
                 CDfsLogicalFileName dlfn;
                 dlfn.setPlaneExternal(planeName, fileName);
                 if (write)
-                    throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for file scope: %s on DropZone: %s", dlfn.get(), planeName);
+                    throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for file scope: %s on DropZone: %s, %s", dlfn.get(), planeName, context.str());
                 else
-                    throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for file scope: %s on DropZone: %s", dlfn.get(), planeName);
+                    throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for file scope: %s on DropZone: %s, %s", dlfn.get(), planeName, context.str());
             }
             if (write)
-                throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s", fileName);
+                throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s, %s", fileName, context.str());
             else
-                throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s", fileName);
+                throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s, %s", fileName, context.str());
         }
     }
 
@@ -618,12 +626,13 @@ class CDFUengine: public CInterface, implements IDFUengine
     void checkPhysicalFilePermissions(IFileDescriptor *fd,IUserDescriptor *user,bool write)
     {
         unsigned auditflags = (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED);
+        logNullUser(user);//stack trace if NULL user
         if (write)
             auditflags |= DALI_LDAP_WRITE_WANTED;
 
         SecAccessFlags perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags);
         StringBuffer name;
-        ensureFilePermissions(nullptr,getFDescName(fd,name),perm,write);
+        ensureFilePermissions(nullptr,getFDescName(fd,name),perm,user,write);
     }
 
     void checkForeignFilePermissions(IConstDFUfileSpec *fSpec,IFileDescriptor *fd,IUserDescriptor *user)
@@ -667,6 +676,7 @@ class CDFUengine: public CInterface, implements IDFUengine
 
     void checkPlaneFilePermissions(IFileDescriptor *fd,IUserDescriptor *user,bool write)
     {
+        logNullUser(user);//stack trace if NULL user
         //This function checks the scope permissions for a file or files that reside in a single directory on a single plane.
         //The IFileDescriptor is used to discover the plane and directory.
         //If the plane is not present, it implies that it is a bare-metal system and useDropZoneRestriction is off, and there
@@ -697,7 +707,8 @@ class CDFUengine: public CInterface, implements IDFUengine
             {
                 if (getGlobalConfigSP()->getPropBool("expert/@failOverToLegacyPhysicalPerms",!isContainerized()))
                     perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags);
-                ensureFilePermissions(planeName,relativePath,perm,write);
+
+                ensureFilePermissions(planeName,relativePath,perm,user,write);
             }
         }
         else

dali/ft/filecopy.cpp

@@ -1023,6 +1023,7 @@ void FileSprayer::beforeTransfer()
                         io->write(lastOutputOffset-sizeof(null), sizeof(null), &null);
                     }
                 }
+                io->close();
             }
         }
     }

dali/ft/ftbase.ipp

@@ -134,6 +134,10 @@ public:
     virtual offset_t tell() override;
     virtual size32_t write(size32_t len, const void * data) override;
     virtual unsigned __int64 getStatistic(StatisticKind kind) override { return stream->getStatistic(kind); }
+    virtual void close() override
+    {
+        stream->close();
+    }
 
     unsigned getCRC()               { return crc; }
     void setCRC(unsigned long _crc)     { crc = _crc; }

dali/ft/fttransform.cpp

@@ -790,6 +790,8 @@ bool TransferServer::pull()
             assertex(curProgress.status != OutputProgress::StatusRenamed);
             if (curProgress.status != OutputProgress::StatusCopied)
             {
+                if (out)
+                    out->close();
                 out.setown(createIOStream(outio));
                 out->seek(progressOffset, IFSbegin);
                 wrapOutInCRC(curProgress.outputCRC);
@@ -885,6 +887,8 @@ bool TransferServer::pull()
                 }
             }
 
+            if (out)
+                out->close();
             out.setown(createIOStream(outio));
             out->seek(0, IFSbegin);
             wrapOutInCRC(0);
@@ -903,7 +907,10 @@ bool TransferServer::pull()
     }
 
     crcOut.clear();
+    if (out)
+        out->close();
     out.clear();
+
     //Once the transfers have completed, rename the files, and sync file times
     //if replicating...
     if (!isSafeMode)
@@ -997,6 +1004,7 @@ bool TransferServer::push()
                 }
                 outio.setown(createCompressedFileWriter(outio, false, 0, true, compressor, COMPRESS_METHOD_LZ4));
             }
+
             out.setown(createIOStream(outio));
             if (!compressOutput)
                 out->seek(curPartition.outputOffset + curProgress.outputLength, IFSbegin);
@@ -1011,6 +1019,7 @@ bool TransferServer::push()
                 sendProgress(curProgress);
             }
             crcOut.clear();
+            out->close();
             out.clear();
         }
     }

dali/server/daserver.cpp

@@ -612,6 +612,7 @@ int main(int argc, const char* argv[])
                     backupCheck.append("bakchk.").append((unsigned)GetCurrentProcessId());
                     OwnedIFile iFileDataDir = createIFile(backupCheck.str());
                     OwnedIFileIO iFileIO = iFileDataDir->open(IFOcreate);
+                    iFileIO->close();
                     iFileIO.clear();
                     try
                     {

ecl/ecl-bundle/ecl-bundle.cpp

@@ -750,6 +750,7 @@ class CBundleInfoSet : public CInterfaceOf<IBundleInfoSet>
         VStringBuffer redirect("IMPORT %s.%s.%s.%s as _%s; EXPORT %s := _%s;", VERSION_SUBDIR, name, version, name, name, name, name);
         Owned<IFileIO> rfile = redirector->open(IFOcreate);
         rfile->write(0, redirect.length(), redirect.str());
+        rfile->close();
         bundle->setActive(true);
     }
     virtual void setActive(const char *version)

ecl/eclcc/eclcc.cpp

@@ -1990,8 +1990,11 @@ void EclCC::outputXmlToOutputFile(EclCompileInstance & instance, IPropertyTree *
     {
         //Work around windows problem writing 64K to stdout if not redirected/piped
         Owned<IIOStream> stream = createIOStream(ifileio.get());
-        Owned<IIOStream> buffered = createBufferedIOStream(stream,0x8000);
-        saveXML(*buffered, xml);
+        {
+            Owned<IIOStream> buffered = createBufferedIOStream(stream,0x8000);
+            saveXML(*buffered, xml);
+        }
+        ifileio->close();
     }
 }
 
@@ -2044,7 +2047,10 @@ void EclCC::generateOutput(EclCompileInstance & instance)
 
             OwnedIFileIO ifileio = createArchiveOutputFile(instance);
             if (ifileio)
+            {
                 ifileio->write(0, filenames.length(), filenames.str());
+                ifileio->close();
+            }
         }
         else
         {

ecl/eclccserver/eclccserver.cpp

@@ -559,6 +559,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
         {
             Owned<IFileIO> dstIO = dstfile->open(IFOwrite);
             dstIO->write(0, output.length(), output.str());
+            dstIO->close();
 
             IArrayOf<IError> errors;
             extractErrorsFromCppLog(errors, output.str(), numFailed != 0);

ecl/eclcmd/queries/ecl-queries.cpp

@@ -1382,7 +1382,10 @@ class EclCmdQueriesExport : public EclCmdCommon
         fprintf(stdout, "\nWriting to file %s\n", file->queryFilename());
 
         if (io.get())
+        {
             io->write(0, strlen(s), s);
+            io->close();
+        }
         else
             fprintf(stderr, "\nFailed to create file %s\n", file->queryFilename());
     }

ecl/hql/hqlcache.cpp

@@ -492,6 +492,7 @@ static void extractFile(const char * path, const char * moduleName, const char *
     Owned<IFileIO> io = file->open(IFOcreate);
     if (text)
         io->write(0, strlen(text), text);
+    io->close();
     io.clear();
     if (ts)
     {

ecl/hql/hqlopt.cpp

@@ -4285,6 +4285,22 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                                     args.replace(*createConstant(true), 2);
                                 }
 
+                                //This test currently duplicates the test in the enclosing if.  This is in preparation for applying the logic
+                                //to other joins later by removing the outer test.
+                                if (transformed->hasAttribute(lookupAtom))
+                                {
+                                    //LOOKUP join implicitly dedups the right hand side by the hard-match-condition (unless MANY is specified).
+                                    //Technically this is not equivalent to KEEP(1) - since that is only applied after any soft-match condition
+                                    //However in this case the match condition is true (or row-invarient in very obscure cases),
+                                    //so they are identical and this transformation is valid.
+                                    if (!transformed->hasAttribute(manyAtom))
+                                    {
+                                        //There should never be a KEEP attribute if MANY not specified - but remove in case.
+                                        removeAttribute(args, keepAtom);
+                                        args.append(*createAttribute(keepAtom, createConstant(1)));
+                                    }
+                                    removeAttribute(args, lookupAtom);
+                                }
                                 args.append(*createAttribute(allAtom));
                             }
                             if (doTrace(traceOptimizations))

ecl/hqlcpp/hqlecl.cpp

@@ -311,7 +311,7 @@ void HqlDllGenerator::expandCode(StringBuffer & filename, const char * codeTempl
     addDirectoryPrefix(fullname, targetDir).append(filename);
 
     Owned<IFile> out = createIFile(fullname.str());
-    Owned<ITemplateExpander> expander = createTemplateExpander(out, codeTemplate);
+    Owned<ITemplateExpander> expander = createTemplateExpander(codeTemplate);
 
     Owned<ISectionWriter> writer = createCppWriter(*code, compiler);
     Owned<IProperties> props = createProperties(true);
@@ -325,7 +325,7 @@ void HqlDllGenerator::expandCode(StringBuffer & filename, const char * codeTempl
     props->setProp("headerName", headerName.str());
     props->setProp("outputName", fullname.str());
 
-    expander->generate(*writer, pass, props);
+    expander->generate(*writer, out, pass, props);
 
     totalGeneratedSize += out->size();
 

ecl/hqlcpp/hqlwcpp.cpp

@@ -103,9 +103,14 @@ CppWriterTemplate::CppWriterTemplate(const char * codeTemplate)
     loadTemplate(codeTemplate);
 }
 
-void CppWriterTemplate::generate(ISectionWriter & writer, unsigned pass, IProperties * properties)
+void CppWriterTemplate::generate(ISectionWriter & writer, IFile * outputFile, unsigned pass, IProperties * properties)
 {
-    writer.setOutput(out, outStream);
+    Owned<IFileIO> io = outputFile->open(IFOcreate);
+    if (!io)
+        throwError1(HQLERR_CouldNotCreateOutputX, outputFile->queryFilename());
+
+    outStream.setown(createIOStream(io));
+    writer.setOutput(outputFile, outStream);
 
     const char * finger = text;
     bool output = true;
@@ -149,6 +154,8 @@ void CppWriterTemplate::generate(ISectionWriter & writer, unsigned pass, IProper
         outputQuoted(writer, end-finger, finger);
 
     writer.setOutput(NULL, NULL);
+    outStream.clear();
+    io->close();
 }
 
 void CppWriterTemplate::loadTemplate(const char * codeTemplate)
@@ -2271,11 +2278,9 @@ void HqlCppSectionWriter::generateSection(unsigned delta, IAtom * section, unsig
 
 //---------------------------------------------------------------------------
 
-ITemplateExpander * createTemplateExpander(IFile * output, const char * codeTemplate)
+ITemplateExpander * createTemplateExpander(const char * codeTemplate)
 {
-    Owned<CppWriterTemplate> expander = new CppWriterTemplate(codeTemplate);
-    expander->setOutput(output);
-    return expander.getClear();
+    return new CppWriterTemplate(codeTemplate);
 }
 
 ISectionWriter * createCppWriter(IHqlCppInstance & _instance, CompilerType compiler)

ecl/hqlcpp/hqlwcpp.hpp

@@ -35,15 +35,15 @@ interface HQLCPP_API ISectionWriter : public IInterface
 interface HQLCPP_API ITemplateExpander : public IInterface
 {
 public:
-    virtual void generate(ISectionWriter & writer, unsigned pass, IProperties * properties = NULL) = 0;
+    virtual void generate(ISectionWriter & writer, IFile * _output, unsigned pass, IProperties * properties = NULL) = 0;
 };
 
 extern HQLCPP_API StringBuffer & generateExprCpp(StringBuffer & out, IHqlExpression * expr, CompilerType compiler);
 extern HQLCPP_API StringBuffer & generateTypeCpp(StringBuffer & out, ITypeInfo * type, const char * name, CompilerType compiler);
 bool generateFunctionPrototype(StringBuffer & out, IHqlExpression * funcdef, CompilerType compiler);
 void generateFunctionReturnType(StringBuffer & prefix, StringBuffer & params, ITypeInfo * retType, IHqlExpression * attrs, CompilerType compiler);
 
-extern HQLCPP_API ITemplateExpander * createTemplateExpander(IFile * output, const char * codeTemplate);
+extern HQLCPP_API ITemplateExpander * createTemplateExpander(const char * codeTemplate);
 extern HQLCPP_API ISectionWriter * createCppWriter(IHqlCppInstance & _instance, CompilerType compiler);
 extern bool isTypePassedByAddress(ITypeInfo * type);